From 2903b0888a8d8591f68dec2afca089bee17bdcfe Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Wed, 7 Oct 2020 15:34:05 -0400 Subject: [PATCH 001/576] Consolidated file metadata on dataset pg by using file-info-fragment from file replace pg [ref #7081] --- src/main/webapp/editdatafiles.xhtml | 1 + src/main/webapp/file-info-fragment.xhtml | 40 +++++++++-- src/main/webapp/filesFragment.xhtml | 92 ++---------------------- 3 files changed, 41 insertions(+), 92 deletions(-) diff --git a/src/main/webapp/editdatafiles.xhtml b/src/main/webapp/editdatafiles.xhtml index d81e6da22a1..0965ec5520c 100644 --- a/src/main/webapp/editdatafiles.xhtml +++ b/src/main/webapp/editdatafiles.xhtml @@ -43,6 +43,7 @@
+ diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index dff487454ea..a8ffd40cb7a 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -31,13 +31,43 @@
- #{fileMetadata.label} + + + #{fileMetadata.label} + + + + #{fileMetadata.label} +
+
- #{fileMetadata.dataFile.friendlyType} - - #{fileMetadata.dataFile.friendlySize} - - #{fileMetadata.fileDateToDisplay} -
#{fileMetadata.dataFile.checksumType}: #{fileMetadata.dataFile.checksumValue}
+ +
+ +
+ + + + + + + + +
+ +
+ +
+ + + +
diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index 62453b09fe5..bb7041bda66 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -365,93 +365,11 @@
-
-
-
- - - #{fileMetadata.label} - - - - - - - - #{fileMetadata.label} - - - - -
- -
-
- -
-
-
- -
- - - - #{fileMetadata.label} - - - - - #{fileMetadata.label} - - - - -
- -
- -
- - - - - - - - - - -
- -
-
- -
- - - -
- -
- -
- -
- - - - - - - - - - -
-
-
+ +             +             +             +            From f094b97dce164f28701492db089de3164c85ef14 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Thu, 8 Oct 2020 11:19:49 -0400 Subject: [PATCH 002/576] Fixed download count error on file-info-fragment, as well as cleaned up thumbnail code [ref #7081] --- .../edu/harvard/iq/dataverse/DatasetPage.java | 9 --------- .../harvard/iq/dataverse/EditDatafilesPage.java | 14 -------------- src/main/webapp/file-info-fragment.xhtml | 15 ++++++--------- 3 files changed, 6 insertions(+), 32 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 458fcf56ab0..43d945c657c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -1311,15 +1311,6 @@ public boolean isSuperUser(){ } return false; } - /* - TODO/OPTIMIZATION: This is still costing us N SELECT FROM GuestbookResponse queries, - where N is the number of files. This could of course be replaced by a query that'll - look up all N at once... Not sure if it's worth it; especially now that N - will always be 10, for the initial page load. -- L.A. 4.2.1 - */ - public Long getGuestbookResponseCount(FileMetadata fileMetadata) { - return guestbookResponseService.getCountGuestbookResponsesByDataFileId(fileMetadata.getDataFile().getId()); - } /** * Check Dataset related permissions * diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 3a975d1018e..040988a6536 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -2562,20 +2562,6 @@ public boolean isLocked() { return false; } - public boolean isThumbnailAvailable(FileMetadata fileMetadata) { - // new and optimized logic: - // - check download permission here (should be cached - so it's free!) - // - only then ask the file service if the thumbnail is available/exists. - // the service itself no longer checks download permissions. - if (!fileDownloadHelper.canDownloadFile(fileMetadata)) { - return false; - } - - return datafileService.isThumbnailAvailable(fileMetadata.getDataFile()); - } - - - private Boolean lockedFromEditsVar; public boolean isLockedFromEdits() { diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index a8ffd40cb7a..4fbfa4606f2 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -14,8 +14,8 @@
- - + @@ -24,7 +24,7 @@
-
+
@@ -48,16 +48,13 @@
- + - + +
From 56dc4c75ba9acf9c66906169bb9baa2dc74a7691 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Thu, 8 Oct 2020 15:52:24 -0400 Subject: [PATCH 003/576] Removed additional editMode render logic not needed in file table on dataset pg [ref #7081] --- src/main/webapp/file-info-fragment.xhtml | 1 + src/main/webapp/filesFragment.xhtml | 35 ++++++------------------ 2 files changed, 10 insertions(+), 26 deletions(-) diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index 4fbfa4606f2..ddc4b97a017 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -32,6 +32,7 @@
+ #{fileMetadata.label} diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index bb7041bda66..64f35a2bab3 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -43,7 +43,7 @@ value="#{DatasetPage.fileMetadatasSearch}" rowIndexVar="rowNum" rowKey="#{fileMetadata.dataFile.storageIdentifier}" selection="#{DatasetPage.selectedFiles}" var="fileMetadata" widgetVar="filesTable" - rendered="#{DatasetPage.fileDisplayTable and empty DatasetPage.editMode and (DatasetPage.workingVersion != null)}" + rendered="#{DatasetPage.fileDisplayTable and (DatasetPage.workingVersion != null)}" emptyMessage="#{DatasetPage.workingVersion.fileMetadatas.size() == 0 ? bundle['file.notFound.tip'] : bundle['file.notFound.search']}"> @@ -118,13 +118,9 @@
- -
1))}"> - - - +
- - + - - - - - - - - #{node.label} - - - - - #{node.label} - - - + + + + #{node.label} + (#{node.dataFile.friendlySize}) From 44722f51a12878d8e1bf01d452eed082f0b6a6e6 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Fri, 9 Oct 2020 12:27:56 -0400 Subject: [PATCH 004/576] Removed commented out isDuplicate render logic from checksum in dataset pg file table [ref #7081] --- src/main/webapp/file-info-fragment.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index ddc4b97a017..31068b7f1ae 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -58,7 +58,7 @@
- +
From 9eb7f9aef2dd38556d025431b840ae42783300e2 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Fri, 9 Oct 2020 12:50:37 -0400 Subject: [PATCH 005/576] Additional render logic clean up on file-info-fragment [ref #7081] --- src/main/webapp/file-info-fragment.xhtml | 36 +++++++++--------------- 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index 31068b7f1ae..64e5da35368 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -13,7 +13,7 @@
-
+
-
+
@@ -44,44 +44,34 @@
-
- -
- - + + - - - +
- - + +
-
- - - - - - - - - - + + + + + +
From 81384baec44f1a100ff84434a511bc7df93d975e Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Thu, 15 Oct 2020 12:56:42 -0400 Subject: [PATCH 006/576] Added checksum truncation function WIP [ref #7081 #6685 #5210] --- src/main/webapp/file-info-fragment.xhtml | 5 ++- .../resources/js/dv_rebind_bootstrap_ui.js | 39 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index 64e5da35368..c14c8f98b60 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -53,7 +53,10 @@
- + #{fileMetadata.dataFile.checksumType}: #{fileMetadata.dataFile.checksumValue} +
diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index 15c14e7d8bd..31b49d5827a 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -29,6 +29,45 @@ function bind_bsui_components(){ // Disabled pagination links disabledLinks(); + // Truncate checksums + // TO-DO DETERMINE IF THIS IS PLACE IS OVERKILL, ON EVERY PG, EVERY REBIND ACTION ... + // PROBABLY NEEDED LIKE ALL THE OTHER FUNCTIONS FOR DYNAMIC AJAX UPDATES ON PG + checksumTruncate(); + + /* + * Truncate file checksums + */ + function checksumTruncate(){ + + // TO-DO MOVE THIS CHUNK OUT OF BIND_BSUI, TO BOTTOM OF THIS FILE + + $('span.checksum-truncate').each(function () { + + var checksumText = $(this).text(); + + console.log("Total Checksums..." + checksumText); + + $(this).css('color', 'red'); + + console.log("Characters..." + checksumText.length); + + if (checksumText.length > 35) { + $(this).text( checksumText.substr(0, 8) + '...' + checksumText.substr(checksumText.length-3, checksumText.length) ); + } + + // TO-DO WIRE UP CLICK TO COPY W/ CLIPBOARD.JS ... SEE JIM'S PR 5211 + // var clipboard = new Clipboard('button.checksum'); + // clipboard.on('success', function(e) { + // console.log(e); + // }); + // clipboard.on('error', function(e) { + // console.log(e); + // }); + + }); + + } + // Sharrre sharrre(); From 17c343fcee69a6f39275e4be6d734d4dd95321a1 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 16 Oct 2020 15:02:45 -0400 Subject: [PATCH 007/576] Allow replacing a draft file --- .../datasetutility/AddReplaceFileHelper.java | 187 +++++++----------- .../datasetutility/FileReplacePageHelper.java | 8 +- .../datasetutility/OptionalFileParams.java | 19 +- .../impl/UpdateDatasetVersionCommand.java | 30 ++- src/main/java/propertyFiles/Bundle.properties | 2 +- src/main/webapp/file.xhtml | 2 +- 6 files changed, 107 insertions(+), 141 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index ab34b5b2675..a9a358f013b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -20,6 +20,7 @@ import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.impl.AbstractCreateDatasetCommand; import edu.harvard.iq.dataverse.engine.command.impl.CreateNewDatasetCommand; +import edu.harvard.iq.dataverse.engine.command.impl.DeleteDataFileCommand; import edu.harvard.iq.dataverse.engine.command.impl.RestrictFileCommand; import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; @@ -502,6 +503,17 @@ public boolean runReplaceFromUI_Phase1(Long oldFileId, if (!this.step_005_loadFileToReplaceById(oldFileId)){ return false; } + //Update params to match existing file (except checksum, which should match the new file) + if(fileToReplace != null) { + String checksum = optionalFileParams.getCheckSum(); + try { + optionalFileParams = new OptionalFileParams(fileToReplace); + optionalFileParams.setCheckSum(checksum); + } catch (DataFileTagException e) { + // Shouldn't happen since fileToReplace should have valid tags + e.printStackTrace(); + } + } return this.runAddReplacePhase1(fileToReplace.getOwner(), newFileName, @@ -571,6 +583,16 @@ private boolean runAddReplacePhase1(Dataset owner, return false; } + //Reuse any file PID during a replace operation (if File PIDs are in use, if the fileToReplace hasn't been released) + if(fileToReplace !=null && !fileToReplace.isReleased() && systemConfig.isFilePIDsEnabled()) { + DataFile df = finalFileList.get(0); //step_055 uses a loop and assumes only one file + df.setGlobalId(fileToReplace.getGlobalId()); + df.setGlobalIdCreateTime(fileToReplace.getGlobalIdCreateTime()); + //Should be true or fileToReplace wouldn't have an identifier (since it's not yet released in this if statement) + df.setIdentifierRegistered(fileToReplace.isIdentifierRegistered()); + fileToReplace.setGlobalId(null); + } + return true; } @@ -1058,16 +1080,6 @@ private boolean step_005_loadFileToReplaceById(Long dataFileId){ if (!step_015_auto_check_permissions(existingFile.getOwner())){ return false; }; - - - - // Is the file published? - // - if (!existingFile.isReleased()){ - addError(getBundleErr("unpublished_file_cannot_be_replaced")); - return false; - } - // Is the file in the latest dataset version? // if (!step_007_auto_isReplacementInLatestVersion(existingFile)){ @@ -1501,7 +1513,7 @@ private boolean step_060_addFilesViaIngestService(){ } int nFiles = finalFileList.size(); - finalFileList = ingestService.saveAndAddFilesToDataset(workingVersion, finalFileList, isFileReplaceOperation()); + finalFileList = ingestService.saveAndAddFilesToDataset(workingVersion, finalFileList, true); if (nFiles != finalFileList.size()) { if (nFiles == 1) { @@ -1528,7 +1540,22 @@ private boolean step_070_run_update_dataset_command(){ } Command update_cmd; - update_cmd = new UpdateDatasetVersionCommand(dataset, dvRequest, clone); + String deleteStorageLocation = null; + long deleteFileId=-1; + if(isFileReplaceOperation()) { + List filesToDelete = new ArrayList(); + filesToDelete.add(fileToReplace.getFileMetadata()); + + if(!fileToReplace.isReleased()) { + //If file is only in draft version, also need to delete the physical file + deleteStorageLocation = fileService.getPhysicalFileToDelete(fileToReplace); + deleteFileId=fileToReplace.getId(); + } + //Adding the file to the delete list for the command will delete this filemetadata and, if the file hasn't been released, the datafile itself. + update_cmd = new UpdateDatasetVersionCommand(dataset, dvRequest, filesToDelete, clone); + } else { + update_cmd = new UpdateDatasetVersionCommand(dataset, dvRequest, clone); + } ((UpdateDatasetVersionCommand) update_cmd).setValidateLenient(true); try { @@ -1550,89 +1577,23 @@ private boolean step_070_run_update_dataset_command(){ this.addErrorSevere("add.add_file_error (see logs)"); logger.severe(ex.getMessage()); return false; - } - return true; - } - - - /** - * Go through the working DatasetVersion and remove the - * FileMetadata of the file to replace - * - * @return - */ - private boolean step_085_auto_remove_filemetadata_to_replace_from_working_version(){ - - msgt("step_085_auto_remove_filemetadata_to_replace_from_working_version 1"); - - if (!isFileReplaceOperation()){ - // Shouldn't happen! - this.addErrorSevere(getBundleErr("only_replace_operation") + " (step_085_auto_remove_filemetadata_to_replace_from_working_version"); - return false; - } - msg("step_085_auto_remove_filemetadata_to_replace_from_working_version 2"); - - if (this.hasError()){ - return false; } - - - msgt("File to replace getId: " + fileToReplace.getId()); - - Iterator fmIt = workingVersion.getFileMetadatas().iterator(); - msgt("Clear file to replace"); - int cnt = 0; - while (fmIt.hasNext()) { - cnt++; - - FileMetadata fm = fmIt.next(); - msg(cnt + ") next file: " + fm); - msg(" getDataFile().getId(): " + fm.getDataFile().getId()); - if (fm.getDataFile().getId() != null) { - if (Objects.equals(fm.getDataFile().getId(), fileToReplace.getId())) { - msg("Let's remove it!"); - - // If this is a tabular data file with a UNF, we'll need - // to recalculate the version UNF, once the file is removed: - - boolean recalculateUNF = !StringUtils.isEmpty(fm.getDataFile().getUnf()); - - if (workingVersion.getId() != null) { - // If this is an existing draft (i.e., this draft version - // is already saved in the dataset, we'll also need to remove this filemetadata - // explicitly: - msg(" this is an existing draft version..."); - fileService.removeFileMetadata(fm); - - // remove the filemetadata from the list of filemetadatas - // attached to the datafile object as well, for a good - // measure: - fileToReplace.getFileMetadatas().remove(fm); - // (and yes, we can do .remove(fm) safely - if this released - // file is part of an existing draft, we know that the - // filemetadata object also exists in the database, and thus - // has the id, and can be identified unambiguously. - } - - // and remove it from the list of filemetadatas attached - // to the version object, via the iterator: - fmIt.remove(); - - if (recalculateUNF) { - msg("recalculating the UNF"); - ingestService.recalculateDatasetVersionUNF(workingVersion); - msg("UNF recalculated: "+workingVersion.getUNF()); - } - - return true; + //Sanity check + if(isFileReplaceOperation()) { + if (deleteStorageLocation != null) { + // Finalize the delete of the physical file + // (File service will double-check that the datafile no + // longer exists in the database, before proceeding to + // delete the physical file) + try { + fileService.finalizeFileDelete(deleteFileId, deleteStorageLocation); + } catch (IOException ioex) { + logger.warning("Failed to delete the physical file associated with the deleted datafile id=" + + deleteFileId + ", storage location: " + deleteStorageLocation); } } } - - msg("No matches found!"); - addErrorSevere(getBundleErr("failed_to_remove_old_file_from_dataset")); - runMajorCleanup(); - return false; + return true; } @@ -1707,13 +1668,6 @@ private boolean step_080_run_update_dataset_command_for_replace(){ return false; } - // ----------------------------------------------------------- - // Remove the "fileToReplace" from the current working version - // ----------------------------------------------------------- - if (!step_085_auto_remove_filemetadata_to_replace_from_working_version()){ - return false; - } - // ----------------------------------------------------------- // Set the "root file ids" and "previous file ids" // THIS IS A KEY STEP - SPLIT IT OUT @@ -1723,26 +1677,27 @@ private boolean step_080_run_update_dataset_command_for_replace(){ // ----------------------------------------------------------- - /* - Check the root file id on fileToReplace, updating it if necessary - */ - if (fileToReplace.getRootDataFileId().equals(DataFile.ROOT_DATAFILE_ID_DEFAULT)){ + if (fileToReplace.isReleased()) { + /* + * Check the root file id on fileToReplace, updating it if necessary + */ + if (fileToReplace.getRootDataFileId().equals(DataFile.ROOT_DATAFILE_ID_DEFAULT)) { - fileToReplace.setRootDataFileId(fileToReplace.getId()); - fileToReplace = fileService.save(fileToReplace); - } - - /* - Go through the final file list, settting the rootFileId and previousFileId - */ - for (DataFile df : finalFileList){ - df.setPreviousDataFileId(fileToReplace.getId()); - - df.setRootDataFileId(fileToReplace.getRootDataFileId()); - - } + fileToReplace.setRootDataFileId(fileToReplace.getId()); + fileToReplace = fileService.save(fileToReplace); + } + + /* + * Go through the final file list, settting the rootFileId and previousFileId + */ + for (DataFile df : finalFileList) { + df.setPreviousDataFileId(fileToReplace.getId()); + + df.setRootDataFileId(fileToReplace.getRootDataFileId()); - // Call the update dataset command + } + } + // Call the update dataset command which will delete the replaced filemetadata and file in needed (if file is not released) // return step_070_run_update_dataset_command(); diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/FileReplacePageHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/FileReplacePageHelper.java index 93bd903130c..555c4d99b0c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/FileReplacePageHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/FileReplacePageHelper.java @@ -110,13 +110,9 @@ public boolean handleNativeFileUpload(InputStream inputStream, String fullStorag } OptionalFileParams ofp = null; + ofp = new OptionalFileParams(); + if(checkSum != null) { - try { - ofp = new OptionalFileParams(null); - } catch (DataFileTagException e) { - //Shouldn't happen with null input - e.printStackTrace(); - } ofp.setCheckSum(checkSum); } // Run 1st phase of replace diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java index e48d96e355d..823a1b56e59 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java @@ -71,7 +71,9 @@ public class OptionalFileParams { private String checkSum; public static final String CHECKSUM_ATTR_NAME = "md5Hash"; - + public OptionalFileParams() { + } + public OptionalFileParams(String jsonData) throws DataFileTagException{ if (jsonData != null){ @@ -102,6 +104,21 @@ public OptionalFileParams(String description, this.restrict = restrict; } + //For use in replace operations - load the file metadata from the file being replaced so it can be applied to the new file + //checksum and mimetype aren't needed + public OptionalFileParams(DataFile df) throws DataFileTagException { + FileMetadata fm = df.getFileMetadata(); + + this.description = fm.getDescription(); + setCategories(fm.getCategoriesByName()); + this.addFileDataTags(df.getTagLabels()); + this.restrict = fm.isRestricted(); + //Explicitly do not replace the file name - replaces with -force may change the mimetype and extension + //this.label = fm.getLabel(); + this.directoryLabel = fm.getDirectoryLabel(); + this.provFreeForm = fm.getProvFreeForm(); + } + /** * Set description diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java index fefa8707c8b..a961b495fab 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java @@ -165,14 +165,9 @@ public Dataset execute(CommandContext ctxt) throws CommandException { } // we have to merge to update the database but not flush because // we don't want to create two draft versions! - // Dataset tempDataset = ctxt.em().merge(theDataset); - //SEK 5/30/2019 - // This interim merge is causing: - // java.lang.IllegalArgumentException: Cannot merge an entity that has been removed: edu.harvard.iq.dvn.core.study.FileMetadata - // at the merge at line 177 - //Is this merge needed to add the lock? - seems to be 'no' so what is it needed for? - - // theDataset = ctxt.em().merge(theDataset); + //Merge is required to avoid problems with file deletion + //See #5847 for info about why it was removed and see below for a fix for that + theDataset = ctxt.em().merge(theDataset); for (FileMetadata fmd : filesToDelete) { if (!fmd.getDataFile().isReleased()) { @@ -181,19 +176,22 @@ public Dataset execute(CommandContext ctxt) throws CommandException { ctxt.engine().submit(new DeleteDataFileCommand(fmd.getDataFile(), getRequest())); theDataset.getFiles().remove(fmd.getDataFile()); theDataset.getEditVersion().getFileMetadatas().remove(fmd); - // added this check to handle issue where you could not deleter a file that - // shared a category with a new file - // the relation ship does not seem to cascade, yet somehow it was trying to - // merge the filemetadata - // todo: clean this up some when we clean the create / update dataset methods - for (DataFileCategory cat : theDataset.getCategories()) { - cat.getFileMetadatas().remove(fmd); - } } else { FileMetadata mergedFmd = ctxt.em().merge(fmd); ctxt.em().remove(mergedFmd); fmd.getDataFile().getFileMetadatas().remove(mergedFmd); theDataset.getEditVersion().getFileMetadatas().remove(mergedFmd); + + } + //Moved check to also handle published files - see #5847 + // added this check to handle issue where you could not delete a file that + // shared a category with a new file + // the relation ship does not seem to cascade, yet somehow it was trying to + // merge the filemetadata + // todo: clean this up some when we clean the create / update dataset methods + + for (DataFileCategory cat : theDataset.getCategories()) { + cat.getFileMetadatas().remove(fmd); } } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 8c70475953c..b75ed73f844 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1508,7 +1508,7 @@ file.noSelectedFiles.tip=There are no selected files to display. file.noUploadedFiles.tip=Files you upload will appear here. file.replace=Replace file.replaced.warning.header=Edit File -file.replaced.warning.draft.warningMessage=You can not replace a file that has been replaced in a dataset draft. In order to replace it with a different file you must delete the dataset draft. Note that doing so will discard any other changes within this draft. +file.replaced.warning.draft.warningMessage=You can not replace a published file that has already been replaced in a dataset draft. You can replace the file that is now in the draft version. file.replaced.warning.previous.warningMessage=You can not edit a file that has been replaced in a previous dataset version. In order to edit it you must go to the most recently published version of the file. file.alreadyDeleted.previous.warningMessage=This file has already been deleted in current version. It may not be edited. file.delete=Delete diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml index 9a68212ab9b..efa873e5ae8 100644 --- a/src/main/webapp/file.xhtml +++ b/src/main/webapp/file.xhtml @@ -267,7 +267,7 @@ - +
  • From 279002dbaaeaf82bfc12ba2905f9e0697386237a Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 19 Oct 2020 12:32:55 -0400 Subject: [PATCH 008/576] per @scolopasta - set previous datafileId when needed when replacing a draft-only file that was already a replacement itself --- .../datasetutility/AddReplaceFileHelper.java | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index a9a358f013b..415aea9ac06 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -583,16 +583,26 @@ private boolean runAddReplacePhase1(Dataset owner, return false; } - //Reuse any file PID during a replace operation (if File PIDs are in use, if the fileToReplace hasn't been released) - if(fileToReplace !=null && !fileToReplace.isReleased() && systemConfig.isFilePIDsEnabled()) { - DataFile df = finalFileList.get(0); //step_055 uses a loop and assumes only one file - df.setGlobalId(fileToReplace.getGlobalId()); - df.setGlobalIdCreateTime(fileToReplace.getGlobalIdCreateTime()); - //Should be true or fileToReplace wouldn't have an identifier (since it's not yet released in this if statement) - df.setIdentifierRegistered(fileToReplace.isIdentifierRegistered()); - fileToReplace.setGlobalId(null); + // if the fileToReplace hasn't been released, + if (fileToReplace != null && !fileToReplace.isReleased()) { + DataFile df = finalFileList.get(0); // step_055 uses a loop and assumes only one file + // set the replacement file's previous and root datafileIds to match (unless + // they are the defaults) + if (fileToReplace.getPreviousDataFileId() != null) { + df.setPreviousDataFileId(fileToReplace.getPreviousDataFileId()); + df.setRootDataFileId(fileToReplace.getRootDataFileId()); + } + // Reuse any file PID during a replace operation (if File PIDs are in use) + if (systemConfig.isFilePIDsEnabled()) { + df.setGlobalId(fileToReplace.getGlobalId()); + df.setGlobalIdCreateTime(fileToReplace.getGlobalIdCreateTime()); + // Should be true or fileToReplace wouldn't have an identifier (since it's not + // yet released in this if statement) + df.setIdentifierRegistered(fileToReplace.isIdentifierRegistered()); + fileToReplace.setGlobalId(null); + } } - + return true; } From 2cd3d3a39387ef57d27b9e8f4568b270a39cf7a5 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 20 Oct 2020 10:25:28 -0400 Subject: [PATCH 009/576] typo --- .../iq/dataverse/datasetutility/AddReplaceFileHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index 415aea9ac06..e74010d5672 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -1523,7 +1523,7 @@ private boolean step_060_addFilesViaIngestService(){ } int nFiles = finalFileList.size(); - finalFileList = ingestService.saveAndAddFilesToDataset(workingVersion, finalFileList, true); + finalFileList = ingestService.saveAndAddFilesToDataset(workingVersion, finalFileList, isFileReplaceOperation()); if (nFiles != finalFileList.size()) { if (nFiles == 1) { From 5ecb64c89406bd07e014a3be328035d45effb9dc Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 20 Oct 2020 10:29:41 -0400 Subject: [PATCH 010/576] adding release note --- doc/release-notes/7337-replace-draft-file.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/release-notes/7337-replace-draft-file.md diff --git a/doc/release-notes/7337-replace-draft-file.md b/doc/release-notes/7337-replace-draft-file.md new file mode 100644 index 00000000000..e85d5556332 --- /dev/null +++ b/doc/release-notes/7337-replace-draft-file.md @@ -0,0 +1,3 @@ +## Major Use Cases + +- Users can now replace files in draft datasets. This functionality was previously only available on published datasets. Issue #7149/PR #7337 \ No newline at end of file From d0f039f8b3354224709166f13a36e247451fc977 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 20 Oct 2020 13:54:41 -0400 Subject: [PATCH 011/576] return 400 with bad json --- .../java/edu/harvard/iq/dataverse/api/Datasets.java | 3 +++ src/main/java/edu/harvard/iq/dataverse/api/Files.java | 4 ++-- .../dataverse/datasetutility/OptionalFileParams.java | 7 +------ .../java/edu/harvard/iq/dataverse/api/FilesIT.java | 11 ++++------- 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 655cdafe04c..ffb5f4fb68b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1772,6 +1772,9 @@ public Response addFileToDataset(@PathParam("id") String idSupplied, } catch (DataFileTagException ex) { return error( Response.Status.BAD_REQUEST, ex.getMessage()); } + catch (ClassCastException | com.google.gson.JsonParseException ex) { + return error(Response.Status.BAD_REQUEST, "Exception parsing provided json"); + } // ------------------------------------- // (3) Get the file name and content type diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 81db7f9dec1..058242833de 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -213,8 +213,8 @@ public Response replaceFileInDataset( } catch (DataFileTagException ex) { return error(Response.Status.BAD_REQUEST, ex.getMessage()); } - } catch (ClassCastException ex) { - logger.info("Exception parsing string '" + jsonData + "': " + ex); + } catch (ClassCastException | com.google.gson.JsonParseException ex) { + return error(Response.Status.BAD_REQUEST, "Exception parsing provided json"); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java index 823a1b56e59..efe52baca79 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java @@ -295,12 +295,7 @@ private void loadParamsFromJson(String jsonData) throws DataFileTagException{ // logger.log(Level.SEVERE, "jsonData is null"); } JsonObject jsonObj; - try { - jsonObj = new Gson().fromJson(jsonData, JsonObject.class); - } catch (ClassCastException ex) { - logger.info("Exception parsing string '" + jsonData + "': " + ex); - return; - } + jsonObj = new Gson().fromJson(jsonData, JsonObject.class); // ------------------------------- // get description as string diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index a538cb54f59..f671c350238 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -732,14 +732,11 @@ public void test_007_ReplaceFileUnpublishedAndBadIds() { String pathToFile2 = "src/main/webapp/resources/images/cc0.png"; Response replaceResp = UtilIT.replaceFile(origFileId.toString(), pathToFile2, apiToken); - String errMsgUnpublished = BundleUtil.getStringFromBundle("file.addreplace.error.unpublished_file_cannot_be_replaced"); - replaceResp.then().assertThat() - .statusCode(BAD_REQUEST.getStatusCode()) - .body("status", equalTo(AbstractApiBean.STATUS_ERROR)) - .body("message", Matchers.startsWith(errMsgUnpublished)) - ; - + .body("data.files[0].dataFile.contentType", equalTo("image/png")) + .body("data.files[0].label", equalTo("cc0.png")) + .statusCode(OK.getStatusCode()); + // ------------------------- // Publish dataset // ------------------------- From 83eab44d3692648179dfd8e600e034cc28c278aa Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 20 Oct 2020 15:21:05 -0400 Subject: [PATCH 012/576] check fail on bad json in tests, use Bundle --- .../harvard/iq/dataverse/api/Datasets.java | 2 +- .../edu/harvard/iq/dataverse/api/Files.java | 4 ++-- src/main/java/propertyFiles/Bundle.properties | 1 + .../edu/harvard/iq/dataverse/api/FilesIT.java | 21 ++++++++----------- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index ffb5f4fb68b..44bbdf0ac19 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1773,7 +1773,7 @@ public Response addFileToDataset(@PathParam("id") String idSupplied, return error( Response.Status.BAD_REQUEST, ex.getMessage()); } catch (ClassCastException | com.google.gson.JsonParseException ex) { - return error(Response.Status.BAD_REQUEST, "Exception parsing provided json"); + return error(Response.Status.BAD_REQUEST, BundleUtil.getStringFromBundle("file.addreplace.error.parsing")); } // ------------------------------------- diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 058242833de..eb967afe261 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -214,7 +214,7 @@ public Response replaceFileInDataset( return error(Response.Status.BAD_REQUEST, ex.getMessage()); } } catch (ClassCastException | com.google.gson.JsonParseException ex) { - return error(Response.Status.BAD_REQUEST, "Exception parsing provided json"); + return error(Response.Status.BAD_REQUEST, BundleUtil.getStringFromBundle("file.addreplace.error.parsing")); } } @@ -358,7 +358,7 @@ public Response updateFileMetadata(@FormDataParam("jsonData") String jsonData, return error(Response.Status.BAD_REQUEST, ex.getMessage()); } } catch (ClassCastException | com.google.gson.JsonParseException ex) { - return error(Response.Status.BAD_REQUEST, "Exception parsing provided json"); + return error(Response.Status.BAD_REQUEST, BundleUtil.getStringFromBundle("file.addreplace.error.parsing")); } } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index b75ed73f844..b0bad2441de 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1906,6 +1906,7 @@ file.addreplace.error.byte_abrev=B file.addreplace.error.file_exceeds_limit=This file size ({0}) exceeds the size limit of {1}. file.addreplace.error.dataset_is_null=The dataset cannot be null. file.addreplace.error.dataset_id_is_null=The dataset ID cannot be null. +file.addreplace.error.parsing=Error in parsing provided json find.dataset.error.dataset_id_is_null=When accessing a dataset based on Persistent ID, a {0} query parameter must be present. find.dataset.error.dataset.not.found.persistentId=Dataset with Persistent ID {0} not found. find.dataset.error.dataset.not.found.id=Dataset with ID {0} not found. diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index f671c350238..1073c650c64 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -240,16 +240,12 @@ public void testAddFileBadJson() { Response addResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, junkJson, apiToken); + String parseError = BundleUtil.getStringFromBundle("file.addreplace.error.parsing"); + addResponse.then().assertThat() - .body("status", equalTo(AbstractApiBean.STATUS_OK)) - .body("data.files[0].categories", nullValue()) - .body("data.files[0].dataFile.contentType", equalTo("image/png")) - .body("data.files[0].dataFile.description", equalTo("")) - .body("data.files[0].dataFile.tabularTags", nullValue()) - .body("data.files[0].label", equalTo("dataverseproject.png")) - // not sure why description appears in two places - .body("data.files[0].description", equalTo("")) - .statusCode(OK.getStatusCode()); + .statusCode(BAD_REQUEST.getStatusCode()) + .body("status", equalTo(AbstractApiBean.STATUS_ERROR)) + .body("message", equalTo(parseError)); } @Test @@ -899,10 +895,11 @@ public void testReplaceFileBadJson() { Response replaceResp = UtilIT.replaceFile(origFileId.toString(), pathToFile2, jsonAsString, apiToken); msgt("replace resp: " + replaceResp.prettyPrint()); - + String parseError = BundleUtil.getStringFromBundle("file.addreplace.error.parsing"); replaceResp.then().assertThat() - .statusCode(OK.getStatusCode()) - .body("status", equalTo(AbstractApiBean.STATUS_OK)); + .statusCode(BAD_REQUEST.getStatusCode()) + .body("status", equalTo(AbstractApiBean.STATUS_ERROR)) + .body("message", equalTo(parseError)); } From 04d36ae0572c33309a175ed2989d1191bdbc6710 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 20 Oct 2020 16:11:04 -0400 Subject: [PATCH 013/576] adding lock sleeps --- src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index 1073c650c64..693ae8713c8 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -367,6 +367,8 @@ public void test_006_ReplaceFileGood() throws InterruptedException { .add("categories", Json.createArrayBuilder() .add("Data") ); + + UtilIT.sleepForLock(datasetId, null, apiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION); Response replaceResp = UtilIT.replaceFile(origFileId.toString(), pathToFile2, json.build(), apiToken); msgt(replaceResp.prettyPrint()); @@ -616,6 +618,9 @@ public void testForceReplaceAndUpdate() { .add("categories", Json.createArrayBuilder() .add("Data") ); + + UtilIT.sleepForLock(datasetId, null, apiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION); + Response replaceResp = UtilIT.replaceFile(origFileId.toString(), pathToFile2, json.build(), apiToken); replaceResp.prettyPrint(); From 77c6bdf8451b9eff3b174aed41bc81413dc7e8b9 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 21 Oct 2020 11:15:25 -0400 Subject: [PATCH 014/576] fix merge issue --- .../impl/UpdateDatasetVersionCommand.java | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java index a961b495fab..f1af598fb6d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java @@ -167,7 +167,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { // we don't want to create two draft versions! //Merge is required to avoid problems with file deletion //See #5847 for info about why it was removed and see below for a fix for that - theDataset = ctxt.em().merge(theDataset); + //theDataset = ctxt.em().merge(theDataset); for (FileMetadata fmd : filesToDelete) { if (!fmd.getDataFile().isReleased()) { @@ -175,21 +175,13 @@ public Dataset execute(CommandContext ctxt) throws CommandException { // filemetadata object) ctxt.engine().submit(new DeleteDataFileCommand(fmd.getDataFile(), getRequest())); theDataset.getFiles().remove(fmd.getDataFile()); - theDataset.getEditVersion().getFileMetadatas().remove(fmd); - } else { - FileMetadata mergedFmd = ctxt.em().merge(fmd); - ctxt.em().remove(mergedFmd); - fmd.getDataFile().getFileMetadatas().remove(mergedFmd); - theDataset.getEditVersion().getFileMetadatas().remove(mergedFmd); + } else { + //just need to remove latest metadata + fmd.getDataFile().getFileMetadatas().remove(fmd); } - //Moved check to also handle published files - see #5847 - // added this check to handle issue where you could not delete a file that - // shared a category with a new file - // the relation ship does not seem to cascade, yet somehow it was trying to - // merge the filemetadata - // todo: clean this up some when we clean the create / update dataset methods - + //and in both cases, remove from the dataset and categories lists + theDataset.getEditVersion().getFileMetadatas().remove(fmd); for (DataFileCategory cat : theDataset.getCategories()) { cat.getFileMetadatas().remove(fmd); } From ff47892a8f7609b582448397ccd7e147c5eed7b9 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 21 Oct 2020 14:15:32 -0400 Subject: [PATCH 015/576] debugging: only remove fmd when deleting from draft --- .../engine/command/impl/UpdateDatasetVersionCommand.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java index f1af598fb6d..7cddf0fb1cf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java @@ -177,8 +177,11 @@ public Dataset execute(CommandContext ctxt) throws CommandException { theDataset.getFiles().remove(fmd.getDataFile()); } else { - //just need to remove latest metadata - fmd.getDataFile().getFileMetadatas().remove(fmd); + //just need to remove latest metadata if the file is in a draft dataset + if(theDataset.getEditVersion().getFileMetadatas().contains(fmd)) { + logger.fine("datafile : " + fmd.getDataFile().getId() + " found in draft dataset"); + fmd.getDataFile().getFileMetadatas().remove(fmd); + } } //and in both cases, remove from the dataset and categories lists theDataset.getEditVersion().getFileMetadatas().remove(fmd); From be1026f9fcf664573c6cff224de59d853dd072e2 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 22 Oct 2020 17:44:30 -0400 Subject: [PATCH 016/576] resolved merge issues - simplifying tbd --- .../iq/dataverse/EditDatafilesPage.java | 6 +- .../impl/UpdateDatasetVersionCommand.java | 68 ++++++++++++++---- .../iq/dataverse/util/FileMetadataUtil.java | 72 +++++++++++++++++++ 3 files changed, 130 insertions(+), 16 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/util/FileMetadataUtil.java diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index b4feecfcdf4..265ec1ee937 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -35,6 +35,8 @@ import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.EjbUtil; +import edu.harvard.iq.dataverse.util.FileMetadataUtil; + import static edu.harvard.iq.dataverse.util.JsfHelper.JH; import java.io.File; import java.io.FileOutputStream; @@ -948,9 +950,9 @@ private void deleteFiles(List filesForDelete) { // fileMetadatas list. (but doing both just adds a no-op and won't cause an // error) // 1. delete the filemetadata from the local display list: - removeFileMetadataFromList(fileMetadatas, markedForDelete); + FileMetadataUtil.removeFileMetadataFromList(fileMetadatas, markedForDelete); // 2. delete the filemetadata from the version: - removeFileMetadataFromList(dataset.getEditVersion().getFileMetadatas(), markedForDelete); + FileMetadataUtil.removeFileMetadataFromList(dataset.getEditVersion().getFileMetadatas(), markedForDelete); } if (markedForDelete.getDataFile().getId() == null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java index 7cddf0fb1cf..c1cdb5546ca 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java @@ -9,6 +9,8 @@ import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; +import edu.harvard.iq.dataverse.util.FileMetadataUtil; + import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -165,31 +167,69 @@ public Dataset execute(CommandContext ctxt) throws CommandException { } // we have to merge to update the database but not flush because // we don't want to create two draft versions! - //Merge is required to avoid problems with file deletion - //See #5847 for info about why it was removed and see below for a fix for that - //theDataset = ctxt.em().merge(theDataset); + // Dataset tempDataset = ctxt.em().merge(theDataset); + //SEK 5/30/2019 + // This interim merge is causing: + // java.lang.IllegalArgumentException: Cannot merge an entity that has been removed: edu.harvard.iq.dvn.core.study.FileMetadata + // at the merge at line 177 + //Is this merge needed to add the lock? - seems to be 'no' so what is it needed for? + + theDataset = ctxt.em().merge(theDataset); for (FileMetadata fmd : filesToDelete) { + logger.fine("Deleting fmd: " + fmd.getId() + " for file: " + fmd.getDataFile().getId()); if (!fmd.getDataFile().isReleased()) { // if file is draft (ie. new to this version, delete; otherwise just remove // filemetadata object) ctxt.engine().submit(new DeleteDataFileCommand(fmd.getDataFile(), getRequest())); theDataset.getFiles().remove(fmd.getDataFile()); - + theDataset.getEditVersion().getFileMetadatas().remove(fmd); + // added this check to handle issue where you could not delete a file that + // shared a category with a new file + // the relation ship does not seem to cascade, yet somehow it was trying to + // merge the filemetadata + // todo: clean this up some when we clean the create / update dataset methods + for (DataFileCategory cat : theDataset.getCategories()) { + cat.getFileMetadatas().remove(fmd); + } } else { - //just need to remove latest metadata if the file is in a draft dataset - if(theDataset.getEditVersion().getFileMetadatas().contains(fmd)) { - logger.fine("datafile : " + fmd.getDataFile().getId() + " found in draft dataset"); - fmd.getDataFile().getFileMetadatas().remove(fmd); + if (fmd.getId() != null) { + if (theDataset.getEditVersion() != fmd.getDatasetVersion()) { + fmd = FileMetadataUtil.getFmdForFileInEditVersion(fmd, theDataset.getEditVersion()); + logger.fine("Now Deleting fmd: " + fmd.getId() + "for file: " + fmd.getDataFile().getId()); + logger.fine("Num fmd on file: " + fmd.getDataFile().getFileMetadatas().size()); + + ctxt.em().remove(fmd); + FileMetadataUtil.removeFileMetadataFromList(fmd.getDataFile().getFileMetadatas(), fmd); + logger.fine("2 Num fmd on file: " + fmd.getDataFile().getFileMetadatas().size()); + FileMetadataUtil.removeFileMetadataFromList(theDataset.getEditVersion().getFileMetadatas(), fmd); + + for (DataFileCategory cat : theDataset.getCategories()) { + FileMetadataUtil.removeFileMetadataFromList(cat.getFileMetadatas(), fmd); + } + } else { + FileMetadata mergedFmd = ctxt.em().merge(fmd); + ctxt.em().remove(mergedFmd); + fmd.getDataFile().getFileMetadatas().remove(mergedFmd); + theDataset.getEditVersion().getFileMetadatas().remove(mergedFmd); + for (DataFileCategory cat : theDataset.getCategories()) { + cat.getFileMetadatas().remove(mergedFmd); + } + } + } else { + ctxt.em().remove(fmd); + FileMetadataUtil.removeFileMetadataFromList(fmd.getDataFile().getFileMetadatas(), fmd); + FileMetadataUtil.removeFileMetadataFromList(theDataset.getEditVersion().getFileMetadatas(), fmd); + for (DataFileCategory cat : theDataset.getCategories()) { + FileMetadataUtil.removeFileMetadataFromList(cat.getFileMetadatas(), fmd); + } } } - //and in both cases, remove from the dataset and categories lists - theDataset.getEditVersion().getFileMetadatas().remove(fmd); - for (DataFileCategory cat : theDataset.getCategories()) { - cat.getFileMetadatas().remove(fmd); - } } - + for(FileMetadata fmd: theDataset.getEditVersion().getFileMetadatas()) { + logger.fine("FMD: " + fmd.getId() + " for file: " + fmd.getDataFile().getId() + "is in final draft version"); + } + if (recalculateUNF) { ctxt.ingest().recalculateDatasetVersionUNF(theDataset.getEditVersion()); } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileMetadataUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileMetadataUtil.java new file mode 100644 index 00000000000..43dbf76e85a --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileMetadataUtil.java @@ -0,0 +1,72 @@ +/* + Copyright (C) 2005-2012, by the President and Fellows of Harvard College. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Dataverse Network - A web application to share, preserve and analyze research data. + Developed at the Institute for Quantitative Social Science, Harvard University. + Version 3.0. +*/ + +package edu.harvard.iq.dataverse.util; + +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.FileMetadata; + +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.logging.Logger; + +public class FileMetadataUtil implements java.io.Serializable { + private static final Logger logger = Logger.getLogger(FileMetadataUtil.class.getCanonicalName()); + + public static void removeFileMetadataFromList(Collection collection, FileMetadata fmToDelete) { + Iterator fmit = collection.iterator(); + while (fmit.hasNext()) { + FileMetadata fmd = fmit.next(); + if (fmToDelete.getDataFile().getStorageIdentifier().equals(fmd.getDataFile().getStorageIdentifier())) { + if (fmToDelete.getDatasetVersion().getId() == null) { + if (fmd.getDatasetVersion().getId() == null) { + fmit.remove(); + break; + } + } else if (fmToDelete.getDatasetVersion().getId().equals(fmd.getDatasetVersion().getId())) { + fmit.remove(); + break; + } + } + } + } + + public static void removeDataFileFromList(List dfs, DataFile dfToDelete) { + Iterator dfit = dfs.iterator(); + while (dfit.hasNext()) { + DataFile df = dfit.next(); + if (dfToDelete.getStorageIdentifier().equals(df.getStorageIdentifier())) { + dfit.remove(); + break; + } + } + } + + public static FileMetadata getFmdForFileInEditVersion(FileMetadata fmd, DatasetVersion editVersion) { + for(FileMetadata editFmd: editVersion.getFileMetadatas()) { + if(editFmd.getDataFile().getId().equals(fmd.getDataFile().getId())) { + return editFmd; + } + } + return null; + } +} From e6a403339fa744a247dd562850628bd84dcb859a Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 22 Oct 2020 20:05:54 -0400 Subject: [PATCH 017/576] support null lock type in sleep --- src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 618b17c765c..22338cea0a3 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2221,7 +2221,7 @@ static Boolean sleepForLock(String idOrPersistentId, String lockType, String api } catch (InterruptedException ex) { Logger.getLogger(UtilIT.class.getName()).log(Level.SEVERE, null, ex); } - } while (lockedForIngest.body().prettyPrint().contains(lockType)); + } while (lockedForIngest.body().jsonPath().getList("data").size() >0 && (lockType==null || lockedForIngest.body().prettyPrint().contains(lockType))); return i <= duration; From d16b3cbf7f937f5b0b6525e587a5e6444f743244 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 23 Oct 2020 14:28:15 -0400 Subject: [PATCH 018/576] hopefully cleanup and documentation - no functional change --- .../iq/dataverse/EditDatafilesPage.java | 30 +---- .../impl/UpdateDatasetVersionCommand.java | 106 ++++++++++-------- .../iq/dataverse/util/FileMetadataUtil.java | 52 ++++++--- .../edu/harvard/iq/dataverse/api/FilesIT.java | 15 ++- 4 files changed, 112 insertions(+), 91 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 265ec1ee937..a42e126cb37 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -936,8 +936,12 @@ private void deleteFiles(List filesForDelete) { // and let the delete be handled in the command (by adding it to the // filesToBeDeleted list): + // ToDo - FileMetadataUtil.removeFileMetadataFromList should handle these two + // removes so they could be put after this if clause and the else clause could + // be removed. dataset.getEditVersion().getFileMetadatas().remove(markedForDelete); fileMetadatas.remove(markedForDelete); + filesToBeDeleted.add(markedForDelete); } else { logger.fine("this is a brand-new (unsaved) filemetadata"); @@ -961,8 +965,8 @@ private void deleteFiles(List filesForDelete) { // removing it from the fileMetadatas lists (above), we also remove it from // the newFiles list and the dataset's files, so it never gets saved. - removeDataFileFromList(dataset.getFiles(), markedForDelete.getDataFile()); - removeDataFileFromList(newFiles, markedForDelete.getDataFile()); + FileMetadataUtil.removeDataFileFromList(dataset.getFiles(), markedForDelete.getDataFile()); + FileMetadataUtil.removeDataFileFromList(newFiles, markedForDelete.getDataFile()); FileUtil.deleteTempFile(markedForDelete.getDataFile(), dataset, ingestService); // Also remove checksum from the list of newly uploaded checksums (perhaps odd // to delete and then try uploading the same file again, but it seems like it @@ -991,28 +995,6 @@ private void deleteFiles(List filesForDelete) { } - private void removeFileMetadataFromList(List fmds, FileMetadata fmToDelete) { - Iterator fmit = fmds.iterator(); - while (fmit.hasNext()) { - FileMetadata fmd = fmit.next(); - if (fmToDelete.getDataFile().getStorageIdentifier().equals(fmd.getDataFile().getStorageIdentifier())) { - fmit.remove(); - break; - } - } - } - - private void removeDataFileFromList(List dfs, DataFile dfToDelete) { - Iterator dfit = dfs.iterator(); - while (dfit.hasNext()) { - DataFile df = dfit.next(); - if (dfToDelete.getStorageIdentifier().equals(df.getStorageIdentifier())) { - dfit.remove(); - break; - } - } - } - public String saveWithTermsOfUse() { logger.fine("saving terms of use, and the dataset version"); return save(); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java index c1cdb5546ca..e23f27cac03 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java @@ -167,63 +167,71 @@ public Dataset execute(CommandContext ctxt) throws CommandException { } // we have to merge to update the database but not flush because // we don't want to create two draft versions! - // Dataset tempDataset = ctxt.em().merge(theDataset); - //SEK 5/30/2019 - // This interim merge is causing: - // java.lang.IllegalArgumentException: Cannot merge an entity that has been removed: edu.harvard.iq.dvn.core.study.FileMetadata - // at the merge at line 177 - //Is this merge needed to add the lock? - seems to be 'no' so what is it needed for? - + // Although not completely tested, it looks like this merge handles the + // thumbnail case - if the filemetadata is removed from the context below and + // the dataset still references it, that could cause an issue. Merging here + // avoids any reference from it being the dataset thumbnail theDataset = ctxt.em().merge(theDataset); + /* + * This code has to handle many cases, and anyone making changes should + * carefully check tests and basic methods that update the dataset version. The + * differences between the cases stem primarily from differences in whether the + * files to add, and there filemetadata, and files to delete, and their + * filemetadata have been persisted at this point, which manifests itself as to + * whether they have id numbers or not, and apparently, whether or not they + * exists in lists, e.g. the getFileMetadatas() list of a datafile. + * + * To handle this, the code is carefully checking to make sure that deletions + * are deleting the right things and not, for example, doing a remove(fmd) when + * the fmd.getId() is null, which just removes the first element found. + */ for (FileMetadata fmd : filesToDelete) { logger.fine("Deleting fmd: " + fmd.getId() + " for file: " + fmd.getDataFile().getId()); + // if file is draft (ie. new to this version), delete it. Otherwise just remove + // filemetadata object) + // There are a few cases to handle: + // * the fmd has an id (has been persisted) and is the one in the current + // (draft) version + // * the fmd has an id (has been persisted) but it is from a published version + // so we need the corresponding one from the draft version (i.e. created during + // a getEditVersion call) + // * the fmd has no id (hasn't been persisted) so we have to use non-id based + // means to identify it and remove it from lists + + if (fmd.getId() != null) { + // If the datasetversion doesn't match, we have the fmd from a published version + // and we need to remove the one for the newly created draft instead, so we find + // it here + if (theDataset.getEditVersion() != fmd.getDatasetVersion()) { + fmd = FileMetadataUtil.getFmdForFileInEditVersion(fmd, theDataset.getEditVersion()); + } + } else { + // Not sure if this is needed now that there is a dataset merge above, but we + // need to assure it is on the context + fmd = ctxt.em().merge(fmd); + } + // There are two datafile cases as well - the file has been released, so we're + // jsut removing it from the current draft version or it is only in the draft + // version and we completely remove the file. if (!fmd.getDataFile().isReleased()) { - // if file is draft (ie. new to this version, delete; otherwise just remove - // filemetadata object) + // remove the file ctxt.engine().submit(new DeleteDataFileCommand(fmd.getDataFile(), getRequest())); + // and remove the file from the dataset's list theDataset.getFiles().remove(fmd.getDataFile()); - theDataset.getEditVersion().getFileMetadatas().remove(fmd); - // added this check to handle issue where you could not delete a file that - // shared a category with a new file - // the relation ship does not seem to cascade, yet somehow it was trying to - // merge the filemetadata - // todo: clean this up some when we clean the create / update dataset methods - for (DataFileCategory cat : theDataset.getCategories()) { - cat.getFileMetadatas().remove(fmd); - } } else { - if (fmd.getId() != null) { - if (theDataset.getEditVersion() != fmd.getDatasetVersion()) { - fmd = FileMetadataUtil.getFmdForFileInEditVersion(fmd, theDataset.getEditVersion()); - logger.fine("Now Deleting fmd: " + fmd.getId() + "for file: " + fmd.getDataFile().getId()); - logger.fine("Num fmd on file: " + fmd.getDataFile().getFileMetadatas().size()); - - ctxt.em().remove(fmd); - FileMetadataUtil.removeFileMetadataFromList(fmd.getDataFile().getFileMetadatas(), fmd); - logger.fine("2 Num fmd on file: " + fmd.getDataFile().getFileMetadatas().size()); - FileMetadataUtil.removeFileMetadataFromList(theDataset.getEditVersion().getFileMetadatas(), fmd); - - for (DataFileCategory cat : theDataset.getCategories()) { - FileMetadataUtil.removeFileMetadataFromList(cat.getFileMetadatas(), fmd); - } - } else { - FileMetadata mergedFmd = ctxt.em().merge(fmd); - ctxt.em().remove(mergedFmd); - fmd.getDataFile().getFileMetadatas().remove(mergedFmd); - theDataset.getEditVersion().getFileMetadatas().remove(mergedFmd); - for (DataFileCategory cat : theDataset.getCategories()) { - cat.getFileMetadatas().remove(mergedFmd); - } - } - } else { - ctxt.em().remove(fmd); - FileMetadataUtil.removeFileMetadataFromList(fmd.getDataFile().getFileMetadatas(), fmd); - FileMetadataUtil.removeFileMetadataFromList(theDataset.getEditVersion().getFileMetadatas(), fmd); - for (DataFileCategory cat : theDataset.getCategories()) { - FileMetadataUtil.removeFileMetadataFromList(cat.getFileMetadatas(), fmd); - } - } + // if we aren't removing the file, we need to explicitly remove the fmd from the + // context and then remove it from the datafile's list + ctxt.em().remove(fmd); + FileMetadataUtil.removeFileMetadataFromList(fmd.getDataFile().getFileMetadatas(), fmd); + } + // In either case, to fully remove the fmd, we have to remove any other possible + // references + // From the datasetversion + FileMetadataUtil.removeFileMetadataFromList(theDataset.getEditVersion().getFileMetadatas(), fmd); + // and from the list associated with each category + for (DataFileCategory cat : theDataset.getCategories()) { + FileMetadataUtil.removeFileMetadataFromList(cat.getFileMetadatas(), fmd); } } for(FileMetadata fmd: theDataset.getEditVersion().getFileMetadatas()) { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileMetadataUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileMetadataUtil.java index 43dbf76e85a..5c48fccb2d6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileMetadataUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileMetadataUtil.java @@ -32,31 +32,51 @@ public class FileMetadataUtil implements java.io.Serializable { private static final Logger logger = Logger.getLogger(FileMetadataUtil.class.getCanonicalName()); + //Delete the filemetadata from the list if and only if it is in the list public static void removeFileMetadataFromList(Collection collection, FileMetadata fmToDelete) { - Iterator fmit = collection.iterator(); - while (fmit.hasNext()) { - FileMetadata fmd = fmit.next(); - if (fmToDelete.getDataFile().getStorageIdentifier().equals(fmd.getDataFile().getStorageIdentifier())) { - if (fmToDelete.getDatasetVersion().getId() == null) { - if (fmd.getDatasetVersion().getId() == null) { + // With an id, the standard remove will work + if (fmToDelete.getId() != null) { + collection.remove(fmToDelete); + } else { + Iterator fmit = collection.iterator(); + while (fmit.hasNext()) { + FileMetadata fmd = fmit.next(); + // If not, we can remove based on a match based on the id of the related + // datafile + if (fmToDelete.getDataFile().getStorageIdentifier().equals(fmd.getDataFile().getStorageIdentifier())) { + // and a match on the datasetversion + if (fmToDelete.getDatasetVersion().getId() == null) { + // If the fmd to delete is for a datasetversion with no id, we assume match to + // any other fmd with a datasetversion with no id (since there should be only + // one) + // Otherwise, we don't delete anything (this fmd hasn't been persisted and isn't + // in the list. + if (fmd.getDatasetVersion().getId() == null) { + fmit.remove(); + break; + } + } else if (fmToDelete.getDatasetVersion().getId().equals(fmd.getDatasetVersion().getId())) { fmit.remove(); break; } - } else if (fmToDelete.getDatasetVersion().getId().equals(fmd.getDatasetVersion().getId())) { - fmit.remove(); - break; } } } } - public static void removeDataFileFromList(List dfs, DataFile dfToDelete) { - Iterator dfit = dfs.iterator(); - while (dfit.hasNext()) { - DataFile df = dfit.next(); - if (dfToDelete.getStorageIdentifier().equals(df.getStorageIdentifier())) { - dfit.remove(); - break; + //Delete datafile from the list even if it's id is null (hasn't yet been persisted) + public static void removeDataFileFromList(List dfList, DataFile dfToDelete) { + // With an id, the standard remove will work + if (dfToDelete.getId() != null) { + dfList.remove(dfToDelete); + } else { + Iterator dfit = dfList.iterator(); + while (dfit.hasNext()) { + DataFile df = dfit.next(); + if (dfToDelete.getStorageIdentifier().equals(df.getStorageIdentifier())) { + dfit.remove(); + break; + } } } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index 693ae8713c8..3173c3fe1f0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -368,13 +368,24 @@ public void test_006_ReplaceFileGood() throws InterruptedException { .add("Data") ); + /* + * ToDo: When the dataset is still locked, the replaceFile call below returns an + * 'OK' status with an empty 'data' array The sleepForLock avoids that so this + * test tests the normal replace functionality directly, but a new test to check + * that, when the dataset is locked, the call fails instead of returning OK + * would be useful (along with making the replace call do that) + */ + /* + * ToDo: make sleep time shorter for this? Add sleepForLock before subsequent + * calls as well? (Or is it only needed here because it is still locked from the + * publish call above?) + */ + UtilIT.sleepForLock(datasetId, null, apiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION); Response replaceResp = UtilIT.replaceFile(origFileId.toString(), pathToFile2, json.build(), apiToken); msgt(replaceResp.prettyPrint()); - String successMsg2 = BundleUtil.getStringFromBundle("file.addreplace.success.replace"); - replaceResp.then().assertThat() /** * @todo We have a need to show human readable success messages From 72475d3db290df2b3b0be673f3b8581324a70639 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 28 Oct 2020 09:26:57 -0400 Subject: [PATCH 019/576] fix duplicate replace button --- src/main/webapp/file.xhtml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml index efa873e5ae8..37ea7f9b31c 100644 --- a/src/main/webapp/file.xhtml +++ b/src/main/webapp/file.xhtml @@ -274,9 +274,9 @@
    • - +
  • - +
    • From e7f38f3d1780f38512907ad1493427b939082569 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 30 Oct 2020 10:31:17 -0400 Subject: [PATCH 020/576] != to .equals() Conflicts: src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java --- .../engine/command/impl/UpdateDatasetVersionCommand.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java index e23f27cac03..3bf5198d8e1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java @@ -203,7 +203,9 @@ public Dataset execute(CommandContext ctxt) throws CommandException { // If the datasetversion doesn't match, we have the fmd from a published version // and we need to remove the one for the newly created draft instead, so we find // it here - if (theDataset.getEditVersion() != fmd.getDatasetVersion()) { + logger.fine("Edit ver: " + theDataset.getEditVersion().getId()); + logger.fine("fmd ver: " + fmd.getDatasetVersion().getId()); + if (!theDataset.getEditVersion().equals(fmd.getDatasetVersion())) { fmd = FileMetadataUtil.getFmdForFileInEditVersion(fmd, theDataset.getEditVersion()); } } else { From 64c9e8838b2049053d3fc2c454cf76714bd4414b Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 30 Oct 2020 10:37:16 -0400 Subject: [PATCH 021/576] always merge --- .../command/impl/UpdateDatasetVersionCommand.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java index 3bf5198d8e1..756e2ff5948 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java @@ -208,13 +208,11 @@ public Dataset execute(CommandContext ctxt) throws CommandException { if (!theDataset.getEditVersion().equals(fmd.getDatasetVersion())) { fmd = FileMetadataUtil.getFmdForFileInEditVersion(fmd, theDataset.getEditVersion()); } - } else { - // Not sure if this is needed now that there is a dataset merge above, but we - // need to assure it is on the context - fmd = ctxt.em().merge(fmd); - } + } + fmd = ctxt.em().merge(fmd); + // There are two datafile cases as well - the file has been released, so we're - // jsut removing it from the current draft version or it is only in the draft + // just removing it from the current draft version or it is only in the draft // version and we completely remove the file. if (!fmd.getDataFile().isReleased()) { // remove the file From c5772febf3ba743856f863d05c3de74ece8fdd85 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 30 Oct 2020 12:41:15 -0400 Subject: [PATCH 022/576] fix delete during upload --- .../java/edu/harvard/iq/dataverse/util/FileMetadataUtil.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileMetadataUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileMetadataUtil.java index 5c48fccb2d6..cd5db359344 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileMetadataUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileMetadataUtil.java @@ -45,7 +45,10 @@ public static void removeFileMetadataFromList(Collection collectio // datafile if (fmToDelete.getDataFile().getStorageIdentifier().equals(fmd.getDataFile().getStorageIdentifier())) { // and a match on the datasetversion - if (fmToDelete.getDatasetVersion().getId() == null) { + if(fmToDelete.getDatasetVersion() == null) { + //not yet associated with a version (i.e. deleting from the upload screen), so the match on datafile is good enough + fmit.remove(); + } else if (fmToDelete.getDatasetVersion().getId() == null) { // If the fmd to delete is for a datasetversion with no id, we assume match to // any other fmd with a datasetversion with no id (since there should be only // one) From 36ede87d12859a310d98e0afdfc57f61588997f7 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 2 Nov 2020 14:56:54 -0500 Subject: [PATCH 023/576] Add ASCIIFoldingFilterFactory to handle non-ascii chars in search --- conf/solr/7.7.2/schema.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conf/solr/7.7.2/schema.xml b/conf/solr/7.7.2/schema.xml index da40a8e99fa..b0ecaba9c99 100644 --- a/conf/solr/7.7.2/schema.xml +++ b/conf/solr/7.7.2/schema.xml @@ -38,7 +38,7 @@ catchall "text" field, and use that for searching. --> - + +
    From 5df5b747cf71773258b31d22fd09b9bfc08a4d5e Mon Sep 17 00:00:00 2001 From: Gustavo Durand Date: Thu, 17 Dec 2020 15:40:05 -0500 Subject: [PATCH 035/576] Update 5.0-release-notes.md --- doc/release-notes/5.0-release-notes.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/5.0-release-notes.md b/doc/release-notes/5.0-release-notes.md index 91e5614ed99..e440642485a 100644 --- a/doc/release-notes/5.0-release-notes.md +++ b/doc/release-notes/5.0-release-notes.md @@ -245,9 +245,11 @@ The instructions below describe the upgrade procedure based on moving an existin `sudo /usr/local/glassfish4/bin/asadmin stop-domain` `sudo cp -ar /usr/local/glassfish4/glassfish/domains/domain1 /usr/local/payara5/glassfish/domains/` -7. Remove the Glassfish cache directories +7. Remove the cache and the ejbtimer database directories - `sudo rm -rf /usr/local/payara5/glassfish/domains/domain1/generated/` `sudo rm -rf /usr/local/payara5/glassfish/domains/domain1/osgi-cache/` + `sudo rm -rf /usr/local/payara5/glassfish/domains/domain1/generated/` + `sudo rm -rf /usr/local/payara5/glassfish/domains/domain1/osgi-cache/` + `sudo rm -rf /usr/local/payara5/glassfish/domains/domain1/lib/databases/ejbtimer` 8. In domain.xml: From 837df641642113abba534c29d757191e2e25acc9 Mon Sep 17 00:00:00 2001 From: Gustavo Durand Date: Thu, 17 Dec 2020 15:41:43 -0500 Subject: [PATCH 036/576] Update 5.0-release-notes.md --- doc/release-notes/5.0-release-notes.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/release-notes/5.0-release-notes.md b/doc/release-notes/5.0-release-notes.md index e440642485a..e3d13eaca7c 100644 --- a/doc/release-notes/5.0-release-notes.md +++ b/doc/release-notes/5.0-release-notes.md @@ -245,11 +245,10 @@ The instructions below describe the upgrade procedure based on moving an existin `sudo /usr/local/glassfish4/bin/asadmin stop-domain` `sudo cp -ar /usr/local/glassfish4/glassfish/domains/domain1 /usr/local/payara5/glassfish/domains/` -7. Remove the cache and the ejbtimer database directories +7. Remove the cache directories `sudo rm -rf /usr/local/payara5/glassfish/domains/domain1/generated/` `sudo rm -rf /usr/local/payara5/glassfish/domains/domain1/osgi-cache/` - `sudo rm -rf /usr/local/payara5/glassfish/domains/domain1/lib/databases/ejbtimer` 8. In domain.xml: From 4e89a3dc9ae9a9224afb25c8c3a4f2122cc050ea Mon Sep 17 00:00:00 2001 From: Gustavo Durand Date: Thu, 17 Dec 2020 15:48:25 -0500 Subject: [PATCH 037/576] Remove ejbtimer database directory --- doc/release-notes/5.3-release-notes.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/release-notes/5.3-release-notes.md b/doc/release-notes/5.3-release-notes.md index d91c23a18b1..b7736c8cd4a 100644 --- a/doc/release-notes/5.3-release-notes.md +++ b/doc/release-notes/5.3-release-notes.md @@ -140,7 +140,7 @@ instead of: ` ` -If this is not done, replace that element. +If this is not done, replace that jdbc-connection-pool element. 5\. Reset the EJB timer database back to default: @@ -151,11 +151,11 @@ If this is not done, replace that element. - `/bin/asadmin delete-jdbc-connection-pool --cascade=true dvnDbPool` - `/bin/asadmin delete-password-alias db_password_alias` -7\. Stop payara and remove the generated directory, start. +7\. Stop payara, remove the generated and ejbtimer database directories, then restart. - `service payara stop` -- remove the generated directory: -`rm -rf /glassfish/domains/domain1/generated` +- `rm -rf /glassfish/domains/domain1/generated` +- `rm -rf /glassfish/domains/domain1/lib/databases/ejbtimer` - `service payara start` 8\. Deploy this version. From dc1f554e13d7e7c1786d90540e648526a28a653f Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 18 Dec 2020 11:58:03 -0500 Subject: [PATCH 038/576] emphasize differences with SWORD auth (vs. native) #7441 --- doc/sphinx-guides/source/api/auth.rst | 2 ++ doc/sphinx-guides/source/api/sword.rst | 21 ++++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/auth.rst b/doc/sphinx-guides/source/api/auth.rst index 6d53e52d3b4..5a914830c88 100644 --- a/doc/sphinx-guides/source/api/auth.rst +++ b/doc/sphinx-guides/source/api/auth.rst @@ -21,6 +21,8 @@ Anyone who has your API Token can add and delete data as you so you should treat Passing Your API Token as an HTTP Header (Preferred) or a Query Parameter ------------------------------------------------------------------------- +Note: The SWORD API uses a different way of passing the API token. Please see :ref:`sword-auth` for details. + See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. There are two ways to pass your API token to Dataverse APIs. The preferred method is to send the token in the ``X-Dataverse-key`` HTTP header, as in the following curl example. diff --git a/doc/sphinx-guides/source/api/sword.rst b/doc/sphinx-guides/source/api/sword.rst index f031f78f822..5182e3dd000 100755 --- a/doc/sphinx-guides/source/api/sword.rst +++ b/doc/sphinx-guides/source/api/sword.rst @@ -23,6 +23,25 @@ As a profile of AtomPub, XML is used throughout SWORD. As of Dataverse 4.0 datas .. _SWORDv2 specification: http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html +.. _sword-auth: + +Authentication for SWORD +------------------------ + +The :doc:`/api/auth` section describes how to pass API tokens for the "native" API (as a header or query parameter) but SWORD uses a different mechanism known as HTTP Basic Authentication (`RFC 7617`_). + +HTTP Basic Authentication commonly makes use of both a username and a password but the SWORD API uses only a username that is populated with the user's API token. The password should be blank or empty. + +Clients such as ``curl`` expect both a username and a password separated by a colon. With ``curl``, the way to indicate that the password should be blank or empty is to include the colon at the end of the username (the API token) like this: + +``curl -u 54b143b5-d001-4254-afc0-a1c0f6a5b5a7:`` + +All the curl examples below take this form but instead of showing an API token like above, a Bash environment variable called ``$API_TOKEN`` is shown instead like this: + +``curl -u $API_TOKEN:`` + +.. _RFC 7617: https://tools.ietf.org/html/rfc7617 + .. _incompatible: Backward incompatible changes @@ -45,7 +64,7 @@ Differences in Dataverse 4 from DVN 3.x lead to a few minor backward incompatibl New features as of v1.1 ----------------------- -- Dataverse 4 supports API tokens and requires them to be used for APIs instead of a username and password. In the ``curl`` examples below, you will see ``curl -u $API_TOKEN:`` showing that you should send your API token as the username and nothing as the password. For example, ``curl -u 54b143b5-d001-4254-afc0-a1c0f6a5b5a7:``. +- Dataverse 4+ supports API tokens and requires them to be used for APIs instead of a username and password. In the ``curl`` examples below, you will see ``curl -u $API_TOKEN:`` showing that you should send your API token as the username and nothing as the password. For example, ``curl -u 54b143b5-d001-4254-afc0-a1c0f6a5b5a7:``. See also :ref:`sword-auth`. - SWORD operations no longer require "admin" permission. In order to use any SWORD operation in DVN 3.x, you had to be an "admin" on a dataverse (the container for your dataset) and similar rules were applied in Dataverse 4.4 and earlier (the ``EditDataverse`` permission was required). The SWORD API has now been fully integrated with the Dataverse 4 permission model such that any action you have permission to perform in the GUI or "native" API you are able to perform via SWORD. This means that even a user with a "Contributor" role can operate on datasets via SWORD. Note that users with the "Contributor" role do not have the ``PublishDataset`` permission and will not be able publish their datasets via any mechanism, GUI or API. From c42bc701bec1ba90a2c6904c166417365464317b Mon Sep 17 00:00:00 2001 From: Gustavo Durand Date: Fri, 18 Dec 2020 13:42:58 -0500 Subject: [PATCH 039/576] fixed timer pool connection pool step --- doc/release-notes/5.3-release-notes.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/doc/release-notes/5.3-release-notes.md b/doc/release-notes/5.3-release-notes.md index b7736c8cd4a..004cdb12e75 100644 --- a/doc/release-notes/5.3-release-notes.md +++ b/doc/release-notes/5.3-release-notes.md @@ -132,16 +132,10 @@ Please configure your connection details, replacing all the `${DB_...}`. - `/bin/asadmin create-password-alias --passwordfile /tmp/password.txt dataverse.db.password` - `rm /tmp/password.txt` -4\. Verify that domain.xml contains: - - ` ` - -instead of: +4\. In domain.xml, verify that the __TimerPool jdbc-connection-pool is using the H2 database, as follows (if you have the old Derby version from Glassfish 4, replace it): ` ` -If this is not done, replace that jdbc-connection-pool element. - 5\. Reset the EJB timer database back to default: - `/bin/asadmin set configs.config.server-config.ejb-container.ejb-timer-service.timer-datasource=jdbc/__TimerPool` From e289309b36d0f4854ec01281da87c7983d088d99 Mon Sep 17 00:00:00 2001 From: landreev Date: Fri, 18 Dec 2020 16:08:43 -0500 Subject: [PATCH 040/576] Update 5.3-release-notes.md --- doc/release-notes/5.3-release-notes.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/5.3-release-notes.md b/doc/release-notes/5.3-release-notes.md index 004cdb12e75..c72588f44db 100644 --- a/doc/release-notes/5.3-release-notes.md +++ b/doc/release-notes/5.3-release-notes.md @@ -108,8 +108,11 @@ If this is a new installation, please see our [Installation Guide](https://guide 1\. Upgrade to Payara 5.2020.6 or higher. -Instructions how to update can be found in the -[Payara documentation](https://docs.payara.fish/community/docs/5.2020.6/documentation/user-guides/upgrade-payara.html) +Instructions on how to update can be found in the +[Payara documentation](https://docs.payara.fish/community/docs/5.2020.6/documentation/user-guides/upgrade-payara.html). + +It would likely be safer to upgrade Payara first, while still running Dataverse 5.2, and then proceed with the steps below. Upgrading from an earlier version of Payara should be a straightforward process: Undeploy Dataverse; stop Payara; move the current Payara directory out of the way; unzip the new Payara version in its place; replace **the brand new `payara/glassfish/domains/domain1` with your old, preserved `domain1`**; start Payara, deploy Dataverse 5.2. We still recommend that you read the detailed upgrade instructions above; and, if you run into any issues with this upgrade, it will help to be able to separate them from any problems with the upgrade of Dataverse proper. +If you are still using pre-5.0 version of Dataverse, and Glassfish version 4, please follow the upgrade instructions in the Dataverse 5.0 release notes; but use the latest version of Payara 5 (5.200.7, as of this writing). 2\. Undeploy the previous version. From 2319e2d856d6cc3183b62705ab71513933419c1c Mon Sep 17 00:00:00 2001 From: landreev Date: Fri, 18 Dec 2020 16:18:03 -0500 Subject: [PATCH 041/576] Update doc/release-notes/5.3-release-notes.md Co-authored-by: Philip Durbin --- doc/release-notes/5.3-release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/5.3-release-notes.md b/doc/release-notes/5.3-release-notes.md index c72588f44db..b78aa742f45 100644 --- a/doc/release-notes/5.3-release-notes.md +++ b/doc/release-notes/5.3-release-notes.md @@ -112,7 +112,7 @@ Instructions on how to update can be found in the [Payara documentation](https://docs.payara.fish/community/docs/5.2020.6/documentation/user-guides/upgrade-payara.html). It would likely be safer to upgrade Payara first, while still running Dataverse 5.2, and then proceed with the steps below. Upgrading from an earlier version of Payara should be a straightforward process: Undeploy Dataverse; stop Payara; move the current Payara directory out of the way; unzip the new Payara version in its place; replace **the brand new `payara/glassfish/domains/domain1` with your old, preserved `domain1`**; start Payara, deploy Dataverse 5.2. We still recommend that you read the detailed upgrade instructions above; and, if you run into any issues with this upgrade, it will help to be able to separate them from any problems with the upgrade of Dataverse proper. -If you are still using pre-5.0 version of Dataverse, and Glassfish version 4, please follow the upgrade instructions in the Dataverse 5.0 release notes; but use the latest version of Payara 5 (5.200.7, as of this writing). +If you are still using pre-5.0 version of Dataverse, and Glassfish version 4, please follow the upgrade instructions in the Dataverse 5.0 release notes; but use the latest version of Payara 5 (5.2020.7, as of this writing). 2\. Undeploy the previous version. From 1f89326d7b5eee6761208294d702383f028d56ca Mon Sep 17 00:00:00 2001 From: Heinz Kramski Date: Sun, 20 Dec 2020 12:12:17 +0100 Subject: [PATCH 042/576] Install missing wget --- scripts/vagrant/setup.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/vagrant/setup.sh b/scripts/vagrant/setup.sh index 8cc2ad466eb..646812b6c26 100644 --- a/scripts/vagrant/setup.sh +++ b/scripts/vagrant/setup.sh @@ -1,6 +1,9 @@ #!/bin/bash echo "Installing dependencies for Dataverse" +# wget seems to be missing in box 'bento/centos-8.2' +dnf install -qy wget + # python3 and psycopg2 for the Dataverse installer dnf install -qy python3 python3-psycopg2 From e381336ab8cd4321d8c325a93b7ea8977ab47426 Mon Sep 17 00:00:00 2001 From: Heinz Kramski Date: Sun, 20 Dec 2020 12:15:05 +0100 Subject: [PATCH 043/576] Fix and extend final setup message --- scripts/vagrant/install-dataverse.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/vagrant/install-dataverse.sh b/scripts/vagrant/install-dataverse.sh index 3acce33eb5f..1d983971c72 100644 --- a/scripts/vagrant/install-dataverse.sh +++ b/scripts/vagrant/install-dataverse.sh @@ -26,4 +26,6 @@ if [ -e tmp-${pid}-default.config ]; then # if we moved it out, move it back mv -f tmp-${pid}-default.config default.config fi -echo "If "vagrant up" was successful (check output above) Dataverse is running on port 8080 of the Linux machine running within Vagrant, but this port has been forwarded to port 8888 of the computer you ran "vagrant up" on. For this reason you should go to http://localhost:8888 to see the Dataverse app running." +echo "If "vagrant up" was successful (check output above) Dataverse is running on port 8080 of the Linux machine running within Vagrant, but this port has been forwarded to port 8088 of the computer you ran "vagrant up" on. For this reason you should go to http://localhost:8088 to see the Dataverse app running." + +echo "Please also note that the installation script has now started Payara, but has not set up an autostart mechanism for it.\nTherefore, the next time this VM is started, Payara must be started manually.\nSee https://guides.dataverse.org/en/latest/installation/prerequisites.html#launching-payara-on-system-boot for details." From e9428623add1ae984dc1eff5bd0668101dec0066 Mon Sep 17 00:00:00 2001 From: Benjamin Peuch Date: Mon, 21 Dec 2020 10:13:02 +0100 Subject: [PATCH 044/576] Update faq.rst --- doc/sphinx-guides/source/api/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/faq.rst b/doc/sphinx-guides/source/api/faq.rst index a3994501f6c..b8e3b97b9dd 100644 --- a/doc/sphinx-guides/source/api/faq.rst +++ b/doc/sphinx-guides/source/api/faq.rst @@ -35,7 +35,7 @@ SWORD only supports a dozen or so operations. The Native API supports many more. To Operate on a Dataset Should I Use Its DOI (or Handle) or Its Database ID? ---------------------------------------------------------------------------- -It is fine to target a datasets using either its Persistent ID (PID such as DOI or Handle) or its database id. +It is fine to target a dataset using either its Persistent ID (PID such as DOI or Handle) or its database id. Here's an example from :ref:`publish-dataset-api` of targeting a dataset using its DOI: From 072500d3c00af2d30ffb7cb482fe287565e156ee Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Sun, 20 Dec 2020 17:14:17 -0500 Subject: [PATCH 045/576] cache endpoint, proxy, use proxy for download, format --- .../iq/dataverse/dataaccess/S3AccessIO.java | 615 +++++++++--------- 1 file changed, 312 insertions(+), 303 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index c0defccfdef..1e42b27f05b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -36,6 +36,8 @@ import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.datavariable.DataVariable; import edu.harvard.iq.dataverse.util.FileUtil; +import opennlp.tools.util.StringUtil; + import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -85,10 +87,15 @@ public S3AccessIO(T dvObject, DataAccessRequest req, String driverId) { this.setIsLocalFile(false); try { - bucketName=getBucketName(driverId); - minPartSize = getMinPartSize(driverId); + bucketName=getBucketName(driverId); + minPartSize = getMinPartSize(driverId); s3=getClient(driverId); tm=getTransferManager(driverId); + endpoint = System.getProperty("dataverse.files." + driverId + ".custom-endpoint-url", ""); + proxy = System.getProperty("dataverse.files." + driverId + ".proxy-url", ""); + if(!StringUtil.isEmpty(proxy)&&StringUtil.isEmpty(endpoint)) { + logger.severe(driverId + " config error: Must specify a custom-endpoint-url if proxy-url is specified"); + } //Not sure this is needed but moving it from the open method for now since it definitely doesn't need to run every time an object is opened. try { if (bucketName == null || !s3.doesBucketExistV2(bucketName)) { @@ -104,18 +111,18 @@ public S3AccessIO(T dvObject, DataAccessRequest req, String driverId) { } } - public S3AccessIO(String storageLocation, String driverId) { - this(null, null, driverId); + public S3AccessIO(String storageLocation, String driverId) { + this(null, null, driverId); // TODO: validate the storage location supplied bucketName = storageLocation.substring(0,storageLocation.indexOf('/')); minPartSize = getMinPartSize(driverId); key = storageLocation.substring(storageLocation.indexOf('/')+1); } - //Used for tests only + //Used for tests only public S3AccessIO(T dvObject, DataAccessRequest req, @NotNull AmazonS3 s3client, String driverId) { super(dvObject, req, driverId); - bucketName = getBucketName(driverId); + bucketName = getBucketName(driverId); this.setIsLocalFile(false); this.s3 = s3client; } @@ -125,6 +132,8 @@ public S3AccessIO(T dvObject, DataAccessRequest req, @NotNull AmazonS3 s3client, private String bucketName = null; private String key = null; private long minPartSize; + private String endpoint = null; + private String proxy= null; @Override public void open(DataAccessOption... options) throws IOException { @@ -160,28 +169,28 @@ public void open(DataAccessOption... options) throws IOException { // So we fix it up here. ToDo: refactor so that storageidentifier is generated by the appropriate StorageIO class and is final from the start. String newStorageIdentifier = null; if (storageIdentifier.startsWith(this.driverId + "://")) { - if(!storageIdentifier.substring((this.driverId + "://").length()).contains(":")) { - //Driver id but no bucket - if(bucketName!=null) { - newStorageIdentifier=this.driverId + "://" + bucketName + ":" + storageIdentifier.substring((this.driverId + "://").length()); - } else { - throw new IOException("S3AccessIO: DataFile (storage identifier " + storageIdentifier + ") is not associated with a bucket."); - } - } // else we're OK (assumes bucket name in storageidentifier matches the driver's bucketname) + if(!storageIdentifier.substring((this.driverId + "://").length()).contains(":")) { + //Driver id but no bucket + if(bucketName!=null) { + newStorageIdentifier=this.driverId + "://" + bucketName + ":" + storageIdentifier.substring((this.driverId + "://").length()); + } else { + throw new IOException("S3AccessIO: DataFile (storage identifier " + storageIdentifier + ") is not associated with a bucket."); + } + } // else we're OK (assumes bucket name in storageidentifier matches the driver's bucketname) } else { - if(!storageIdentifier.substring((this.driverId + "://").length()).contains(":")) { - //No driver id or bucket - newStorageIdentifier= this.driverId + "://" + bucketName + ":" + storageIdentifier; - } else { - //Just the bucketname - newStorageIdentifier= this.driverId + "://" + storageIdentifier; - } + if(!storageIdentifier.substring((this.driverId + "://").length()).contains(":")) { + //No driver id or bucket + newStorageIdentifier= this.driverId + "://" + bucketName + ":" + storageIdentifier; + } else { + //Just the bucketname + newStorageIdentifier= this.driverId + "://" + storageIdentifier; + } } if(newStorageIdentifier != null) { - //Fixup needed: - storageIdentifier = newStorageIdentifier; - dvObject.setStorageIdentifier(newStorageIdentifier); - } + //Fixup needed: + storageIdentifier = newStorageIdentifier; + dvObject.setStorageIdentifier(newStorageIdentifier); + } if (isReadAccess) { @@ -224,36 +233,36 @@ public void open(DataAccessOption... options) throws IOException { } else if (dvObject instanceof Dataverse) { throw new IOException("Data Access: Storage driver does not support dvObject type Dataverse yet"); } else { - // Direct access, e.g. for external upload - no associated DVobject yet, but we want to be able to get the size - // With small files, it looks like we may call before S3 says it exists, so try some retries before failing - if(key!=null) { - ObjectMetadata objectMetadata = null; - int retries = 20; - while(retries > 0) { - try { - objectMetadata = s3.getObjectMetadata(bucketName, key); - if(retries != 20) { - logger.warning("Success for key: " + key + " after " + ((20-retries)*3) + " seconds"); - } - retries = 0; - } catch (SdkClientException sce) { - if(retries > 1) { - retries--; - try { - Thread.sleep(3000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - logger.warning("Retrying after: " + sce.getMessage()); - } else { - throw new IOException("Cannot get S3 object " + key + " ("+sce.getMessage()+")"); - } - } - } + // Direct access, e.g. for external upload - no associated DVobject yet, but we want to be able to get the size + // With small files, it looks like we may call before S3 says it exists, so try some retries before failing + if(key!=null) { + ObjectMetadata objectMetadata = null; + int retries = 20; + while(retries > 0) { + try { + objectMetadata = s3.getObjectMetadata(bucketName, key); + if(retries != 20) { + logger.warning("Success for key: " + key + " after " + ((20-retries)*3) + " seconds"); + } + retries = 0; + } catch (SdkClientException sce) { + if(retries > 1) { + retries--; + try { + Thread.sleep(3000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + logger.warning("Retrying after: " + sce.getMessage()); + } else { + throw new IOException("Cannot get S3 object " + key + " ("+sce.getMessage()+")"); + } + } + } this.setSize(objectMetadata.getContentLength()); - }else { + }else { throw new IOException("Data Access: Invalid DvObject type"); - } + } } } @@ -578,7 +587,7 @@ public void saveInputStreamAsAux(InputStream inputStream, String auxItemTag) thr //Helper method for supporting saving streams with unknown length to S3 //We save those streams to a file and then upload the file - private File createTempFile(Path path, InputStream inputStream) throws IOException { + private File createTempFile(Path path, InputStream inputStream) throws IOException { File targetFile = new File(path.toUri()); // File needs a name try (OutputStream outStream = new FileOutputStream(targetFile);) { @@ -722,8 +731,8 @@ public boolean exists() { if (dvObject instanceof DataFile) { destinationKey = key; } else if((dvObject==null) && (key !=null)) { - //direct access - destinationKey = key; + //direct access + destinationKey = key; } else { logger.warning("Trying to check if a path exists is only supported for a data file."); } @@ -784,42 +793,42 @@ String getDestinationKey(String auxItemTag) throws IOException { */ String getMainFileKey() throws IOException { if (key == null) { - DataFile df = this.getDataFile(); - // TODO: (?) - should we worry here about the datafile having null for the owner here? - key = getMainFileKey(df.getOwner(), df.getStorageIdentifier(), driverId); + DataFile df = this.getDataFile(); + // TODO: (?) - should we worry here about the datafile having null for the owner here? + key = getMainFileKey(df.getOwner(), df.getStorageIdentifier(), driverId); } return key; } static String getMainFileKey(Dataset owner, String storageIdentifier, String driverId) throws IOException { - - // or about the owner dataset having null for the authority and/or identifier? - // we should probably check for that and throw an exception. (unless we are - // super positive that this condition would have been intercepted by now) - String baseKey = owner.getAuthorityForFileStorage() + "/" + owner.getIdentifierForFileStorage(); - return getMainFileKey(baseKey, storageIdentifier, driverId); + + // or about the owner dataset having null for the authority and/or identifier? + // we should probably check for that and throw an exception. (unless we are + // super positive that this condition would have been intercepted by now) + String baseKey = owner.getAuthorityForFileStorage() + "/" + owner.getIdentifierForFileStorage(); + return getMainFileKey(baseKey, storageIdentifier, driverId); } private static String getMainFileKey(String baseKey, String storageIdentifier, String driverId) throws IOException { - String key = null; - if (storageIdentifier == null || "".equals(storageIdentifier)) { - throw new FileNotFoundException("Data Access: No local storage identifier defined for this datafile."); - } - - if (storageIdentifier.indexOf(driverId + "://")>=0) { - //String driverId = storageIdentifier.substring(0, storageIdentifier.indexOf("://")+3); - //As currently implemented (v4.20), the bucket is part of the identifier and we could extract it and compare it with getBucketName() as a check - - //Only one bucket per driver is supported (though things might work if the profile creds work with multiple buckets, then again it's not clear when logic is reading from the driver property or from the DataFile). - //String bucketName = storageIdentifier.substring(driverId.length() + 3, storageIdentifier.lastIndexOf(":")); - key = baseKey + "/" + storageIdentifier.substring(storageIdentifier.lastIndexOf(":") + 1); - } else { - throw new IOException("S3AccessIO: DataFile (storage identifier " + storageIdentifier + ") does not appear to be an S3 object associated with driver: " + driverId); - } - return key; + String key = null; + if (storageIdentifier == null || "".equals(storageIdentifier)) { + throw new FileNotFoundException("Data Access: No local storage identifier defined for this datafile."); + } + + if (storageIdentifier.indexOf(driverId + "://")>=0) { + //String driverId = storageIdentifier.substring(0, storageIdentifier.indexOf("://")+3); + //As currently implemented (v4.20), the bucket is part of the identifier and we could extract it and compare it with getBucketName() as a check - + //Only one bucket per driver is supported (though things might work if the profile creds work with multiple buckets, then again it's not clear when logic is reading from the driver property or from the DataFile). + //String bucketName = storageIdentifier.substring(driverId.length() + 3, storageIdentifier.lastIndexOf(":")); + key = baseKey + "/" + storageIdentifier.substring(storageIdentifier.lastIndexOf(":") + 1); + } else { + throw new IOException("S3AccessIO: DataFile (storage identifier " + storageIdentifier + ") does not appear to be an S3 object associated with driver: " + driverId); + } + return key; } public boolean downloadRedirectEnabled() { - String optionValue = System.getProperty("dataverse.files." + this.driverId + ".download-redirect"); + String optionValue = System.getProperty("dataverse.files." + this.driverId + ".download-redirect"); if ("true".equalsIgnoreCase(optionValue)) { return true; } @@ -873,7 +882,11 @@ public String generateTemporaryS3Url() throws IOException { } if (s != null) { - return s.toString(); + if(!StringUtil.isEmpty(proxy)) { + return s.toString().replace(endpoint, proxy); + } else { + return s.toString(); + } } //throw new IOException("Failed to generate temporary S3 url for "+key); @@ -889,7 +902,7 @@ public String generateTemporaryS3Url() throws IOException { @Deprecated public String generateTemporaryS3UploadUrl() throws IOException { - + key = getMainFileKey(); Date expiration = new Date(); long msec = expiration.getTime(); @@ -899,86 +912,82 @@ public String generateTemporaryS3UploadUrl() throws IOException { return generateTemporaryS3UploadUrl(key, expiration); } - private String generateTemporaryS3UploadUrl(String key, Date expiration) throws IOException { + private String generateTemporaryS3UploadUrl(String key, Date expiration) throws IOException { GeneratePresignedUrlRequest generatePresignedUrlRequest = - new GeneratePresignedUrlRequest(bucketName, key).withMethod(HttpMethod.PUT).withExpiration(expiration); + new GeneratePresignedUrlRequest(bucketName, key).withMethod(HttpMethod.PUT).withExpiration(expiration); //Require user to add this header to indicate a temporary file generatePresignedUrlRequest.putCustomRequestHeader(Headers.S3_TAGGING, "dv-state=temp"); URL presignedUrl; try { - presignedUrl = s3.generatePresignedUrl(generatePresignedUrlRequest); + presignedUrl = s3.generatePresignedUrl(generatePresignedUrlRequest); } catch (SdkClientException sce) { - logger.warning("SdkClientException generating temporary S3 url for "+key+" ("+sce.getMessage()+")"); - presignedUrl = null; + logger.warning("SdkClientException generating temporary S3 url for "+key+" ("+sce.getMessage()+")"); + presignedUrl = null; } String urlString = null; if (presignedUrl != null) { - String endpoint = System.getProperty("dataverse.files." + driverId + ".custom-endpoint-url"); - String proxy = System.getProperty("dataverse.files." + driverId + ".proxy-url"); - if(proxy!=null) { - urlString = presignedUrl.toString().replace(endpoint, proxy); - } else { - urlString = presignedUrl.toString(); - } + if(!StringUtil.isEmpty(proxy)) { + urlString = presignedUrl.toString().replace(endpoint, proxy); + } else { + urlString = presignedUrl.toString(); + } } return urlString; } - public JsonObjectBuilder generateTemporaryS3UploadUrls(String globalId, String storageIdentifier, long fileSize) throws IOException { - - JsonObjectBuilder response = Json.createObjectBuilder(); - key = getMainFileKey(); - java.util.Date expiration = new java.util.Date(); - long msec = expiration.getTime(); - msec += 60 * 1000 * getUrlExpirationMinutes(); - expiration.setTime(msec); - - if (fileSize <= minPartSize) { - response.add("url", generateTemporaryS3UploadUrl(key, expiration)); - } else { - JsonObjectBuilder urls = Json.createObjectBuilder(); - InitiateMultipartUploadRequest initiationRequest = new InitiateMultipartUploadRequest(bucketName, key); - initiationRequest.putCustomRequestHeader(Headers.S3_TAGGING, "dv-state=temp"); - InitiateMultipartUploadResult initiationResponse = s3.initiateMultipartUpload(initiationRequest); - String uploadId = initiationResponse.getUploadId(); - for (int i = 1; i <= (fileSize / minPartSize) + (fileSize % minPartSize > 0 ? 1 : 0); i++) { - GeneratePresignedUrlRequest uploadPartUrlRequest = new GeneratePresignedUrlRequest(bucketName, key) - .withMethod(HttpMethod.PUT).withExpiration(expiration); - uploadPartUrlRequest.addRequestParameter("uploadId", uploadId); - uploadPartUrlRequest.addRequestParameter("partNumber", Integer.toString(i)); - URL presignedUrl; - try { - presignedUrl = s3.generatePresignedUrl(uploadPartUrlRequest); - } catch (SdkClientException sce) { - logger.warning("SdkClientException generating temporary S3 url for " + key + " (" + sce.getMessage() - + ")"); - presignedUrl = null; - } - String urlString = null; - if (presignedUrl != null) { - String endpoint = System.getProperty("dataverse.files." + driverId + ".custom-endpoint-url"); - String proxy = System.getProperty("dataverse.files." + driverId + ".proxy-url"); - if (proxy != null) { - urlString = presignedUrl.toString().replace(endpoint, proxy); - } else { - urlString = presignedUrl.toString(); - } - } - urls.add(Integer.toString(i), urlString); - } - response.add("urls", urls); - response.add("abort", "/api/datasets/mpupload?globalid=" + globalId + "&uploadid=" + uploadId - + "&storageidentifier=" + storageIdentifier); - response.add("complete", "/api/datasets/mpupload?globalid=" + globalId + "&uploadid=" + uploadId - + "&storageidentifier=" + storageIdentifier); - - } - response.add("partSize", minPartSize); - - return response; - } + public JsonObjectBuilder generateTemporaryS3UploadUrls(String globalId, String storageIdentifier, long fileSize) throws IOException { + + JsonObjectBuilder response = Json.createObjectBuilder(); + key = getMainFileKey(); + java.util.Date expiration = new java.util.Date(); + long msec = expiration.getTime(); + msec += 60 * 1000 * getUrlExpirationMinutes(); + expiration.setTime(msec); + + if (fileSize <= minPartSize) { + response.add("url", generateTemporaryS3UploadUrl(key, expiration)); + } else { + JsonObjectBuilder urls = Json.createObjectBuilder(); + InitiateMultipartUploadRequest initiationRequest = new InitiateMultipartUploadRequest(bucketName, key); + initiationRequest.putCustomRequestHeader(Headers.S3_TAGGING, "dv-state=temp"); + InitiateMultipartUploadResult initiationResponse = s3.initiateMultipartUpload(initiationRequest); + String uploadId = initiationResponse.getUploadId(); + for (int i = 1; i <= (fileSize / minPartSize) + (fileSize % minPartSize > 0 ? 1 : 0); i++) { + GeneratePresignedUrlRequest uploadPartUrlRequest = new GeneratePresignedUrlRequest(bucketName, key) + .withMethod(HttpMethod.PUT).withExpiration(expiration); + uploadPartUrlRequest.addRequestParameter("uploadId", uploadId); + uploadPartUrlRequest.addRequestParameter("partNumber", Integer.toString(i)); + URL presignedUrl; + try { + presignedUrl = s3.generatePresignedUrl(uploadPartUrlRequest); + } catch (SdkClientException sce) { + logger.warning("SdkClientException generating temporary S3 url for " + key + " (" + sce.getMessage() + + ")"); + presignedUrl = null; + } + String urlString = null; + if (presignedUrl != null) { + if(!StringUtil.isEmpty(proxy)) { + urlString = presignedUrl.toString().replace(endpoint, proxy); + } else { + urlString = presignedUrl.toString(); + } + } + urls.add(Integer.toString(i), urlString); + } + response.add("urls", urls); + response.add("abort", "/api/datasets/mpupload?globalid=" + globalId + "&uploadid=" + uploadId + + "&storageidentifier=" + storageIdentifier); + response.add("complete", "/api/datasets/mpupload?globalid=" + globalId + "&uploadid=" + uploadId + + "&storageidentifier=" + storageIdentifier); + + } + response.add("partSize", minPartSize); + + return response; + } int getUrlExpirationMinutes() { String optionValue = System.getProperty("dataverse.files." + this.driverId + ".url-expiration-minutes"); @@ -997,174 +1006,174 @@ int getUrlExpirationMinutes() { } private static String getBucketName(String driverId) { - return System.getProperty("dataverse.files." + driverId + ".bucket-name"); + return System.getProperty("dataverse.files." + driverId + ".bucket-name"); } - private static long getMinPartSize(String driverId) { - // as a default, pick 1 GB minimum part size for AWS S3 - // (minimum allowed is 5*1024**2 but it probably isn't worth the complexity starting at ~5MB. Also - confirmed that they use base 2 definitions) - long min = 5 * 1024 * 1024l; - - String partLength = System.getProperty("dataverse.files." + driverId + ".min-part-size"); - try { - if (partLength != null) { - long val = Long.parseLong(partLength); - if(val>=min) { - min=val; - } else { - logger.warning(min + " is the minimum part size allowed for jvm option dataverse.files." + driverId + ".min-part-size" ); - } - } else { - min = 1024 * 1024 * 1024l; - } - } catch (NumberFormatException nfe) { - logger.warning("Unable to parse dataverse.files." + driverId + ".min-part-size as long: " + partLength); - } - return min; - } + private static long getMinPartSize(String driverId) { + // as a default, pick 1 GB minimum part size for AWS S3 + // (minimum allowed is 5*1024**2 but it probably isn't worth the complexity starting at ~5MB. Also - confirmed that they use base 2 definitions) + long min = 5 * 1024 * 1024l; + + String partLength = System.getProperty("dataverse.files." + driverId + ".min-part-size"); + try { + if (partLength != null) { + long val = Long.parseLong(partLength); + if(val>=min) { + min=val; + } else { + logger.warning(min + " is the minimum part size allowed for jvm option dataverse.files." + driverId + ".min-part-size" ); + } + } else { + min = 1024 * 1024 * 1024l; + } + } catch (NumberFormatException nfe) { + logger.warning("Unable to parse dataverse.files." + driverId + ".min-part-size as long: " + partLength); + } + return min; + } private static TransferManager getTransferManager(String driverId) { - if(driverTMMap.containsKey(driverId)) { - return driverTMMap.get(driverId); - } else { - // building a TransferManager instance to support multipart uploading for files over 4gb. - TransferManager manager = TransferManagerBuilder.standard() - .withS3Client(getClient(driverId)) - .build(); - driverTMMap.put(driverId, manager); - return manager; - } + if(driverTMMap.containsKey(driverId)) { + return driverTMMap.get(driverId); + } else { + // building a TransferManager instance to support multipart uploading for files over 4gb. + TransferManager manager = TransferManagerBuilder.standard() + .withS3Client(getClient(driverId)) + .build(); + driverTMMap.put(driverId, manager); + return manager; + } } private static AmazonS3 getClient(String driverId) { - if(driverClientMap.containsKey(driverId)) { - return driverClientMap.get(driverId); - } else { - // get a standard client, using the standard way of configuration the credentials, etc. - AmazonS3ClientBuilder s3CB = AmazonS3ClientBuilder.standard(); - - ClientConfiguration cc = new ClientConfiguration(); - Integer poolSize = Integer.getInteger("dataverse.files." + driverId + ".connection-pool-size", 256); - cc.setMaxConnections(poolSize); - s3CB.setClientConfiguration(cc); - - /** - * Pass in a URL pointing to your S3 compatible storage. - * For possible values see https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/client/builder/AwsClientBuilder.EndpointConfiguration.html - */ - String s3CEUrl = System.getProperty("dataverse.files." + driverId + ".custom-endpoint-url", ""); - /** - * Pass in a region to use for SigV4 signing of requests. - * Defaults to "dataverse" as it is not relevant for custom S3 implementations. - */ - String s3CERegion = System.getProperty("dataverse.files." + driverId + ".custom-endpoint-region", "dataverse"); - - // if the admin has set a system property (see below) we use this endpoint URL instead of the standard ones. - if (!s3CEUrl.isEmpty()) { - s3CB.setEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(s3CEUrl, s3CERegion)); - } - /** - * Pass in a boolean value if path style access should be used within the S3 client. - * Anything but case-insensitive "true" will lead to value of false, which is default value, too. - */ - Boolean s3pathStyleAccess = Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".path-style-access", "false")); - // some custom S3 implementations require "PathStyleAccess" as they us a path, not a subdomain. default = false - s3CB.withPathStyleAccessEnabled(s3pathStyleAccess); - - /** - * Pass in a boolean value if payload signing should be used within the S3 client. - * Anything but case-insensitive "true" will lead to value of false, which is default value, too. - */ - Boolean s3payloadSigning = Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".payload-signing","false")); - /** - * Pass in a boolean value if chunked encoding should not be used within the S3 client. - * Anything but case-insensitive "false" will lead to value of true, which is default value, too. - */ - Boolean s3chunkedEncoding = Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".chunked-encoding","true")); - // Openstack SWIFT S3 implementations require "PayloadSigning" set to true. default = false - s3CB.setPayloadSigningEnabled(s3payloadSigning); - // Openstack SWIFT S3 implementations require "ChunkedEncoding" set to false. default = true - // Boolean is inverted, otherwise setting dataverse.files..chunked-encoding=false would result in leaving Chunked Encoding enabled - s3CB.setChunkedEncodingDisabled(!s3chunkedEncoding); - - /** - * Pass in a string value if this storage driver should use a non-default AWS S3 profile. - * The default is "default" which should work when only one profile exists. - */ - String s3profile = System.getProperty("dataverse.files." + driverId + ".profile","default"); - - s3CB.setCredentials(new ProfileCredentialsProvider(s3profile)); - // let's build the client :-) - AmazonS3 client = s3CB.build(); - driverClientMap.put(driverId, client); - return client; - } + if(driverClientMap.containsKey(driverId)) { + return driverClientMap.get(driverId); + } else { + // get a standard client, using the standard way of configuration the credentials, etc. + AmazonS3ClientBuilder s3CB = AmazonS3ClientBuilder.standard(); + + ClientConfiguration cc = new ClientConfiguration(); + Integer poolSize = Integer.getInteger("dataverse.files." + driverId + ".connection-pool-size", 256); + cc.setMaxConnections(poolSize); + s3CB.setClientConfiguration(cc); + + /** + * Pass in a URL pointing to your S3 compatible storage. + * For possible values see https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/client/builder/AwsClientBuilder.EndpointConfiguration.html + */ + String s3CEUrl = System.getProperty("dataverse.files." + driverId + ".custom-endpoint-url", ""); + /** + * Pass in a region to use for SigV4 signing of requests. + * Defaults to "dataverse" as it is not relevant for custom S3 implementations. + */ + String s3CERegion = System.getProperty("dataverse.files." + driverId + ".custom-endpoint-region", "dataverse"); + + // if the admin has set a system property (see below) we use this endpoint URL instead of the standard ones. + if (!s3CEUrl.isEmpty()) { + s3CB.setEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(s3CEUrl, s3CERegion)); + } + /** + * Pass in a boolean value if path style access should be used within the S3 client. + * Anything but case-insensitive "true" will lead to value of false, which is default value, too. + */ + Boolean s3pathStyleAccess = Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".path-style-access", "false")); + // some custom S3 implementations require "PathStyleAccess" as they us a path, not a subdomain. default = false + s3CB.withPathStyleAccessEnabled(s3pathStyleAccess); + + /** + * Pass in a boolean value if payload signing should be used within the S3 client. + * Anything but case-insensitive "true" will lead to value of false, which is default value, too. + */ + Boolean s3payloadSigning = Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".payload-signing","false")); + /** + * Pass in a boolean value if chunked encoding should not be used within the S3 client. + * Anything but case-insensitive "false" will lead to value of true, which is default value, too. + */ + Boolean s3chunkedEncoding = Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".chunked-encoding","true")); + // Openstack SWIFT S3 implementations require "PayloadSigning" set to true. default = false + s3CB.setPayloadSigningEnabled(s3payloadSigning); + // Openstack SWIFT S3 implementations require "ChunkedEncoding" set to false. default = true + // Boolean is inverted, otherwise setting dataverse.files..chunked-encoding=false would result in leaving Chunked Encoding enabled + s3CB.setChunkedEncodingDisabled(!s3chunkedEncoding); + + /** + * Pass in a string value if this storage driver should use a non-default AWS S3 profile. + * The default is "default" which should work when only one profile exists. + */ + String s3profile = System.getProperty("dataverse.files." + driverId + ".profile","default"); + + s3CB.setCredentials(new ProfileCredentialsProvider(s3profile)); + // let's build the client :-) + AmazonS3 client = s3CB.build(); + driverClientMap.put(driverId, client); + return client; + } } - public void removeTempTag() throws IOException { - if (!(dvObject instanceof DataFile)) { - logger.warning("Attempt to remove tag from non-file DVObject id: " + dvObject.getId()); - throw new IOException("Attempt to remove temp tag from non-file S3 Object"); - } - try { - - key = getMainFileKey(); - DeleteObjectTaggingRequest deleteObjectTaggingRequest = new DeleteObjectTaggingRequest(bucketName, key); - //NOte - currently we only use one tag so delete is the fastest and cheapest way to get rid of that one tag - //Otherwise you have to get tags, remove the one you don't want and post new tags and get charged for the operations + public void removeTempTag() throws IOException { + if (!(dvObject instanceof DataFile)) { + logger.warning("Attempt to remove tag from non-file DVObject id: " + dvObject.getId()); + throw new IOException("Attempt to remove temp tag from non-file S3 Object"); + } + try { + + key = getMainFileKey(); + DeleteObjectTaggingRequest deleteObjectTaggingRequest = new DeleteObjectTaggingRequest(bucketName, key); + //NOte - currently we only use one tag so delete is the fastest and cheapest way to get rid of that one tag + //Otherwise you have to get tags, remove the one you don't want and post new tags and get charged for the operations s3.deleteObjectTagging(deleteObjectTaggingRequest); } catch (SdkClientException sce) { - if(sce.getMessage().contains("Status Code: 501")) { - // In this case, it's likely that tags are not implemented at all (e.g. by Minio) so no tag was set either and it's just something to be aware of - logger.warning("Temp tag not deleted: Object tags not supported by storage: " + driverId); - } else { - // In this case, the assumption is that adding tags has worked, so not removing it is a problem that should be looked into. - logger.severe("Unable to remove temp tag from : " + bucketName + " : " + key); - } + if(sce.getMessage().contains("Status Code: 501")) { + // In this case, it's likely that tags are not implemented at all (e.g. by Minio) so no tag was set either and it's just something to be aware of + logger.warning("Temp tag not deleted: Object tags not supported by storage: " + driverId); + } else { + // In this case, the assumption is that adding tags has worked, so not removing it is a problem that should be looked into. + logger.severe("Unable to remove temp tag from : " + bucketName + " : " + key); + } } catch (IOException e) { - logger.warning("Could not create key for S3 object." ); - e.printStackTrace(); - } - - } - - public static void abortMultipartUpload(String globalId, String storageIdentifier, String uploadId) - throws IOException { - String baseKey = null; - int index = globalId.indexOf(":"); - if (index >= 0) { - baseKey = globalId.substring(index + 1); - } else { - throw new IOException("Invalid Global ID (expected form with ':' prefix)"); - } - String[] info = DataAccess.getDriverIdAndStorageLocation(storageIdentifier); - String driverId = info[0]; - AmazonS3 s3Client = getClient(driverId); - String bucketName = getBucketName(driverId); - String key = getMainFileKey(baseKey, storageIdentifier, driverId); - AbortMultipartUploadRequest req = new AbortMultipartUploadRequest(bucketName, key, uploadId); - s3Client.abortMultipartUpload(req); - } - - public static void completeMultipartUpload(String globalId, String storageIdentifier, String uploadId, - List etags) throws IOException { - String baseKey = null; - int index = globalId.indexOf(":"); - if (index >= 0) { - baseKey = globalId.substring(index + 1); - } else { - throw new IOException("Invalid Global ID (expected form with ':' prefix)"); - } - - String[] info = DataAccess.getDriverIdAndStorageLocation(storageIdentifier); - String driverId = info[0]; - AmazonS3 s3Client = getClient(driverId); - String bucketName = getBucketName(driverId); - String key = getMainFileKey(baseKey, storageIdentifier, driverId); - CompleteMultipartUploadRequest req = new CompleteMultipartUploadRequest(bucketName, key, uploadId, etags); - s3Client.completeMultipartUpload(req); - } + logger.warning("Could not create key for S3 object." ); + e.printStackTrace(); + } + + } + + public static void abortMultipartUpload(String globalId, String storageIdentifier, String uploadId) + throws IOException { + String baseKey = null; + int index = globalId.indexOf(":"); + if (index >= 0) { + baseKey = globalId.substring(index + 1); + } else { + throw new IOException("Invalid Global ID (expected form with ':' prefix)"); + } + String[] info = DataAccess.getDriverIdAndStorageLocation(storageIdentifier); + String driverId = info[0]; + AmazonS3 s3Client = getClient(driverId); + String bucketName = getBucketName(driverId); + String key = getMainFileKey(baseKey, storageIdentifier, driverId); + AbortMultipartUploadRequest req = new AbortMultipartUploadRequest(bucketName, key, uploadId); + s3Client.abortMultipartUpload(req); + } + + public static void completeMultipartUpload(String globalId, String storageIdentifier, String uploadId, + List etags) throws IOException { + String baseKey = null; + int index = globalId.indexOf(":"); + if (index >= 0) { + baseKey = globalId.substring(index + 1); + } else { + throw new IOException("Invalid Global ID (expected form with ':' prefix)"); + } + + String[] info = DataAccess.getDriverIdAndStorageLocation(storageIdentifier); + String driverId = info[0]; + AmazonS3 s3Client = getClient(driverId); + String bucketName = getBucketName(driverId); + String key = getMainFileKey(baseKey, storageIdentifier, driverId); + CompleteMultipartUploadRequest req = new CompleteMultipartUploadRequest(bucketName, key, uploadId, etags); + s3Client.completeMultipartUpload(req); + } } From 74c652c76e6da7ecbb476e679b7cfa519f75a601 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Sun, 20 Dec 2020 17:24:29 -0500 Subject: [PATCH 046/576] document proxy setting FWIW - proxy setup can be complex - not sure that it makes sense to give step by step directions (especially given the different proxy options that exist), but this commit notes the types of settings to worry about. --- doc/sphinx-guides/source/installation/config.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 4a877eabff7..e74419f8527 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -541,6 +541,7 @@ dataverse.files..url-expiration-minutes If direct uploa dataverse.files..min-part-size Multipart direct uploads will occur for files larger than this. Optional. ``1024**3`` dataverse.files..custom-endpoint-url Use custom S3 endpoint. Needs URL either with or without protocol. (none) dataverse.files..custom-endpoint-region Only used when using custom endpoint. Optional. ``dataverse`` +dataverse.files..proxy-url URL of a proxy protecting the S3 store. Optional. (none) dataverse.files..path-style-access ``true``/``false`` Use path style buckets instead of subdomains. Optional. ``false`` dataverse.files..payload-signing ``true``/``false`` Enable payload signing. Optional ``false`` dataverse.files..chunked-encoding ``true``/``false`` Disable chunked encoding. Optional ``true`` @@ -559,6 +560,9 @@ Reported Working S3-Compatible Storage Set ``dataverse.files..payload-signing=true`` and ``dataverse.files..chunked-encoding=false`` to use Surf Object Store. +Note that the ``dataverse.files..proxy-url`` setting can be used in installations where the object store is proxied. For direct uploads and downloads, Dataverse redirects to the proxy-url but presigns the urls using the ``dataverse.files..custom-endpoint-url``. +Additional configuration (appropriate CORS settings, proxy caching/timeout configuration, and proxy settings to pass headers to/from S3 and to avoid adding additional headers) will also be needed to enable use of a proxy with direct upload and download. + **HINT:** If you are successfully using an S3 storage implementation not yet listed above, please feel free to `open an issue at Github `_ and describe your setup. We will be glad to add it here. From c74d6b84e34207714fd2107a7f4a4e0d0f35565f Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 23 Dec 2020 11:17:36 -0500 Subject: [PATCH 047/576] updates per review comments --- .../dataverse/externaltools/ExternalTool.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java index c42aa200378..0ae8c9c496d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java @@ -345,25 +345,25 @@ public String toString() { } public String getDescriptionLang() { - String toolName = ""; + String description = ""; if (this.toolName != null) { - toolName = "externaltools." + this.toolName + ".description"; - return (BundleUtil.getStringFromBundle(toolName)); - } else { - return this.getDescription(); + description = (BundleUtil.getStringFromBundle("externaltools." + this.toolName + ".description")); + } + if (StringUtil.isEmpty(description)) { + description = this.getDescription(); } + return description; } public String getDisplayNameLang() { - String toolName = ""; + String displayName = ""; if (this.toolName != null) { - toolName = "externaltools." + this.toolName + ".displayname"; - toolName = (BundleUtil.getStringFromBundle(toolName)); + displayName = (BundleUtil.getStringFromBundle("externaltools." + this.toolName + ".displayname")); } - if (StringUtil.isEmpty(toolName)) { - toolName = this.getDisplayName(); + if (StringUtil.isEmpty(displayName)) { + displayName = this.getDisplayName(); } - return toolName; + return displayName; } From 00cf232c752b5c625a0f03dbe0b1f9fa4f94da2e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 5 Jan 2021 15:21:01 -0500 Subject: [PATCH 048/576] Revert "change 404 msg to handle cases where the specific thing doesn't exist" This reverts commit ee4af2a346444b23e67dab878004c562faf7b249. We've had this message since the beginning and I'm not comfortable changing it without more discussion with the group. It seems out of scope for the current effort for the PID API. --- .../api/errorhandlers/WebApplicationExceptionHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java index 14e830a8ca0..b6229e58192 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java @@ -60,7 +60,7 @@ public Response toResponse(WebApplicationException ex) { if ( (ex.getMessage()+"").toLowerCase().startsWith("datafile")) { jrb.message(ex.getMessage()); } else { - jrb.message("The resource does not exist on this server. Please check your code for typos, or consult our API guide at http://guides.dataverse.org."); + jrb.message("API endpoint does not exist on this server. Please check your code for typos, or consult our API guide at http://guides.dataverse.org."); jrb.request(request); } break; From 35982b764575553c5a3fdc5f8f0f18f2274dbd8e Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Tue, 5 Jan 2021 15:23:27 -0500 Subject: [PATCH 049/576] Checksum truncate and click to copy added to file table on dataset pg [ref #5210] --- src/main/java/propertyFiles/Bundle.properties | 2 +- src/main/webapp/editFilesFragment.xhtml | 26 +++++----- src/main/webapp/file-info-fragment.xhtml | 12 ++--- src/main/webapp/filesFragment.xhtml | 2 +- .../resources/js/dv_rebind_bootstrap_ui.js | 50 ++++++++----------- 5 files changed, 41 insertions(+), 51 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 79af8899cc9..b27ab777ba6 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1557,7 +1557,7 @@ file.rsyncUpload.httpUploadDisabledDueToRsyncFileExisting=HTTP upload is disable file.rsyncUpload.httpUploadDisabledDueToRsyncFileExistingAndPublished=HTTP upload is disabled for this dataset because you have already uploaded files via rsync and published the dataset. file.rsyncUpload.rsyncUploadDisabledDueFileUploadedViaHttp=Upload with rsync + SSH is disabled for this dataset because you have already uploaded files via HTTP. If you would like to switch to rsync upload, then you must first remove all uploaded files from this dataset. Once this dataset is published, the chosen upload method is permanently locked in. file.rsyncUpload.rsyncUploadDisabledDueFileUploadedViaHttpAndPublished=Upload with rsync + SSH is disabled for this dataset because you have already uploaded files via HTTP and published the dataset. - +file.metaData.checksum.copy=Click to copy file.metaData.dataFile.dataTab.variables=Variables file.metaData.dataFile.dataTab.observations=Observations file.addDescription=Add file description... diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index 833a6c2fd07..5ad2dc6dab0 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -326,8 +326,8 @@ -
    -
    +
    +
    -
    +
    @@ -369,20 +369,20 @@ style="width:60%; margin-bottom:.5em;" disabled="#{fileMetadata.dataFile.filePackage}" widgetVar="fileDirectoryName"/> -
    + -
    - +
    + +
    + +
    + + +
    -
    - -
    - - -
    - +
    -
    +
    @@ -52,15 +52,15 @@ -
    +
    #{fileMetadata.dataFile.checksumType}: #{fileMetadata.dataFile.checksumValue} - - + data-clipboard-action="copy" + data-clipboard-text="#{fileMetadata.dataFile.checksumValue}" + title="#{bundle['file.metaData.checksum.copy']} #{fileMetadata.dataFile.checksumValue}">#{fileMetadata.dataFile.checksumType}: #{fileMetadata.dataFile.checksumValue}
    -
    +
    diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index b83d53f89a4..9f958cd451c 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -370,7 +370,7 @@             - +
    '); + var clipboard = new Clipboard('span.checksum-truncate'); + clipboard.on('success', function(e) { + console.log(e); + }); + clipboard.on('error', function(e) { + console.log(e); + }); + }); +} /* * Select dataset/file citation onclick event From c359cbe5ff69fed6197cd5b1098f85fe61f1e986 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 5 Jan 2021 15:56:20 -0500 Subject: [PATCH 050/576] expose more detail of 404 from DataCite via API #7187 --- .../java/edu/harvard/iq/dataverse/api/Pids.java | 3 +++ .../harvard/iq/dataverse/pidproviders/PidUtil.java | 13 ++++++++++++- .../iq/dataverse/pidproviders/PidUtilTest.java | 11 ++++++++--- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Pids.java b/src/main/java/edu/harvard/iq/dataverse/api/Pids.java index 85b4928d521..5a2acf3209f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Pids.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Pids.java @@ -15,6 +15,7 @@ import javax.ws.rs.DELETE; import javax.ws.rs.GET; import javax.ws.rs.InternalServerErrorException; +import javax.ws.rs.NotFoundException; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -51,6 +52,8 @@ public Response getPid(@QueryParam("persistentId") String persistentId) { try { JsonObjectBuilder result = PidUtil.queryDoi(persistentId, baseUrl, username, password); return ok(result); + } catch (NotFoundException ex) { + return error(ex.getResponse().getStatusInfo().toEnum(), ex.getLocalizedMessage()); } catch (InternalServerErrorException ex) { return error(ex.getResponse().getStatusInfo().toEnum(), ex.getLocalizedMessage()); } diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidUtil.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidUtil.java index 3365ced4c38..f9b451ecab2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidUtil.java @@ -25,6 +25,17 @@ public class PidUtil { private static final Logger logger = Logger.getLogger(PidUtil.class.getCanonicalName()); + /** + * @throws BadRequestException if user didn't supply a DOI. + * + * @throws NotFoundException if DOI not found in DataCite. + * + * @throws ServiceUnavailableException if non 200 or non 404 response from + * DataCite. + * + * @throws InternalServerErrorException on local misconfiguration such as + * DataCite hostname not in DNS. + */ public static JsonObjectBuilder queryDoi(String persistentId, String baseUrl, String username, String password) { try { // This throws an exception if this is not a DOI, which is the only @@ -53,7 +64,7 @@ public static JsonObjectBuilder queryDoi(String persistentId, String baseUrl, St } if (status == 404) { //Could check to see if Dataverse expects the DOI to be registered - that would result in a 404 from Dataverse before having to contact DataCite, and DataCite could still return a 404 - throw new NotFoundException(); + throw new NotFoundException("404 (NOT FOUND) from DataCite for DOI " + persistentId); } if (status != 200) { /* We could just send back whatever status code DataCite sends, but we've seen diff --git a/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java index b3b1f328d39..fc90d846bf6 100644 --- a/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java @@ -3,6 +3,7 @@ import edu.harvard.iq.dataverse.util.json.JsonUtil; import java.io.IOException; import javax.json.JsonObjectBuilder; +import javax.ws.rs.NotFoundException; import org.junit.Test; import org.junit.Ignore; @@ -19,9 +20,13 @@ public void testGetDoi() throws IOException { String baseUrl = "https://api.test.datacite.org"; String pid = ""; pid = "doi:10.70122/QE5A-XN55"; - JsonObjectBuilder result = PidUtil.queryDoi(pid, baseUrl, username, password); - String out = JsonUtil.prettyPrint(result.build()); - System.out.println("out: " + out); + try { + JsonObjectBuilder result = PidUtil.queryDoi(pid, baseUrl, username, password); + String out = JsonUtil.prettyPrint(result.build()); + System.out.println("out: " + out); + } catch (NotFoundException ex) { + System.out.println("ex: " + ex); + } } } From 8b9bb2c8c8c9dfd3ed036516566b54fee9045f5c Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 5 Jan 2021 17:10:31 -0500 Subject: [PATCH 051/576] updates for proper usage of "Dataverse" --- doc/sphinx-guides/source/admin/backups.rst | 4 +- doc/sphinx-guides/source/admin/dashboard.rst | 6 +-- .../source/admin/dataverses-datasets.rst | 48 +++++++++---------- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/doc/sphinx-guides/source/admin/backups.rst b/doc/sphinx-guides/source/admin/backups.rst index e11aa84ebfc..49cc9760622 100644 --- a/doc/sphinx-guides/source/admin/backups.rst +++ b/doc/sphinx-guides/source/admin/backups.rst @@ -4,8 +4,8 @@ Backups .. contents:: Contents: :local: -Running tape, or similar backups to ensure the long term preservation of all the data stored in the Dataverse is an implied responsibility that should be taken most seriously. +Running tape, or similar backups to ensure the long term preservation of all the data stored in the Dataverse Repository is an implied responsibility that should be taken most seriously. -*In addition* to running these disk-level backups, we have provided an experimental script that can be run on schedule (via a cron job or something similar) to create extra archival copies of all the Datafiles stored in the Dataverse on a remote storage server, accessible via an ssh connection. The script and some documentation can be found in ``scripts/backup/run_backup`` in the Dataverse source tree at https://github.com/IQSS/dataverse . Some degree of knowledge of system administration and Python is required. +*In addition* to running these disk-level backups, we have provided an experimental script that can be run on schedule (via a cron job or something similar) to create extra archival copies of all the Datafiles stored in the Dataverse Repository on a remote storage server, accessible via an ssh connection. The script and some documentation can be found in ``scripts/backup/run_backup`` in the Dataverse Project source tree at https://github.com/IQSS/dataverse . Some degree of knowledge of system administration and Python is required. Once again, the script is experimental and NOT a replacement of regular and reliable disk backups! diff --git a/doc/sphinx-guides/source/admin/dashboard.rst b/doc/sphinx-guides/source/admin/dashboard.rst index 5ee53790471..a3ab90110ea 100644 --- a/doc/sphinx-guides/source/admin/dashboard.rst +++ b/doc/sphinx-guides/source/admin/dashboard.rst @@ -1,7 +1,7 @@ Dashboard ========= -Dataverse offers a dashboard of administrative tools for superusers only. If you are a logged-in superuser, you can access it by clicking your username in the navbar, and then clicking "Dashboard" from the dropdown. You can verify that you are a superuser by checking the color of your username in the navbar. If it's red, you have the right permissions to use the Dashboard. Superusers can give other users the superuser status via :doc:`user-administration`. +The Dataverse Platform offers a dashboard of administrative tools for superusers only. If you are a logged-in superuser, you can access it by clicking your username in the navbar, and then clicking "Dashboard" from the dropdown. You can verify that you are a superuser by checking the color of your username in the navbar. If it's red, you have the right permissions to use the Dashboard. Superusers can give other users the superuser status via :doc:`user-administration`. .. contents:: Contents: :local: @@ -22,7 +22,7 @@ This dashboard tool allows you to define sets of local datasets to make availabl Metadata Export --------------- -This part of the Dashboard is simply a reminder message that metadata export happens through the Harvard Dataverse API. See the :doc:`metadataexport` section and the :doc:`/api/native-api` section of the API Guide for more details. +This part of the Dashboard is simply a reminder message that metadata export happens through the Dataverse Platform API. See the :doc:`metadataexport` section and the :doc:`/api/native-api` section of the API Guide for more details. Users ----- @@ -32,4 +32,4 @@ This dashboard tool allows you to search a list of all users of your Dataverse i Move Data --------- -This tool allows you to move datasets. To move dataverses, see the :doc:`dataverses-datasets` section. +This tool allows you to move datasets. To move Dataverse collections, see the :doc:`dataverses-datasets` section. diff --git a/doc/sphinx-guides/source/admin/dataverses-datasets.rst b/doc/sphinx-guides/source/admin/dataverses-datasets.rst index 9c122c25abc..1fe70e34997 100644 --- a/doc/sphinx-guides/source/admin/dataverses-datasets.rst +++ b/doc/sphinx-guides/source/admin/dataverses-datasets.rst @@ -4,46 +4,46 @@ Managing Datasets and Dataverses .. contents:: |toctitle| :local: -Dataverses ----------- +Dataverse Collections +--------------------- -Delete a Dataverse -^^^^^^^^^^^^^^^^^^ +Delete a Dataverse Collection +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Dataverses have to be empty to delete them. Navigate to the dataverse and click "Edit" and then "Delete Dataverse" to delete it. To delete a dataverse via API, see the :doc:`/api/native-api` section of the API Guide. +Dataverse collections have to be empty to delete them. Navigate to the Dataverse collection and click "Edit" and then "Delete Dataverse" to delete it. To delete a Dataverse collection via API, see the :doc:`/api/native-api` section of the API Guide. -Move a Dataverse -^^^^^^^^^^^^^^^^ +Move a Dataverse Collection +^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Moves a dataverse whose id is passed to a new dataverse whose id is passed. The dataverse alias also may be used instead of the id. If the moved dataverse has a guestbook, template, metadata block, link, or featured dataverse that is not compatible with the destination dataverse, you will be informed and given the option to force the move and remove the association. Only accessible to superusers. :: +Moves a Dataverse collection whose id is passed to a new Dataverse collection whose id is passed. The Dataverse collection alias also may be used instead of the id. If the moved Dataverse collection has a guestbook, template, metadata block, link, or featured Dataverse collection that is not compatible with the destination Dataverse collection, you will be informed and given the option to force the move and remove the association. Only accessible to superusers. :: curl -H "X-Dataverse-key: $API_TOKEN" -X POST http://$SERVER/api/dataverses/$id/move/$destination-id -Link a Dataverse -^^^^^^^^^^^^^^^^ +Link a Dataverse Collection +^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Creates a link between a dataverse and another dataverse (see the :ref:`dataverse-linking` section of the User Guide for more information). Only accessible to superusers. :: +Creates a link between a Dataverse collection and another Dataverse collection (see the :ref:`dataverse-linking` section of the User Guide for more information). Only accessible to superusers. :: curl -H "X-Dataverse-key: $API_TOKEN" -X PUT http://$SERVER/api/dataverses/$linked-dataverse-alias/link/$linking-dataverse-alias -Unlink a Dataverse -^^^^^^^^^^^^^^^^^^ +Unlink a Dataverse Collection +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Removes a link between a dataverse and another dataverse. Only accessible to superusers. :: +Removes a link between a Dataverse collection and another Dataverse collection. Only accessible to superusers. :: curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE http://$SERVER/api/dataverses/$linked-dataverse-alias/deleteLink/$linking-dataverse-alias -Add Dataverse RoleAssignments to Child Dataverses -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Add Dataverse Collection RoleAssignments to Dataverse Subcollections +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Recursively assigns the users and groups having a role(s),that are in the set configured to be inheritable via the :InheritParentRoleAssignments setting, on a specified dataverse to have the same role assignments on all of the dataverses that have been created within it. The response indicates success or failure and lists the individuals/groups and dataverses involved in the update. Only accessible to superusers. :: +Recursively assigns the users and groups having a role(s),that are in the set configured to be inheritable via the :InheritParentRoleAssignments setting, on a specified Dataverse collections to have the same role assignments on all of the Dataverse collections that have been created within it. The response indicates success or failure and lists the individuals/groups and Dataverse collections involved in the update. Only accessible to superusers. :: curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/admin/dataverse/$dataverse-alias/addRoleAssignmentsToChildren -Configure a Dataverse to store all new files in a specific file store -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Configure a Dataverse Collection to store all new files in a specific file store +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -To direct new files (uploaded when datasets are created or edited) for all datasets in a given dataverse, the store can be specified via the API as shown below, or by editing the 'General Information' for a Dataverse on the Dataverse page. Only accessible to superusers. :: +To direct new files (uploaded when datasets are created or edited) for all datasets in a given Dataverse collection, the store can be specified via the API as shown below, or by editing the 'General Information' for a Dataverse collection on the Dataverse collection page. Only accessible to superusers. :: curl -H "X-Dataverse-key: $API_TOKEN" -X PUT -d $storageDriverLabel http://$SERVER/api/admin/dataverse/$dataverse-alias/storageDriver @@ -70,21 +70,21 @@ Move a Dataset Superusers can move datasets using the dashboard. See also :doc:`dashboard`. -Moves a dataset whose id is passed to a dataverse whose alias is passed. If the moved dataset has a guestbook or a dataverse link that is not compatible with the destination dataverse, you will be informed and given the option to force the move (with ``forceMove=true`` as a query parameter) and remove the guestbook or link (or both). Only accessible to users with permission to publish the dataset in the original and destination dataverse. :: +Moves a dataset whose id is passed to a Dataverse collection whose alias is passed. If the moved dataset has a guestbook or a Dataverse collection link that is not compatible with the destination Dataverse collection, you will be informed and given the option to force the move (with ``forceMove=true`` as a query parameter) and remove the guestbook or link (or both). Only accessible to users with permission to publish the dataset in the original and destination Dataverse collection. :: curl -H "X-Dataverse-key: $API_TOKEN" -X POST http://$SERVER/api/datasets/$id/move/$alias Link a Dataset ^^^^^^^^^^^^^^ -Creates a link between a dataset and a dataverse (see the :ref:`dataset-linking` section of the User Guide for more information). :: +Creates a link between a dataset and a Dataverse collection (see the :ref:`dataset-linking` section of the User Guide for more information). :: curl -H "X-Dataverse-key: $API_TOKEN" -X PUT http://$SERVER/api/datasets/$linked-dataset-id/link/$linking-dataverse-alias Unlink a Dataset ^^^^^^^^^^^^^^^^ -Removes a link between a dataset and a dataverse. Only accessible to superusers. :: +Removes a link between a dataset and a Dataverse collection. Only accessible to superusers. :: curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE http://$SERVER/api/datasets/$linked-dataset-id/deleteLink/$linking-dataverse-alias @@ -130,7 +130,7 @@ As a superuser, click "Update Current Version" when publishing. (This option is Diagnose Constraint Violations Issues in Datasets ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -To identify invalid data values in specific datasets (if, for example, an attempt to edit a dataset results in a ConstraintViolationException in the server log), or to check all the datasets in the Dataverse for constraint violations, see :ref:`Dataset Validation ` in the :doc:`/api/native-api` section of the User Guide. +To identify invalid data values in specific datasets (if, for example, an attempt to edit a dataset results in a ConstraintViolationException in the server log), or to check all the datasets in the Dataverse installation for constraint violations, see :ref:`Dataset Validation ` in the :doc:`/api/native-api` section of the User Guide. Configure a Dataset to store all new files in a specific file store ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 694364f9bdefb3ca36513bb0c0c73a27a67bb7fd Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 5 Jan 2021 17:17:55 -0500 Subject: [PATCH 052/576] updates for proper usage of Dataverse --- .../source/admin/external-tools.rst | 28 +++++++++---------- .../source/admin/geoconnect-worldmap.rst | 10 +++---- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/doc/sphinx-guides/source/admin/external-tools.rst b/doc/sphinx-guides/source/admin/external-tools.rst index 3e1965c48ea..4426ab2d37f 100644 --- a/doc/sphinx-guides/source/admin/external-tools.rst +++ b/doc/sphinx-guides/source/admin/external-tools.rst @@ -1,7 +1,7 @@ External Tools ============== -External tools can provide additional features that are not part of Dataverse itself, such as data file previews, visualization, and curation. +External tools can provide additional features that are not part of the Dataverse Platform itself, such as data file previews, visualization, and curation. .. contents:: |toctitle| :local: @@ -22,10 +22,10 @@ Inventory of External Tools Managing External Tools ----------------------- -Adding External Tools to Dataverse -+++++++++++++++++++++++++++++++++++ +Adding External Tools to a Dataverse Installation ++++++++++++++++++++++++++++++++++++++++++++++++++ -To add an external tool to your installation of Dataverse you must first download a JSON file for that tool, which we refer to as a "manifest". It should look something like this: +To add an external tool to your Dataverse installation you must first download a JSON file for that tool, which we refer to as a "manifest". It should look something like this: .. literalinclude:: ../_static/installation/files/root/external-tools/fabulousFileTool.json @@ -37,27 +37,27 @@ Configure the tool with the curl command below, making sure to replace the ``fab curl -X POST -H 'Content-type: application/json' http://localhost:8080/api/admin/externalTools --upload-file fabulousFileTool.json -Listing All External Tools in Dataverse -+++++++++++++++++++++++++++++++++++++++ +Listing All External Tools in a Dataverse Installation +++++++++++++++++++++++++++++++++++++++++++++++++++++++ -To list all the external tools that are available in Dataverse: +To list all the external tools that are available in a Dataverse installation: .. code-block:: bash curl http://localhost:8080/api/admin/externalTools -Showing an External Tool in Dataverse -+++++++++++++++++++++++++++++++++++++ +Showing an External Tool in a Dataverse Installation +++++++++++++++++++++++++++++++++++++++++++++++++++++ -To show one of the external tools that are available in Dataverse, pass its database id: +To show one of the external tools that are available in a Dataverse installation, pass its database id: .. code-block:: bash export TOOL_ID=1 curl http://localhost:8080/api/admin/externalTools/$TOOL_ID -Removing an External Tool From Dataverse -++++++++++++++++++++++++++++++++++++++++ +Removing an External Tool From a Dataverse Installation ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Assuming the external tool database id is "1", remove it with the following command: @@ -71,7 +71,7 @@ Assuming the external tool database id is "1", remove it with the following comm Testing External Tools ---------------------- -Once you have added an external tool to your installation of Dataverse, you will probably want to test it to make sure it is functioning properly. +Once you have added an external tool to your Dataverse installation, you will probably want to test it to make sure it is functioning properly. File Level vs. Dataset Level ++++++++++++++++++++++++++++ @@ -114,4 +114,4 @@ Writing Your Own External Tool If you plan to write a external tool, see the :doc:`/api/external-tools` section of the API Guide. -If you have an idea for an external tool, please let the Dataverse community know by posting about it on the dataverse-community mailing list: https://groups.google.com/forum/#!forum/dataverse-community +If you have an idea for an external tool, please let the Dataverse Project community know by posting about it on the dataverse-community mailing list: https://groups.google.com/forum/#!forum/dataverse-community diff --git a/doc/sphinx-guides/source/admin/geoconnect-worldmap.rst b/doc/sphinx-guides/source/admin/geoconnect-worldmap.rst index fb8250c0650..87dad651c3e 100644 --- a/doc/sphinx-guides/source/admin/geoconnect-worldmap.rst +++ b/doc/sphinx-guides/source/admin/geoconnect-worldmap.rst @@ -1,7 +1,7 @@ Geoconnect and WorldMap ======================= -One of the optional components listed under "Architecture and Components" in the :doc:`/installation/prep` section of the Installation Guide is `Geoconnect `_, a piece of middleware that allows Dataverse users to create maps in `WorldMap `_ based on geospatial data stored in Dataverse. For more details on the feature from the user perspective, see the :doc:`/user/data-exploration/worldmap` section of the User Guide. +One of the optional components listed under "Architecture and Components" in the :doc:`/installation/prep` section of the Installation Guide is `Geoconnect `_, a piece of middleware that allows users to create maps in `WorldMap `_ based on geospatial data stored in a Dataverse installation. For more details on the feature from the user perspective, see the :doc:`/user/data-exploration/worldmap` section of the User Guide. .. contents:: |toctitle| :local: @@ -35,18 +35,18 @@ SQL commands to point a Dataverse installation at different Geoconnect servers: Removing Dead Explore Links --------------------------- -After a map has been created in WorldMap (assuming all the setup has been done), in Dataverse the file will display WorldMap as an explore option. In rare occasions, the map has been deleted on the WorldMap resulting in a dead link, a 404 page not found error. +After a map has been created in WorldMap (assuming all the setup has been done), in a Dataverse installation the file will display WorldMap as an explore option. In rare occasions, the map has been deleted on the WorldMap resulting in a dead link, a 404 page not found error. -Functionality has been added on the Dataverse side to iterate through all the maps Dataverse knows about (stored in the ``maplayermetadata`` database table) and to check for the existence of each map in WorldMap. The status code returned from WorldMap (200, 404, etc.) is recorded in Dataverse along with a timestamp of when the check was performed. To perform this check, you can execute the following ``curl`` command: +Functionality has been added on the Dataverse Platform side to iterate through all the maps the Dataverse installation knows about (stored in the ``maplayermetadata`` database table) and to check for the existence of each map in WorldMap. The status code returned from WorldMap (200, 404, etc.) is recorded in the Dataverse installation along with a timestamp of when the check was performed. To perform this check, you can execute the following ``curl`` command: ``curl -X POST http://localhost:8080/api/admin/geoconnect/mapLayerMetadatas/check`` The output above will contain the ``layerLink`` being checked as well as the HTTP response status code (200, 404, etc.) in the ``lastVerifiedStatus`` field. 200 means OK and 404 means not found. 500 might indicate that the map is only temporarily unavailable. The ``lastVerifiedStatus`` and ``lastVerifiedTime`` will be persisted to the ``maplayermetadata`` database table. -Armed with this information about WorldMap returning a 404 for a map, you may want to delete any record of the map on the Dataverse side removing WorldMap as an explore option (and so that thumbnail files are cleaned up). To accomplish this, use the following ``curl`` command, substituting the id of the file: +Armed with this information about WorldMap returning a 404 for a map, you may want to delete any record of the map on the Dataverse installation side removing WorldMap as an explore option (and so that thumbnail files are cleaned up). To accomplish this, use the following ``curl`` command, substituting the id of the file: ``curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE http://localhost:8080/api/files/{file_id}/map`` -End users can also run the ``DELETE`` command above (if they have permission to edit the dataset) but it's more likely that the sysadmin reading this guide will run it for them. It is recommended that you add the "check" command above to cron so that Dataverse periodically checks on the status of maps in WorldMap. In addition to the "check all maps" command above, you can also check an individual map with the following ``curl`` command, substituting the id of the row from the ``maplayermetadata`` table: +End users can also run the ``DELETE`` command above (if they have permission to edit the dataset) but it's more likely that the sysadmin reading this guide will run it for them. It is recommended that you add the "check" command above to cron so that the Dataverse installation periodically checks on the status of maps in WorldMap. In addition to the "check all maps" command above, you can also check an individual map with the following ``curl`` command, substituting the id of the row from the ``maplayermetadata`` table: ``curl -X POST http://localhost:8080/api/admin/geoconnect/mapLayerMetadatas/check/{id}`` From 49e9f8e24bb843506c356c7a7fefd3e296a9e44b Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 5 Jan 2021 19:40:19 -0500 Subject: [PATCH 053/576] Update harvestserver.rst --- .../source/admin/harvestserver.rst | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/doc/sphinx-guides/source/admin/harvestserver.rst b/doc/sphinx-guides/source/admin/harvestserver.rst index f1436926ea2..0936ba4bd36 100644 --- a/doc/sphinx-guides/source/admin/harvestserver.rst +++ b/doc/sphinx-guides/source/admin/harvestserver.rst @@ -4,12 +4,11 @@ Managing Harvesting Server and Sets .. contents:: |toctitle| :local: -Your Dataverse as an OAI server +Your Dataverse Installation as an OAI server ------------------------------- -As a harvesting *server*, your Dataverse can make some of the local -dataset metadata available to remote harvesting clients. These can be -other Dataverse instances, or any other clients that support OAI-PMH +As a harvesting *server*, your Dataverse installation can make some of the local dataset metadata available to remote harvesting clients. These can be +other Dataverse installations, or any other clients that support OAI-PMH harvesting protocol. Note that the terms "Harvesting Server" and "OAI Server" are being used interchangeably throughout this guide and in the inline help text. @@ -18,7 +17,7 @@ If you want to learn more about OAI-PMH, you could take a look at `DataCite OAI-PMH guide `_ or the `OAI-PMH protocol definition `_. -You might consider adding your OAI-enabled production instance of Dataverse to +You might consider adding your OAI-enabled Dataverse installation to `this shared list `_ of such instances. @@ -27,7 +26,7 @@ The email portion of :ref:`systemEmail` will be visible via OAI-PMH (from the "I How does it work? ----------------- -Only the published, unrestricted datasets in your Dataverse can +Only the published, unrestricted datasets in your Dataverse installation can be made harvestable. Remote clients normally keep their records in sync through scheduled incremental updates, daily or weekly, thus minimizing the load on your server. Note that it is only the metadata @@ -36,9 +35,9 @@ download the data files associated with the harvested datasets. Harvesting server can be enabled or disabled on the "Harvesting Server" page accessible via the :doc:`dashboard`. Harvesting server is by -default disabled on a brand new, "out of the box" Dataverse. +default disabled on a brand new, "out of the box" Dataverse installation. -The OAI-PMH endpoint can be accessed at ``http(s):///oai``. +The OAI-PMH endpoint can be accessed at ``http(s):///oai``. If you want other services to harvest your repository, point them to this URL. *Example URL for 'Identify' verb*: `demo.dataverse.org OAI `_ @@ -50,8 +49,7 @@ that will be available to remote harvesters as *OAI Sets*. Once again, the terms "OAI Set" and "Harvesting Set" are used interchangeably. Sets are defined by search queries. Any such query that finds any number of published, local (non-harvested) datasets can -be used to create an OAI set. Sets can overlap local dataverses, and -can include as few or as many of your local datasets as you wish. A +be used to create an OAI set. Sets can overlap local Dataverse collections, and can include as few or as many of your local datasets as you wish. A good way to master the Dataverse search query language is to experiment with the Advanced Search page. We also recommend that you consult the :doc:`/api/search` section of the API Guide. From 337998d0e5154dad662f64229acc6cde3669ee02 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 5 Jan 2021 23:01:20 -0500 Subject: [PATCH 054/576] Update harvestserver.rst --- .../source/admin/harvestserver.rst | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/doc/sphinx-guides/source/admin/harvestserver.rst b/doc/sphinx-guides/source/admin/harvestserver.rst index 0936ba4bd36..3550599a67c 100644 --- a/doc/sphinx-guides/source/admin/harvestserver.rst +++ b/doc/sphinx-guides/source/admin/harvestserver.rst @@ -50,7 +50,7 @@ the terms "OAI Set" and "Harvesting Set" are used interchangeably. Sets are defined by search queries. Any such query that finds any number of published, local (non-harvested) datasets can be used to create an OAI set. Sets can overlap local Dataverse collections, and can include as few or as many of your local datasets as you wish. A -good way to master the Dataverse search query language is to +good way to master the Dataverse Platform search query language is to experiment with the Advanced Search page. We also recommend that you consult the :doc:`/api/search` section of the API Guide. @@ -61,7 +61,7 @@ can go back and try to re-define the query. Some useful examples of search queries to define OAI sets: -- A good way to create a set that would include all your local, published datasets is to do so by the Unique Identifier authority registered to your Dataverse, for example: +- A good way to create a set that would include all your local, published datasets is to do so by the Unique Identifier authority registered to your Dataverse installation, for example: ``dsPersistentId:"doi:1234/"`` @@ -69,11 +69,11 @@ Some useful examples of search queries to define OAI sets: Note also that the search terms limiting the results to published and local datasets **are added to the query automatically**, so you don't need to worry about that. -- A query to create a set to include the datasets from a specific local dataverse: +- A query to create a set to include the datasets from a specific Dataverse collection: ``parentId:NNN`` - where NNN is the database id of the dataverse object (consult the Dataverse table of the SQL database used by the application to verify the database id). + where NNN is the database id of the Dataverse collection object (consult the Dataverse table of the SQL database used by the application to verify the database id). - A query to find all the dataset by a certain author: @@ -95,7 +95,7 @@ Some useful examples of search queries to define OAI sets: ``dsPersistentId:10.5000/XXYYZZ`` - For all dataverses with subjects of Astronomy and Astrophysics or Earth and Environmental Sciences: + For all Dataverse collections with subjects of Astronomy and Astrophysics or Earth and Environmental Sciences: ``(dvSubject:"Astronomy and Astrophysics" OR dvSubject:"Earth and Environmental Sciences")`` @@ -106,14 +106,14 @@ Some useful examples of search queries to define OAI sets: Important: New SOLR schema required! ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In order to be able to define OAI sets, your SOLR server must be upgraded with the search schema that came with the Dataverse release 4.5 (or later), and all your local datasets must be re-indexed, once the new schema is installed. +In order to be able to define OAI sets, your SOLR server must be upgraded with the search schema that came with release 4.5 (or later), and all your local datasets must be re-indexed, once the new schema is installed. OAI Set updates --------------- Every time a new harvesting set is created, or changes are made to an existing set, the contents of the set are automatically updated - the -Dataverse application will find the datasets defined by the query, and +Dataverse installation will find the datasets defined by the query, and attempt to run the metadata export on the ones that haven't been exported yet. Only the datasets for which the export has completed successfully, and the results cached on the filesystem are included in @@ -134,9 +134,7 @@ the sets every time any changes to the metadata are made. The OAI set will however be updated automatically by a scheduled metadata export job that runs every night (at 2AM, by default). This export timer is created and activated automatically every time the application is deployed -or restarted. Once again, this is new in Dataverse 4, and unlike DVN -v3, where export jobs had to be scheduled and activated by the admin -user. See the :doc:`/admin/metadataexport` section of the Admin Guide, for more information on the automated metadata exports. +or restarted. See the :doc:`/admin/metadataexport` section of the Admin Guide, for more information on the automated metadata exports. It is still possible however to make changes like this be immediately reflected in the OAI server, by going to the *Harvesting Server* page From d72a25c606049ad6c2baada8c194f9babc538fb8 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 5 Jan 2021 23:02:51 -0500 Subject: [PATCH 055/576] Update harvestclients.rst --- doc/sphinx-guides/source/admin/harvestclients.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/admin/harvestclients.rst b/doc/sphinx-guides/source/admin/harvestclients.rst index 26af8eee32c..a3de6690b9f 100644 --- a/doc/sphinx-guides/source/admin/harvestclients.rst +++ b/doc/sphinx-guides/source/admin/harvestclients.rst @@ -4,10 +4,10 @@ Managing Harvesting Clients .. contents:: |toctitle| :local: -Your Dataverse as a Metadata Harvester --------------------------------------- +Your Dataverse Installation as a Metadata Harvester +--------------------------------------------------- -Harvesting is a process of exchanging metadata with other repositories. As a harvesting *client*, your Dataverse can gather metadata records from remote sources. These can be other Dataverse instances or other archives that support OAI-PMH, the standard harvesting protocol. Harvested metadata records will be indexed and made searchable by your users. Clicking on a harvested dataset in the search results takes the user to the original repository. Harvested datasets cannot be edited in your Dataverse installation. +Harvesting is a process of exchanging metadata with other repositories. As a harvesting *client*, your Dataverse installation can gather metadata records from remote sources. These can be other Dataverse installations or other archives that support OAI-PMH, the standard harvesting protocol. Harvested metadata records will be indexed and made searchable by your users. Clicking on a harvested dataset in the search results takes the user to the original repository. Harvested datasets cannot be edited in your Dataverse installation. Harvested records can be kept in sync with the original repository through scheduled incremental updates, daily or weekly. Alternatively, harvests can be run on demand, by the Admin. @@ -26,4 +26,4 @@ What if a Run Fails? Each harvesting client run logs a separate file per run to the app server's default logging directory (``/usr/local/payara5/glassfish/domains/domain1/logs/`` unless you've changed it). Look for filenames in the format ``harvest_TARGET_YYYY_MM_DD_timestamp.log`` to get a better idea of what's going wrong. -Note that you'll want to run a minimum Dataverse version of 4.6, optimally 4.18, for the best OAI-PMH interoperability. +Note that you'll want to run a minimum Dataverse Platform version of 4.6, optimally 4.18, for the best OAI-PMH interoperability. From 5a1b4a5400c4df11c4b45cc187e9c19ebd540d00 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 6 Jan 2021 12:04:05 -0500 Subject: [PATCH 056/576] Update integrations.rst --- .../source/admin/integrations.rst | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/doc/sphinx-guides/source/admin/integrations.rst b/doc/sphinx-guides/source/admin/integrations.rst index 9e43081bb45..65917d24891 100644 --- a/doc/sphinx-guides/source/admin/integrations.rst +++ b/doc/sphinx-guides/source/admin/integrations.rst @@ -1,7 +1,7 @@ Integrations ============ -Now that you've installed Dataverse, you might want to set up some integrations with other systems. Many of these integrations are open source and are cross listed in the :doc:`/api/apps` section of the API Guide. +Now that you've installed a Dataverse installation, you might want to set up some integrations with other systems. Many of these integrations are open source and are cross listed in the :doc:`/api/apps` section of the API Guide. .. contents:: Contents: :local: @@ -9,37 +9,37 @@ Now that you've installed Dataverse, you might want to set up some integrations Getting Data In --------------- -A variety of integrations are oriented toward making it easier for your researchers to deposit data into your installation of Dataverse. +A variety of integrations are oriented toward making it easier for your researchers to deposit data into your Dataverse installation. Dropbox +++++++ -If your researchers have data on Dropbox, you can make it easier for them to get it into Dataverse by setting the :ref:`dataverse.dropbox.key` JVM option described in the :doc:`/installation/config` section of the Installation Guide. +If your researchers have data on Dropbox, you can make it easier for them to get it into your Dataverse installation by setting the :ref:`dataverse.dropbox.key` JVM option described in the :doc:`/installation/config` section of the Installation Guide. Open Science Framework (OSF) ++++++++++++++++++++++++++++ The Center for Open Science's Open Science Framework (OSF) is an open source software project that facilitates open collaboration in science research across the lifespan of a scientific project. -For instructions on depositing data from OSF to your installation of Dataverse, your researchers can visit https://help.osf.io/hc/en-us/articles/360019737314-Connect-Dataverse-to-a-Project +For instructions on depositing data from OSF to your Dataverse installation, your researchers can visit https://help.osf.io/hc/en-us/articles/360019737314-Connect-Dataverse-to-a-Project RSpace ++++++ RSpace is an affordable and secure enterprise grade electronic lab notebook (ELN) for researchers to capture and organize data. -For instructions on depositing data from RSpace to your installation of Dataverse, your researchers can visit https://www.researchspace.com/help-and-support-resources/dataverse-integration/ +For instructions on depositing data from RSpace to your Dataverse installation, your researchers can visit https://www.researchspace.com/help-and-support-resources/dataverse-integration/ Open Journal Systems (OJS) ++++++++++++++++++++++++++ Open Journal Systems (OJS) is a journal management and publishing system that has been developed by the Public Knowledge Project to expand and improve access to research. -The OJS Dataverse Plugin adds data sharing and preservation to the OJS publication process. +The OJS Dataverse Project Plugin adds data sharing and preservation to the OJS publication process. As of this writing only OJS 2.x is supported and instructions for getting started can be found at https://github.com/pkp/ojs/tree/ojs-stable-2_4_8/plugins/generic/dataverse -If you are interested in OJS 3.x supporting deposit from Dataverse, please leave a comment on https://github.com/pkp/pkp-lib/issues/1822 +If you are interested in OJS 3.x supporting deposit to Dataverse installations, please leave a comment on https://github.com/pkp/pkp-lib/issues/1822 Renku +++++ @@ -47,7 +47,7 @@ Renku Renku is a platform that enables collaborative, reproducible and reusable (data)science. It allows researchers to automatically record the provenance of their research results and retain links to imported and exported data. Users -can organize their data in "Datasets", which can be exported to Dataverse via +can organize their data in "Datasets", which can be exported to a Dataverse installation via the command-line interface (CLI). Renku dataset documentation: https://renku-python.readthedocs.io/en/latest/commands.html#module-renku.cli.dataset @@ -63,9 +63,9 @@ Embedding Data on Websites OpenScholar +++++++++++ -`OpenScholar `_ is oriented toward hosting websites for academic institutions and offers `Dataverse Widgets `_ that can be added to web pages. See also: +`OpenScholar `_ is oriented toward hosting websites for academic institutions and offers `Dataverse Project Widgets `_ that can be added to web pages. See also: -- :ref:`openscholar-dataverse-level` (dataverse level) +- :ref:`openscholar-dataverse-level` (Dataverse collection level) - :ref:`openscholar-dataset-level` (dataset level) Analysis and Computation @@ -101,51 +101,51 @@ Whole Tale ++++++++++ `Whole Tale `_ enables researchers to analyze data using popular tools including Jupyter and RStudio with the ultimate goal of supporting publishing of reproducible research packages. Users can -`import data from Dataverse +`import data from a Dataverse installation `_ via identifier (e.g., DOI, URI, etc) or through the External Tools integration. For installation instructions, see the :doc:`external-tools` section or the `Integration `_ section of the Whole Tale User Guide. Binder ++++++ -Researchers can launch Jupyter Notebooks, RStudio, and other computational environments by entering the DOI of a Dataverse dataset on https://mybinder.org +Researchers can launch Jupyter Notebooks, RStudio, and other computational environments by entering the DOI of a dataset in a Dataverse installation on https://mybinder.org -Institutions can self host BinderHub. Dataverse is one of the supported `repository providers `_. +Institutions can self host BinderHub. The Dataverse Project is one of the supported `repository providers `_. Renku +++++ -Researchers can import Dataverse datasets into their Renku projects via the -command-line interface (CLI) by using the Dataverse DOI. See the `renku Dataset +Researchers can import datasets from a Dataverse installation into their Renku projects via the +command-line interface (CLI) by using the dataset's DOI. See the `renku Dataset documentation `_ -for details. Currently Dataverse ``>=4.8.x`` is required for the import to work. If you need -support for an earlier version of Dataverse, please get in touch with the Renku team at +for details. Currently Dataverse Platform ``>=4.8.x`` is required for the import to work. If you need +support for an earlier version of the Dataverse Platform, please get in touch with the Renku team at `Discourse `_ or `GitHub `_. Avgidea Data Search +++++++++++++++++++ -Researchers can use a Google Sheets add-on to search for Dataverse CSV data and then import that data into a sheet. See `Avgidea Data Search `_ for details. +Researchers can use a Google Sheets add-on to search for Dataverse installation's CSV data and then import that data into a sheet. See `Avgidea Data Search `_ for details. Discoverability --------------- -Integration with `DataCite `_ is built in to Dataverse. When datasets are published, metadata is sent to DataCite. You can futher increase the discoverability of your datasets by setting up additional integrations. +Integration with `DataCite `_ is built in to the Dataverse Platform. When datasets are published, metadata is sent to DataCite. You can further increase the discoverability of your datasets by setting up additional integrations. OAI-PMH (Harvesting) ++++++++++++++++++++ -Dataverse supports a protocol called OAI-PMH that facilitates harvesting datasets from one system into another. For details on harvesting, see the :doc:`harvestserver` section. +The Dataverse Platform supports a protocol called OAI-PMH that facilitates harvesting datasets from one system into another. For details on harvesting, see the :doc:`harvestserver` section. SHARE +++++ -`SHARE `_ is building a free, open, data set about research and scholarly activities across their life cycle. It's possible to add an installation of Dataverse as one of the `sources `_ they include if you contact the SHARE team. +`SHARE `_ is building a free, open, data set about research and scholarly activities across their life cycle. It's possible to add a Dataverse installation as one of the `sources `_ they include if you contact the SHARE team. Geodisy +++++++ -`Geodisy `_ will take your Dataverse Installation’s data, search for geospatial metadata and files, and copy them to a new system that allows for visual searching. Your original data and search methods are untouched; you have the benefit of both. For more information, please refer to `Geodisy's GitHub Repository. `_ +`Geodisy `_ will take your Dataverse installation’s data, search for geospatial metadata and files, and copy them to a new system that allows for visual searching. Your original data and search methods are untouched; you have the benefit of both. For more information, please refer to `Geodisy's GitHub Repository. `_ Research Data Preservation -------------------------- @@ -155,21 +155,21 @@ Archivematica `Archivematica `_ is an integrated suite of open-source tools for processing digital objects for long-term preservation, developed and maintained by Artefactual Systems Inc. Its configurable workflow is designed to produce system-independent, standards-based Archival Information Packages (AIPs) suitable for long-term storage and management. -Sponsored by the `Ontario Council of University Libraries (OCUL) `_, this technical integration enables users of Archivematica to select datasets from connected Dataverse instances and process them for long-term access and digital preservation. For more information and list of known issues, please refer to Artefactual's `release notes `_, `integration documentation `_, and the `project wiki `_. +Sponsored by the `Ontario Council of University Libraries (OCUL) `_, this technical integration enables users of Archivematica to select datasets from connected Dataverse installations and process them for long-term access and digital preservation. For more information and list of known issues, please refer to Artefactual's `release notes `_, `integration documentation `_, and the `project wiki `_. DuraCloud/Chronopolis +++++++++++++++++++++ -Dataverse can be configured to submit a copy of published Datasets, packaged as `Research Data Alliance conformant `_ zipped `BagIt `_ bags to the `Chronopolis `_ via `DuraCloud `_ +A Dataverse installation can be configured to submit a copy of published Datasets, packaged as `Research Data Alliance conformant `_ zipped `BagIt `_ bags to the `Chronopolis `_ via `DuraCloud `_ For details on how to configure this integration, look for "DuraCloud/Chronopolis" in the :doc:`/installation/config` section of the Installation Guide. Future Integrations ------------------- -The `Dataverse roadmap `_ is a good place to see integrations that the core Dataverse team is working on. +The `Dataverse Project Roadmap `_ is a good place to see integrations that the core Dataverse Project team is working on. -The `Community Dev `_ column of our project board is a good way to track integrations that are being worked on by the Dataverse community but many are not listed and if you have an idea for an integration, please ask on the `dataverse-community `_ mailing list if someone is already working on it. +The `Community Dev `_ column of our project board is a good way to track integrations that are being worked on by the Dataverse Community but many are not listed and if you have an idea for an integration, please ask on the `dataverse-community `_ mailing list if someone is already working on it. Many integrations take the form of "external tools". See the :doc:`external-tools` section for details. External tool makers should check out the :doc:`/api/external-tools` section of the API Guide. From 3eebc6c2a77223b99fd66143b0fb8296942addf0 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 6 Jan 2021 12:05:32 -0500 Subject: [PATCH 057/576] proper use of Dataverse updates --- doc/sphinx-guides/source/admin/ip-groups.rst | 2 +- doc/sphinx-guides/source/admin/mail-groups.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/admin/ip-groups.rst b/doc/sphinx-guides/source/admin/ip-groups.rst index 642b19878f5..6d9f520cb4d 100644 --- a/doc/sphinx-guides/source/admin/ip-groups.rst +++ b/doc/sphinx-guides/source/admin/ip-groups.rst @@ -20,7 +20,7 @@ IP Groups must be expressed as ranges in IPv4 or IPv6 format. For illustrative p .. literalinclude:: ../_static/admin/ipGroupAll.json -Let's say you download the example above and edit it to give it a range used by your library, giving it a filename of ``ipGroup1.json`` and putting it in the ``/tmp`` directory. Next, load it into Dataverse using the following curl command: +Let's say you download the example above and edit it to give it a range used by your library, giving it a filename of ``ipGroup1.json`` and putting it in the ``/tmp`` directory. Next, load it into your Dataverse installation using the following curl command: ``curl -X POST -H 'Content-type: application/json' http://localhost:8080/api/admin/groups/ip --upload-file /tmp/ipGroup1.json`` diff --git a/doc/sphinx-guides/source/admin/mail-groups.rst b/doc/sphinx-guides/source/admin/mail-groups.rst index 3df529e4112..86fb4272dd3 100644 --- a/doc/sphinx-guides/source/admin/mail-groups.rst +++ b/doc/sphinx-guides/source/admin/mail-groups.rst @@ -66,7 +66,7 @@ Some hints: - Due to their excessive CPU usage, regular expressions should be used rarely. - Remember to properly escape "\" in your regular expression. Both Java and JSON are a bit picky about this. E.g. a character class "\d" would have to be escaped as "\\d". Plenty of tutorials on the web explain this in more detail. -- There is no way Dataverse can detect a wrong regular expression for you. Be sure to do extensive testing, as a misconfigured group could result in privilege escalation or an unexpected influx of support contacts. +- There is no way the Dataverse software can detect a wrong regular expression for you. Be sure to do extensive testing, as a misconfigured group could result in privilege escalation or an unexpected influx of support contacts. - Remember to enable the regular expression support for a group by adding ``"regex": true``! A short example for a group using regular expressions: From 3e2a620000634d7faa9cfa005c648156c7a68aed Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 6 Jan 2021 12:13:18 -0500 Subject: [PATCH 058/576] Update make-data-count.rst --- .../source/admin/make-data-count.rst | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/doc/sphinx-guides/source/admin/make-data-count.rst b/doc/sphinx-guides/source/admin/make-data-count.rst index a432a27fbe7..ac37d9eb11f 100644 --- a/doc/sphinx-guides/source/admin/make-data-count.rst +++ b/doc/sphinx-guides/source/admin/make-data-count.rst @@ -1,7 +1,7 @@ Make Data Count =============== -`Make Data Count`_ is a project to collect and standardize metrics on data use, especially views, downloads, and citations. Dataverse can integrate Make Data Count to collect and display usage metrics including counts of dataset views, file downloads, and dataset citations. +`Make Data Count`_ is a project to collect and standardize metrics on data use, especially views, downloads, and citations. The Dataverse Platform can integrate Make Data Count to collect and display usage metrics including counts of dataset views, file downloads, and dataset citations. .. contents:: Contents: :local: @@ -9,7 +9,7 @@ Make Data Count Introduction ------------ -`Make Data Count`_ is part of a broader Research Data Alliance (RDA) `Data Usage Metrics Working Group`_ which helped to produce a specification called the `COUNTER Code of Practice for Research Data`_ (`PDF`_, `HTML`_) that Dataverse makes every effort to comply with. The Code of Practice (CoP) is built on top of existing standards such as COUNTER and SUSHI that come out of the article publishing world. The Make Data Count project has emphasized that they would like feedback on the code of practice. You can keep up to date on the Make Data Count project by subscribing to their `newsletter`_. +`Make Data Count`_ is part of a broader Research Data Alliance (RDA) `Data Usage Metrics Working Group`_ which helped to produce a specification called the `COUNTER Code of Practice for Research Data`_ (`PDF`_, `HTML`_) that the Dataverse Platform makes every effort to comply with. The Code of Practice (CoP) is built on top of existing standards such as COUNTER and SUSHI that come out of the article publishing world. The Make Data Count project has emphasized that they would like feedback on the code of practice. You can keep up to date on the Make Data Count project by subscribing to their `newsletter`_. Architecture ------------ @@ -19,17 +19,17 @@ Dataverse installations who would like support for Make Data Count must install .. _Counter Processor: https://github.com/CDLUC3/counter-processor .. _DASH: https://cdluc3.github.io/dash/ -The diagram below shows how Counter Processor interacts with Dataverse and the DataCite hub, once configured. Installations of Dataverse using Handles rather than DOIs should note the limitations in the next section of this page. +The diagram below shows how Counter Processor interacts with your Dataverse installation and the DataCite hub, once configured. Dataverse installations using Handles rather than DOIs should note the limitations in the next section of this page. |makedatacount_components| The most important takeaways from the diagram are: -- Once enabled, Dataverse will log activity (views and downloads) to a specialized date-stamped file. -- You should run Counter Processor once a day to create reports in SUSHI (JSON) format that are saved to disk for Dataverse to process and that are sent to the DataCite hub. -- You should set up a cron job to have Dataverse process the daily SUSHI reports, updating the Dataverse database with the latest metrics. -- You should set up a cron job to have Dataverse pull the latest list of citations for each dataset on a periodic basis, perhaps weekly or daily. These citations come from Crossref via the DataCite hub. -- APIs are available in Dataverse to retrieve Make Data Count metrics: views, downloads, and citations. +- Once enabled, your Dataverse installation will log activity (views and downloads) to a specialized date-stamped file. +- You should run Counter Processor once a day to create reports in SUSHI (JSON) format that are saved to disk for your Dataverse installation to process and that are sent to the DataCite hub. +- You should set up a cron job to have your Dataverse installation process the daily SUSHI reports, updating the Dataverse installation database with the latest metrics. +- You should set up a cron job to have your Dataverse installation pull the latest list of citations for each dataset on a periodic basis, perhaps weekly or daily. These citations come from Crossref via the DataCite hub. +- APIs are available in the Dataverse Platform to retrieve Make Data Count metrics: views, downloads, and citations. Limitations for Dataverse Installations Using Handles Rather Than DOIs @@ -49,19 +49,19 @@ Data repositories using Handles and other identifiers are not supported by Make | | MDC citations | | +---------------------+---------------+------------------+ -This being said, the Dataverse usage logging can still generate logs and process those logs with Counter Processor to create json that details usage on a dataset level. Dataverse can ingest this locally generated json. +This being said, the Dataverse Platform usage logging can still generate logs and process those logs with Counter Processor to create json that details usage on a dataset level. Dataverse installations can ingest this locally generated json. When editing the ``counter-processor-config.yaml`` file mentioned below, make sure that the ``upload_to_hub`` boolean is set to ``False``. -Configuring Dataverse for Make Data Count Views and Downloads -------------------------------------------------------------- +Configuring Your Dataverse Installation for Make Data Count Views and Downloads +------------------------------------------------------------------------------- If you haven't already, follow the steps for installing Counter Processor in the :doc:`/installation/prerequisites` section of the Installation Guide. Enable Logging for Make Data Count ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To make Dataverse log dataset usage (views and downloads) for Make Data Count, you must set the ``:MDCLogPath`` database setting. See :ref:`:MDCLogPath` for details. +To make your Dataverse installation log dataset usage (views and downloads) for Make Data Count, you must set the ``:MDCLogPath`` database setting. See :ref:`:MDCLogPath` for details. If you wish to start logging in advance of setting up other components, or wish to log without display MDC metrics for any other reason, you can set the optional ``:DisplayMDCMetrics`` database setting to false. See :ref:`:DisplayMDCMetrics` for details. @@ -70,7 +70,7 @@ After you have your first day of logs, you can process them the next day. Enable or Disable Display of Make Data Count Metrics ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -By default, when MDC logging is enabled (when ``:MDCLogPath`` is set), Dataverse will display MDC metrics instead of it's internal (legacy) metrics. You can avoid this (e.g. to collect MDC metrics for some period of time before starting to display them) by setting ``:DisplayMDCMetrics`` to false. +By default, when MDC logging is enabled (when ``:MDCLogPath`` is set), your Dataverse installation will display MDC metrics instead of it's internal (legacy) metrics. You can avoid this (e.g. to collect MDC metrics for some period of time before starting to display them) by setting ``:DisplayMDCMetrics`` to false. The following discussion assumes ``:MDCLogPath`` has been set to ``/usr/local/payara5/glassfish/domains/domain1/logs/mdc`` @@ -94,7 +94,7 @@ Configure Counter Processor Populate Views and Downloads for the First Time ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Soon we will be setting up a cron job to run nightly but we start with a single successful configuration and manual run of Counter Processor and calls to Dataverse APIs. (The scripts discussed in the next section automate the steps described here, including creating empty log files if you're starting mid-month.) +Soon we will be setting up a cron job to run nightly but we start with a single successful configuration and manual run of Counter Processor and calls to your Dataverse installation's APIs. (The scripts discussed in the next section automate the steps described here, including creating empty log files if you're starting mid-month.) * Change to the directory where you installed Counter Processor. @@ -122,14 +122,14 @@ Soon we will be setting up a cron job to run nightly but we start with a single * Verify that views and downloads are available via API. - * Now that views and downloads have been recorded in the Dataverse database, you should make sure you can retrieve them from a dataset or two. Use the :ref:`Dataset Metrics ` endpoints in the :doc:`/api/native-api` section of the API Guide. + * Now that views and downloads have been recorded in the Dataverse installation's database, you should make sure you can retrieve them from a dataset or two. Use the :ref:`Dataset Metrics ` endpoints in the :doc:`/api/native-api` section of the API Guide. Populate Views and Downloads Nightly ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Running ``main.py`` to create the SUSHI JSON file and the subsequent calling of the Dataverse API to process it should be added as a cron job. +Running ``main.py`` to create the SUSHI JSON file and the subsequent calling of the Dataverse Platform API to process it should be added as a cron job. -Dataverse provides example scripts that run the steps to process new accesses and uploads and update Dataverse's database (`counter_daily.sh`) and to retrieve citations for all Datasets from DataCite (`counter_weekly.sh`). These scripts should be configured for your environment and can be run manually or as cron jobs. +The Dataverse Platform provides example scripts that run the steps to process new accesses and uploads and update your Dataverse installation's database (`counter_daily.sh`) and to retrieve citations for all Datasets from DataCite (`counter_weekly.sh`). These scripts should be configured for your environment and can be run manually or as cron jobs. Sending Usage Metrics to the DataCite Hub ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -141,12 +141,12 @@ Once you are satisfied with your testing, you should contact support@datacite.or .. _MDC-updateCitationsForDataset: -Configuring Dataverse for Make Data Count Citations ---------------------------------------------------- +Configuring Your Dataverse Installation for Make Data Count Citations +--------------------------------------------------------------------- -Please note: as explained in the note above about limitations, this feature is not available to installations of Dataverse that use Handles. +Please note: as explained in the note above about limitations, this feature is not available to Dataverse installations that use Handles. -To configure Dataverse to pull citations from the test vs. production DataCite server see :ref:`doi.dataciterestapiurlstring` in the Installation Guide. +To configure your Dataverse installation to pull citations from the test vs. production DataCite server see :ref:`doi.dataciterestapiurlstring` in the Installation Guide. Please note that in the curl example, Bash environment variables are used with the idea that you can set a few environment variables and copy and paste the examples as is. For example, "$DOI" could become "doi:10.5072/FK2/BL2IBM" by issuing the following export command from Bash: @@ -160,11 +160,11 @@ On some periodic basis (perhaps weekly) you should call the following curl comma ``curl -X POST "http://localhost:8080/api/admin/makeDataCount/:persistentId/updateCitationsForDataset?persistentId=$DOI"`` -Citations will be retrieved for each published dataset and recorded in the Dataverse database. +Citations will be retrieved for each published dataset and recorded in the your Dataverse installation's database. -For how to get the citations out of Dataverse, see "Retrieving Citations for a Dataset" under :ref:`Dataset Metrics ` in the :doc:`/api/native-api` section of the API Guide. +For how to get the citations out of your Dataverse installation, see "Retrieving Citations for a Dataset" under :ref:`Dataset Metrics ` in the :doc:`/api/native-api` section of the API Guide. -Please note that while Dataverse has a metadata field for "Related Dataset" this information is not currently sent as a citation to Crossref. +Please note that while the Dataverse Platform has a metadata field for "Related Dataset" this information is not currently sent as a citation to Crossref. Retrieving Make Data Count Metrics from the DataCite Hub -------------------------------------------------------- @@ -177,10 +177,10 @@ The following metrics can be downloaded directly from the DataCite hub (see http - Downloads for a Dataset - Citations for a Dataset (via Crossref) -Retrieving Make Data Count Metrics from Dataverse -------------------------------------------------- +Retrieving Make Data Count Metrics from a Dataverse Installation +---------------------------------------------------------------- -The Dataverse API endpoints for retrieving Make Data Count metrics are described below under :ref:`Dataset Metrics ` in the :doc:`/api/native-api` section of the API Guide. +The Dataverse Platform API endpoints for retrieving Make Data Count metrics are described below under :ref:`Dataset Metrics ` in the :doc:`/api/native-api` section of the API Guide. Please note that it is also possible to retrieve metrics from the DataCite hub itself via https://api.datacite.org From ea3fc02190ca88730fe7dd00b055a50e267c95da Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 6 Jan 2021 12:32:23 -0500 Subject: [PATCH 059/576] updating tables is especially fun. --- .../source/admin/metadatacustomization.rst | 59 ++++++++++--------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index a7c7e48cdd6..a2efa95deed 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -1,7 +1,7 @@ Metadata Customization ====================== -Dataverse has a flexible data-driven metadata system powered by "metadata blocks" that are listed in the :doc:`/user/appendix` section of the User Guide. In this section we explain the customization options. +The Dataverse software has a flexible data-driven metadata system powered by "metadata blocks" that are listed in the :doc:`/user/appendix` section of the User Guide. In this section we explain the customization options. .. contents:: |toctitle| :local: @@ -9,7 +9,7 @@ Dataverse has a flexible data-driven metadata system powered by "metadata blocks Introduction ------------ -Before you embark on customizing metadata in Dataverse you should make sure you are aware of the modest amount of customization that is available with the Dataverse web interface. It's possible to hide fields and make field required by clicking "Edit" at the dataverse level, clicking "General Information" and making adjustments under "Metadata Fields" as described in the :ref:`create-dataverse` section of the Dataverse Management page in the User Guide. +Before you embark on customizing metadata in your Dataverse installation you should make sure you are aware of the modest amount of customization that is available with your Dataverse installation's web interface. It's possible to hide fields and make field required by clicking "Edit" at the Dataverse collection level, clicking "General Information" and making adjustments under "Metadata Fields" as described in the :ref:`create-dataverse` section of the Dataverse Collection Management page in the User Guide. Much more customization of metadata is possible, but this is an advanced topic so feedback on what is written below is very welcome. The possibilities for customization include: @@ -23,11 +23,11 @@ Much more customization of metadata is possible, but this is an advanced topic s - Changing how saved metadata values are displayed in the UI -Generally speaking it is safer to create your own custom metadata block rather than editing metadata blocks that ship with Dataverse, because changes to these blocks may be made in future releases of Dataverse. If you'd like to make improvements to any of the metadata blocks shipped with Dataverse, please open an issue at https://github.com/IQSS/dataverse/issues so it can be discussed before a pull request is made. Please note that the metadata blocks shipped with Dataverse are based on standards (e.g. DDI for social science) and you can learn more about these standards in the :doc:`/user/appendix` section of the User Guide. If you have developed your own custom metadata block that you think may be of interest to the Dataverse community, please create an issue and consider making a pull request as described in the :doc:`/developers/version-control` section of the Developer Guide. +Generally speaking it is safer to create your own custom metadata block rather than editing metadata blocks that ship with the Dataverse software, because changes to these blocks may be made in future releases. If you'd like to make improvements to any of the metadata blocks shipped with the Dataverse software, please open an issue at https://github.com/IQSS/dataverse/issues so it can be discussed before a pull request is made. Please note that the metadata blocks shipped with the Dataverse software are based on standards (e.g. DDI for social science) and you can learn more about these standards in the :doc:`/user/appendix` section of the User Guide. If you have developed your own custom metadata block that you think may be of interest to the Dataverse community, please create an issue and consider making a pull request as described in the :doc:`/developers/version-control` section of the Developer Guide. -In Dataverse 4, custom metadata are no longer defined as individual +In current versions of the Dataverse software, custom metadata are no longer defined as individual fields, as they were in Dataverse Network (DVN) 3.x, but in metadata blocks. -Dataverse 4 ships with a citation metadata block, which includes +The Dataverse software now ships with a citation metadata block, which includes mandatory fields needed for assigning persistent IDs to datasets, and domain specific metadata blocks. For a complete list, see the :doc:`/user/appendix` section of the User Guide. @@ -37,9 +37,9 @@ tab-separated value (TSV). [1]_\ :sup:`,`\ [2]_ While it is technically possible to define more than one metadata block in a TSV file, it is good organizational practice to define only one in each file. -The metadata block TSVs shipped with Dataverse are in `/tree/develop/scripts/api/data/metadatablocks -`__ and the corresponding ResourceBundle property files `/tree/develop/src/main/java `__ of the Dataverse GitHub repo. Human-readable copies are available in `this Google Sheets -document `__ but they tend to get out of sync with the TSV files, which should be considered authoritative. The Dataverse installation process operates on the TSVs, not the Google spreadsheet. +The metadata block TSVs shipped with the Dataverse software are in `/tree/develop/scripts/api/data/metadatablocks +`__ and the corresponding ResourceBundle property files `/tree/develop/src/main/java `__ of the Dataverse Software GitHub repo. Human-readable copies are available in `this Google Sheets +document `__ but they tend to get out of sync with the TSV files, which should be considered authoritative. The Dataverse Software installation process operates on the TSVs, not the Google spreadsheet. About the metadata block TSV ---------------------------- @@ -53,7 +53,7 @@ the metadata block TSV. - Cardinality: - - 0 or more per Dataverse + - 0 or more per Dataverse installation - 1 per Metadata Block definition @@ -104,11 +104,12 @@ Each of the three main sections own sets of properties: | dataverseAlias | If specified, this | Free text. For an | | | metadata block will | example, see | | | be available only to | custom_hbgdki.tsv. | -| | the dataverse | | +| | the Dataverse | +| | collection | | | designated here by | | | | its alias and to | | | | children of that | | -| | dataverse. | | +| | Dataverse collection. | | +-----------------------+-----------------------+-----------------------+ | displayName | Acts as a brief label | Should be relatively | | | for display related | brief. The limit is | @@ -254,7 +255,8 @@ Each of the three main sections own sets of properties: | | "Browse/Search | | | | Facets" when you edit | | | | "General Information" | | -| | for a dataverse. | | +| | for a Dataverse | | +| |collection. | | | | Setting this value to | | | | TRUE generally makes | | | | sense for enumerated | | @@ -407,7 +409,8 @@ FieldType definitions | | be entered into this field. | +-----------------------------------+-----------------------------------+ | textbox | Any text may be entered. For | -| | input, Dataverse presents a | +| | input, the Dataverse software | +| | presents a | | | multi-line area that accepts | | | newlines. While any HTML is | | | permitted, only a subset of HTML | @@ -503,12 +506,12 @@ These are common ways to use the displayFormat to control how values are display Metadata Block Setup -------------------- -Now that you understand the TSV format used for metadata blocks, the next step is to attempt to make improvements to existing metadata blocks or create entirely new metadata blocks. For either task, you should have a Dataverse environment set up for testing where you can drop the database frequently while you make edits to TSV files. Once you have tested your TSV files, you should consider making a pull request to contribute your improvement back to the community. +Now that you understand the TSV format used for metadata blocks, the next step is to attempt to make improvements to existing metadata blocks or create entirely new metadata blocks. For either task, you should have a Dataverse Software development environment set up for testing where you can drop the database frequently while you make edits to TSV files. Once you have tested your TSV files, you should consider making a pull request to contribute your improvement back to the community. Exploring Metadata Blocks ~~~~~~~~~~~~~~~~~~~~~~~~~ -In addition to studying the TSV files themselves you might find the following highly experimental and subject-to-change API endpoints useful to understand the metadata blocks that have already been loaded into your installation of Dataverse: +In addition to studying the TSV files themselves you might find the following highly experimental and subject-to-change API endpoints useful to understand the metadata blocks that have already been loaded into your Dataverse installation: You can get a dump of metadata fields (yes, the output is odd, please open a issue) like this: @@ -533,7 +536,7 @@ To get a clean environment in Vagrant, you'll be running ``vagrant destroy``. In Editing TSV files ~~~~~~~~~~~~~~~~~ -Early in Dataverse 4 development metadata blocks were edited in the Google spreadsheet mentioned above and then exported in TSV format. This worked fine when there was only one person editing the Google spreadsheet but now that contributions are coming in from all over, the TSV files are edited directly. We are somewhat painfully aware that another format such as XML might make more sense these days. Please see https://github.com/IQSS/dataverse/issues/4451 for a discussion of non-TSV formats. +Early in Dataverse Software version 4 development, metadata blocks were edited in the Google spreadsheet mentioned above and then exported in TSV format. This worked fine when there was only one person editing the Google spreadsheet but now that contributions are coming in from all over, the TSV files are edited directly. We are somewhat painfully aware that another format such as XML might make more sense these days. Please see https://github.com/IQSS/dataverse/issues/4451 for a discussion of non-TSV formats. Please note that metadata fields share a common namespace so they must be unique. The following curl command will print list of metadata fields already available in the system: @@ -541,21 +544,21 @@ Please note that metadata fields share a common namespace so they must be unique We'll use this command again below to update the Solr schema to accomodate metadata fields we've added. -Loading TSV files into Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Loading TSV files into a Dataverse Installation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -A number of TSV files are loaded into Dataverse on every new installation, becoming the metadata blocks you see in the UI. For the list of metadata blocks that are included with Dataverse out of the box, see the :doc:`/user/appendix` section of the User Guide. +A number of TSV files are loaded into a newly-installed Dataverse installation, becoming the metadata blocks you see in the UI. For the list of metadata blocks that are included with the Dataverse Software out of the box, see the :doc:`/user/appendix` section of the User Guide. Along with TSV file, there are corresponding ResourceBundle property files with key=value pair `here `__. To add other language files, see the :doc:`/installation/config` for dataverse.lang.directory JVM Options section, and add a file, for example: "citation_lang.properties" to the path you specified for the ``dataverse.lang.directory`` JVM option, and then restart the app server. -If you are improving an existing metadata block, the Dataverse installation process will load the TSV for you, assuming you edited the TSV file in place. The TSV file for the Citation metadata block, for example, can be found at ``scripts/api/data/metadatablocks/citation.tsv``. +If you are improving an existing metadata block, the Dataverse Software installation process will load the TSV for you, assuming you edited the TSV file in place. The TSV file for the Citation metadata block, for example, can be found at ``scripts/api/data/metadatablocks/citation.tsv``. If any of the below mentioned property values are changed, corresponsing ResourceBundle property file has to be edited and stored under ``dataverse.lang.directory`` location - name, displayName property under #metadataBlock - name, title, description, watermark properties under #datasetfield - DatasetField, Value property under #controlledVocabulary -If you are creating a new custom metadata block (hopefully with the idea of contributing it back to the community if you feel like it would provide value to others), the Dataverse installation process won't know about your new TSV file so you must load it manually. The script that loads the TSV files into the system is ``scripts/api/setup-datasetfields.sh`` and contains a series of curl commands. Here's an example of the necessary curl command with the new custom metadata block in the "/tmp" directory. +If you are creating a new custom metadata block (hopefully with the idea of contributing it back to the community if you feel like it would provide value to others), the Dataverse Software installation process won't know about your new TSV file so you must load it manually. The script that loads the TSV files into the system is ``scripts/api/setup-datasetfields.sh`` and contains a series of curl commands. Here's an example of the necessary curl command with the new custom metadata block in the "/tmp" directory. ``curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file /tmp/new-metadata-block.tsv`` @@ -605,7 +608,7 @@ controlledvocabulary.language.marathi_(marathi)=Marathi (Mar\u0101\u1E6Dh\u012B) Enabling a Metadata Block ~~~~~~~~~~~~~~~~~~~~~~~~~ -Running a curl command like "load" example above should make the new custom metadata block available within the system but in order to start using the fields you must either enable it from the UI (see :ref:`general-information` section of Dataverse Management in the User Guide) or by running a curl command like the one below using a superuser API token. In the example below we are enabling the "journal" and "geospatial" metadata blocks for the root dataverse: +Running a curl command like "load" example above should make the new custom metadata block available within the system but in order to start using the fields you must either enable it from the UI (see :ref:`general-information` section of Dataverse Collection Management in the User Guide) or by running a curl command like the one below using a superuser API token. In the example below we are enabling the "journal" and "geospatial" metadata blocks for the root Dataverse collection: ``curl -H "X-Dataverse-key:$API_TOKEN" -X POST -H "Content-type:application/json" -d "[\"journal\",\"geospatial\"]" http://localhost:8080/api/dataverses/:root/metadatablocks`` @@ -615,14 +618,14 @@ Updating the Solr Schema Once you have enabled a new metadata block you should be able to see the new fields in the GUI but before you can save the dataset, you must add additional fields to your Solr schema. -An API endpoint of Dataverse provides you with a generated set of all fields that need to be added to the Solr schema +An API endpoint of your Dataverse installation provides you with a generated set of all fields that need to be added to the Solr schema configuration, including any enabled metadata schemas: ``curl http://localhost:8080/api/admin/index/solr/schema`` For convenience and automation you can download and consider running :download:`updateSchemaMDB.sh <../../../../conf/solr/7.7.2/updateSchemaMDB.sh>`. It uses the API endpoint above and writes schema files to the filesystem (so be sure to run it on the Solr server itself as the Unix user who owns the Solr files) and then triggers a Solr reload. -By default, it will download from Dataverse at `http://localhost:8080` and reload Solr at `http://localhost:8983`. +By default, it will download from your Dataverse installation at `http://localhost:8080` and reload Solr at `http://localhost:8983`. You may use the following environment variables with this script or mix'n'match with options: ==================== ====== =============================================== ========================================================= @@ -644,7 +647,7 @@ Please note that if you are going to make a pull request updating ``conf/solr/7. Reloading a Metadata Block -------------------------- -As mentioned above, changes to metadata blocks that ship with Dataverse will be made over time to improve them and release notes will sometimes instruct you to reload an existing metadata block. The syntax for reloading is the same as reloading. Here's an example with the "citation" metadata block: +As mentioned above, changes to metadata blocks that ship with the Dataverse Software will be made over time to improve them and release notes will sometimes instruct you to reload an existing metadata block. The syntax for reloading is the same as reloading. Here's an example with the "citation" metadata block: ``curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file citation.tsv`` @@ -657,9 +660,9 @@ Tips from the Dataverse Community If there are tips that you feel are omitted from this document, please open an issue at https://github.com/IQSS/dataverse/issues and consider making a pull request to make improvements. You can find this document at https://github.com/IQSS/dataverse/blob/develop/doc/sphinx-guides/source/admin/metadatacustomization.rst -Alternatively, you are welcome to request "edit" access to this "Tips for Dataverse metadata blocks from the community" Google doc: https://docs.google.com/document/d/1XpblRw0v0SvV-Bq6njlN96WyHJ7tqG0WWejqBdl7hE0/edit?usp=sharing +Alternatively, you are welcome to request "edit" access to this "Tips for Dataverse Software metadata blocks from the community" Google doc: https://docs.google.com/document/d/1XpblRw0v0SvV-Bq6njlN96WyHJ7tqG0WWejqBdl7hE0/edit?usp=sharing -The thinking is that the tips can become issues and the issues can eventually be worked on as features to improve the Dataverse metadata system. +The thinking is that the tips can become issues and the issues can eventually be worked on as features to improve the Dataverse Software metadata system. Footnotes --------- @@ -679,7 +682,7 @@ Footnotes duplicated. See "Editing TSV files" for how to check for duplication. .. [5] - "displayoncreate" was "showabovefold" in Dataverse versions before 4.3.1 (see + "displayoncreate" was "showabovefold" in Dataverse Software versions before 4.3.1 (see `#3073 `__) but parsing is done based on column order rather than name so this only matters to the person reading the TSV file. From 4ec924e3af36c3e47890f35be18288a3152b793d Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 6 Jan 2021 12:34:24 -0500 Subject: [PATCH 060/576] more updates --- doc/sphinx-guides/source/admin/metadataexport.rst | 6 +++--- doc/sphinx-guides/source/admin/monitoring.rst | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/sphinx-guides/source/admin/metadataexport.rst b/doc/sphinx-guides/source/admin/metadataexport.rst index 60f5c96a4e2..55b1ee3617c 100644 --- a/doc/sphinx-guides/source/admin/metadataexport.rst +++ b/doc/sphinx-guides/source/admin/metadataexport.rst @@ -14,7 +14,7 @@ A scheduled timer job that runs nightly will attempt to export any published dat Batch exports through the API ----------------------------- -In addition to the automated exports, a Dataverse admin can start a batch job through the API. The following 2 API calls are provided: +In addition to the automated exports, a Dataverse installation admin can start a batch job through the API. The following 2 API calls are provided: ``curl http://localhost:8080/api/admin/metadata/exportAll`` @@ -28,12 +28,12 @@ Note, that creating, modifying, or re-exporting an OAI set will also attempt to Export Failures --------------- -An export batch job, whether started via the API, or by the application timer, will leave a detailed log in your configured logs directory. This is the same location where your main app server logs are found. The name of the log file is ``export_[timestamp].log`` - for example, *export_2016-08-23T03-35-23.log*. The log will contain the numbers of datasets processed successfully and those for which metadata export failed, with some information on the failures detected. Please attach this log file if you need to contact Dataverse support about metadata export problems. +An export batch job, whether started via the API, or by the application timer, will leave a detailed log in your configured logs directory. This is the same location where your main app server logs are found. The name of the log file is ``export_[timestamp].log`` - for example, *export_2016-08-23T03-35-23.log*. The log will contain the numbers of datasets processed successfully and those for which metadata export failed, with some information on the failures detected. Please attach this log file if you need to contact Dataverse Project support about metadata export problems. Downloading Metadata via GUI ---------------------------- -The :doc:`/user/dataset-management` section of the User Guide explains how end users can download the metadata formats above from the Dataverse GUI. +The :doc:`/user/dataset-management` section of the User Guide explains how end users can download the metadata formats above from your Dataverse installation's GUI. Downloading Metadata via API ---------------------------- diff --git a/doc/sphinx-guides/source/admin/monitoring.rst b/doc/sphinx-guides/source/admin/monitoring.rst index 74b41c1578f..3d97c7ea8f7 100644 --- a/doc/sphinx-guides/source/admin/monitoring.rst +++ b/doc/sphinx-guides/source/admin/monitoring.rst @@ -90,7 +90,7 @@ Process the logs: Please note that load balancers (such as Amazon's ELB) might interfer with the ``LogFormat`` mentioned above. To start troubleshooting errors such as ``AWStats did not find any valid log lines that match your LogFormat parameter``, you might need to bump up the value of ``NbOfLinesForCorruptedLog`` in the config files above and re-try while you interate on your Apache and AWStats config. -Please note that the Dataverse team has attempted to parse Glassfish/Payara logs using AWStats but it didn't seem to just work and posts have been made at https://stackoverflow.com/questions/49134154/what-logformat-definition-does-awstats-require-to-parse-glassfish-http-access-logs and https://sourceforge.net/p/awstats/discussion/43428/thread/9b1befda/ that can be followed up on some day. +Please note that the Dataverse Project team has attempted to parse Glassfish/Payara logs using AWStats but it didn't seem to just work and posts have been made at https://stackoverflow.com/questions/49134154/what-logformat-definition-does-awstats-require-to-parse-glassfish-http-access-logs and https://sourceforge.net/p/awstats/discussion/43428/thread/9b1befda/ that can be followed up on some day. Database Connection Pool Used by App Server ------------------------------------------- @@ -113,7 +113,7 @@ Changes made to draft versions of datasets are logged in a folder called logs/ed Solr Indexing Failures Logging ------------------------------ -Failures occurring during the indexing of dataverses and datasets are logged in a folder called logs/process-failures. This logging will include instructions for manually re-running the failed processes. It may be advantageous to set up a automatic job to monitor new entries into this log folder so that indexes could be re-run. +Failures occurring during the indexing of Dataverse collections and datasets are logged in a folder called logs/process-failures. This logging will include instructions for manually re-running the failed processes. It may be advantageous to set up a automatic job to monitor new entries into this log folder so that indexes could be re-run. EJB Timers ---------- From 349afa758aad202263189c3617cc1b5ffbab03bd Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 6 Jan 2021 12:39:19 -0500 Subject: [PATCH 061/576] more text updates --- .../admin/reporting-tools-and-queries.rst | 8 ++++---- .../source/admin/solr-search-index.rst | 20 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/sphinx-guides/source/admin/reporting-tools-and-queries.rst b/doc/sphinx-guides/source/admin/reporting-tools-and-queries.rst index 197339d767d..21539b9e41f 100644 --- a/doc/sphinx-guides/source/admin/reporting-tools-and-queries.rst +++ b/doc/sphinx-guides/source/admin/reporting-tools-and-queries.rst @@ -10,10 +10,10 @@ Reporting tools and queries created by members of the Dataverse community. * Matrix (): Collaboration Matrix is a visualization showing the connectedness and collaboration between authors and their affiliations. Visit https://rin.lipi.go.id/matrix/ to play with a production installation. -* Dataverse Web Report (): Creates interactive charts showing data extracted from the Dataverse Excel Report +* Dataverse Installation Web Report (): Creates interactive charts showing data extracted from the Dataverse installation Excel Report -* Dataverse Reports for Texas Digital Library (): A python3-based tool to generate and email statistical reports from Dataverse (https://dataverse.org/) using the native API and database queries. +* Dataverse Installation Reports for Texas Digital Library (): A python3-based tool to generate and email statistical reports from Dataverse (https://dataverse.org/) using the native API and database queries. -* dataverse-metrics (): Aggregates and visualizes metrics for installations of Dataverse around the world or a single Dataverse installation. +* dataverse-metrics (): Aggregates and visualizes metrics for Dataverse installations around the world or a single Dataverse installation. -* Useful queries from the Dataverse Community (): A community-generated and maintained document of postgresql queries for getting information about users and dataverses/datasets/files in your Dataverse installation. If you are trying to find out some information from Dataverse, chances are that someone else has had the same questions and it's now listed in this document. If it's not listed, please feel free to add it to the document. \ No newline at end of file +* Useful queries from the Dataverse Community (): A community-generated and maintained document of postgresql queries for getting information about users and dataverse collections,datasets, and files in your Dataverse installation. If you are trying to find out some information from your Dataverse installation, chances are that someone else has had the same questions and it's now listed in this document. If it's not listed, please feel free to add it to the document. \ No newline at end of file diff --git a/doc/sphinx-guides/source/admin/solr-search-index.rst b/doc/sphinx-guides/source/admin/solr-search-index.rst index d37b7eedb26..5dcc866b137 100644 --- a/doc/sphinx-guides/source/admin/solr-search-index.rst +++ b/doc/sphinx-guides/source/admin/solr-search-index.rst @@ -1,7 +1,7 @@ Solr Search Index ================= -Dataverse requires Solr to be operational at all times. If you stop Solr, you should see a error about this on the root dataverse page, which is powered by the search index Solr provides. You can set up Solr by following the steps in our Installation Guide's :doc:`/installation/prerequisites` and :doc:`/installation/config` sections explaining how to configure it. This section you're reading now is about the care and feeding of the search index. PostgreSQL is the "source of truth" and the Dataverse application will copy data from PostgreSQL into Solr. For this reason, the search index can be rebuilt at any time. Depending on the amount of data you have, this can be a slow process. You are encouraged to experiment with production data to get a sense of how long a full reindexing will take. +A Dataverse installation requires Solr to be operational at all times. If you stop Solr, you should see a error about this on the root Dataverse installation page, which is powered by the search index Solr provides. You can set up Solr by following the steps in our Installation Guide's :doc:`/installation/prerequisites` and :doc:`/installation/config` sections explaining how to configure it. This section you're reading now is about the care and feeding of the search index. PostgreSQL is the "source of truth" and the Dataverse installation will copy data from PostgreSQL into Solr. For this reason, the search index can be rebuilt at any time. Depending on the amount of data you have, this can be a slow process. You are encouraged to experiment with production data to get a sense of how long a full reindexing will take. .. contents:: Contents: :local: @@ -9,7 +9,7 @@ Dataverse requires Solr to be operational at all times. If you stop Solr, you sh Full Reindex ------------- -There are two ways to perform a full reindex of the Dataverse search index. Starting with a "clear" ensures a completely clean index but involves downtime. Reindexing in place doesn't involve downtime but does not ensure a completely clean index. +There are two ways to perform a full reindex of the Dataverse installation search index. Starting with a "clear" ensures a completely clean index but involves downtime. Reindexing in place doesn't involve downtime but does not ensure a completely clean index. Clear and Reindex +++++++++++++++++ @@ -29,7 +29,7 @@ Remove all Solr documents that are orphaned (ie not associated with objects in t Clearing Data from Solr ~~~~~~~~~~~~~~~~~~~~~~~ -Please note that the moment you issue this command, it will appear to end users looking at the root dataverse page that all data is gone! This is because the root dataverse page is powered by the search index. +Please note that the moment you issue this command, it will appear to end users looking at the root Dataverse installation page that all data is gone! This is because the root Dataverse installation page is powered by the search index. ``curl http://localhost:8080/api/admin/index/clear`` @@ -60,16 +60,16 @@ If indexing stops, this command should pick up where it left off based on which Manual Reindexing ----------------- -If you have made manual changes to a dataset in the database or wish to reindex a dataset that solr didn't want to index properly, it is possible to manually reindex dataverses and datasets. +If you have made manual changes to a dataset in the database or wish to reindex a dataset that solr didn't want to index properly, it is possible to manually reindex Dataverse collections and datasets. -Reindexing Dataverses -++++++++++++++++++++++ +Reindexing Dataverse Collections +++++++++++++++++++++++++++++++++ -Dataverses must be referenced by database object ID. If you have direct database access an SQL query such as +Dataverse collections must be referenced by database object ID. If you have direct database access an SQL query such as ``select id from dataverse where alias='datavarsealias';`` -should work, or you may click the Dataverse's "Edit" menu and look for dataverseId= in the URLs produced by the drop-down. Then, to re-index: +should work, or you may click the Dataverse Platform's "Edit" menu and look for dataverseId= in the URLs produced by the drop-down. Then, to re-index: ``curl http://localhost:8080/api/admin/index/dataverses/135`` @@ -89,8 +89,8 @@ To re-index a dataset by its database ID: Manually Querying Solr ---------------------- -If you suspect something isn't indexed properly in solr, you may bypass the Dataverse web interface and query the command line directly to verify what solr returns: +If you suspect something isn't indexed properly in solr, you may bypass the Dataverse installation's web interface and query the command line directly to verify what solr returns: ``curl "http://localhost:8983/solr/collection1/select?q=dsPersistentId:doi:10.15139/S3/HFV0AO"`` -to see the JSON you were hopefully expecting to see passed along to Dataverse. +to see the JSON you were hopefully expecting to see passed along to the Dataverse installation. From d2e1cdb120a5a82e18a8fae9d3804d0c458d7f87 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 6 Jan 2021 12:47:19 -0500 Subject: [PATCH 062/576] more updates --- doc/sphinx-guides/source/admin/timers.rst | 16 ++++---- .../source/admin/troubleshooting.rst | 38 +++++++++---------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/doc/sphinx-guides/source/admin/timers.rst b/doc/sphinx-guides/source/admin/timers.rst index d94f22da2a5..afb4f83515b 100644 --- a/doc/sphinx-guides/source/admin/timers.rst +++ b/doc/sphinx-guides/source/admin/timers.rst @@ -1,17 +1,17 @@ .. role:: fixedwidthplain -Dataverse Application Timers -============================ +Dataverse Installation Application Timers +========================================= -Dataverse uses timers to automatically run scheduled Harvest and Metadata export jobs. +Your Dataverse installation uses timers to automatically run scheduled Harvest and Metadata export jobs. .. contents:: |toctitle| :local: -Dedicated timer server in a Dataverse server cluster ----------------------------------------------------- +Dedicated timer server in a Dataverse Installation server cluster +----------------------------------------------------------------- -When running a Dataverse cluster - i.e. multiple Dataverse application +When running a Dataverse installation cluster - i.e. multiple Dataverse installation application servers talking to the same database - **only one** of them must act as the *dedicated timer server*. This is to avoid starting conflicting batch jobs on multiple nodes at the same time. @@ -22,7 +22,7 @@ The following JVM option instructs the application to act as the dedicated timer ``-Ddataverse.timerServer=true`` -**IMPORTANT:** Note that this option is automatically set by the Dataverse installer script. That means that when **configuring a multi-server cluster**, it will be the responsibility of the installer to remove the option from the :fixedwidthplain:`domain.xml` of every node except the one intended to be the timer server. +**IMPORTANT:** Note that this option is automatically set by the Dataverse Software installer script. That means that when **configuring a multi-server cluster**, it will be the responsibility of the installer to remove the option from the :fixedwidthplain:`domain.xml` of every node except the one intended to be the timer server. Harvesting Timers ----------------- @@ -65,7 +65,7 @@ This job is automatically scheduled to run once a week at 12:30AM local time on * Build and deploy the application -* Alternatively, you can insert an ejb-jar.xml file into a provided dataverse war file without building the application. +* Alternatively, you can insert an ejb-jar.xml file into a provided Dataverse Software war file without building the application. * Check if there is already an ejb-jar.xml file in the war file diff --git a/doc/sphinx-guides/source/admin/troubleshooting.rst b/doc/sphinx-guides/source/admin/troubleshooting.rst index d115554f7bc..99bf00c5647 100644 --- a/doc/sphinx-guides/source/admin/troubleshooting.rst +++ b/doc/sphinx-guides/source/admin/troubleshooting.rst @@ -3,15 +3,15 @@ Troubleshooting =============== -Sometimes Dataverse users get into trouble. Sometimes Dataverse itself gets into trouble. If something has gone wrong, this section is for you. +Sometimes a Dataverse installation's users get into trouble. Sometimes a Dataverse installation itself gets into trouble. If something has gone wrong, this section is for you. .. contents:: Contents: :local: -Using Dataverse APIs to Troubleshoot and Fix Problems ------------------------------------------------------ +Using Dataverse Installation APIs to Troubleshoot and Fix Problems +------------------------------------------------------------------ -See the :doc:`/api/intro` section of the API Guide for a high level overview of Dataverse APIs. Below are listed problems that support teams might encounter that can be handled via API (sometimes only via API). +See the :doc:`/api/intro` section of the API Guide for a high level overview of Dataverse Software APIs. Below are listed problems that support teams might encounter that can be handled via API (sometimes only via API). A Dataset Is Locked And Cannot Be Edited or Published ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -22,19 +22,19 @@ It's normal for the ingest process described in the :doc:`/user/tabulardatainges A dataset is locked with a lock of type ``finalizePublication`` while the persistent identifiers for the datafiles in the dataset are registered or updated, and/or while the physical files are being validated by recalculating the checksums and verifying them against the values stored in the database, before the publication process can be completed (Note that either of the two tasks can be disabled via database options - see :doc:`/installation/config`). If a dataset has been in this state for a long period of time, for hours or longer, it is somewhat safe to assume that it is stuck (for example, the process may have been interrupted by an application server restart, or a system crash), so you may want to remove the lock (to be safe, do restart the application server, to ensure that the job is no longer running in the background) and advise the user to try publishing again. See :doc:`dataverses-datasets` for more information on publishing. -If any files in the dataset fail the validation above the dataset will be left locked with a ``DatasetLock.Reason=FileValidationFailed``. The user will be notified that they need to contact their Dataverse support in order to address the issue before another attempt to publish can be made. The admin will have to address and fix the underlying problems (by either restoring the missing or corrupted files, or by purging the affected files from the dataset) before deleting the lock and advising the user to try to publish again. The goal of the validation framework is to catch these types of conditions while the dataset is still in DRAFT. +If any files in the dataset fail the validation above the dataset will be left locked with a ``DatasetLock.Reason=FileValidationFailed``. The user will be notified that they need to contact their Dataverse installation's support in order to address the issue before another attempt to publish can be made. The admin will have to address and fix the underlying problems (by either restoring the missing or corrupted files, or by purging the affected files from the dataset) before deleting the lock and advising the user to try to publish again. The goal of the validation framework is to catch these types of conditions while the dataset is still in DRAFT. During an attempt to publish a dataset, the validation will stop after encountering the first file that fails it. It is strongly recommended for the admin to review and verify *all* the files in the dataset, so that all the compromised files can be fixed before the lock is removed. We recommend using the ``/api/validate/dataset/files/{id}`` API. It will go through all the files for the dataset specified, and will report which ones have failed validation. see :ref:`Physical Files Validation in a Dataset ` in the :doc:`/api/native-api` section of the User Guide. -The following are two real life examples of problems that have resulted in corrupted datafiles during normal operation of Dataverse: +The following are two real life examples of problems that have resulted in corrupted datafiles during normal operation of a Dataverse installation: -1. Botched file deletes - while a datafile is in DRAFT, attempting to delete it from the dataset involves deleting both the ``DataFile`` database table entry, and the physical file. (Deleting a datafile from a *published* version merely removes it from the future versions - but keeps the file in the dataset). The problem we've observed in the early versions of Dataverse was a *partially successful* delete, where the database tansaction would fail (for whatever reason), but only after the physical file had already been deleted from the filesystem. Thus resulting in a datafile entry remaining in the dataset, but with the corresponding physical file missing. We believe we have addressed the issue that was making this condition possible, so it shouldn't happen again - but there may be a datafile in this state in your database. Assuming the user's intent was in fact to delete the file, the easiest solution is simply to confirm it and purge the datafile entity from the database. Otherwise the file needs to be restored from backups, or obtained from the user and copied back into storage. -2. Another issue we've observed: a failed tabular data ingest that leaves the datafile un-ingested, BUT with the physical file already replaced by the generated tab-delimited version of the data. This datafile will fail the validation because the checksum in the database matches the file in the original format (Stata, SPSS, etc.) as uploaded by the user. To fix: luckily, this is easily reversable, since the uploaded original should be saved in your storage, with the .orig extension. Simply swapping the .orig copy with the main file associated with the datafile will fix it. Similarly, we believe this condition should not happen again in Dataverse versions 4.20+, but you may have some legacy cases on your server. +1. Botched file deletes - while a datafile is in DRAFT, attempting to delete it from the dataset involves deleting both the ``DataFile`` database table entry, and the physical file. (Deleting a datafile from a *published* version merely removes it from the future versions - but keeps the file in the dataset). The problem we've observed in the early versions of the Dataverse Software was a *partially successful* delete, where the database transaction would fail (for whatever reason), but only after the physical file had already been deleted from the filesystem. Thus resulting in a datafile entry remaining in the dataset, but with the corresponding physical file missing. We believe we have addressed the issue that was making this condition possible, so it shouldn't happen again - but there may be a datafile in this state in your database. Assuming the user's intent was in fact to delete the file, the easiest solution is simply to confirm it and purge the datafile entity from the database. Otherwise the file needs to be restored from backups, or obtained from the user and copied back into storage. +2. Another issue we've observed: a failed tabular data ingest that leaves the datafile un-ingested, BUT with the physical file already replaced by the generated tab-delimited version of the data. This datafile will fail the validation because the checksum in the database matches the file in the original format (Stata, SPSS, etc.) as uploaded by the user. To fix: luckily, this is easily reversable, since the uploaded original should be saved in your storage, with the .orig extension. Simply swapping the .orig copy with the main file associated with the datafile will fix it. Similarly, we believe this condition should not happen again in Dataverse Software versions 4.20+, but you may have some legacy cases on your server. Someone Created Spam Datasets and I Need to Delete Them ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Depending on how open your installation of Dataverse is to the general public creating datasets, you may sometimes need to deal with spam datasets. +Depending on how open your Dataverse installation is to the general public creating datasets, you may sometimes need to deal with spam datasets. Look for "destroy" in the :doc:`/api/native-api` section of the API Guide. @@ -51,7 +51,7 @@ Ingest Long-Running Ingest Jobs Have Exhausted System Resources ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Ingest is both CPU- and memory-intensive, and depending on your system resources and the size and format of tabular data files uploaded, may render Dataverse unresponsive or nearly inoperable. It is possible to cancel these jobs by purging the ingest queue. +Ingest is both CPU- and memory-intensive, and depending on your system resources and the size and format of tabular data files uploaded, may render your Dataverse installation unresponsive or nearly inoperable. It is possible to cancel these jobs by purging the ingest queue. ``/usr/local/payara5/mq/bin/imqcmd -u admin query dst -t q -n DataverseIngest`` will query the DataverseIngest destination. The password, unless you have changed it, matches the username. @@ -61,7 +61,7 @@ Finally, list destinations to verify that the purge was successful:: ``/usr/local/payara5/mq/bin/imqcmd -u admin list dst`` -If you are still running Glassfish, substitute glassfish4 for payara5 above. If you have installed Dataverse in some other location, adjust the above paths accordingly. +If you are still running Glassfish, substitute glassfish4 for payara5 above. If you have installed your Dataverse installation in some other location, adjust the above paths accordingly. .. _troubleshooting-payara: @@ -73,7 +73,7 @@ Payara Finding the Payara Log File ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``/usr/local/payara5/glassfish/domains/domain1/logs/server.log`` is the main place to look when you encounter problems (assuming you installed Payara in the default directory). Hopefully an error message has been logged. If there's a stack trace, it may be of interest to developers, especially they can trace line numbers back to a tagged version or commit. Send more of the stack trace (the entire file if possible) to developers who can help (see "Getting Help", below) and be sure to say which version of Dataverse you are running. +``/usr/local/payara5/glassfish/domains/domain1/logs/server.log`` is the main place to look when you encounter problems (assuming you installed Payara in the default directory). Hopefully an error message has been logged. If there's a stack trace, it may be of interest to developers, especially they can trace line numbers back to a tagged version or commit. Send more of the stack trace (the entire file if possible) to developers who can help (see "Getting Help", below) and be sure to say which version of the Dataverse Software you have installed. .. _increase-payara-logging: @@ -92,7 +92,7 @@ Our guides focus on using the command line to manage Payara but you might be int Deployment fails, "EJB Timer Service not available" --------------------------------------------------- -Sometimes the Dataverse application fails to deploy, or Payara fails to restart once the application is deployed, with the following error message: :fixedwidthplain:`"remote failure: Error occurred during deployment: Exception while loading the app : EJB Timer Service is not available. Please see server.log for more details."` +Sometimes your Dataverse installation fails to deploy, or Payara fails to restart once the application is deployed, with the following error message: :fixedwidthplain:`"remote failure: Error occurred during deployment: Exception while loading the app : EJB Timer Service is not available. Please see server.log for more details."` We don't know what's causing this issue, but here's a known workaround: @@ -112,7 +112,7 @@ Note that it may or may not work on your system, so it is provided as an example Timer Not Working ----------------- -Dataverse relies on EJB timers to perform scheduled tasks: harvesting from remote servers, updating the local OAI sets and running metadata exports. (See :doc:`timers` for details.) If these scheduled jobs are not running on your server, you might experience the following symptoms: +Your Dataverse installation relies on EJB timers to perform scheduled tasks: harvesting from remote servers, updating the local OAI sets and running metadata exports. (See :doc:`timers` for details.) If these scheduled jobs are not running on your server, you might experience the following symptoms: If you are seeing the following in your server.log... @@ -125,15 +125,15 @@ followed by an Exception stack trace with these lines in it: :fixedwidthplain:`Exception Description: Could not deserialize object from byte array` ... -... you should reach out by opening an issue. In the good old days of running Dataverse 4 running on Glassfish 4, this -was a hint for an unsupported JDBC driver. In Dataverse 5 this would be a new regression and its cause would need to be +... you should reach out by opening an issue. In the good ol' days of running Dataverse Software version 4 running on Glassfish 4, this +was a hint for an unsupported JDBC driver. In Dataverse Software version 5 this would be a new regression and its cause would need to be investigated. Constraint Violations Issues ---------------------------- -In real life production use, it may be possible to end up in a situation where some values associated with the datasets in your database are no longer valid under the constraints enforced by the latest version of Dataverse. This is not very likely to happen, but if it does, the symptoms will be as follows: Some datasets can no longer be edited, long exception stack traces logged in the app server log, caused by:: +In real life production use, it may be possible to end up in a situation where some values associated with the datasets in your database are no longer valid under the constraints enforced by the later versions of the Dataverse Software. This is not very likely to happen, but if it does, the symptoms will be as follows: Some datasets can no longer be edited, long exception stack traces logged in the app server log, caused by:: javax.validation.ConstraintViolationException: Bean Validation constraint(s) violated while executing Automatic Bean Validation on callback event:'preUpdate'. @@ -141,12 +141,12 @@ In real life production use, it may be possible to end up in a situation where s (contrary to what the message suggests, there are no specific "details" anywhere in the stack trace that would explain what values violate which constraints) -To identifiy the specific invalid values in the affected datasets, or to check all the datasets in the Dataverse for constraint violations, see :ref:`Dataset Validation ` in the :doc:`/api/native-api` section of the User Guide. +To identify the specific invalid values in the affected datasets, or to check all the datasets in the Dataverse installation for constraint violations, see :ref:`Dataset Validation ` in the :doc:`/api/native-api` section of the User Guide. Many Files with a File Type of "Unknown", "Application", or "Binary" -------------------------------------------------------------------- -From the home page of a Dataverse installation you can get a count of files by file type by clicking "Files" and then scrolling down to "File Type". If you see a lot of files that are "Unknown", "Application", or "Binary" you can have Dataverse attempt to redetect the file type by using the :ref:`Redetect File Type ` API endpoint. +From the home page of a Dataverse installation you can get a count of files by file type by clicking "Files" and then scrolling down to "File Type". If you see a lot of files that are "Unknown", "Application", or "Binary" you can have the Dataverse instalation attempt to redetect the file type by using the :ref:`Redetect File Type ` API endpoint. Getting Help ------------ From 5652040259ae62e1a4afc94a6acc5fa8a54f0baa Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 6 Jan 2021 12:48:34 -0500 Subject: [PATCH 063/576] last update of admin guide --- doc/sphinx-guides/source/admin/user-administration.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/admin/user-administration.rst b/doc/sphinx-guides/source/admin/user-administration.rst index d9907a94f43..bc9be64775f 100644 --- a/doc/sphinx-guides/source/admin/user-administration.rst +++ b/doc/sphinx-guides/source/admin/user-administration.rst @@ -9,7 +9,7 @@ This section focuses on user administration tools and tasks. Manage Users ------------ -The Manage Users table gives the network administrator a list of all user accounts in table form. You can access it by clicking the "Manage Users" button on the :doc:`dashboard`, which is linked from the header of all Dataverse pages (if you're logged in as an administrator). It lists username, full name, email address, affiliation, the authentication method they use, the roles their account has been granted, and whether or not they have Superuser status. +The Manage Users table gives the network administrator a list of all user accounts in table form. You can access it by clicking the "Manage Users" button on the :doc:`dashboard`, which is linked from the header of all Dataverse installation pages (if you're logged in as an administrator). It lists username, full name, email address, affiliation, the authentication method they use, the roles their account has been granted, and whether or not they have Superuser status. Users are listed alphabetically by username. The search bar above the table allows you to search for a specific user. It performs a right-truncated wildcard search of the Username, Name, and Email columns. This means, if you search "baseba" then it will search those three columns for any string of text that begins with "baseba", e.g. "baseball" or "baseballfan". @@ -47,9 +47,9 @@ See :ref:`change-identifier-label` Confirm Email ------------- -Dataverse encourages builtin/local users to verify their email address upon signup or email change so that sysadmins can be assured that users can be contacted. +A Dataverse installation encourages builtin/local users to verify their email address upon signup or email change so that sysadmins can be assured that users can be contacted. -The app will send a standard welcome email with a URL the user can click, which, when activated, will store a ``lastconfirmed`` timestamp in the ``authenticateduser`` table of the database. Any time this is "null" for a user (immediately after signup and/or changing of their Dataverse email address), their current email on file is considered to not be verified. The link that is sent expires after a time (the default is 24 hours), but this is configurable by a superuser via the ``:MinutesUntilConfirmEmailTokenExpires`` config option. +The app will send a standard welcome email with a URL the user can click, which, when activated, will store a ``lastconfirmed`` timestamp in the ``authenticateduser`` table of the database. Any time this is "null" for a user (immediately after signup and/or changing of their Dataverse installation email address), their current email on file is considered to not be verified. The link that is sent expires after a time (the default is 24 hours), but this is configurable by a superuser via the ``:MinutesUntilConfirmEmailTokenExpires`` config option. Should users' URL token expire, they will see a "Verify Email" button on the account information page to send another URL. From 9416735e7ad478569cdee1ffc2777fd58aa56f46 Mon Sep 17 00:00:00 2001 From: Ana Trisovic Date: Wed, 6 Jan 2021 14:35:54 -0500 Subject: [PATCH 064/576] add more mime types based on use frequency --- .../propertyFiles/MimeTypeFacets.properties | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/main/java/propertyFiles/MimeTypeFacets.properties b/src/main/java/propertyFiles/MimeTypeFacets.properties index 54c5e01d317..2f91ca042b4 100644 --- a/src/main/java/propertyFiles/MimeTypeFacets.properties +++ b/src/main/java/propertyFiles/MimeTypeFacets.properties @@ -5,6 +5,7 @@ application/pdf=Document image/pdf=Document text/pdf=Document application/x-pdf=Document +application/cnt=Document application/msword=Document application/vnd.ms-excel=Document application/vnd.openxmlformats-officedocument.spreadsheetml.sheet=Document @@ -12,14 +13,17 @@ application/vnd.ms-powerpoint=Document application/vnd.openxmlformats-officedocument.presentationml.presentation=Document application/vnd.openxmlformats-officedocument.wordprocessingml.document=Document application/vnd.oasis.opendocument.spreadsheet=Document +application/vnd.ms-excel.sheet.macroenabled.12=Document # Text text/plain=Text +text/x-log=Text text/html=Text application/x-tex=Text text/x-tex=Text text/markdown=Text text/x-markdown=Text text/x-r-markdown=Text +text/x-rst=Text application/rtf=Text text/rtf=Text text/richtext=Text @@ -28,16 +32,24 @@ application/xml=Text text/xml=Text # Code text/x-c=Code +text/x-c++src=Code text/css=Code +text/x-objcsrc=Code +application/java-vm=Code +text/x-java-source=Code text/javascript=Code application/javascript=Code application/x-javascript=Code +text/x-perl-script=Code text/x-matlab=Code text/x-mathematica=Code text/php=Code +text/x-fortran=Code +text/x-pascal=Code text/x-python=Code text/x-python-script=Code text/x-r-source=Code +text/x-sh=Code application/x-sh=Code application/x-shellscript=Code application/x-sql=Code @@ -53,12 +65,17 @@ application/x-spss-syntax=Code text/x-sas-syntax=Code application/x-sas-syntax=Code type/x-r-syntax=Code +application/postscript=Code +application/vnd.wolfram.mathematica.package=Code +application/vnd.wolfram.mathematica=Code # Ingested text/tab-separated-values=Tabular Data # Data text/tsv=Data text/comma-separated-values=Data +text/x-comma-separated-values=Data text/csv=Data +text/x-vcard=Data text/x-fixed-field=Data application/x-rlang-transport=Data application/x-R-2=Data @@ -81,6 +98,7 @@ application/x-sas-data=Data application/x-sas-catalog=Data application/x-sas-log=Data application/x-sas-output=Data +application/x-r-data=Data application/softgrid-do=Data application/x-dvn-csvspss-zip=Data application/x-dvn-tabddi-zip=Data @@ -100,7 +118,12 @@ application/netcdf=Data application/x-netcdf=Data application/vnd.lotus-notes=Data application/x-nsdstat=Data +application/vnd.flographit=Data application/vnd.realvnc.bed=Data +application/vnd.ms-pki.stl=Data +application/vnd.isac.fcs=Data +application/java-serialized-object=Data +chemical/x-xyz=Data # FITS image/fits=FITS application/fits=FITS @@ -112,6 +135,8 @@ application/sbn=Shape application/sbx=Shape application/shp=Shape application/shx=Shape +application/x-esri-shape=Shape +application/vnd.google-earth.kml+xml=Shape application/zipped-shapefile=Shape # Archive application/zip=Archive @@ -122,15 +147,18 @@ application/x-bzip2=Archive application/vnd.google-earth.kmz=Archive application/gzip=Archive application/x-gzip=Archive +application/x-gzip-compressed=Archive application/rar=Archive application/x-rar=Archive application/x-rar-compressed=Archive application/tar=Archive application/x-tar=Archive +application/x-compressed=Archive application/x-compressed-tar=Archive application/x-7z-compressed=Archive application/x-xz=Archive application/warc=Archive +application/x-iso9660-image=Archive # Image image/gif=Image image/jpeg=Image @@ -151,8 +179,13 @@ image/tiff=Image image/bmp=Image image/x-xbitmap=Image image/RAW=Image +image/raw=Image +application/x-tgif=Image image/x-xpixmap=Image image/x-xwindowdump=Image +application/photoshop=Image +image/vnd.adobe.photoshop=Image +application/x-photoshop=Image # (anything else that looks like image/* will also be indexed as facet type "Image") # Audio audio/x-aiff=Audio From 870fcfac1764d9082605758970653ececd3dfa1b Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 7 Jan 2021 10:46:38 -0500 Subject: [PATCH 065/576] more updates --- doc/sphinx-guides/source/api/apps.rst | 26 +++++++++++++------------- doc/sphinx-guides/source/api/auth.rst | 6 +++--- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/doc/sphinx-guides/source/api/apps.rst b/doc/sphinx-guides/source/api/apps.rst index dca641b1feb..0428dee9278 100755 --- a/doc/sphinx-guides/source/api/apps.rst +++ b/doc/sphinx-guides/source/api/apps.rst @@ -1,9 +1,9 @@ Apps ==== -The introduction of Dataverse APIs has fostered the development of a variety of software applications that are listed in the :doc:`/admin/integrations`, :doc:`/admin/external-tools`, and :doc:`/admin/reporting-tools-and-queries` sections of the Admin Guide. +The introduction of Dataverse Platform APIs has fostered the development of a variety of software applications that are listed in the :doc:`/admin/integrations`, :doc:`/admin/external-tools`, and :doc:`/admin/reporting-tools-and-queries` sections of the Admin Guide. -The apps below are open source and demonstrate how to use Dataverse APIs. Some of these apps are built on :doc:`/api/client-libraries` that are available for Dataverse APIs in Python, Javascript, R, and Java. +The apps below are open source and demonstrate how to use Dataverse Platform APIs. Some of these apps are built on :doc:`/api/client-libraries` that are available for Dataverse Platform APIs in Python, Javascript, R, and Java. .. contents:: |toctitle| :local: @@ -42,7 +42,7 @@ https://github.com/IQSS/TwoRavens Python ------ -Please note that there are multiple Python modules for Dataverse APIs listed in the :doc:`client-libraries` section. +Please note that there are multiple Python modules for Dataverse Platform APIs listed in the :doc:`client-libraries` section. dataverse-sample-data ~~~~~~~~~~~~~~~~~~~~~ @@ -54,28 +54,28 @@ https://github.com/IQSS/dataverse-sample-data Texas Digital Library dataverse-reports ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Dataverse Reports for Texas Digital Library generates and emails statistical reports for an installation of Dataverse using the native API and database queries. +Dataverse Installation Reports from Texas Digital Library generates and emails statistical reports for a Dataverse installation using the native API and database queries. https://github.com/TexasDigitalLibrary/dataverse-reports OSF ~~~ -OSF allows you to view, download, and upload files to and from a Dataverse dataset from an Open Science Framework (OSF) project. +OSF allows you to view, download, and upload files to and from a dataset in a Dataverse installation from an Open Science Framework (OSF) project. https://github.com/CenterForOpenScience/osf.io/tree/develop/addons/dataverse Geoconnect ~~~~~~~~~~ -Geoconnect allows Dataverse files to be visualized and explored on `WorldMap `_. Read more about it in the :doc:`/user/data-exploration/worldmap` section of the User Guide. +Geoconnect allows files from a Dataverse installation to be visualized and explored on `WorldMap `_. Read more about it in the :doc:`/user/data-exploration/worldmap` section of the User Guide. https://github.com/IQSS/geoconnect dataverse-metrics ~~~~~~~~~~~~~~~~~ -dataverse-metrics aggregates and visualizes metrics across multiple Dataverse installations but can also be used with a single installation +dataverse-metrics aggregates and visualizes metrics across multiple Dataverse installations but can also be used with a single installation. https://github.com/IQSS/dataverse-metrics @@ -97,26 +97,26 @@ repo2docker ~~~~~~~~~~~ repo2docker is a command line tool that allows you to create and start a -Docker image from a code repository that follows the [reproducible executable environment specification](https://repo2docker.readthedocs.io/en/latest/specification.html). repo2docker supports Dataverse DOIs to find and retrieve datasets. +Docker image from a code repository that follows the [reproducible executable environment specification](https://repo2docker.readthedocs.io/en/latest/specification.html). repo2docker supports Dataverse installation DOIs to find and retrieve datasets. https://github.com/jupyter/repo2docker/blob/master/repo2docker/contentproviders/dataverse.py Java ---- -Please note that there is a Java library for Dataverse APIs listed in the :doc:`client-libraries` section. +Please note that there is a Java library for Dataverse Platform APIs listed in the :doc:`client-libraries` section. DVUploader ~~~~~~~~~~ -The open-source DVUploader tool is a stand-alone command-line Java application that uses the Dataverse API to upload files to a specified Dataset. Files can be specified by name, or the DVUploader can upload all files in a directory or recursively from a directory tree. The DVUploader can also verify that uploaded files match their local sources by comparing the local and remote fixity checksums. Source code, release 1.0.0- jar file, and documentation are available on GitHub. DVUploader's creation was supported by the Texas Digital Library. +The open-source DVUploader tool is a stand-alone command-line Java application that uses the Dataverse Platform API to upload files to a specified Dataset. Files can be specified by name, or the DVUploader can upload all files in a directory or recursively from a directory tree. The DVUploader can also verify that uploaded files match their local sources by comparing the local and remote fixity checksums. Source code, release 1.0.0- jar file, and documentation are available on GitHub. DVUploader's creation was supported by the Texas Digital Library. https://github.com/IQSS/dataverse-uploader Dataverse for Android ~~~~~~~~~~~~~~~~~~~~~ -Dataverse for Android makes use of Dataverse's Search API. +Dataverse Software on Android makes use of a Dataverse installation's Search API. https://github.com/IQSS/dataverse-android @@ -126,13 +126,13 @@ PHP OJS ~~~ -The Open Journal Systems (OJS) Dataverse Plugin adds data sharing and preservation to the OJS publication process. +The Open Journal Systems (OJS) Dataverse Software Plugin adds data sharing and preservation to the OJS publication process. https://github.com/pkp/ojs/tree/ojs-stable-2_4_8/plugins/generic/dataverse OpenScholar ~~~~~~~~~~~ -The Dataverse module from OpenScholar allows Dataverse widgets to be easily embedded in its web pages: +The Dataverse Software module from OpenScholar allows a Dataverse installation's widgets to be easily embedded in its web pages: https://github.com/openscholar/openscholar/tree/SCHOLAR-3.x/openscholar/modules/os_features/os_dataverse diff --git a/doc/sphinx-guides/source/api/auth.rst b/doc/sphinx-guides/source/api/auth.rst index 6d53e52d3b4..99a8a6a843b 100644 --- a/doc/sphinx-guides/source/api/auth.rst +++ b/doc/sphinx-guides/source/api/auth.rst @@ -1,7 +1,7 @@ API Tokens and Authentication ============================= -An API token is similar to a password and allows you to authenticate to Dataverse APIs to perform actions as you. Many Dataverse APIs require the use of an API token. +An API token is similar to a password and allows you to authenticate to Dataverse Platform APIs to perform actions as you. Many Dataverse Platform APIs require the use of an API token. .. contents:: |toctitle| :local: @@ -23,7 +23,7 @@ Passing Your API Token as an HTTP Header (Preferred) or a Query Parameter See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. -There are two ways to pass your API token to Dataverse APIs. The preferred method is to send the token in the ``X-Dataverse-key`` HTTP header, as in the following curl example. +There are two ways to pass your API token to Dataverse Platform APIs. The preferred method is to send the token in the ``X-Dataverse-key`` HTTP header, as in the following curl example. .. code-block:: bash @@ -60,4 +60,4 @@ Use of the ``X-Dataverse-key`` HTTP header form is preferred to passing ``key`` Resetting Your API Token ------------------------ -You can reset your API Token from your account page in Dataverse as described in the :doc:`/user/account` section of the User Guide. +You can reset your API Token from your account page in your Dataverse installation as described in the :doc:`/user/account` section of the User Guide. From 8e7ea59ab185b57f2cf72b6e820788bcd23ed3b6 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 7 Jan 2021 10:51:18 -0500 Subject: [PATCH 066/576] more updates --- .../source/api/client-libraries.rst | 16 ++++++++-------- doc/sphinx-guides/source/api/dataaccess.rst | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/sphinx-guides/source/api/client-libraries.rst b/doc/sphinx-guides/source/api/client-libraries.rst index 64726c77f58..406064424cb 100755 --- a/doc/sphinx-guides/source/api/client-libraries.rst +++ b/doc/sphinx-guides/source/api/client-libraries.rst @@ -1,9 +1,9 @@ Client Libraries ================ -Currently there are client libraries for Python, Javascript, R, and Java that can be used to develop against Dataverse APIs. We use the term "client library" on this page but "Dataverse SDK" (software development kit) is another way of describing these resources. They are designed to help developers express Dataverse concepts more easily in the languages listed below. For support on any of these client libraries, please consult each project's README. +Currently there are client libraries for Python, Javascript, R, and Java that can be used to develop against Dataverse Software APIs. We use the term "client library" on this page but "Dataverse Software SDK" (software development kit) is another way of describing these resources. They are designed to help developers express Dataverse Software concepts more easily in the languages listed below. For support on any of these client libraries, please consult each project's README. -Because Dataverse is a SWORD server, additional client libraries exist for Java, Ruby, and PHP per the :doc:`/api/sword` page. +Because a Dataverse installation is a SWORD server, additional client libraries exist for Java, Ruby, and PHP per the :doc:`/api/sword` page. .. contents:: |toctitle| :local: @@ -11,29 +11,29 @@ Because Dataverse is a SWORD server, additional client libraries exist for Java, Python ------ -There are two Python modules for interacting with Dataverse APIs. +There are two Python modules for interacting with Dataverse Software APIs. `pyDataverse `_ had its initial release in 2019 and can be installed with ``pip install pyDataverse``. The module is developed by `Stefan Kasberger `_ from `AUSSDA - The Austrian Social Science Data Archive `_. -`dataverse-client-python `_ had its initial release in 2015. `Robert Liebowitz `_ created this library while at the `Center for Open Science (COS) `_ and the COS uses it to integrate the `Open Science Framework (OSF) `_ with Dataverse via an add-on which itself is open source and listed on the :doc:`/api/apps` page. +`dataverse-client-python `_ had its initial release in 2015. `Robert Liebowitz `_ created this library while at the `Center for Open Science (COS) `_ and the COS uses it to integrate the `Open Science Framework (OSF) `_ with a Dataverse installation via an add-on which itself is open source and listed on the :doc:`/api/apps` page. Javascript ---------- -https://github.com/IQSS/dataverse-client-javascript is the official Javascript package for Dataverse APIs. It can be found on npm at https://www.npmjs.com/package/js-dataverse +https://github.com/IQSS/dataverse-client-javascript is the official Javascript package for Dataverse Software APIs. It can be found on npm at https://www.npmjs.com/package/js-dataverse It was created and is maintained by `The Agile Monkeys `_. R - -https://github.com/IQSS/dataverse-client-r is the official R package for Dataverse APIs. The latest release can be installed from `CRAN `_. +https://github.com/IQSS/dataverse-client-r is the official R package for Dataverse Software APIs. The latest release can be installed from `CRAN `_. -The package is currently maintained by `Will Beasley `_. It was created by `Thomas Leeper `_ whose dataverse can be found at https://dataverse.harvard.edu/dataverse/leeper +The package is currently maintained by `Will Beasley `_. It was created by `Thomas Leeper `_ whose Dataverse collection can be found at https://dataverse.harvard.edu/dataverse/leeper Java ---- -https://github.com/IQSS/dataverse-client-java is the official Java library for Dataverse APIs. +https://github.com/IQSS/dataverse-client-java is the official Java library for Dataverse Platform APIs. `Richard Adams `_ from `ResearchSpace `_ created and maintains this library. diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst index 3c20233f6e6..9bafde1d819 100755 --- a/doc/sphinx-guides/source/api/dataaccess.rst +++ b/doc/sphinx-guides/source/api/dataaccess.rst @@ -1,7 +1,7 @@ Data Access API =============== -The Data Access API provides programmatic download access to the files stored under Dataverse. +The Data Access API provides programmatic download access to the files stored in a Dataverse installation. More advanced features of the Access API include format-specific transformations (thumbnail generation/resizing for images; converting tabular data into alternative file formats) and access to the data-level metadata that describes the contents of the tabular files. .. contents:: |toctitle| @@ -286,7 +286,7 @@ Preprocessed Data ``/api/access/datafile/$id/metadata/preprocessed`` -This method provides the "preprocessed data" - a summary record that describes the values of the data vectors in the tabular file, in JSON. These metadata values are used by TwoRavens, the companion data exploration utility of the Dataverse application. Please note that this format might change in the future. +This method provides the "preprocessed data" - a summary record that describes the values of the data vectors in the tabular file, in JSON. These metadata values are used by TwoRavens, an external tool that integrates with a Dataverse installation. Please note that this format might change in the future. Authentication and Authorization -------------------------------- From b2f3bfaf6c382cc822edcdf04828234f7d71d881 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 7 Jan 2021 11:30:00 -0500 Subject: [PATCH 067/576] yet more updates --- .../source/api/external-tools.rst | 40 +++++++++---------- doc/sphinx-guides/source/api/faq.rst | 18 ++++----- .../source/api/getting-started.rst | 26 ++++++------ 3 files changed, 42 insertions(+), 42 deletions(-) diff --git a/doc/sphinx-guides/source/api/external-tools.rst b/doc/sphinx-guides/source/api/external-tools.rst index 9c261154150..5ea4a1aa83b 100644 --- a/doc/sphinx-guides/source/api/external-tools.rst +++ b/doc/sphinx-guides/source/api/external-tools.rst @@ -1,7 +1,7 @@ Building External Tools ======================= -External tools can provide additional features that are not part of Dataverse itself, such as data exploration. Thank you for your interest in building an external tool for Dataverse! +External tools can provide additional features that are not part of the Dataverse Platform itself, such as data exploration. Thank you for your interest in building an external tool for the Dataverse Platform! .. contents:: |toctitle| :local: @@ -9,11 +9,11 @@ External tools can provide additional features that are not part of Dataverse it Introduction ------------ -External tools are additional applications the user can access or open from Dataverse to preview, explore, and manipulate data files and datasets. The term "external" is used to indicate that the tool is not part of the main Dataverse application. +External tools are additional applications the user can access or open from your Dataverse installation to preview, explore, and manipulate data files and datasets. The term "external" is used to indicate that the tool is not part of the main Dataverse Platform. -Once you have created the external tool itself (which is most of the work!), you need to teach Dataverse how to construct URLs that your tool needs to operate. For example, if you've deployed your tool to fabulousfiletool.com your tool might want the ID of a file and the siteUrl of the Dataverse installation like this: https://fabulousfiletool.com?fileId=42&siteUrl=http://demo.dataverse.org +Once you have created the external tool itself (which is most of the work!), you need to teach a Dataverse installation how to construct URLs that your tool needs to operate. For example, if you've deployed your tool to fabulousfiletool.com your tool might want the ID of a file and the siteUrl of the Dataverse installation like this: https://fabulousfiletool.com?fileId=42&siteUrl=http://demo.dataverse.org -In short, you will be creating a manifest in JSON format that describes not only how to construct URLs for your tool, but also what types of files your tool operates on, where it should appear in the Dataverse web interfaces, etc. +In short, you will be creating a manifest in JSON format that describes not only how to construct URLs for your tool, but also what types of files your tool operates on, where it should appear in the Dataverse installation web interfaces, etc. The possibilities for external tools are endless. Let's look at some examples to get your creative juices flowing. Then we'll look at a complete list of parameters you can use when creating the manifest file for your tool. @@ -37,18 +37,18 @@ Note: This is the same list that appears in the :doc:`/admin/external-tools` sec How External Tools Are Presented to Users ----------------------------------------- -An external tool can appear in Dataverse in a variety of ways: +An external tool can appear in your Dataverse installation in a variety of ways: - as an explore, preview, or configure option for a file - as an explore option for a dataset - as an embedded preview on the file landing page -See also the :ref:`testing-external-tools` section of the Admin Guide for some perspective on how installations of Dataverse will expect to test your tool before announcing it to their users. +See also the :ref:`testing-external-tools` section of the Admin Guide for some perspective on how Dataverse installations will expect to test your tool before announcing it to their users. Creating an External Tool Manifest ---------------------------------- -External tools must be expressed in an external tool manifest file, a specific JSON format Dataverse requires. As the author of an external tool, you are expected to provide this JSON file and installation instructions on a web page for your tool. +External tools must be expressed in an external tool manifest file, a specific JSON format a Dataverse installation requires. As the author of an external tool, you are expected to provide this JSON file and installation instructions on a web page for your tool. Examples of Manifests +++++++++++++++++++++ @@ -78,9 +78,9 @@ Terminology =========================== ========== Term Definition =========================== ========== - external tool manifest A **JSON file** the defines the URL constructed by Dataverse when users click explore or configure tool options. External tool makers are asked to host this JSON file on a website (no app store yet, sorry) and explain how to use install and use the tool. Examples include :download:`fabulousFileTool.json <../_static/installation/files/root/external-tools/fabulousFileTool.json>` and :download:`dynamicDatasetTool.json <../_static/installation/files/root/external-tools/dynamicDatasetTool.json>` as well as the real world examples above such as Data Explorer. + external tool manifest A **JSON file** the defines the URL constructed by a Dataverse installation when users click explore or configure tool options. External tool makers are asked to host this JSON file on a website (no app store yet, sorry) and explain how to use install and use the tool. Examples include :download:`fabulousFileTool.json <../_static/installation/files/root/external-tools/fabulousFileTool.json>` and :download:`dynamicDatasetTool.json <../_static/installation/files/root/external-tools/dynamicDatasetTool.json>` as well as the real world examples above such as Data Explorer. - displayName The **name** of the tool in the Dataverse web interface. For example, "Data Explorer". + displayName The **name** of the tool in the Dataverse installation web interface. For example, "Data Explorer". description The **description** of the tool, which appears in a popup (for configure tools only) so the user who clicked the tool can learn about the tool before being redirected the tool in a new tab in their browser. HTML is supported. @@ -102,7 +102,7 @@ Terminology reserved words A **set of strings surrounded by curly braces** such as ``{fileId}`` or ``{datasetId}`` that will be inserted into query parameters. See the table below for a complete list. - toolName A **name** of an external tool that is used to differentiate between external tools and also used in bundle.properties for localization in the Dataverse web interface. For example, the toolName for Data Explorer is ``explorer``. For the Data Curation Tool the toolName is ``dct``. This is an optional parameter in the manifest JSON file. + toolName A **name** of an external tool that is used to differentiate between external tools and also used in bundle.properties for localization in the Dataverse installation web interface. For example, the toolName for Data Explorer is ``explorer``. For the Data Curation Tool the toolName is ``dct``. This is an optional parameter in the manifest JSON file. =========================== ========== Reserved Words @@ -118,9 +118,9 @@ Reserved Words ``{fileId}`` depends The database ID of a file the user clicks "Explore" or "Configure" on. For example, ``42``. This reserved word is **required for file level tools** unless you use ``{filePid}`` instead. - ``{filePid}`` depends The Persistent ID (DOI or Handle) of a file the user clicks "Explore" or "Configure" on. For example, ``doi:10.7910/DVN/TJCLKP/3VSTKY``. Note that not all installations of Dataverse have Persistent IDs (PIDs) enabled at the file level. This reserved word is **required for file level tools** unless you use ``{fileId}`` instead. + ``{filePid}`` depends The Persistent ID (DOI or Handle) of a file the user clicks "Explore" or "Configure" on. For example, ``doi:10.7910/DVN/TJCLKP/3VSTKY``. Note that not all Dataverse installations have Persistent IDs (PIDs) enabled at the file level. This reserved word is **required for file level tools** unless you use ``{fileId}`` instead. - ``{apiToken}`` optional The Dataverse API token of the user launching the external tool, if available. Please note that API tokens should be treated with the same care as a password. For example, ``f3465b0c-f830-4bc7-879f-06c0745a5a5c``. + ``{apiToken}`` optional The Dataverse installation's API token of the user launching the external tool, if available. Please note that API tokens should be treated with the same care as a password. For example, ``f3465b0c-f830-4bc7-879f-06c0745a5a5c``. ``{datasetId}`` depends The database ID of the dataset. For example, ``42``. This reseved word is **required for dataset level tools** unless you use ``{datasetPid}`` instead. @@ -128,13 +128,13 @@ Reserved Words ``{datasetVersion}`` optional The friendly version number ( or \:draft ) of the dataset version the file level tool is being launched from. For example, ``1.0`` or ``:draft``. - ``{localeCode}`` optional The code for the language ("en" for English, "fr" for French, etc.) that user has selected from the language toggle in Dataverse. See also :ref:`i18n`. + ``{localeCode}`` optional The code for the language ("en" for English, "fr" for French, etc.) that user has selected from the language toggle in a Dataverse installation. See also :ref:`i18n`. =========================== ========== =========== Internationalization of Your External Tool ++++++++++++++++++++++++++++++++++++++++++ -The name and description of your tool can be localized and made available in different languages in the Dataverse web interface. Use the ``toolName`` parameter in the manifest JSON file and then add that toolName to bundle.properties. +The name and description of your tool can be localized and made available in different languages in your Dataverse installation's web interface. Use the ``toolName`` parameter in the manifest JSON file and then add that toolName to bundle.properties. For example, if the ``toolName`` of your external tool is ``fabulous`` then the lines in Bundle.properties should be: @@ -150,9 +150,9 @@ Again, you can use :download:`fabulousFileTool.json <../_static/installation/fil Testing Your External Tool -------------------------- -As the author of an external tool, you are not expected to learn how to install and operate Dataverse. There's a very good chance your tool can be added to a server Dataverse developers use for testing if you reach out on any of the channels listed under :ref:`getting-help-developers` in the Developer Guide. +As the author of an external tool, you are not expected to learn how to install and operate a Dataverse installation. There's a very good chance your tool can be added to a server Dataverse Community developers use for testing if you reach out on any of the channels listed under :ref:`getting-help-developers` in the Developer Guide. -By all means, if you'd like to install Dataverse yourself, a number of developer-centric options are available. For example, there's a script to spin up Dataverse on EC2 at https://github.com/IQSS/dataverse-sample-data . The process for using curl to add your external tool to a Dataverse installation is documented under :ref:`managing-external-tools` in the Admin Guide. +By all means, if you'd like to install a Dataverse installation yourself, a number of developer-centric options are available. For example, there's a script to spin up a Dataverse installation on EC2 at https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible . The process for using curl to add your external tool to your Dataverse installation is documented under :ref:`managing-external-tools` in the Admin Guide. Spreading the Word About Your External Tool ------------------------------------------- @@ -162,19 +162,19 @@ Adding Your Tool to the Inventory of External Tools Once you've gotten your tool working, please make a pull request to update the list of tools above! You are also welcome to download :download:`dataverse-external-tools.tsv <../_static/admin/dataverse-external-tools.tsv>`, add your tool to the TSV file, create and issue at https://github.com/IQSS/dataverse/issues , and then upload your TSV file there. -Unless your tool runs entirely in a browser, you may have integrated server-side software with Dataverse. If so, please double check that your software is listed in the :doc:`/admin/integrations` section of the Admin Guide and if not, please open an issue or pull request to add it. Thanks! +Unless your tool runs entirely in a browser, you may have integrated server-side software with your Dataverse installation. If so, please double check that your software is listed in the :doc:`/admin/integrations` section of the Admin Guide and if not, please open an issue or pull request to add it. Thanks! -If you've thought to yourself that there ought to be an app store for Dataverse external tools, you're not alone. Please see https://github.com/IQSS/dataverse/issues/5688 :) +If you've thought to yourself that there ought to be an app store for Dataverse Platform external tools, you're not alone. Please see https://github.com/IQSS/dataverse/issues/5688 :) Demoing Your External Tool ++++++++++++++++++++++++++ -https://demo.dataverse.org is the place to play around with Dataverse and your tool can be included. Please email support@dataverse.org to start the conversation about adding your tool. Additionally, you are welcome to open an issue at https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible which already includes a number of the tools listed above. +https://demo.dataverse.org is the place to play around with the Dataverse Platform and your tool can be included. Please email support@dataverse.org to start the conversation about adding your tool. Additionally, you are welcome to open an issue at https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible which already includes a number of the tools listed above. Announcing Your External Tool +++++++++++++++++++++++++++++ You are welcome to announce your external tool at https://groups.google.com/forum/#!forum/dataverse-community -If you're too shy, we'll do it for you. We'll probably tweet about it too. Thank you for your contribution to Dataverse! +If you're too shy, we'll do it for you. We'll probably tweet about it too. Thank you for your contribution to the Dataverse Project! diff --git a/doc/sphinx-guides/source/api/faq.rst b/doc/sphinx-guides/source/api/faq.rst index a3994501f6c..d580692a9f5 100644 --- a/doc/sphinx-guides/source/api/faq.rst +++ b/doc/sphinx-guides/source/api/faq.rst @@ -11,13 +11,13 @@ What is an API? See "What is an API?" in the :doc:`intro` section. -What Are Common Use Cases for Dataverse APIs? ---------------------------------------------- +What Are Common Use Cases for Dataverse Platform APIs? +---------------------------------------------------------- See the :doc:`getting-started` section for common use cases for researchers and curators. Other types of API users should find starting points at :ref:`types-of-api-users`. -Where Can I Find Examples of Using Dataverse APIs? --------------------------------------------------- +Where Can I Find Examples of Using Dataverse Platform APIs? +----------------------------------------------------------- See the :doc:`getting-started` section links to examples using curl. @@ -26,7 +26,7 @@ For examples in Javascript, Python, R, and Java, and PHP, see the :doc:`apps` an When Should I Use the Native API vs. the SWORD API? --------------------------------------------------- -The :doc:`sword` is based on a standard, works fine, and is fully supported, but much more development effort has been going into the :doc:`native-api`, which is not based on a standard. It is specific to Dataverse. +The :doc:`sword` is based on a standard, works fine, and is fully supported, but much more development effort has been going into the :doc:`native-api`, which is not based on a standard. It is specific to the Dataverse Platform. SWORD uses XML. The Native API uses JSON. @@ -54,9 +54,9 @@ Note that when multiple query parameters are used (such as ``persistentId`` and Where is the Comprehensive List of All API Functionality? --------------------------------------------------------- -There are so many Dataverse APIs that a single page in this guide would probably be overwhelming. See :ref:`list-of-dataverse-apis` for links to various pages. +There are so many Dataverse Platform APIs that a single page in this guide would probably be overwhelming. See :ref:`list-of-dataverse-apis` for links to various pages. -It is possible to get a complete list of API functionality in Swagger/OpenAPI format if you deploy Dataverse to Payara 5+. For details, see https://github.com/IQSS/dataverse/issues/5794 +It is possible to get a complete list of API functionality in Swagger/OpenAPI format if you deploy Dataverse Software version 5+. For details, see https://github.com/IQSS/dataverse/issues/5794 Is There a Changelog of API Functionality That Has Been Added Over Time? ------------------------------------------------------------------------ @@ -70,8 +70,8 @@ What Functionality is GUI Only and Not Available Via API The following tasks cannot currently be automated via API because no API exists for them. The web interface should be used instead for these GUI-only features: -- Setting a logo image, URL, and tagline when creating a dataverse. -- Editing properties of an existing dataverse. +- Setting a logo image, URL, and tagline when creating a Dataverse collection. +- Editing properties of an existing Dataverse collection. - Set "Enable Access Request" for Terms of Use: https://groups.google.com/d/msg/dataverse-community/oKdesT9rFGc/qM6wrsnnBAAJ - Downloading a guestbook. - Set guestbook_id for a dataset: https://groups.google.com/d/msg/dataverse-community/oKdesT9rFGc/qM6wrsnnBAAJ diff --git a/doc/sphinx-guides/source/api/getting-started.rst b/doc/sphinx-guides/source/api/getting-started.rst index 1c96a0af542..0b4fd42ad4a 100644 --- a/doc/sphinx-guides/source/api/getting-started.rst +++ b/doc/sphinx-guides/source/api/getting-started.rst @@ -1,7 +1,7 @@ Getting Started with APIs ========================= -If you are a researcher or curator who wants to automate parts of your workflow, this section should help you get started. The :doc:`intro` section lists resources for other groups who may be interested in Dataverse APIs such as developers of integrations and support teams. +If you are a researcher or curator who wants to automate parts of your workflow, this section should help you get started. The :doc:`intro` section lists resources for other groups who may be interested in Dataverse Platform APIs such as developers of integrations and support teams. .. contents:: |toctitle| :local: @@ -9,14 +9,14 @@ If you are a researcher or curator who wants to automate parts of your workflow, Servers You Can Test With ------------------------- -Rather than using a production installation of Dataverse, API users are welcome to use http://demo.dataverse.org for testing. You can email support@dataverse.org if you have any trouble with this server. +Rather than using a production Dataverse installation, API users are welcome to use http://demo.dataverse.org for testing. You can email support@dataverse.org if you have any trouble with this server. If you would rather have full control over your own test server, deployments to AWS, Docker, Vagrant, and more are covered in the :doc:`/developers/index` and the :doc:`/installation/index`. Getting an API Token -------------------- -Many Dataverse APIs require an API token. +Many Dataverse Platform APIs require an API token. Once you have identified a server to test with, create an account, click on your name, and get your API token. For more details, see the :doc:`auth` section. @@ -59,8 +59,8 @@ If you don't like curl, don't have curl, or want to use a different programming Depositing Data --------------- -Creating a Dataverse -~~~~~~~~~~~~~~~~~~~~ +Creating a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ See :ref:`create-dataverse-api`. @@ -74,8 +74,8 @@ Uploading Files See :ref:`add-file-api`. -Publishing a Dataverse -~~~~~~~~~~~~~~~~~~~~~~ +Publishing a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ See :ref:`publish-dataverse-api`. @@ -94,8 +94,8 @@ A quick example search for the word "data" is https://demo.dataverse.org/api/sea See the :doc:`search` section for details. -Finding Recently Published Dataverses, Datasets, and Files -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Finding Recently Published Dataverse Collections, Datasets, and Files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ See :ref:`search-date-range`. @@ -117,8 +117,8 @@ Dataset metadata is available in a variety of formats listed at :ref:`metadata-e See :ref:`export-dataset-metadata-api`. -Listing the Contents of a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Listing the Contents of a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ See :ref:`show-contents-of-a-dataverse-api`. @@ -143,11 +143,11 @@ See :ref:`list-role-assignments-on-a-dataverse-api`. Beyond "Getting Started" Tasks ------------------------------ -In addition to the tasks listed above, Dataverse supports many other operations via API. +In addition to the tasks listed above, your Dataverse installation supports many other operations via API. See :ref:`list-of-dataverse-apis` and :ref:`types-of-api-users` to get oriented. -If you're looking for some inspiration for how you can use Dataverse APIs, there are open source projects that integrate with Dataverse listed in the :doc:`apps` section. +If you're looking for some inspiration for how you can use the Dataverse Platform APIs, there are open source projects that integrate with the Dataverse Platform listed in the :doc:`apps` section. Getting Help ------------- From b6e1edb5a32c8bce86d3b0bc30b1199eacee7095 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 7 Jan 2021 12:53:08 -0500 Subject: [PATCH 068/576] annnnd more updates --- doc/sphinx-guides/source/api/intro.rst | 34 ++++++++++++------------ doc/sphinx-guides/source/api/metrics.rst | 16 +++++------ doc/sphinx-guides/source/api/search.rst | 22 +++++++-------- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/doc/sphinx-guides/source/api/intro.rst b/doc/sphinx-guides/source/api/intro.rst index 6ce77f5fe7d..122f99854b9 100755 --- a/doc/sphinx-guides/source/api/intro.rst +++ b/doc/sphinx-guides/source/api/intro.rst @@ -1,16 +1,16 @@ Introduction ============ -Dataverse APIs allow users to accomplish many tasks such as... +Dataverse Platform APIs allow users to accomplish many tasks such as... - creating datasets - uploading files - publishing datasets - and much, much more -... all without using the Dataverse web interface. +... all without using the Dataverse installation's web interface. -APIs open the door for integrations between Dataverse and other software. For a list, see the :doc:`/admin/integrations` section of the Admin Guide. +APIs open the door for integrations between the Dataverse Platform and other software. For a list, see the :doc:`/admin/integrations` section of the Admin Guide. .. contents:: |toctitle| :local: @@ -18,7 +18,7 @@ APIs open the door for integrations between Dataverse and other software. For a What is an API? --------------- -API stands for "Application Programming Interface" and an example is Dataverse's "file upload" API. In the diagram below, we can see that while users can click a button within Dataverse's web interface to upload a file, there are many other ways to get files into Dataverse, all using an API that allows for uploading of files. +API stands for "Application Programming Interface" and an example is the Dataverse Platform's "file upload" API. In the diagram below, we can see that while users can click a button within Dataverse installation's web interface to upload a file, there are many other ways to get files into a Dataverse installation, all using an API that allows for uploading of files. .. graphviz:: @@ -65,34 +65,34 @@ API stands for "Application Programming Interface" and an example is Dataverse's The components above that use the "file" upload API are: - DvUploader is terminal-based application for uploading files that is described in the :doc:`/user/dataset-management` section of the User Guide. -- OJS, OSF, and RSpace are all web applications that can integrate with Dataverse and are described in "Getting Data In" in the :doc:`/admin/integrations` section of the Admin Guide. +- OJS, OSF, and RSpace are all web applications that can integrate with a Dataverse installation and are described in "Getting Data In" in the :doc:`/admin/integrations` section of the Admin Guide. - The script in the diagram can be as simple as a single line of code that is run in a terminal. You can copy and paste "one-liners" like this from the guide. See the :doc:`getting-started` section for examples using a tool called "curl". The diagram above shows only a few examples of software using a specific API but many more APIs are available. .. _types-of-api-users: -Types of Dataverse API Users ----------------------------- +Types of Dataverse Platform API Users +------------------------------------- This guide is intended to serve multiple audiences but pointers various sections of the guide are provided below based on the type of API user you are. -API Users Within a Single Installation of Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +API Users Within a Single Dataverse Installation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Each installation of Dataverse will have its own groups of people interested in APIs. +Each Dataverse installation will have its own groups of people interested in APIs. Users of Integrations and Apps ++++++++++++++++++++++++++++++ Integrations and apps can take many forms but two examples are: -- Using Open Science Framework (OSF), a web application, to deposit and publish data into Dataverse. -- Using DVUploader, a terminal-based desktop application, to upload files into Dataverse. +- Using Open Science Framework (OSF), a web application, to deposit and publish data into a Dataverse installation. +- Using DVUploader, a terminal-based desktop application, to upload files into a Dataverse installation. -In both examples, users need to obtain an API Token to authenticate with Dataverse. +In both examples, users need to obtain an API Token to authenticate with a Dataverse installation. -|Start| A good starting point is "API Tokens" in the :doc:`/user/account` section of the User Guide. DvUploader is documented in the :doc:`/user/dataset-management` section of the User Guide. The integrations that are enabled depend on your installation of Dataverse. You can find a list in the :doc:`/admin/integrations` section of the Admin Guide. +|Start| A good starting point is "API Tokens" in the :doc:`/user/account` section of the User Guide. DvUploader is documented in the :doc:`/user/dataset-management` section of the User Guide. The integrations that are enabled depend on your Dataverse installation. You can find a list in the :doc:`/admin/integrations` section of the Admin Guide. Power Users +++++++++++ @@ -104,16 +104,16 @@ Power users may be researchers or curators who are comfortable with automating p Support Teams and Superusers ++++++++++++++++++++++++++++ -Support teams that answer questions about their installation of Dataverse should familiarize themselves with the :doc:`getting-started` section to get a sense of common tasks that researchers and curators might be trying to accomplish by using Dataverse APIs. +Support teams that answer questions about their Dataverse installation should familiarize themselves with the :doc:`getting-started` section to get a sense of common tasks that researchers and curators might be trying to accomplish by using Dataverse Platform APIs. -Superusers of an installation of Dataverse have access a superuser dashboard described in the :doc:`/admin/dashboard` section of the Admin Guide but some operations can only be done via API. +Superusers of a Dataverse installation have access a superuser dashboard described in the :doc:`/admin/dashboard` section of the Admin Guide but some operations can only be done via API. |Start| A good starting point for both groups is the :doc:`getting-started` section of this guide followed by the :doc:`/admin/troubleshooting` section of the Admin Guide. Sysadmins +++++++++ -Sysadmins often write scripts to automate tasks and Dataverse APIs make this possible. Sysadmins have control over the server that Dataverse is running on and may be called upon to execute API commands that are limited to "localhost" (the server itself) for security reasons. +Sysadmins often write scripts to automate tasks and Dataverse Platform APIs make this possible. Sysadmins have control over the server that the Dataverse installation is running on and may be called upon to execute API commands that are limited to "localhost" (the server itself) for security reasons. |Start| A good starting point for sysadmins is "Blocking API Endpoints" in the :doc:`/installation/config` section of the Installation Guide, followed by the :doc:`getting-started` section of this guide, followed by the :doc:`/admin/troubleshooting` section of the Admin Guide. diff --git a/doc/sphinx-guides/source/api/metrics.rst b/doc/sphinx-guides/source/api/metrics.rst index 70a55a27cc8..e29cb3a2c9e 100755 --- a/doc/sphinx-guides/source/api/metrics.rst +++ b/doc/sphinx-guides/source/api/metrics.rst @@ -1,7 +1,7 @@ Metrics API =========== -The Metrics API provides counts of downloads, datasets created, files uploaded, and more, as described below. Dataverse also supports Make Data Count, which is described in the :doc:`/admin/make-data-count` section of the Admin Guide. +The Metrics API provides counts of downloads, datasets created, files uploaded, and more, as described below. The Dataverse Platform also supports Make Data Count, which is described in the :doc:`/admin/make-data-count` section of the Admin Guide. .. contents:: |toctitle| :local: @@ -15,7 +15,7 @@ The Metrics API provides counts of downloads, datasets created, files uploaded, Total ----- -Returns a count of various objects in dataverse over all-time:: +Returns a count of various objects in a Dataverse installation over all-time:: GET https://$SERVER/api/info/metrics/$type @@ -26,7 +26,7 @@ Example: ``curl https://demo.dataverse.org/api/info/metrics/downloads`` To-Month -------- -Returns a count of various objects in dataverse up to a specified month ``$YYYY-DD`` in YYYY-MM format (e.g. ``2018-01``):: +Returns a count of various objects in a Dataverse installation up to a specified month ``$YYYY-DD`` in YYYY-MM format (e.g. ``2018-01``):: GET https://$SERVER/api/info/metrics/$type/toMonth/$YYYY-DD @@ -38,7 +38,7 @@ Example: ``curl https://demo.dataverse.org/api/info/metrics/dataverses/toMonth/2 Past Days --------- -Returns a count of various objects in dataverse for the past ``$days`` (e.g. ``30``):: +Returns a count of various objects in a Dataverse installation for the past ``$days`` (e.g. ``30``):: GET https://$SERVER/api/info/metrics/$type/pastDays/$days @@ -47,13 +47,13 @@ Returns a count of various objects in dataverse for the past ``$days`` (e.g. ``3 Example: ``curl https://demo.dataverse.org/api/info/metrics/datasets/pastDays/30`` -Dataverse Specific Metrics --------------------------- +Dataverse Collection Specific Metrics +------------------------------------- By Subject ~~~~~~~~~~~~~~~ -Returns the number of dataverses by each subject:: +Returns the number of Dataverse collections in a Dataverse installation by each subject:: GET https://$SERVER/api/info/metrics/dataverses/bySubject @@ -61,7 +61,7 @@ Returns the number of dataverses by each subject:: By Category ~~~~~~~~~~~~~~~~~~~~~~ -Returns the number of dataverses by each category:: +Returns the number of Dataverse collections by each category:: GET https://$SERVER/api/info/metrics/dataverses/byCategory diff --git a/doc/sphinx-guides/source/api/search.rst b/doc/sphinx-guides/source/api/search.rst index 25bad2b8091..60e799f00fc 100755 --- a/doc/sphinx-guides/source/api/search.rst +++ b/doc/sphinx-guides/source/api/search.rst @@ -4,7 +4,7 @@ Search API .. contents:: |toctitle| :local: -The Search API supports the same searching, sorting, and faceting operations as the Dataverse web interface. +The Search API supports the same searching, sorting, and faceting operations as the Dataverse Platform's web interface. To search unpublished content, you must pass in an API token as described in the :doc:`auth` section. @@ -15,7 +15,7 @@ The parameters and JSON response are partly inspired by the `GitHub Search API < .. _CORS: https://www.w3.org/TR/cors/ -Please note that in Dataverse 4.3 and older the "citation" field wrapped the persistent ID URL in an ```` tag but this has been changed to plaintext. If you want the old value with HTML in it, a new field called "citationHtml" can be used. +Please note that in Dataverse software version 4.3 and older the "citation" field wrapped the persistent ID URL in an ```` tag but this has been changed to plaintext. If you want the old value with HTML in it, a new field called "citationHtml" can be used. Parameters @@ -25,8 +25,8 @@ Parameters Name Type Description =============== ======= =========== q string The search term or terms. Using "title:data" will search only the "title" field. "*" can be used as a wildcard either alone or adjacent to a term (i.e. "bird*"). For example, https://demo.dataverse.org/api/search?q=title:data . For a list of fields to search, please see https://github.com/IQSS/dataverse/issues/2558 (for now). -type string Can be either "dataverse", "dataset", or "file". Multiple "type" parameters can be used to include multiple types (i.e. ``type=dataset&type=file``). If omitted, all types will be returned. For example, https://demo.dataverse.org/api/search?q=*&type=dataset -subtree string The identifier of the dataverse to which the search should be narrowed. The subtree of this dataverse and all its children will be searched. Multiple "subtree" parameters can be used to include multiple Dataverses. For example, https://demo.dataverse.org/api/search?q=data&subtree=birds&subtree=cats . +type string Can be either "Dataverse", "dataset", or "file". Multiple "type" parameters can be used to include multiple types (i.e. ``type=dataset&type=file``). If omitted, all types will be returned. For example, https://demo.dataverse.org/api/search?q=*&type=dataset +subtree string The identifier of the Dataverse collection to which the search should be narrowed. The subtree of this Dataverse collection and all its children will be searched. Multiple "subtree" parameters can be used to include multiple Dataverse collections. For example, https://demo.dataverse.org/api/search?q=data&subtree=birds&subtree=cats . sort string The sort field. Supported values include "name" and "date". See example under "order". order string The order in which to sort. Can either be "asc" or "desc". For example, https://demo.dataverse.org/api/search?q=data&sort=name&order=asc per_page int The number of results to return per request. The default is 10. The max is 1000. See :ref:`iteration example `. @@ -96,7 +96,7 @@ https://demo.dataverse.org/api/search?q=trees "url":"https://demo.dataverse.org/dataverse/birds", "image_url":"https://demo.dataverse.org/api/access/dvCardImage/2", "identifier":"birds", - "description":"A bird dataverse with some trees", + "description":"A bird Dataverse collection with some trees", "published_at":"2016-05-10T12:57:27Z" }, { @@ -152,7 +152,7 @@ Advanced Search Examples https://demo.dataverse.org/api/search?q=finch&show_relevance=true&show_facets=true&fq=publicationDate:2016&subtree=birds -In this example, ``show_relevance=true`` matches per field are shown. Available facets are shown with ``show_facets=true`` and of the facets is being used with ``fq=publicationDate:2016``. The search is being narrowed to the dataverse with the identifier "birds" with the parameter ``subtree=birds``. +In this example, ``show_relevance=true`` matches per field are shown. Available facets are shown with ``show_facets=true`` and of the facets is being used with ``fq=publicationDate:2016``. The search is being narrowed to the Dataverse collection with the identifier "birds" with the parameter ``subtree=birds``. .. code-block:: json @@ -171,13 +171,13 @@ In this example, ``show_relevance=true`` matches per field are shown. Available "url":"https://demo.dataverse.org/dataverse/finches", "image_url":"https://demo.dataverse.org/api/access/dvCardImage/3", "identifier":"finches", - "description":"A dataverse with finches", + "description":"A Dataverse collection with finches", "published_at":"2016-05-10T12:57:38Z", "matches":[ { "description":{ "snippets":[ - "A dataverse with finches" + "A Dataverse collection with finches" ] } }, @@ -283,10 +283,10 @@ The above example ``fq=publicationStatus:Published`` retrieves only "RELEASED" v "global_id": "doi:10.70122/FK2/GUAS41", "description": "Darwin's finches (also known as the Galápagos finches) are a group of about fifteen species of passerine birds.", "published_at": "2019-12-24T08:05:02Z", - "publisher": "mdmizanur rahman Dataverse", + "publisher": "mdmizanur rahman Dataverse collection", "citationHtml": "Finch, Fiona, 2019, \"Darwin's Finches\", https://doi.org/10.70122/FK2/GUAS41, Demo Dataverse, V1", "identifier_of_dataverse": "rahman", - "name_of_dataverse": "mdmizanur rahman Dataverse", + "name_of_dataverse": "mdmizanur rahman Dataverse collection", "citation": "Finch, Fiona, 2019, \"Darwin's Finches\", https://doi.org/10.70122/FK2/GUAS41, Demo Dataverse, V1", "storageIdentifier": "file://10.70122/FK2/GUAS41", "subjects": [ @@ -352,7 +352,7 @@ The above example ``fq=publicationStatus:Published`` retrieves only "RELEASED" v Date Range Search Example ------------------------- -Below is an example of searching across a date range of dataverses, datasets, and files that were published in 2018. +Below is an example of searching across a date range of Dataverse collections, datasets, and files that were published in 2018. `https://demo.dataverse.org/api/search?q=*&per_page=1000&sort=date&order=asc&q=*&fq=dateSort:[2018-01-01T00\:00\:00Z+TO+2019-01-01T00\:00\:00Z] `_ From 166b6358838a1685aeeed81beecc7d35afda3b92 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 7 Jan 2021 15:55:33 -0500 Subject: [PATCH 069/576] support download of UTF-8 filenames #7188 --- .../dataverse/api/DownloadInstanceWriter.java | 5 +- .../iq/dataverse/api/DownloadFilesIT.java | 83 +++++++++++++++++++ 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java index 1624e9932e2..4c143002c29 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java @@ -33,6 +33,7 @@ import java.io.FileInputStream; import java.net.URI; import java.net.URISyntaxException; +import java.net.URLEncoder; import java.util.ArrayList; import java.util.List; import java.util.logging.Level; @@ -302,8 +303,8 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] // Provide both the "Content-disposition" and "Content-Type" headers, // to satisfy the widest selection of browsers out there. - httpHeaders.add("Content-disposition", "attachment; filename=\"" + fileName + "\""); - httpHeaders.add("Content-Type", mimeType + "; name=\"" + fileName + "\""); + httpHeaders.add("Content-disposition", "attachment; filename=\"" + URLEncoder.encode(fileName, "UTF-8") + "\""); + httpHeaders.add("Content-Type", mimeType + "; name=\"" + URLEncoder.encode(fileName, "UTF-8") + "\""); long contentSize; boolean useChunkedTransfer = false; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DownloadFilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DownloadFilesIT.java index dde79574b87..60e6f45b71c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DownloadFilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DownloadFilesIT.java @@ -2,8 +2,10 @@ import com.jayway.restassured.RestAssured; import com.jayway.restassured.path.json.JsonPath; +import com.jayway.restassured.response.Headers; import com.jayway.restassured.response.Response; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; @@ -13,6 +15,7 @@ import java.util.HashSet; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; import static javax.ws.rs.core.Response.Status.CREATED; import static javax.ws.rs.core.Response.Status.FORBIDDEN; import static javax.ws.rs.core.Response.Status.OK; @@ -367,6 +370,86 @@ public void downloadAllFilesTabular() throws IOException { Assert.assertEquals(new HashSet<>(Arrays.asList("50by1000.dta", "MANIFEST.TXT")), gatherFilenames(downloadFiles2.getBody().asInputStream())); } + /** + * Download a file with a UTF-8 filename. + */ + @Test + public void downloadFilenameUtf8() throws IOException { + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + createUser.then().assertThat() + .statusCode(OK.getStatusCode()); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + createDataverseResponse.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDataset.prettyPrint(); + createDataset.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset); + String datasetPid = UtilIT.getDatasetPersistentIdFromResponse(createDataset); + + // Put a filename with an en-dash (READ–ME.md) into a zip file. + StringBuilder sb = new StringBuilder(); + sb.append("This is my README."); + Path pathtoTempDir = Paths.get(Files.createTempDirectory(null).toString()); + String pathToZipFile = pathtoTempDir + File.separator + "test.zip"; + File f = new File(pathToZipFile); + ZipOutputStream out = new ZipOutputStream(new FileOutputStream(f)); + ZipEntry e = new ZipEntry("READ–ME.md"); + out.putNextEntry(e); + byte[] data = sb.toString().getBytes(); + out.write(data, 0, data.length); + out.closeEntry(); + out.close(); + + // We upload via SWORD (as a zip) because the native API gives this error: + // "Constraint violation found in FileMetadata. File Name cannot contain any + // of the following characters: / : * ? " < > | ; # . The invalid value is "READ?ME.md"." + // This error probably has something to do with the way REST Assured sends the filename + // to the native API. The en-dash is turned into question mark, which is disallowed. + Response uploadViaSword = UtilIT.uploadZipFileViaSword(datasetPid, pathToZipFile, apiToken); + uploadViaSword.prettyPrint(); + uploadViaSword.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + Response getDatasetJson = UtilIT.nativeGet(datasetId, apiToken); + getDatasetJson.then().assertThat() + .statusCode(OK.getStatusCode()); + + int fileId = JsonPath.from(getDatasetJson.getBody().asString()).getInt("data.latestVersion.files[0].dataFile.id"); + + // Download the file individually and assert READ–ME.md has an en-dash. + Response downloadFile = UtilIT.downloadFile(new Integer(fileId), apiToken); + downloadFile.then().assertThat() + .statusCode(OK.getStatusCode()); + Headers headers = downloadFile.getHeaders(); + // In READ–ME.md the en-dash ("–") is "%E2%80%93" (e2 80 93 in hex). + Assert.assertEquals("attachment; filename=\"READ%E2%80%93ME.md\"", headers.getValue("Content-disposition")); + Assert.assertEquals("text/markdown; name=\"READ%E2%80%93ME.md\";charset=UTF-8", headers.getValue("Content-Type")); + + // Download all files as a zip and assert READ–ME.md has an en-dash. + Response downloadFiles = UtilIT.downloadFiles(datasetPid, apiToken); + downloadFiles.then().assertThat() + .statusCode(OK.getStatusCode()); + + HashSet filenamesFound = gatherFilenames(downloadFiles.getBody().asInputStream()); + + // Note that a MANIFEST.TXT file is added. + // READ–ME.md (with an en-dash) is correctly extracted from the downloaded zip + HashSet expectedFiles = new HashSet<>(Arrays.asList("MANIFEST.TXT", "READ–ME.md")); + Assert.assertEquals(expectedFiles, filenamesFound); + } + private HashSet gatherFilenames(InputStream inputStream) throws IOException { HashSet filenamesFound = new HashSet<>(); try (ZipInputStream zipStream = new ZipInputStream(inputStream)) { From 2c20ff09b45a85231de7405dca2cde9e13b46b3d Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 7 Jan 2021 18:14:03 -0500 Subject: [PATCH 070/576] more edits. --- doc/sphinx-guides/source/api/intro.rst | 34 ++-- doc/sphinx-guides/source/api/native-api.rst | 176 ++++++++++---------- 2 files changed, 105 insertions(+), 105 deletions(-) diff --git a/doc/sphinx-guides/source/api/intro.rst b/doc/sphinx-guides/source/api/intro.rst index 122f99854b9..988d401eddf 100755 --- a/doc/sphinx-guides/source/api/intro.rst +++ b/doc/sphinx-guides/source/api/intro.rst @@ -1,7 +1,7 @@ Introduction ============ -Dataverse Platform APIs allow users to accomplish many tasks such as... +The Dataverse Platform APIs allow users to accomplish many tasks such as... - creating datasets - uploading files @@ -120,37 +120,37 @@ Sysadmins often write scripts to automate tasks and Dataverse Platform APIs make In House Developers +++++++++++++++++++ -Some organizations that run Dataverse employ developers who are tasked with using Dataverse APIs to accomplish specific tasks such as building custom integrations with in house systems or creating reports specific to the organization's needs. +Some organizations that run a Dataverse installation employ developers who are tasked with using the Dataverse installation's APIs to accomplish specific tasks such as building custom integrations with in house systems or creating reports specific to the organization's needs. |Start| A good starting point for in house developers is the :doc:`getting-started` section. API Users Across the Dataverse Project ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The Dataverse project loves contributors! Depending on your interests and skills, you might fall into one or more of the groups below. +The Dataverse Project loves contributors! Depending on your interests and skills, you might fall into one or more of the groups below. Developers of Integrations, External Tools, and Apps ++++++++++++++++++++++++++++++++++++++++++++++++++++ -One of the primary purposes for Dataverse APIs in the first place is to enable integrations with third party software. Integrations are listed in the following places: +One of the primary purposes for Dataverse Platform APIs in the first place is to enable integrations with third party software. Integrations are listed in the following places: - The :doc:`/admin/integrations` section of the Admin Guide. - The :doc:`/api/external-tools` section this guide. - The :doc:`apps` section of this guide. -|Start| Good starting points are the three sections above to get a sense of third-party software that already integrates with Dataverse, followed by the :doc:`getting-started` section. +|Start| Good starting points are the three sections above to get a sense of third-party software that already integrates with the Dataverse Platform, followed by the :doc:`getting-started` section. -Developers of Dataverse API Client Libraries -++++++++++++++++++++++++++++++++++++++++++++ +Developers of Dataverse Platform API Client Libraries ++++++++++++++++++++++++++++++++++++++++++++++++++++++ -A client library helps developers using a specific programming language such as Python, Javascript, R, or Java interact with Dataverse APIs in a manner that is idiomatic for their language. For example, a Python programmer may want to +A client library helps developers using a specific programming language such as Python, Javascript, R, or Java interact with Dataverse Platform APIs in a manner that is idiomatic for their language. For example, a Python programmer may want to |Start| A good starting point is the :doc:`client-libraries` section, followed by the :doc:`getting-started` section. -Developers of Dataverse Itself -++++++++++++++++++++++++++++++ +Developers of The Dataverse Software Itself ++++++++++++++++++++++++++++++++++++++++++++ -Developers working on Dataverse itself use Dataverse APIs when adding features, fixing bugs, and testing those features and bug fixes. +Developers working on the Dataverse Software itself use the APIs when adding features, fixing bugs, and testing those features and bug fixes. |Start| A good starting point is the :doc:`/developers/testing` section of the Developer Guide. @@ -178,7 +178,7 @@ See :doc:`auth`. Lists of Dataverse APIs ~~~~~~~~~~~~~~~~~~~~~~~ -- :doc:`search`: For searching dataverses, datasets, and files. +- :doc:`search`: For searching dataverse collections, datasets, and files. - :doc:`dataaccess`: For downloading and subsetting data. - :doc:`native-api`: For performing most tasks that are possible in the GUI. See :doc:`getting-started` for the most common commands which operate on endpoints with names like: @@ -208,7 +208,7 @@ Please note that some APIs are only documented in other guides that are more sui Client Libraries ~~~~~~~~~~~~~~~~ -See :doc:`client-libraries` for how to use Dataverse APIs from Python, R, and Java. +See :doc:`client-libraries` for how to use Dataverse Platform APIs from Python, R, Java, and Javascript. Examples ~~~~~~~~ @@ -225,11 +225,11 @@ See :doc:`faq`. Getting Help ------------ -Dataverse API questions are on topic in all the usual places: +Dataverse Platform API questions are on topic in all the usual places: - The dataverse-community Google Group: https://groups.google.com/forum/#!forum/dataverse-community -- Dataverse community calls: https://dataverse.org/community-calls -- The Dataverse chat room: http://chat.dataverse.org -- The Dataverse ticketing system: support@dataverse.org +- Dataverse Project community calls: https://dataverse.org/community-calls +- The Dataverse Project chat room: http://chat.dataverse.org +- The Dataverse Project ticketing system: support@dataverse.org After your question has been answered, you are welcome to help improve the :doc:`faq` section of this guide. diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index c9301ae5999..d420a41568f 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1,7 +1,7 @@ Native API ========== -Dataverse 4 exposes most of its GUI functionality via a REST-based API. This section describes that functionality. Most API endpoints require an API token that can be passed as the ``X-Dataverse-key`` HTTP header or in the URL as the ``key`` query parameter. +The Dataverse Platform exposes most of its GUI functionality via a REST-based API. This section describes that functionality. Most API endpoints require an API token that can be passed as the ``X-Dataverse-key`` HTTP header or in the URL as the ``key`` query parameter. .. note:: |CORS| Some API endpoint allow CORS_ (cross-origin resource sharing), which makes them usable from scripts running in web browsers. These endpoints are marked with a *CORS* badge. @@ -9,25 +9,25 @@ Dataverse 4 exposes most of its GUI functionality via a REST-based API. This sec .. _CORS: https://www.w3.org/TR/cors/ -.. warning:: Dataverse 4's API is versioned at the URI - all API calls may include the version number like so: ``http://server-address/api/v1/...``. Omitting the ``v1`` part would default to the latest API version (currently 1). When writing scripts/applications that will be used for a long time, make sure to specify the API version, so they don't break when the API is upgraded. +.. warning:: The Dataverse Platform's API is versioned at the URI - all API calls may include the version number like so: ``http://server-address/api/v1/...``. Omitting the ``v1`` part would default to the latest API version (currently 1). When writing scripts/applications that will be used for a long time, make sure to specify the API version, so they don't break when the API is upgraded. .. contents:: |toctitle| :local: -Dataverses ----------- +Dataverse Collections +--------------------- .. _create-dataverse-api: -Create a Dataverse -~~~~~~~~~~~~~~~~~~ +Create a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -A dataverse is a container for datasets and other dataverses as explained in the :doc:`/user/dataverse-management` section of the User Guide. +A Dataverse collection is a container for datasets and other Dataverse collections as explained in the :doc:`/user/dataverse-management` section of the User Guide. -The steps for creating a dataverse are: +The steps for creating a Dataverse collection are: -- Prepare a JSON file containing the name, description, etc, of the dataverse you'd like to create. -- Figure out the alias or database id of the "parent" dataverse into which you will be creating your new dataverse. +- Prepare a JSON file containing the name, description, etc, of the Dataverse collection you'd like to create. +- Figure out the alias or database id of the "parent" Dataverse collection into which you will be creating your new Dataverse collection. - Execute a curl command or equivalent. Download :download:`dataverse-complete.json <../_static/api/dataverse-complete.json>` file and modify it to suit your needs. The fields ``name``, ``alias``, and ``dataverseContacts`` are required. The controlled vocabulary for ``dataverseType`` is the following: @@ -46,7 +46,7 @@ Download :download:`dataverse-complete.json <../_static/api/dataverse-complete.j The curl command below assumes you have kept the name "dataverse-complete.json" and that this file is in your current working directory. -Next you need to figure out the alias or database id of the "parent" dataverse into which you will be creating your new dataverse. Out of the box the top level dataverse has an alias of "root" and a database id of "1" but your installation may vary. The easiest way to determine the alias of your root dataverse is to click "Advanced Search" and look at the URL. You may also choose a parent under the root. +Next you need to figure out the alias or database id of the "parent" Dataverse collection into which you will be creating your new Dataverse collection. Out of the box the top level Dataverse collection has an alias of "root" and a database id of "1" but your installation may vary. The easiest way to determine the alias of your root Dataverse collection is to click "Advanced Search" and look at the URL. You may also choose a parent under the root. .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. @@ -64,16 +64,16 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST https://demo.dataverse.org/api/dataverses/root --upload-file dataverse-complete.json -You should expect an HTTP 200 response and JSON beginning with "status":"OK" followed by a representation of the newly-created dataverse. +You should expect an HTTP 200 response and JSON beginning with "status":"OK" followed by a representation of the newly-created Dataverse collection. .. _view-dataverse: -View a Dataverse -~~~~~~~~~~~~~~~~ +View a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~ -|CORS| View a JSON representation of the dataverse identified by ``$id``. ``$id`` can be the database ID of the dataverse, its alias, or the special value ``:root`` for the root dataverse. +|CORS| View a JSON representation of the Dataverse collection identified by ``$id``. ``$id`` can be the database ID of the Dataverse collection, its alias, or the special value ``:root`` for the root Dataverse collection. -To view a published dataverse: +To view a published Dataverse collection: .. code-block:: bash @@ -88,7 +88,7 @@ The fully expanded example above (without environment variables) looks like this curl https://demo.dataverse.org/api/dataverses/root -To view an unpublished dataverse: +To view an unpublished Dataverse collection: .. code-block:: bash @@ -104,12 +104,12 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root -Delete a Dataverse -~~~~~~~~~~~~~~~~~~ +Delete a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Before you may delete a dataverse you must first delete or move all of its contents elsewhere. +Before you may delete a Dataverse collection you must first delete or move all of its contents elsewhere. -Deletes the dataverse whose database ID or alias is given: +Deletes the Dataverse collection whose database ID or alias is given: .. code-block:: bash @@ -127,10 +127,10 @@ The fully expanded example above (without environment variables) looks like this .. _show-contents-of-a-dataverse-api: -Show Contents of a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Show Contents of a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -|CORS| Lists all the dataverses and datasets directly under a dataverse (direct children only, not recursive) specified by database id or alias. If you pass your API token and have access, unpublished dataverses and datasets will be included in the response. +|CORS| Lists all the Dataverse collections and datasets directly under a Dataverse collection (direct children only, not recursive) specified by database id or alias. If you pass your API token and have access, unpublished Dataverse collections and datasets will be included in the response. .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. @@ -148,10 +148,10 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/contents -Report the data (file) size of a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Report the data (file) size of a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Shows the combined size in bytes of all the files uploaded into the dataverse ``id``: +Shows the combined size in bytes of all the files uploaded into the Dataverse collection ``id``: .. code-block:: bash @@ -167,13 +167,13 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/storagesize -The size of published and unpublished files will be summed both in the dataverse specified and beneath all its sub-dataverses, recursively. -By default, only the archival files are counted - i.e., the files uploaded by users (plus the tab-delimited versions generated for tabular data files on ingest). If the optional argument ``includeCached=true`` is specified, the API will also add the sizes of all the extra files generated and cached by Dataverse - the resized thumbnail versions for image files, the metadata exports for published datasets, etc. +The size of published and unpublished files will be summed both in the Dataverse collection specified and beneath all its sub-collections, recursively. +By default, only the archival files are counted - i.e., the files uploaded by users (plus the tab-delimited versions generated for tabular data files on ingest). If the optional argument ``includeCached=true`` is specified, the API will also add the sizes of all the extra files generated and cached by the Dataverse installation - the resized thumbnail versions for image files, the metadata exports for published datasets, etc. -List Roles Defined in a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +List Roles Defined in a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -All the roles defined directly in the dataverse identified by ``id``: +All the roles defined directly in the Dataverse collection identified by ``id``: .. code-block:: bash @@ -189,10 +189,10 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/roles -List Facets Configured for a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +List Facets Configured for a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -|CORS| List all the facets for a given dataverse ``id``: +|CORS| List all the facets for a given Dataverse collection ``id``: .. code-block:: bash @@ -208,10 +208,10 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/facets -Set Facets for a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~ +Set Facets for a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Assign search facets for a given dataverse identified by ``id``: +Assign search facets for a given Dataverse collection identified by ``id``: .. code-block:: bash @@ -229,10 +229,10 @@ The fully expanded example above (without environment variables) looks like this Where ``facets.json`` contains a JSON encoded list of metadata keys (e.g. ``["authorName","authorAffiliation"]``). -Create a New Role in a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Create a New Role in a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Creates a new role under dataverse ``id``. Needs a json file with the role description: +Creates a new role under Dataverse collection ``id``. Needs a json file with the role description: .. code-block:: bash @@ -261,10 +261,10 @@ Where ``roles.json`` looks like this:: .. _list-role-assignments-on-a-dataverse-api: -List Role Assignments in a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +List Role Assignments in a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -List all the role assignments at the given dataverse: +List all the role assignments at the given Dataverse collection: .. code-block:: bash @@ -280,10 +280,10 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/assignments -Assign Default Role to User Creating a Dataset in a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Assign Default Role to User Creating a Dataset in a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Assign a default role to a user creating a dataset in a dataverse ``id`` where ``roleAlias`` is the database alias of the role to be assigned: +Assign a default role to a user creating a dataset in a Dataverse collection ``id`` where ``roleAlias`` is the database alias of the role to be assigned: .. code-block:: bash @@ -300,12 +300,12 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X PUT https://demo.dataverse.org/api/dataverses/root/defaultContributorRole/curator -Note: You may use "none" as the ``ROLE_ALIAS``. This will prevent a user who creates a dataset from having any role on that dataset. It is not recommended for dataverses with human contributors. +Note: You may use "none" as the ``ROLE_ALIAS``. This will prevent a user who creates a dataset from having any role on that dataset. It is not recommended for Dataverse collections with human contributors. .. _assign-role-on-a-dataverse-api: -Assign a New Role on a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Assign a New Role on a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Assigns a new role, based on the POSTed JSON: @@ -332,8 +332,8 @@ POSTed JSON example (the content of ``role.json`` file):: .. _revoke-role-on-a-dataverse-api: -Delete Role Assignment from a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Delete Role Assignment from a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Delete the assignment whose id is ``$id``: @@ -352,12 +352,12 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X DELETE https://demo.dataverse.org/api/dataverses/root/assignments/6 -List Metadata Blocks Defined on a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +List Metadata Blocks Defined on a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -|CORS| Get the metadata blocks defined on a dataverse which determine which field are available to authors when they create and edit datasets within that dataverse. This feature is described in :ref:`general-information` section of Dataverse Management of the User Guide. +|CORS| Get the metadata blocks defined on a Dataverse collection which determine which field are available to authors when they create and edit datasets within that Dataverse collection. This feature is described in :ref:`general-information` section of Dataverse Collection Management of the User Guide. -Please note that an API token is only required if the dataverse has not been published. +Please note that an API token is only required if the Dataverse collection has not been published. .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. @@ -375,12 +375,12 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/metadatablocks -Define Metadata Blocks for a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Define Metadata Blocks for a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -You can define the metadata blocks available to authors within a dataverse. +You can define the metadata blocks available to authors within a Dataverse collection. -The metadata blocks that are available with a default installation of Dataverse are in :download:`define-metadatablocks.json <../_static/api/define-metadatablocks.json>` (also shown below) and you should download this file and edit it to meet your needs. Please note that the "citation" metadata block is required. You must have "EditDataverse" permission on the dataverse. +The metadata blocks that are available with a default Dataverse installation are in :download:`define-metadatablocks.json <../_static/api/define-metadatablocks.json>` (also shown below) and you should download this file and edit it to meet your needs. Please note that the "citation" metadata block is required. You must have "EditDataverse" permission on the Dataverse collection. .. literalinclude:: ../_static/api/define-metadatablocks.json @@ -400,10 +400,10 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST -H "Content-type:application/json" --upload-file define-metadatablocks.json https://demo.dataverse.org/api/dataverses/root/metadatablocks -Determine if a Dataverse Inherits Its Metadata Blocks from Its Parent -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Determine if a Dataverse Collection Inherits Its Metadata Blocks from Its Parent +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Get whether the dataverse is a metadata block root, or does it uses its parent blocks: +Get whether the Dataverse collection is a metadata block root, or does it uses its parent blocks: .. code-block:: bash @@ -419,10 +419,10 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/metadatablocks/isRoot -Configure a Dataverse to Inherit Its Metadata Blocks from Its Parent -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Configure a Dataverse Collection to Inherit Its Metadata Blocks from Its Parent +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Set whether the dataverse is a metadata block root, or does it uses its parent blocks. Possible +Set whether the Dataverse collection is a metadata block root, or does it uses its parent blocks. Possible values are ``true`` and ``false`` (both are valid JSON expressions): .. code-block:: bash @@ -444,8 +444,8 @@ The fully expanded example above (without environment variables) looks like this .. _create-dataset-command: -Create a Dataset in a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Create a Dataset in a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ A dataset is a container for files as explained in the :doc:`/user/dataset-management` section of the User Guide. @@ -457,11 +457,11 @@ To create a dataset, you must supply a JSON file that contains at least the foll - Description - Subject -As a starting point, you can download :download:`dataset-finch1.json <../../../../scripts/search/tests/data/dataset-finch1.json>` and modify it to meet your needs. (In addition to this minimal example, you can download :download:`dataset-create-new-all-default-fields.json <../../../../scripts/api/data/dataset-create-new-all-default-fields.json>` which populates all of the metadata fields that ship with Dataverse.) +As a starting point, you can download :download:`dataset-finch1.json <../../../../scripts/search/tests/data/dataset-finch1.json>` and modify it to meet your needs. (In addition to this minimal example, you can download :download:`dataset-create-new-all-default-fields.json <../../../../scripts/api/data/dataset-create-new-all-default-fields.json>` which populates all of the metadata fields that ship with a Dataverse installation.) The curl command below assumes you have kept the name "dataset-finch1.json" and that this file is in your current working directory. -Next you need to figure out the alias or database id of the "parent" dataverse into which you will be creating your new dataset. Out of the box the top level dataverse has an alias of "root" and a database id of "1" but your installation may vary. The easiest way to determine the alias of your root dataverse is to click "Advanced Search" and look at the URL. You may also choose a parent dataverse under the root dataverse. +Next you need to figure out the alias or database id of the "parent" Dataverse collection into which you will be creating your new dataset. Out of the box the top level Dataverse collection has an alias of "root" and a database id of "1" but your installation may vary. The easiest way to determine the alias of your root Dataverse collection is to click "Advanced Search" and look at the URL. You may also choose a parent Dataverse collection under the root Dataverse collection. .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. @@ -481,14 +481,14 @@ The fully expanded example above (without the environment variables) looks like You should expect an HTTP 200 ("OK") response and JSON indicating the database ID and Persistent ID (PID such as DOI or Handle) that has been assigned to your newly created dataset. -.. note:: Only a Dataverse account with superuser permissions is allowed to include files when creating a dataset via this API. Adding files this way only adds their file metadata to the database, you will need to manually add the physical files to the file system. +.. note:: Only a Dataverse installation account with superuser permissions is allowed to include files when creating a dataset via this API. Adding files this way only adds their file metadata to the database, you will need to manually add the physical files to the file system. -Import a Dataset into a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Import a Dataset into a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. note:: This action requires a Dataverse account with super-user permissions. +.. note:: This action requires a Dataverse installation account with super-user permissions. -To import a dataset with an existing persistent identifier (PID), the dataset's metadata should be prepared in Dataverse's native JSON format. The PID is provided as a parameter at the URL. The following line imports a dataset with the PID ``PERSISTENT_IDENTIFIER`` to Dataverse, and then releases it: +To import a dataset with an existing persistent identifier (PID), the dataset's metadata should be prepared in Dataverse installation's native JSON format. The PID is provided as a parameter at the URL. The following line imports a dataset with the PID ``PERSISTENT_IDENTIFIER`` to the Dataverse installation, and then releases it: .. code-block:: bash @@ -507,7 +507,7 @@ The fully expanded example above (without environment variables) looks like this The ``pid`` parameter holds a persistent identifier (such as a DOI or Handle). The import will fail if no PID is provided, or if the provided PID fails validation. -The optional ``release`` parameter tells Dataverse to immediately publish the dataset. If the parameter is changed to ``no``, the imported dataset will remain in ``DRAFT`` status. +The optional ``release`` parameter tells the Dataverse installation to immediately publish the dataset. If the parameter is changed to ``no``, the imported dataset will remain in ``DRAFT`` status. The JSON format is the same as that supported by the native API's :ref:`create dataset command`, although it also allows packages. For example: @@ -519,17 +519,17 @@ Before calling the API, make sure the data files referenced by the ``POST``\ ed .. warning:: - * This API does not cover staging files (with correct contents, checksums, sizes, etc.) in the corresponding places in the Dataverse filestore. + * This API does not cover staging files (with correct contents, checksums, sizes, etc.) in the corresponding places in the Dataverse installation's filestore. * This API endpoint does not support importing *files'* persistent identifiers. - * A Dataverse server can import datasets with a valid PID that uses a different protocol or authority than said server is configured for. However, the server will not update the PID metadata on subsequent update and publish actions. + * A Dataverse installation can import datasets with a valid PID that uses a different protocol or authority than said server is configured for. However, the server will not update the PID metadata on subsequent update and publish actions. -Import a Dataset into a Dataverse with a DDI file -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Import a Dataset into a Dataverse Installation with a DDI file +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. note:: This action requires a Dataverse account with super-user permissions. +.. note:: This action requires a Dataverse installation account with super-user permissions. -To import a dataset with an existing persistent identifier (PID), you have to provide the PID as a parameter at the URL. The following line imports a dataset with the PID ``PERSISTENT_IDENTIFIER`` to Dataverse, and then releases it: +To import a dataset with an existing persistent identifier (PID), you have to provide the PID as a parameter at the URL. The following line imports a dataset with the PID ``PERSISTENT_IDENTIFIER`` to the Dataverse installation, and then releases it: .. code-block:: bash @@ -548,21 +548,21 @@ The fully expanded example above (without environment variables) looks like this The optional ``pid`` parameter holds a persistent identifier (such as a DOI or Handle). The import will fail if the provided PID fails validation. -The optional ``release`` parameter tells Dataverse to immediately publish the dataset. If the parameter is changed to ``no``, the imported dataset will remain in ``DRAFT`` status. +The optional ``release`` parameter tells the Dataverse installation to immediately publish the dataset. If the parameter is changed to ``no``, the imported dataset will remain in ``DRAFT`` status. The file is a DDI xml file. .. warning:: * This API does not handle files related to the DDI file. - * A Dataverse server can import datasets with a valid PID that uses a different protocol or authority than said server is configured for. However, the server will not update the PID metadata on subsequent update and publish actions. + * A Dataverse installation can import datasets with a valid PID that uses a different protocol or authority than said server is configured for. However, the server will not update the PID metadata on subsequent update and publish actions. .. _publish-dataverse-api: -Publish a Dataverse -~~~~~~~~~~~~~~~~~~~ +Publish a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In order to publish a dataverse, you must know either its "alias" (which the GUI calls an "identifier") or its database ID. +In order to publish a Dataverse collection, you must know either its "alias" (which the GUI calls an "identifier") or its database ID. .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. @@ -585,7 +585,7 @@ You should expect a 200 ("OK") response and JSON output. Datasets -------- -**Note** Creation of new datasets is done with a ``POST`` onto dataverses. See Dataverses_ section. +**Note** Creation of new datasets is done with a ``POST`` onto a Dataverse collection. See Dataverse_ section. **Note** In all commands below, dataset versions can be referred to as: From 18ab2c1ec8854f03dac7210e037a9b14d5e7e470 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 8 Jan 2021 09:56:20 -0500 Subject: [PATCH 071/576] adding release note --- doc/release-notes/7502-more-mime-types.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 doc/release-notes/7502-more-mime-types.md diff --git a/doc/release-notes/7502-more-mime-types.md b/doc/release-notes/7502-more-mime-types.md new file mode 100644 index 00000000000..8fc3f33f92f --- /dev/null +++ b/doc/release-notes/7502-more-mime-types.md @@ -0,0 +1,4 @@ +## Upgrade Steps + +- Redetect file types using the Redetect File Types API: +https://guides.dataverse.org/en/latest/api/native-api.html#redetect-file-type \ No newline at end of file From 702b505bca914c18835563226fc2fd2fc246c9be Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 8 Jan 2021 12:17:59 -0500 Subject: [PATCH 072/576] ensure spaces aren't turned to plus signs #7188 --- .../dataverse/api/DownloadInstanceWriter.java | 8 +++++--- .../iq/dataverse/api/DownloadFilesIT.java | 18 +++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java index 4c143002c29..39bcbabbc99 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java @@ -302,9 +302,11 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] // Provide both the "Content-disposition" and "Content-Type" headers, // to satisfy the widest selection of browsers out there. - - httpHeaders.add("Content-disposition", "attachment; filename=\"" + URLEncoder.encode(fileName, "UTF-8") + "\""); - httpHeaders.add("Content-Type", mimeType + "; name=\"" + URLEncoder.encode(fileName, "UTF-8") + "\""); + // Encode the filename as UTF-8, then deal with spaces. "encode" changes + // a space to + so we change it back to a space (%20). + String finalFileName = URLEncoder.encode(fileName, "UTF-8").replaceAll("\\+", "%20"); + httpHeaders.add("Content-disposition", "attachment; filename=\"" + finalFileName + "\""); + httpHeaders.add("Content-Type", mimeType + "; name=\"" + finalFileName + "\""); long contentSize; boolean useChunkedTransfer = false; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DownloadFilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DownloadFilesIT.java index 60e6f45b71c..96bd5be6ca5 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DownloadFilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DownloadFilesIT.java @@ -371,7 +371,7 @@ public void downloadAllFilesTabular() throws IOException { } /** - * Download a file with a UTF-8 filename. + * Download a file with a UTF-8 filename with a space. */ @Test public void downloadFilenameUtf8() throws IOException { @@ -398,14 +398,14 @@ public void downloadFilenameUtf8() throws IOException { Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset); String datasetPid = UtilIT.getDatasetPersistentIdFromResponse(createDataset); - // Put a filename with an en-dash (READ–ME.md) into a zip file. + // Put a filename with an en-dash ("MY READ–ME.md") into a zip file. StringBuilder sb = new StringBuilder(); sb.append("This is my README."); Path pathtoTempDir = Paths.get(Files.createTempDirectory(null).toString()); String pathToZipFile = pathtoTempDir + File.separator + "test.zip"; File f = new File(pathToZipFile); ZipOutputStream out = new ZipOutputStream(new FileOutputStream(f)); - ZipEntry e = new ZipEntry("READ–ME.md"); + ZipEntry e = new ZipEntry("MY READ–ME.md"); out.putNextEntry(e); byte[] data = sb.toString().getBytes(); out.write(data, 0, data.length); @@ -433,11 +433,11 @@ public void downloadFilenameUtf8() throws IOException { downloadFile.then().assertThat() .statusCode(OK.getStatusCode()); Headers headers = downloadFile.getHeaders(); - // In READ–ME.md the en-dash ("–") is "%E2%80%93" (e2 80 93 in hex). - Assert.assertEquals("attachment; filename=\"READ%E2%80%93ME.md\"", headers.getValue("Content-disposition")); - Assert.assertEquals("text/markdown; name=\"READ%E2%80%93ME.md\";charset=UTF-8", headers.getValue("Content-Type")); + // In "MY READ–ME.md" below the space is %20 and the en-dash ("–") is "%E2%80%93" (e2 80 93 in hex). + Assert.assertEquals("attachment; filename=\"MY%20READ%E2%80%93ME.md\"", headers.getValue("Content-disposition")); + Assert.assertEquals("text/markdown; name=\"MY%20READ%E2%80%93ME.md\";charset=UTF-8", headers.getValue("Content-Type")); - // Download all files as a zip and assert READ–ME.md has an en-dash. + // Download all files as a zip and assert "MY READ–ME.md" has an en-dash. Response downloadFiles = UtilIT.downloadFiles(datasetPid, apiToken); downloadFiles.then().assertThat() .statusCode(OK.getStatusCode()); @@ -445,8 +445,8 @@ public void downloadFilenameUtf8() throws IOException { HashSet filenamesFound = gatherFilenames(downloadFiles.getBody().asInputStream()); // Note that a MANIFEST.TXT file is added. - // READ–ME.md (with an en-dash) is correctly extracted from the downloaded zip - HashSet expectedFiles = new HashSet<>(Arrays.asList("MANIFEST.TXT", "READ–ME.md")); + // "MY READ–ME.md" (with an en-dash) is correctly extracted from the downloaded zip + HashSet expectedFiles = new HashSet<>(Arrays.asList("MANIFEST.TXT", "MY READ–ME.md")); Assert.assertEquals(expectedFiles, filenamesFound); } From 3ea71dd0c3fbfd008831d82e84142ad498ab5ca9 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Fri, 8 Jan 2021 13:25:08 -0500 Subject: [PATCH 073/576] Checksum/unf truncation, click to copy, for dataset pg and dataverse pg, plus file metadata code clean up [ref #5210] --- .../search/SearchIncludeFragment.java | 14 ---------- src/main/webapp/editFilesFragment.xhtml | 4 +-- src/main/webapp/file-info-fragment.xhtml | 15 ++++++----- src/main/webapp/filesFragment.xhtml | 19 +++++++------ .../resources/js/dv_rebind_bootstrap_ui.js | 27 ++++++++++++------- src/main/webapp/search-include-fragment.xhtml | 25 +++++++++-------- 6 files changed, 51 insertions(+), 53 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java index ac70072b7bb..9219e8b3664 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java @@ -1236,20 +1236,6 @@ public String dataFileSizeDisplay(DataFile datafile) { } - public String dataFileChecksumDisplay(DataFile datafile) { - if (datafile == null) { - return ""; - } - - if (datafile.getChecksumValue() != null && !StringUtils.isEmpty(datafile.getChecksumValue())) { - if (datafile.getChecksumType() != null) { - return " " + datafile.getChecksumType() + ": " + datafile.getChecksumValue() + " "; - } - } - - return ""; - } - public void setDisplayCardValues() { Set harvestedDatasetIds = null; diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index 5ad2dc6dab0..e4e6950c87b 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -374,8 +374,8 @@ -
    - +
    +
    diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index b10383b55ed..4a96327c736 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -52,18 +52,21 @@ -
    - + #{fileMetadata.dataFile.checksumType}: #{fileMetadata.dataFile.checksumType}: #{fileMetadata.dataFile.checksumValue} + title="#{bundle['file.metaData.checksum.copy']} #{fileMetadata.dataFile.checksumValue}">#{fileMetadata.dataFile.checksumValue}
    - +
    - + #{fileMetadata.dataFile.unf}
    diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index 9f958cd451c..0a77a40d881 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -372,6 +372,15 @@ +
    @@ -471,16 +480,6 @@ disabled="#{DatasetPage.locked}"> #{bundle['file.requestAccess']} - - diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index 76d6d02a906..131f3634b60 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -156,22 +156,29 @@ function sharrre(){ } /* - * Truncate file checksums + * Truncate file checksums + click to copy */ function checksumTruncate(){ $('span.checksum-truncate').each(function () { var checksumText = $(this).text(); - if (checksumText.length > 35) { - $(this).text(checksumText.substr(0, 8) + '...' + checksumText.substr(checksumText.length-3, checksumText.length)); + if (checksumText.length > 25) { + // COUNT ":" IN UNF VERSION LABEL + var labelCount = (checksumText.match(/:/g) || []).length; + // COUNT "=" IN UNF SUFFIX + var suffixCount = (checksumText.match(/=/g) || []).length; + // TRUNCATE MIDDLE W/ "..." + FIRST/LAST 3 CHARACTERS + $(this).text(checksumText.substr(0, labelCount + 3) + '...' + checksumText.substr((checksumText.length - suffixCount - 3), checksumText.length)); } + // ADD CLIPBOARD ICON $(this).append(' '); - var clipboard = new Clipboard('span.checksum-truncate'); - clipboard.on('success', function(e) { - console.log(e); - }); - clipboard.on('error', function(e) { - console.log(e); - }); + }); + // USE CLIPBOARD.JS TO COPY + var clipboard = new Clipboard('span.checksum-truncate'); + clipboard.on('success', function(e) { + console.log(e); + }); + clipboard.on('error', function(e) { + console.log(e); }); } diff --git a/src/main/webapp/search-include-fragment.xhtml b/src/main/webapp/search-include-fragment.xhtml index 3260495c4e7..956e57242af 100644 --- a/src/main/webapp/search-include-fragment.xhtml +++ b/src/main/webapp/search-include-fragment.xhtml @@ -601,23 +601,26 @@
    -
    - +
    +
    -
    - - - - - - - +
    + + + + + + #{result.fileChecksumValue} -
    From 5efe3f0b14d61453ce649d5a1958a000aee162b7 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 8 Jan 2021 15:49:51 -0500 Subject: [PATCH 074/576] placeholder release notes --- doc/release-notes/820-non-ascii-chars-in-search.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/release-notes/820-non-ascii-chars-in-search.md diff --git a/doc/release-notes/820-non-ascii-chars-in-search.md b/doc/release-notes/820-non-ascii-chars-in-search.md new file mode 100644 index 00000000000..c449a68cb11 --- /dev/null +++ b/doc/release-notes/820-non-ascii-chars-in-search.md @@ -0,0 +1,8 @@ +### Search with non-ascii characters + +Many languages include characters that have close analogs in ascii, e.g. (á, à, â, ç, é, è, ê, ë, í, ó, ö, ú, ù, û, ü…). This release changes the default Solr configuration to allow search to match words based on these associations, e.g. a search for Mercè would match the word Merce in a Dataset, and vice versa. This should generally be helpful, but can result in false positives.,e.g. "canon" will be found searching for "cañon". + +## Upgrade Instructions + +1. You will need to replace or modify your `schema.xml` and restart solr. Re-indexing is required to get full-functionality from this change - the standard instructions for an incremental reindex could be added here. + \ No newline at end of file From 7ad07b6871647c06dc2c4effa6f16a213ece44c3 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 8 Jan 2021 16:47:18 -0500 Subject: [PATCH 075/576] lots of updates in a very long file --- doc/sphinx-guides/source/api/native-api.rst | 106 ++++++++++---------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index d420a41568f..9abe5dea3b4 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -585,7 +585,7 @@ You should expect a 200 ("OK") response and JSON output. Datasets -------- -**Note** Creation of new datasets is done with a ``POST`` onto a Dataverse collection. See Dataverse_ section. +**Note** Creation of new datasets is done with a ``POST`` onto a Dataverse collection. See the Dataverse Collections section above. **Note** In all commands below, dataset versions can be referred to as: @@ -760,12 +760,12 @@ The fully expanded example above (without environment variables) looks like this Schema.org JSON-LD ^^^^^^^^^^^^^^^^^^ -Please note that the ``schema.org`` format has changed in backwards-incompatible ways after Dataverse 4.9.4: +Please note that the ``schema.org`` format has changed in backwards-incompatible ways after Dataverse software version 4.9.4: - "description" was a single string and now it is an array of strings. - "citation" was an array of strings and now it is an array of objects. -Both forms are valid according to Google's Structured Data Testing Tool at https://search.google.com/structured-data/testing-tool . (This tool will report "The property affiliation is not recognized by Google for an object of type Thing" and this known issue is being tracked at https://github.com/IQSS/dataverse/issues/5029 .) Schema.org JSON-LD is an evolving standard that permits a great deal of flexibility. For example, https://schema.org/docs/gs.html#schemaorg_expected indicates that even when objects are expected, it's ok to just use text. As with all metadata export formats, we will try to keep the Schema.org JSON-LD format Dataverse emits backward-compatible to made integrations more stable, despite the flexibility that's afforded by the standard. +Both forms are valid according to Google's Structured Data Testing Tool at https://search.google.com/structured-data/testing-tool . (This tool will report "The property affiliation is not recognized by Google for an object of type Thing" and this known issue is being tracked at https://github.com/IQSS/dataverse/issues/5029 .) Schema.org JSON-LD is an evolving standard that permits a great deal of flexibility. For example, https://schema.org/docs/gs.html#schemaorg_expected indicates that even when objects are expected, it's ok to just use text. As with all metadata export formats, we will try to keep the Schema.org JSON-LD format your Dataverse installation emits backward-compatible to made integrations more stable, despite the flexibility that's afforded by the standard. List Files in a Dataset ~~~~~~~~~~~~~~~~~~~~~~~ @@ -830,9 +830,9 @@ Update Metadata For a Dataset Updates the metadata for a dataset. If a draft of the dataset already exists, the metadata of that draft is overwritten; otherwise, a new draft is created with this metadata. -You must download a JSON representation of the dataset, edit the JSON you download, and then send the updated JSON to the Dataverse server. +You must download a JSON representation of the dataset, edit the JSON you download, and then send the updated JSON to the Dataverse installation. -For example, after making your edits, your JSON file might look like :download:`dataset-update-metadata.json <../_static/api/dataset-update-metadata.json>` which you would send to Dataverse like this: +For example, after making your edits, your JSON file might look like :download:`dataset-update-metadata.json <../_static/api/dataset-update-metadata.json>` which you would send to the Dataverse installation like this: .. code-block:: bash @@ -868,7 +868,7 @@ Now that the resulting JSON file only contains the ``metadataBlocks`` key, you c vi dataset-update-metadata.json -Now that you've made edits to the metadata in your JSON file, you can send it to Dataverse as described above. +Now that you've made edits to the metadata in your JSON file, you can send it to a Dataverse installation as described above. Edit Dataset Metadata ~~~~~~~~~~~~~~~~~~~~~ @@ -933,7 +933,7 @@ For these deletes your JSON file must include an exact match of those dataset fi Publish a Dataset ~~~~~~~~~~~~~~~~~ -When publishing a dataset it's good to be aware of Dataverse's versioning system, which is described in the :doc:`/user/dataset-management` section of the User Guide. +When publishing a dataset it's good to be aware of the Dataverse Platform's versioning system, which is described in the :doc:`/user/dataset-management` section of the User Guide. If this is the first version of the dataset, its version number will be set to ``1.0``. Otherwise, the new dataset version number is determined by the most recent version number and the ``type`` parameter. Passing ``type=minor`` increases the minor version number (2.3 is updated to 2.4). Passing ``type=major`` increases the major version number (2.3 is updated to 3.0). (Superusers can pass ``type=updatecurrent`` to update metadata without changing the version number.) @@ -1180,7 +1180,7 @@ The fully expanded example above (without environment variables) looks like this You should expect a 201 ("CREATED") response and JSON indicating the database id that has been assigned to your newly uploaded file. -Please note that it's possible to "trick" Dataverse into giving a file a content type (MIME type) of your choosing. For example, you can make a text file be treated like a video file with ``-F 'file=@README.txt;type=video/mpeg4'``, for example. If Dataverse does not properly detect a file type, specifying the content type via API like this a potential workaround. +Please note that it's possible to "trick" a Dataverse installation into giving a file a content type (MIME type) of your choosing. For example, you can make a text file be treated like a video file with ``-F 'file=@README.txt;type=video/mpeg4'``, for example. If the Dataverse installation does not properly detect a file type, specifying the content type via API like this a potential workaround. The curl syntax above to upload a file is tricky and a Python version is provided below. (Please note that it depends on libraries such as "requests" that you may need to install but this task is out of scope for this guide.) Here are some parameters you can set in the script: @@ -1200,7 +1200,7 @@ In practice, you only need one the ``dataset_id`` or the ``persistentId``. The e # -------------------------------------------------- # Update the 4 params below to run this code # -------------------------------------------------- - dataverse_server = 'https://your dataverse server' # no trailing slash + dataverse_server = 'https://your dataverse installation' # no trailing slash api_key = 'api key' dataset_id = 1 # database id of the dataset persistentId = 'doi:10.5072/FK2/6XACVA' # doi or hdl of the dataset @@ -1286,7 +1286,7 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/datasets/24/storagesize The size of published and unpublished files will be summed in the dataset specified. -By default, only the archival files are counted - i.e., the files uploaded by users (plus the tab-delimited versions generated for tabular data files on ingest). If the optional argument ``includeCached=true`` is specified, the API will also add the sizes of all the extra files generated and cached by Dataverse - the resized thumbnail versions for image files, the metadata exports for published datasets, etc. Because this deals with unpublished files the token supplied must have permission to view unpublished drafts. +By default, only the archival files are counted - i.e., the files uploaded by users (plus the tab-delimited versions generated for tabular data files on ingest). If the optional argument ``includeCached=true`` is specified, the API will also add the sizes of all the extra files generated and cached by the Dataverse installation - the resized thumbnail versions for image files, the metadata exports for published datasets, etc. Because this deals with unpublished files the token supplied must have permission to view unpublished drafts. Get the size of Downloading all the files of a Dataset Version @@ -1363,7 +1363,7 @@ The review process can sometimes resemble a tennis match, with the authors submi Link a Dataset ~~~~~~~~~~~~~~ -Creates a link between a dataset and a dataverse (see :ref:`dataset-linking` section of Dataverse Management in the User Guide for more information): +Creates a link between a dataset and a Dataverse collection (see :ref:`dataset-linking` section of Dataverse Collection Management in the User Guide for more information): .. code-block:: bash @@ -1494,7 +1494,7 @@ If the dataset is not locked (or if there is no lock of the specified type), the Dataset Metrics ~~~~~~~~~~~~~~~ -Please note that these dataset level metrics are only available if support for Make Data Count has been enabled in your installation of Dataverse. See the :ref:`Dataset Metrics ` in the :doc:`/user/dataset-management` section of the User Guide and the :doc:`/admin/make-data-count` section of the Admin Guide for details. +Please note that these dataset level metrics are only available if support for Make Data Count has been enabled in your Dataverse installation. See the :ref:`Dataset Metrics ` in the :doc:`/user/dataset-management` section of the User Guide and the :doc:`/admin/make-data-count` section of the Admin Guide for details. .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. @@ -1652,7 +1652,7 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE https://demo.dataverse.org/api/datasets/24/destroy -Calling the destroy endpoint is permanent and irreversible. It will remove the dataset and its datafiles, then re-index the parent dataverse in Solr. This endpoint requires the API token of a superuser. +Calling the destroy endpoint is permanent and irreversible. It will remove the dataset and its datafiles, then re-index the parent Dataverse collection in Solr. This endpoint requires the API token of a superuser. Configure a Dataset to Use a Specific File Store ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1773,7 +1773,7 @@ The fully expanded example above (without environment variables) looks like this Reingest a File ~~~~~~~~~~~~~~~ -Attempt to ingest an existing datafile as tabular data. This API can be used on a file that was not ingested as tabular back when it was uploaded. For example, a Stata v.14 file that was uploaded before ingest support for Stata 14 was added (in Dataverse v.4.9). It can also be used on a file that failed to ingest due to a bug in the ingest plugin that has since been fixed (hence the name "reingest"). +Attempt to ingest an existing datafile as tabular data. This API can be used on a file that was not ingested as tabular back when it was uploaded. For example, a Stata v.14 file that was uploaded before ingest support for Stata 14 was added (in Dataverse software v.4.9). It can also be used on a file that failed to ingest due to a bug in the ingest plugin that has since been fixed (hence the name "reingest"). Note that this requires "superuser" credentials. @@ -1816,7 +1816,7 @@ Note: at present, the API cannot be used on a file that's already successfully i Redetect File Type ~~~~~~~~~~~~~~~~~~ -Dataverse uses a variety of methods for determining file types (MIME types or content types) and these methods (listed below) are updated periodically. If you have files that have an unknown file type, you can have Dataverse attempt to redetect the file type. +The Dataverse Platform uses a variety of methods for determining file types (MIME types or content types) and these methods (listed below) are updated periodically. If you have files that have an unknown file type, you can have the Dataverse Platform attempt to redetect the file type. When using the curl command below, you can pass ``dryRun=true`` if you don't want any changes to be saved to the database. Change this to ``dryRun=false`` (or omit it) to save the change. @@ -2301,7 +2301,7 @@ In order to delete a token use:: Builtin Users ------------- -Builtin users are known as "Username/Email and Password" users in the :doc:`/user/account` of the User Guide. Dataverse stores a password (encrypted, of course) for these users, which differs from "remote" users such as Shibboleth or OAuth users where the password is stored elsewhere. See also :ref:`auth-modes` section of Configuration in the Installation Guide. It's a valid configuration of Dataverse to not use builtin users at all. +Builtin users are known as "Username/Email and Password" users in the :doc:`/user/account` of the User Guide. The Dataverse installation stores a password (encrypted, of course) for these users, which differs from "remote" users such as Shibboleth or OAuth users where the password is stored elsewhere. See also :ref:`auth-modes` section of Configuration in the Installation Guide. It's a valid configuration of a Dataverse installation to not use builtin users at all. Create a Builtin User ~~~~~~~~~~~~~~~~~~~~~ @@ -2321,10 +2321,10 @@ Optionally, you may use a third query parameter "sendEmailNotification=false" to Roles ----- -Create a New Role in a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Create a New Role in a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Creates a new role in dataverse object whose Id is ``dataverseIdtf`` (that's an id/alias):: +Creates a new role in Dataverse collection object whose Id is ``dataverseIdtf`` (that's an id/alias):: POST http://$SERVER/api/roles?dvo=$dataverseIdtf&key=$apiKey @@ -2348,9 +2348,9 @@ Explicit Groups Create New Explicit Group ~~~~~~~~~~~~~~~~~~~~~~~~~ -Explicit groups list their members explicitly. These groups are defined in dataverses, which is why their API endpoint is under ``api/dataverses/$id/``, where ``$id`` is the id of the dataverse. +Explicit groups list their members explicitly. These groups are defined in Dataverse collections, which is why their API endpoint is under ``api/dataverses/$id/``, where ``$id`` is the id of the Dataverse collection. -Create a new explicit group under dataverse ``$id``:: +Create a new explicit group under Dataverse collection ``$id``:: POST http://$server/api/dataverses/$id/groups @@ -2362,31 +2362,31 @@ Data being POSTed is json-formatted description of the group:: "aliasInOwner":"ccs" } -List Explicit Groups in a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +List Explicit Groups in a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -List explicit groups under dataverse ``$id``:: +List explicit groups under Dataverse collection ``$id``:: GET http://$server/api/dataverses/$id/groups -Show Single Group in a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Show Single Group in a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Show group ``$groupAlias`` under dataverse ``$dv``:: GET http://$server/api/dataverses/$dv/groups/$groupAlias -Update Group in a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Update Group in a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Update group ``$groupAlias`` under dataverse ``$dv``. The request body is the same as the create group one, except that the group alias cannot be changed. Thus, the field ``aliasInOwner`` is ignored. :: +Update group ``$groupAlias`` under Dataverse collection ``$dv``. The request body is the same as the create group one, except that the group alias cannot be changed. Thus, the field ``aliasInOwner`` is ignored. :: PUT http://$server/api/dataverses/$dv/groups/$groupAlias -Delete Group from a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Delete Group from a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Delete group ``$groupAlias`` under dataverse ``$dv``:: +Delete group ``$groupAlias`` under Dataverse collection ``$dv``:: DELETE http://$server/api/dataverses/$dv/groups/$groupAlias @@ -2421,10 +2421,10 @@ Management of Shibboleth groups via API is documented in the :doc:`/installation Info ---- -Show Dataverse Version and Build Number -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Show Dataverse Software Version and Build Number +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -|CORS| Get the Dataverse version. The response contains the version and build numbers: +|CORS| Get the Dataverse installation version. The response contains the version and build numbers: .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below. @@ -2440,10 +2440,10 @@ The fully expanded example above (without environment variables) looks like this curl https://demo.dataverse.org/api/info/version -Show Dataverse Server Name -~~~~~~~~~~~~~~~~~~~~~~~~~~ +Show Dataverse Installation Server Name +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Get the server name. This is useful when a Dataverse system is composed of multiple app servers behind a load balancer: +Get the server name. This is useful when a Dataverse installation is composed of multiple app servers behind a load balancer: .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below. @@ -2611,7 +2611,7 @@ The fully expanded example above (without environment variables) looks like this Delete a PID ~~~~~~~~~~~~ -Delete PID (this is only possible for PIDs that are in the "draft" state) and within Dataverse, set ``globalidcreatetime`` to null and ``identifierregistered`` to false. A superuser API token is required. +Delete PID (this is only possible for PIDs that are in the "draft" state) and within a Dataverse installation, set ``globalidcreatetime`` to null and ``identifierregistered`` to false. A superuser API token is required. .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below. @@ -3005,14 +3005,14 @@ List all role assignments of a role assignee (i.e. a user or a group):: Note that ``identifier`` can contain slashes (e.g. ``&ip/localhost-users``). -List Permissions a User Has on a Dataverse or Dataset -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +List Permissions a User Has on a Dataverse Collection or Dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -List permissions a user (based on API Token used) has on a dataverse or dataset:: +List permissions a user (based on API Token used) has on a Dataverse collection or dataset:: GET http://$SERVER/api/admin/permissions/$identifier -The ``$identifier`` can be a dataverse alias or database id or a dataset persistent ID or database id. +The ``$identifier`` can be a Dataverse collection alias or database id or a dataset persistent ID or database id. Show Role Assignee ~~~~~~~~~~~~~~~~~~ @@ -3028,7 +3028,7 @@ The ``$identifier`` should start with an ``@`` if it's a user. Groups start with Saved Search ~~~~~~~~~~~~ -The Saved Search, Linked Dataverses, and Linked Datasets features shipped with Dataverse 4.0, but as a "`superuser-only `_" because they are **experimental** (see `#1364 `_, `#1813 `_, `#1840 `_, `#1890 `_, `#1939 `_, `#2167 `_, `#2186 `_, `#2053 `_, and `#2543 `_). The following API endpoints were added to help people with access to the "admin" API make use of these features in their current form. There is a known issue (`#1364 `_) that once a link to a dataverse or dataset is created, it cannot be removed (apart from database manipulation and reindexing) which is why a ``DELETE`` endpoint for saved searches is neither documented nor functional. The Linked Dataverses feature is `powered by Saved Search `_ and therefore requires that the "makelinks" endpoint be executed on a periodic basis as well. +The Saved Search, Linked Dataverses, and Linked Datasets features shipped with Dataverse 4.0, but as a "`superuser-only `_" because they are **experimental** (see `#1364 `_, `#1813 `_, `#1840 `_, `#1890 `_, `#1939 `_, `#2167 `_, `#2186 `_, `#2053 `_, and `#2543 `_). The following API endpoints were added to help people with access to the "admin" API make use of these features in their current form. There is a known issue (`#1364 `_) that once a link to a Dataverse collection or dataset is created, it cannot be removed (apart from database manipulation and reindexing) which is why a ``DELETE`` endpoint for saved searches is neither documented nor functional. The Linked Dataverse collections feature is `powered by Saved Search `_ and therefore requires that the "makelinks" endpoint be executed on a periodic basis as well. List all saved searches. :: @@ -3038,11 +3038,11 @@ List a saved search by database id. :: GET http://$SERVER/api/admin/savedsearches/$id -Execute a saved search by database id and make links to dataverses and datasets that are found. The JSON response indicates which dataverses and datasets were newly linked versus already linked. The ``debug=true`` query parameter adds to the JSON response extra information about the saved search being executed (which you could also get by listing the saved search). :: +Execute a saved search by database id and make links to Dataverse collections and datasets that are found. The JSON response indicates which Dataverse collections and datasets were newly linked versus already linked. The ``debug=true`` query parameter adds to the JSON response extra information about the saved search being executed (which you could also get by listing the saved search). :: PUT http://$SERVER/api/admin/savedsearches/makelinks/$id?debug=true -Execute all saved searches and make links to dataverses and datasets that are found. ``debug`` works as described above. This happens automatically with a timer. For details, see :ref:`saved-search-timer` in the Admin Guide. :: +Execute all saved searches and make links to Dataverse collections and datasets that are found. ``debug`` works as described above. This happens automatically with a timer. For details, see :ref:`saved-search-timer` in the Admin Guide. :: PUT http://$SERVER/api/admin/savedsearches/makelinks/all?debug=true @@ -3100,7 +3100,7 @@ if validation fails, will report the specific database entity and the offending If the optional argument ``variables=true`` is specified, the API will also validate the metadata associated with any tabular data files found in the dataset specified. (For example: an invalid or empty variable name). -Validate all the datasets in the Dataverse, report any constraint violations found:: +Validate all the datasets in the Dataverse installation, report any constraint violations found:: curl $SERVER_URL/api/admin/validate/datasets @@ -3119,7 +3119,7 @@ This API streams its output in real time, i.e. it will start producing the outpu ] } -Note that if you are attempting to validate a very large number of datasets in your Dataverse, this API may time out - subject to the timeout limit set in your app server configuration. If this is a production Dataverse instance serving large amounts of data, you most likely have that timeout set to some high value already. But if you need to increase it, it can be done with the asadmin command. For example:: +Note that if you are attempting to validate a very large number of datasets in your Dataverse installation, this API may time out - subject to the timeout limit set in your app server configuration. If this is a production Dataverse installation serving large amounts of data, you most likely have that timeout set to some high value already. But if you need to increase it, it can be done with the asadmin command. For example:: asadmin set server-config.network-config.protocols.protocol.http-listener-1.http.request-timeout-seconds=3600 @@ -3161,7 +3161,7 @@ Unset the default workflow for ``triggerType``. After this call, dataset release DELETE http://$SERVER/api/admin/workflows/default/$triggerType -Set the whitelist of IP addresses separated by a semicolon (``;``) allowed to resume workflows. Request body is a list of IP addresses allowed to send "resume workflow" messages to this Dataverse instance:: +Set the whitelist of IP addresses separated by a semicolon (``;``) allowed to resume workflows. Request body is a list of IP addresses allowed to send "resume workflow" messages to this Dataverse installation:: PUT http://$SERVER/api/admin/workflows/ip-whitelist @@ -3190,13 +3190,13 @@ Clear a specific metric cache. Currently this must match the name of the row in CORS -Inherit Dataverse Role Assignments -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Inherit Dataverse Collection Role Assignments +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Recursively applies the role assignments of the specified dataverse, for the roles specified by the ``:InheritParentRoleAssignments`` setting, to all dataverses contained within it:: +Recursively applies the role assignments of the specified Dataverse collection, for the roles specified by the ``:InheritParentRoleAssignments`` setting, to all Dataverse collections contained within it:: GET http://$SERVER/api/admin/dataverse/{dataverse alias}/addRoleAssignmentsToChildren -Note: setting ``:InheritParentRoleAssignments`` will automatically trigger inheritance of the parent dataverse's role assignments for a newly created dataverse. Hence this API call is intended as a way to update existing child dataverses or to update children after a change in role assignments has been made on a parent dataverse. +Note: setting ``:InheritParentRoleAssignments`` will automatically trigger inheritance of the parent Dataverse collection's role assignments for a newly created Dataverse collection. Hence this API call is intended as a way to update existing child Dataverse collections or to update children after a change in role assignments has been made on a parent Dataverse collection. From 283769218eb7ddebc54c6478047caf798cd5afeb Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 8 Jan 2021 16:54:35 -0500 Subject: [PATCH 076/576] updates to SWORD doc --- doc/sphinx-guides/source/api/sword.rst | 48 +++++++++++++------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/doc/sphinx-guides/source/api/sword.rst b/doc/sphinx-guides/source/api/sword.rst index f031f78f822..0d09f9ac047 100755 --- a/doc/sphinx-guides/source/api/sword.rst +++ b/doc/sphinx-guides/source/api/sword.rst @@ -9,11 +9,11 @@ SWORD_ stands for "Simple Web-service Offering Repository Deposit" and is a "pro About ----- -Introduced in Dataverse Network (DVN) `3.6 `_, the SWORD API was formerly known as the "Data Deposit API" and ``data-deposit/v1`` appeared in the URLs. For backwards compatibility these URLs continue to work (with deprecation warnings). Due to architectural changes and security improvements (especially the introduction of API tokens) in Dataverse 4.0, a few backward incompatible changes were necessarily introduced and for this reason the version has been increased to ``v1.1``. For details, see :ref:`incompatible`. +Introduced in Dataverse Network (DVN) `3.6 `_, the SWORD API was formerly known as the "Data Deposit API" and ``data-deposit/v1`` appeared in the URLs. For backwards compatibility these URLs continue to work (with deprecation warnings). Due to architectural changes and security improvements (especially the introduction of API tokens) in Dataverse Software 4.0, a few backward incompatible changes were necessarily introduced and for this reason the version has been increased to ``v1.1``. For details, see :ref:`incompatible`. -Dataverse implements most of SWORDv2_, which is specified at http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html . Please reference the `SWORDv2 specification`_ for expected HTTP status codes (i.e. 201, 204, 404, etc.), headers (i.e. "Location"), etc. +The Dataverse Platform implements most of SWORDv2_, which is specified at http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html . Please reference the `SWORDv2 specification`_ for expected HTTP status codes (i.e. 201, 204, 404, etc.), headers (i.e. "Location"), etc. -As a profile of AtomPub, XML is used throughout SWORD. As of Dataverse 4.0 datasets can also be created via JSON using the "native" API. SWORD is limited to the dozen or so fields listed below in the crosswalk, but the native API allows you to populate all metadata fields available in Dataverse. +As a profile of AtomPub, XML is used throughout SWORD. As of Dataverse Software 4.0 datasets can also be created via JSON using the "native" API. SWORD is limited to the dozen or so fields listed below in the crosswalk, but the native API allows you to populate all metadata fields available in a Dataverse installation. .. _SWORD: http://en.wikipedia.org/wiki/SWORD_%28protocol%29 @@ -30,7 +30,7 @@ Backward incompatible changes For better security than in DVN 3.x, usernames and passwords are no longer accepted. The use of an API token is required. -Differences in Dataverse 4 from DVN 3.x lead to a few minor backward incompatible changes in the Dataverse implementation of SWORD, which are listed below. Old ``v1`` URLs should continue to work but the ``Service Document`` will contain a deprecation warning and responses will contain ``v1.1`` URLs. See also :ref:`known-issues`. +Differences in Dataverse Platform 4 from DVN 3.x lead to a few minor backward incompatible changes in the Dataverse Platform implementation of SWORD, which are listed below. Old ``v1`` URLs should continue to work but the ``Service Document`` will contain a deprecation warning and responses will contain ``v1.1`` URLs. See also :ref:`known-issues`. - Newly required fields when creating/editing datasets for compliance with the `Joint Declaration for Data Citation principles `_. @@ -40,16 +40,16 @@ Differences in Dataverse 4 from DVN 3.x lead to a few minor backward incompatibl - Deaccessioning is no longer supported. An alternative will be developed at https://github.com/IQSS/dataverse/issues/778 -- The Service Document will show a single API Terms of Use rather than root level and dataverse level Deposit Terms of Use. +- The Service Document will show a single API Terms of Use rather than root level and Dataverse collection level Deposit Terms of Use. New features as of v1.1 ----------------------- -- Dataverse 4 supports API tokens and requires them to be used for APIs instead of a username and password. In the ``curl`` examples below, you will see ``curl -u $API_TOKEN:`` showing that you should send your API token as the username and nothing as the password. For example, ``curl -u 54b143b5-d001-4254-afc0-a1c0f6a5b5a7:``. +- The Dataverse Software supports API tokens and requires them to be used for APIs instead of a username and password. In the ``curl`` examples below, you will see ``curl -u $API_TOKEN:`` showing that you should send your API token as the username and nothing as the password. For example, ``curl -u 54b143b5-d001-4254-afc0-a1c0f6a5b5a7:``. -- SWORD operations no longer require "admin" permission. In order to use any SWORD operation in DVN 3.x, you had to be an "admin" on a dataverse (the container for your dataset) and similar rules were applied in Dataverse 4.4 and earlier (the ``EditDataverse`` permission was required). The SWORD API has now been fully integrated with the Dataverse 4 permission model such that any action you have permission to perform in the GUI or "native" API you are able to perform via SWORD. This means that even a user with a "Contributor" role can operate on datasets via SWORD. Note that users with the "Contributor" role do not have the ``PublishDataset`` permission and will not be able publish their datasets via any mechanism, GUI or API. +- SWORD operations no longer require "admin" permission. In order to use any SWORD operation in DVN 3.x, you had to be an "admin" on a Dataverse collection (the container for your dataset) and similar rules were applied in Dataverse Software 4.4 and earlier (the ``EditDataverse`` permission was required). The SWORD API has now been fully integrated with the Dataverse Software permission model such that any action you have permission to perform in the GUI or "native" API you are able to perform via SWORD. This means that even a user with a "Contributor" role can operate on datasets via SWORD. Note that users with the "Contributor" role do not have the ``PublishDataset`` permission and will not be able publish their datasets via any mechanism, GUI or API. -- Dataverses can be published via SWORD. +- Dataverse collections can be published via SWORD. - Datasets versions will only be increased to the next minor version (i.e. 1.1) rather than a major version (2.0) if possible. This depends on the nature of the change. Adding or removing a file, for example, requires a major version bump. @@ -73,14 +73,14 @@ curl examples Retrieve SWORD service document ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The service document enumerates the dataverses ("collections" from a SWORD perspective) the user can deposit data into. The "collectionPolicy" element for each dataverse contains the Terms of Use. Any user with an API token can use this API endpoint. Institution-wide Shibboleth groups are not respected because membership in such a group can only be set via a browser. +The service document enumerates the Dataverse collections (also "collections" from a SWORD perspective) the user can deposit data into. The "collectionPolicy" element for each Dataverse collections contains the Terms of Use. Any user with an API token can use this API endpoint. Institution-wide Shibboleth groups are not respected because membership in such a group can only be set via a browser. ``curl -u $API_TOKEN: https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/service-document`` Create a dataset with an Atom entry ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To create a dataset, you must have the "Dataset Creator" role (the ``AddDataset`` permission) on a dataverse. Practically speaking, you should first retrieve the service document to list the dataverses into which you are authorized to deposit data. +To create a dataset, you must have the "Dataset Creator" role (the ``AddDataset`` permission) on a Dataverse collection. Practically speaking, you should first retrieve the service document to list the Dataverse collections into which you are authorized to deposit data. ``curl -u $API_TOKEN: --data-binary "@path/to/atom-entry-study.xml" -H "Content-Type: application/atom+xml" https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/collection/dataverse/$DATAVERSE_ALIAS`` @@ -88,10 +88,10 @@ Example Atom entry (XML) .. literalinclude:: sword-atom-entry.xml -Dublin Core Terms (DC Terms) Qualified Mapping - Dataverse DB Element Crosswalk -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Dublin Core Terms (DC Terms) Qualified Mapping - Dataverse Project DB Element Crosswalk +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+ -|DC (terms: namespace) | Dataverse DB Element | Required | Note | +|DC (terms: namespace) | Dataverse Software DB Element | Required | Note | +=============================+==============================================+==============+=============================================================================================================================================================+ |dcterms:title | title | Y | Title of the Dataset. | +-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -115,17 +115,17 @@ Dublin Core Terms (DC Terms) Qualified Mapping - Dataverse DB Element Crosswalk +-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+ |dcterms:coverage | otherGeographicCoverage | | General information on the geographic coverage of the Dataset. | +-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+ -|dcterms:license | license | | Set the license to CC0 (default in Dataverse for new Datasets), otherwise enter "NONE" and fill in the dcterms:rights field. | +|dcterms:license | license | | Set the license to CC0 (default in a Dataverse installation for new Datasets), otherwise enter "NONE" and fill in the dcterms:rights field. | +-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+ |dcterms:rights | termsofuse | | If not using CC0, enter any terms of use or restrictions for the Dataset. | +-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+ |dcterms:isReferencedBy | publicationCitation | | The publication (journal article, book, other work) that uses this dataset (include citation, permanent identifier (DOI), and permanent URL). | +-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+ -List datasets in a dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +List datasets in a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -You must have permission to add datasets in a dataverse (the dataverse should appear in the service document) to list the datasets inside. Institution-wide Shibboleth groups are not respected because membership in such a group can only be set via a browser. +You must have permission to add datasets in a Dataverse collection (the Dataverse collection should appear in the service document) to list the datasets inside. Institution-wide Shibboleth groups are not respected because membership in such a group can only be set via a browser. ``curl -u $API_TOKEN: https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/collection/dataverse/$DATAVERSE_ALIAS`` @@ -173,17 +173,17 @@ You must have the ``DeleteDatasetDraft`` permission (Contributor role or above s ``curl -u $API_TOKEN: -i -X DELETE https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/edit/study/doi:TEST/12345`` -Determine if a dataverse has been published -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Determine if a Dataverse Collection has been published +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This API endpoint is the same as the "list datasets in a dataverse" endpoint documented above and the same permissions apply but it is documented here separately to point out that you can look for a boolean called ``dataverseHasBeenReleased`` to know if a dataverse has been released, which is required for publishing a dataset. +This API endpoint is the same as the "list datasets in a Dataverse collection" endpoint documented above and the same permissions apply but it is documented here separately to point out that you can look for a boolean called ``dataverseHasBeenReleased`` to know if a Dataverse collection has been released, which is required for publishing a dataset. ``curl -u $API_TOKEN: https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/collection/dataverse/$DATAVERSE_ALIAS`` -Publish a dataverse -~~~~~~~~~~~~~~~~~~~ +Publish a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The ``cat /dev/null`` and ``--data-binary @-`` arguments are used to send zero-length content to the API, which is required by the upstream library to process the ``In-Progress: false`` header. You must have the ``PublishDataverse`` permission (Admin role) on the dataverse to publish it. +The ``cat /dev/null`` and ``--data-binary @-`` arguments are used to send zero-length content to the API, which is required by the upstream library to process the ``In-Progress: false`` header. You must have the ``PublishDataverse`` permission (Admin role) on the Dataverse collection to publish it. ``cat /dev/null | curl -u $API_TOKEN: -X POST -H "In-Progress: false" --data-binary @- https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/edit/dataverse/$DATAVERSE_ALIAS`` @@ -203,7 +203,7 @@ Known issues - The Service Document does not honor groups within groups: https://github.com/IQSS/dataverse/issues/3056 -- Should see all the fields filled in for a dataset regardless of what the parent dataverse specifies: https://github.com/IQSS/dataverse/issues/756 +- Should see all the fields filled in for a dataset regardless of what the parent Dataverse collection specifies: https://github.com/IQSS/dataverse/issues/756 - SWORD 2.0 Profile 6.4 "Retrieving the content" has not been implemented: https://github.com/IQSS/dataverse/issues/183 From d96265bb2c3d1c6c8974f2d5913168cbd0fd8e34 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 8 Jan 2021 18:04:30 -0500 Subject: [PATCH 077/576] handle aws network aliasing for proxy testing AWS uses a custom-endpoint-url+https://s3.amazonaws.com but then signs URLs of the form https://.s3.amazonaws.com/* --- .../source/installation/config.rst | 4 ++- .../iq/dataverse/dataaccess/S3AccessIO.java | 31 ++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index e74419f8527..caa63b1b1cd 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -560,8 +560,10 @@ Reported Working S3-Compatible Storage Set ``dataverse.files..payload-signing=true`` and ``dataverse.files..chunked-encoding=false`` to use Surf Object Store. -Note that the ``dataverse.files..proxy-url`` setting can be used in installations where the object store is proxied. For direct uploads and downloads, Dataverse redirects to the proxy-url but presigns the urls using the ``dataverse.files..custom-endpoint-url``. +Note that the ``dataverse.files..proxy-url`` setting can be used in installations where the object store is proxied, but it should be considered an advanced option that will require significant expertise to properly configure. +For direct uploads and downloads, Dataverse redirects to the proxy-url but presigns the urls based on the ``dataverse.files..custom-endpoint-url``. Additional configuration (appropriate CORS settings, proxy caching/timeout configuration, and proxy settings to pass headers to/from S3 and to avoid adding additional headers) will also be needed to enable use of a proxy with direct upload and download. +For Amazon AWS, see comments in the edu.harvard.iq.dataverse.dataaccess.S3AccessIO class about support for AWS's bucket-specific DNS names. **HINT:** If you are successfully using an S3 storage implementation not yet listed above, please feel free to `open an issue at Github `_ and describe your setup. diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index 1e42b27f05b..12b57dfde21 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -883,7 +883,36 @@ public String generateTemporaryS3Url() throws IOException { if (s != null) { if(!StringUtil.isEmpty(proxy)) { - return s.toString().replace(endpoint, proxy); + /* + * AWS actually uses two URLs for its endpoint - for example + * https://s3.amazonaws.com is what's used to configure the custom-endpoint-url + * in Dataverse, but presigned URLs are of the form + * https://.s3.amazonaws.com + * + * Since we only record the first form, we'll use a regexp to match endpoints + * that have an additional 'bucket prefix' in the servername. + * + * Institutional S3 servers, e.g. based on MinIO, don't need to do this (i.e. + * it's just AWS network setup, not part of the S3 protocol itself), so + * supporting this may only be used in testing. + * + * Further, since the signatures only validate for the correct URLs, the risk in + * a bad match appears to be limited to breaking things, but if the potential + * for substitutions gets more complex, it might be better to just add another + * config setting. + */ + // endpoint-urls for AWS don't have to have the protocol, so while we expect + // them for some servers, we check whether the protocol is in the url and then + // normalizing to use the part without the protocol + String endpointServer = endpoint; + int protocolEnd = endpoint.indexOf("://"); + if (protocolEnd >=0 ) { + endpointServer = endpoint.substring(protocolEnd + 3); + } + // We're then replacing + // http or https followed by :// and an optional . before the normalized endpoint url + // with the proxy info (which is protocol + machine name and optional port) + return s.toString().replaceFirst("http[s]*:\\/\\/([^\\/]+\\.)"+endpointServer, proxy); } else { return s.toString(); } From d0397848abc540bc92345a38449c8cf74710d0de Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Mon, 11 Jan 2021 10:12:15 -0500 Subject: [PATCH 078/576] Applied truncation to UNF, moved tabular metadata labels to bundle [ref #5210 #6685] --- .../iq/dataverse/search/SearchIncludeFragment.java | 6 +++--- src/main/java/propertyFiles/Bundle.properties | 1 + src/main/webapp/filesFragment.xhtml | 3 ++- .../webapp/resources/js/dv_rebind_bootstrap_ui.js | 11 +++++++++-- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java index 9219e8b3664..61859700077 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java @@ -1213,14 +1213,14 @@ public String tabularDataDisplayInfo(DataFile datafile) { Long varNumber = datatable.getVarQuantity(); Long obsNumber = datatable.getCaseQuantity(); if (varNumber != null && varNumber.intValue() != 0) { - ret = ret.concat(varNumber + " Variables"); + ret = ret.concat(varNumber + " " + BundleUtil.getStringFromBundle("file.metaData.dataFile.dataTab.variables")); if (obsNumber != null && obsNumber.intValue() != 0) { - ret = ret.concat(", " + obsNumber + " Observations"); + ret = ret.concat(", " + obsNumber + " " + BundleUtil.getStringFromBundle("file.metaData.dataFile.dataTab.observations")); } ret = ret.concat(" - "); } if (unf != null && !unf.equals("")) { - ret = ret.concat("UNF: " + unf); + ret = ret.concat(BundleUtil.getStringFromBundle("file.metaData.dataFile.dataTab.unf") + ": " + unf); } } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index b27ab777ba6..8c8a9e5310a 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1558,6 +1558,7 @@ file.rsyncUpload.httpUploadDisabledDueToRsyncFileExistingAndPublished=HTTP uploa file.rsyncUpload.rsyncUploadDisabledDueFileUploadedViaHttp=Upload with rsync + SSH is disabled for this dataset because you have already uploaded files via HTTP. If you would like to switch to rsync upload, then you must first remove all uploaded files from this dataset. Once this dataset is published, the chosen upload method is permanently locked in. file.rsyncUpload.rsyncUploadDisabledDueFileUploadedViaHttpAndPublished=Upload with rsync + SSH is disabled for this dataset because you have already uploaded files via HTTP and published the dataset. file.metaData.checksum.copy=Click to copy +file.metaData.dataFile.dataTab.unf=UNF file.metaData.dataFile.dataTab.variables=Variables file.metaData.dataFile.dataTab.observations=Observations file.addDescription=Add file description... diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index 0a77a40d881..5c3cdcdc24f 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -35,11 +35,12 @@ + 25) { // COUNT ":" IN UNF VERSION LABEL - var labelCount = (checksumText.match(/:/g) || []).length; + var labelIndex = checksumText.indexOf(':'); // COUNT "=" IN UNF SUFFIX var suffixCount = (checksumText.match(/=/g) || []).length; + // TRUNCATE MIDDLE W/ "..." + FIRST/LAST 3 CHARACTERS - $(this).text(checksumText.substr(0, labelCount + 3) + '...' + checksumText.substr((checksumText.length - suffixCount - 3), checksumText.length)); + // CHECK IF UNF LABEL + if (labelIndex > 0) { + $(this).text(checksumText.substr(0, (labelIndex + 3) + 3) + '...' + checksumText.substr((checksumText.length - suffixCount - 3), checksumText.length)); + } + else { + $(this).text(checksumText.substr(0, 3) + '...' + checksumText.substr((checksumText.length - 3), checksumText.length)); + } } // ADD CLIPBOARD ICON $(this).append(' '); From e97e75791bcc0d8c6c882baa65513134465b3636 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 11 Jan 2021 11:28:06 -0500 Subject: [PATCH 079/576] implement regex AWS S3 for uploads as well, add fine logging --- .../iq/dataverse/dataaccess/S3AccessIO.java | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index 12b57dfde21..f68cc24d399 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -909,10 +909,13 @@ public String generateTemporaryS3Url() throws IOException { if (protocolEnd >=0 ) { endpointServer = endpoint.substring(protocolEnd + 3); } + logger.fine("Endpoint: " + endpointServer); // We're then replacing // http or https followed by :// and an optional . before the normalized endpoint url // with the proxy info (which is protocol + machine name and optional port) - return s.toString().replaceFirst("http[s]*:\\/\\/([^\\/]+\\.)"+endpointServer, proxy); + String finalUrl = s.toString().replaceFirst("http[s]*:\\/\\/([^\\/]+\\.)"+endpointServer, proxy); + logger.fine("ProxiedURL: " + finalUrl); + return finalUrl; } else { return s.toString(); } @@ -957,7 +960,21 @@ private String generateTemporaryS3UploadUrl(String key, Date expiration) throws String urlString = null; if (presignedUrl != null) { if(!StringUtil.isEmpty(proxy)) { - urlString = presignedUrl.toString().replace(endpoint, proxy); + //See discussion in getTemporaryS3Url + // endpoint-urls for AWS don't have to have the protocol, so while we expect + // them for some servers, we check whether the protocol is in the url and then + // normalizing to use the part without the protocol + String endpointServer = endpoint; + int protocolEnd = endpoint.indexOf("://"); + if (protocolEnd >=0 ) { + endpointServer = endpoint.substring(protocolEnd + 3); + } + logger.fine("Endpoint: " + endpointServer); + // We're then replacing + // http or https followed by :// and an optional . before the normalized endpoint url + // with the proxy info (which is protocol + machine name and optional port) + urlString = presignedUrl.toString().replaceFirst("http[s]*:\\/\\/([^\\/]+\\.)"+endpointServer, proxy); + logger.fine("ProxiedURL: " + urlString); } else { urlString = presignedUrl.toString(); } From b538acdb1ea5ed982c51b85a525ab9e36803bf72 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 11 Jan 2021 11:31:25 -0500 Subject: [PATCH 080/576] one more log statement --- .../java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index f68cc24d399..412d1fdb2f1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -913,6 +913,7 @@ public String generateTemporaryS3Url() throws IOException { // We're then replacing // http or https followed by :// and an optional . before the normalized endpoint url // with the proxy info (which is protocol + machine name and optional port) + logger.fine("Original Url: " + s.toString()); String finalUrl = s.toString().replaceFirst("http[s]*:\\/\\/([^\\/]+\\.)"+endpointServer, proxy); logger.fine("ProxiedURL: " + finalUrl); return finalUrl; From d16b6ee4bdb8b5fb282200391bcf78963d8f7247 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 11 Jan 2021 12:15:36 -0500 Subject: [PATCH 081/576] This is the reason search cards were not fully initialized for tabular files. (#7312) --- src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java index a9a16e60ae2..094e20edf28 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java @@ -497,7 +497,7 @@ public DataFile findCheapAndEasy(Long id) { // If content type indicates it's tabular data, spend 2 extra queries // looking up the data table and tabular tags objects: - if (MIME_TYPE_TSV.equalsIgnoreCase(contentType)) { + if (MIME_TYPE_TSV.equalsIgnoreCase(contentType) || MIME_TYPE_TSV_ALT.equalsIgnoreCase(contentType)) { Object[] dtResult; try { dtResult = (Object[]) em.createNativeQuery("SELECT ID, UNF, CASEQUANTITY, VARQUANTITY, ORIGINALFILEFORMAT, ORIGINALFILESIZE FROM dataTable WHERE DATAFILE_ID = " + id).getSingleResult(); From 064edbae1c88109a1d54dcad5385983e9cfc8c1c Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Mon, 11 Jan 2021 13:29:42 -0500 Subject: [PATCH 082/576] unf updates --- doc/sphinx-guides/source/developers/unf/index.rst | 6 +++--- doc/sphinx-guides/source/developers/unf/unf-v5.rst | 4 ++-- doc/sphinx-guides/source/developers/unf/unf-v6.rst | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/sphinx-guides/source/developers/unf/index.rst b/doc/sphinx-guides/source/developers/unf/index.rst index af9265a7fc0..237d1e9f138 100644 --- a/doc/sphinx-guides/source/developers/unf/index.rst +++ b/doc/sphinx-guides/source/developers/unf/index.rst @@ -20,10 +20,10 @@ computed. The signature is thus independent of the storage format. E.g., the same data object stored in, say, SPSS and Stata, will have the same UNF. -Early versions of Dataverse were using the first released +Early versions of the Dataverse Software were using the first released implementation of the UNF algorithm (v.3, implemented in R). Starting -with Dataverse 2.0 and throughout the 3.* lifecycle, UNF v.5 -(implemented in Java) was used. Dataverse 4.0 uses the latest release, +with Dataverse Software version 2.0 and throughout the 3.* lifecycle, UNF v.5 +(implemented in Java) was used. The Dataverse Software version 4.0 uses the latest release, UNF v.6. Two parallel implementation, in R and Java, will be available, for cross-validation. diff --git a/doc/sphinx-guides/source/developers/unf/unf-v5.rst b/doc/sphinx-guides/source/developers/unf/unf-v5.rst index 29b4556b1f9..cbbde2e18f8 100644 --- a/doc/sphinx-guides/source/developers/unf/unf-v5.rst +++ b/doc/sphinx-guides/source/developers/unf/unf-v5.rst @@ -8,8 +8,8 @@ UNF Version 5 **Important Update:** -UNF Version 5 has been in use by the Dataverse project since 2009. It was built into every version of the DVN, starting with 2.0 and up to 3.6.2. However, some problems were recently found in that implementation. Namely, in certain cases data normalization is not implemented fully to the spec. UNF signatures it generates are still reasonably strong statistically; however, this means that at least some of our signatures are not independently verifiable. I.e., if somebody fully implements their own version of UNF calculator, for certain datasets it would calculate signatures different from those generated by the DVN. Unless of course they implement it with the exact same bugs as ours. +UNF Version 5 has been in use by the Dataverse Project since 2009. It was built into every version of the DVN, starting with 2.0 and up to 3.6.2. However, some problems were recently found in that implementation. Namely, in certain cases data normalization is not implemented fully to the spec. UNF signatures it generates are still reasonably strong statistically; however, this means that at least some of our signatures are not independently verifiable. I.e., if somebody fully implements their own version of UNF calculator, for certain datasets it would calculate signatures different from those generated by the DVN. Unless of course they implement it with the exact same bugs as ours. -To address this, the Project is about to release UNF Version 6. The release date is still being discussed. It may coincide with the release of Dataverse 4.0. Alternatively, the production version of DVN 3.6.3 may get upgraded to use UNF v6 prior to that. This will be announced shortly. In the process, we are solving another problem with UNF v5 - this time we've made an effort to offer very implementer-friendly documentation that describes the algorithm fully and unambiguously. So if you are interested in implementing your own version of a UNF calculator, (something we would like to encourage!) please proceed directly to the Version 6 documentation. +To address this, the Project is about to release UNF Version 6. The release date is still being discussed. It may coincide with the release of Dataverse Software version 4.0. Alternatively, the production version of DVN 3.6.3 may get upgraded to use UNF v6 prior to that. This will be announced shortly. In the process, we are solving another problem with UNF v5 - this time we've made an effort to offer very implementer-friendly documentation that describes the algorithm fully and unambiguously. So if you are interested in implementing your own version of a UNF calculator, (something we would like to encourage!) please proceed directly to the Version 6 documentation. Going forward, we are going to offer a preserved version of the Version 5 library and, possibly, an online UNF v5 calculator, for the purposes of validating vectors and data sets for which published Version 5 UNFs exist. diff --git a/doc/sphinx-guides/source/developers/unf/unf-v6.rst b/doc/sphinx-guides/source/developers/unf/unf-v6.rst index b5c898a9a08..9648bae47c8 100644 --- a/doc/sphinx-guides/source/developers/unf/unf-v6.rst +++ b/doc/sphinx-guides/source/developers/unf/unf-v6.rst @@ -160,7 +160,7 @@ For example, to specify a non-default precision the parameter it is specified us Note: we do not recommend truncating character strings at fewer bytes than the default ``128`` (the ``X`` parameter). At the very least this number **must** be high enough so that the printable UNFs of individual variables or files are not truncated, when calculating combined UNFs of files or datasets, respectively. -It should also be noted that the Dataverse application never calculates UNFs with any non-default parameters. And we are not aware of anyone else actually doing so. If you are considering creating your own implementation of the UNF, it may be worth trying to create a simplified, defaults-only version first. Such an implementation would be sufficient to independently verify Dataverse-produced UNFs, among other things. +It should also be noted that the Dataverse Software never calculates UNFs with any non-default parameters. And we are not aware of anyone else actually doing so. If you are considering creating your own implementation of the UNF, it may be worth trying to create a simplified, defaults-only version first. Such an implementation would be sufficient to independently verify Dataverse Software-produced UNFs, among other things. .. _note2: From 1a87191b978d3f06b0f7ff0d9b23bebfc8b8511e Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Mon, 11 Jan 2021 13:39:23 -0500 Subject: [PATCH 083/576] more updates --- .../source/developers/aux-file-support.rst | 2 +- .../source/developers/big-data-support.rst | 58 +++++++++---------- .../source/developers/coding-style.rst | 6 +- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/doc/sphinx-guides/source/developers/aux-file-support.rst b/doc/sphinx-guides/source/developers/aux-file-support.rst index be21b56c245..260c4493590 100644 --- a/doc/sphinx-guides/source/developers/aux-file-support.rst +++ b/doc/sphinx-guides/source/developers/aux-file-support.rst @@ -1,7 +1,7 @@ Auxiliary File Support ====================== -Auxiliary file support is experimental. Auxiliary files in Dataverse are being added to support depositing and downloading differentially private metadata, as part of the OpenDP project (OpenDP.io). In future versions, this approach may become more broadly used and supported. +Auxiliary file support is experimental. Auxiliary files in the Dataverse Software are being added to support depositing and downloading differentially private metadata, as part of the OpenDP project (OpenDP.io). In future versions, this approach may become more broadly used and supported. Adding an Auxiliary File to a Datafile -------------------------------------- diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index 11cfbfcde08..37a14b63f34 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -11,18 +11,18 @@ Various components need to be installed and/or configured for big data support. S3 Direct Upload and Download ----------------------------- -A lightweight option for supporting file sizes beyond a few gigabytes - a size that can cause performance issues when uploaded through the Dataverse server itself - is to configure an S3 store to provide direct upload and download via 'pre-signed URLs'. When these options are configured, file uploads and downloads are made directly to and from a configured S3 store using secure (https) connections that enforce Dataverse's access controls. (The upload and download URLs are signed with a unique key that only allows access for a short time period and Dataverse will only generate such a URL if the user has permission to upload/download the specific file in question.) +A lightweight option for supporting file sizes beyond a few gigabytes - a size that can cause performance issues when uploaded through a Dataverse installation itself - is to configure an S3 store to provide direct upload and download via 'pre-signed URLs'. When these options are configured, file uploads and downloads are made directly to and from a configured S3 store using secure (https) connections that enforce a Dataverse installation's access controls. (The upload and download URLs are signed with a unique key that only allows access for a short time period and a Dataverse installation will only generate such a URL if the user has permission to upload/download the specific file in question.) -This option can handle files >40GB and could be appropriate for files up to a TB. Other options can scale farther, but this option has the advantages that it is simple to configure and does not require any user training - uploads and downloads are done via the same interface as normal uploads to Dataverse. +This option can handle files >40GB and could be appropriate for files up to a TB. Other options can scale farther, but this option has the advantages that it is simple to configure and does not require any user training - uploads and downloads are done via the same interface as normal uploads to a Dataverse installation. -To configure these options, an administrator must set two JVM options for the Dataverse server using the same process as for other configuration options: +To configure these options, an administrator must set two JVM options for the Dataverse installation using the same process as for other configuration options: ``./asadmin create-jvm-options "-Ddataverse.files..download-redirect=true"`` ``./asadmin create-jvm-options "-Ddataverse.files..upload-redirect=true"`` -With multiple stores configured, it is possible to configure one S3 store with direct upload and/or download to support large files (in general or for specific dataverses) while configuring only direct download, or no direct access for another store. +With multiple stores configured, it is possible to configure one S3 store with direct upload and/or download to support large files (in general or for specific Dataverse collections) while configuring only direct download, or no direct access for another store. The direct upload option now switches between uploading the file in one piece (up to 1 GB by default) and sending it as multiple parts. The default can be changed by setting: @@ -32,12 +32,12 @@ For AWS, the minimum allowed part size is 5*1024*1024 bytes and the maximum is 5 It is also possible to set file upload size limits per store. See the :MaxFileUploadSizeInBytes setting described in the :doc:`/installation/config` guide. -At present, one potential drawback for direct-upload is that files are only partially 'ingested', tabular and FITS files are processed, but zip files are not unzipped, and the file contents are not inspected to evaluate their mimetype. This could be appropriate for large files, or it may be useful to completely turn off ingest processing for performance reasons (ingest processing requires a copy of the file to be retrieved by Dataverse from the S3 store). A store using direct upload can be configured to disable all ingest processing for files above a given size limit: +At present, one potential drawback for direct-upload is that files are only partially 'ingested', tabular and FITS files are processed, but zip files are not unzipped, and the file contents are not inspected to evaluate their mimetype. This could be appropriate for large files, or it may be useful to completely turn off ingest processing for performance reasons (ingest processing requires a copy of the file to be retrieved by the Dataverse installation from the S3 store). A store using direct upload can be configured to disable all ingest processing for files above a given size limit: ``./asadmin create-jvm-options "-Ddataverse.files..ingestsizelimit="`` -**IMPORTANT:** One additional step that is required to enable direct uploads via Dataverse and for direct download to work with previewers is to allow cross site (CORS) requests on your S3 store. +**IMPORTANT:** One additional step that is required to enable direct uploads via a Dataverse installation and for direct download to work with previewers is to allow cross site (CORS) requests on your S3 store. The example below shows how to enable CORS rules (to support upload and download) on a bucket using the AWS CLI command line tool. Note that you may want to limit the AllowedOrigins and/or AllowedHeaders further. https://github.com/GlobalDataverseCommunityConsortium/dataverse-previewers/wiki/Using-Previewers-with-download-redirects-from-S3 has some additional information about doing this. ``aws s3api put-bucket-cors --bucket --cors-configuration file://cors.json`` @@ -59,7 +59,7 @@ with the contents of the file cors.json as follows: Alternatively, you can enable CORS using the AWS S3 web interface, using json-encoded rules as in the example above. -Since the direct upload mechanism creates the final file rather than an intermediate temporary file, user actions, such as neither saving or canceling an upload session before closing the browser page, can leave an abandoned file in the store. The direct upload mechanism attempts to use S3 Tags to aid in identifying/removing such files. Upon upload, files are given a "dv-status":"temp" tag which is removed when the dataset changes are saved and the new file(s) are added in Dataverse. Note that not all S3 implementations support Tags: Minio does not. WIth such stores, direct upload works, but Tags are not used. +Since the direct upload mechanism creates the final file rather than an intermediate temporary file, user actions, such as neither saving or canceling an upload session before closing the browser page, can leave an abandoned file in the store. The direct upload mechanism attempts to use S3 Tags to aid in identifying/removing such files. Upon upload, files are given a "dv-status":"temp" tag which is removed when the dataset changes are saved and the new file(s) are added in the Dataverse installation. Note that not all S3 implementations support Tags: Minio does not. WIth such stores, direct upload works, but Tags are not used. Data Capture Module (DCM) ------------------------- @@ -71,37 +71,37 @@ DCM was developed and tested using Glassfish but these docs have been updated wi Install a DCM ~~~~~~~~~~~~~ -Installation instructions can be found at https://github.com/sbgrid/data-capture-module/blob/master/doc/installation.md. Note that shared storage (posix or AWS S3) between Dataverse and your DCM is required. You cannot use a DCM with Swift at this point in time. +Installation instructions can be found at https://github.com/sbgrid/data-capture-module/blob/master/doc/installation.md. Note that shared storage (posix or AWS S3) between your Dataverse installation and your DCM is required. You cannot use a DCM with Swift at this point in time. .. FIXME: Explain what ``dataverse.files.dcm-s3-bucket-name`` is for and what it has to do with ``dataverse.files.s3.bucket-name``. -Once you have installed a DCM, you will need to configure two database settings on the Dataverse side. These settings are documented in the :doc:`/installation/config` section of the Installation Guide: +Once you have installed a DCM, you will need to configure two database settings on the Dataverse installation side. These settings are documented in the :doc:`/installation/config` section of the Installation Guide: - ``:DataCaptureModuleUrl`` should be set to the URL of a DCM you installed. - ``:UploadMethods`` should include ``dcm/rsync+ssh``. -This will allow your Dataverse installation to communicate with your DCM, so that Dataverse can download rsync scripts for your users. +This will allow your Dataverse installation to communicate with your DCM, so that your Dataverse installation can download rsync scripts for your users. -Downloading rsync scripts via Dataverse API -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Downloading rsync scripts via Your Dataverse Installation's API +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The rsync script can be downloaded from Dataverse via API using an authorized API token. In the curl example below, substitute ``$PERSISTENT_ID`` with a DOI or Handle: +The rsync script can be downloaded from your Dataverse installation via API using an authorized API token. In the curl example below, substitute ``$PERSISTENT_ID`` with a DOI or Handle: ``curl -H "X-Dataverse-key: $API_TOKEN" $DV_BASE_URL/api/datasets/:persistentId/dataCaptureModule/rsync?persistentId=$PERSISTENT_ID`` -How a DCM reports checksum success or failure to Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +How a DCM reports checksum success or failure to your Dataverse Installation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Once the user uploads files to a DCM, that DCM will perform checksum validation and report to Dataverse the results of that validation. The DCM must be configured to pass the API token of a superuser. The implementation details, which are subject to change, are below. +Once the user uploads files to a DCM, that DCM will perform checksum validation and report to your Dataverse installation the results of that validation. The DCM must be configured to pass the API token of a superuser. The implementation details, which are subject to change, are below. -The JSON that a DCM sends to Dataverse on successful checksum validation looks something like the contents of :download:`checksumValidationSuccess.json <../_static/installation/files/root/big-data-support/checksumValidationSuccess.json>` below: +The JSON that a DCM sends to your Dataverse installation on successful checksum validation looks something like the contents of :download:`checksumValidationSuccess.json <../_static/installation/files/root/big-data-support/checksumValidationSuccess.json>` below: .. literalinclude:: ../_static/installation/files/root/big-data-support/checksumValidationSuccess.json :language: json - ``status`` - The valid strings to send are ``validation passed`` and ``validation failed``. -- ``uploadFolder`` - This is the directory on disk where Dataverse should attempt to find the files that a DCM has moved into place. There should always be a ``files.sha`` file and a least one data file. ``files.sha`` is a manifest of all the data files and their checksums. The ``uploadFolder`` directory is inside the directory where data is stored for the dataset and may have the same name as the "identifier" of the persistent id (DOI or Handle). For example, you would send ``"uploadFolder": "DNXV2H"`` in the JSON file when the absolute path to this directory is ``/usr/local/payara5/glassfish/domains/domain1/files/10.5072/FK2/DNXV2H/DNXV2H``. -- ``totalSize`` - Dataverse will use this value to represent the total size in bytes of all the files in the "package" that's created. If 360 data files and one ``files.sha`` manifest file are in the ``uploadFolder``, this value is the sum of the 360 data files. +- ``uploadFolder`` - This is the directory on disk where your Dataverse installation should attempt to find the files that a DCM has moved into place. There should always be a ``files.sha`` file and a least one data file. ``files.sha`` is a manifest of all the data files and their checksums. The ``uploadFolder`` directory is inside the directory where data is stored for the dataset and may have the same name as the "identifier" of the persistent id (DOI or Handle). For example, you would send ``"uploadFolder": "DNXV2H"`` in the JSON file when the absolute path to this directory is ``/usr/local/payara5/glassfish/domains/domain1/files/10.5072/FK2/DNXV2H/DNXV2H``. +- ``totalSize`` - Your Dataverse installation will use this value to represent the total size in bytes of all the files in the "package" that's created. If 360 data files and one ``files.sha`` manifest file are in the ``uploadFolder``, this value is the sum of the 360 data files. Here's the syntax for sending the JSON. @@ -115,12 +115,12 @@ Steps to set up a DCM mock for Development See instructions at https://github.com/sbgrid/data-capture-module/blob/master/doc/mock.md -Add Dataverse settings to use mock (same as using DCM, noted above): +Add Dataverse Installation settings to use mock (same as using DCM, noted above): - ``curl http://localhost:8080/api/admin/settings/:DataCaptureModuleUrl -X PUT -d "http://localhost:5000"`` - ``curl http://localhost:8080/api/admin/settings/:UploadMethods -X PUT -d "dcm/rsync+ssh"`` -At this point you should be able to download a placeholder rsync script. Dataverse is then waiting for news from the DCM about if checksum validation has succeeded or not. First, you have to put files in place, which is usually the job of the DCM. You should substitute "X1METO" for the "identifier" of the dataset you create. You must also use the proper path for where you store files in your dev environment. +At this point you should be able to download a placeholder rsync script. Your Dataverse installation is then waiting for news from the DCM about if checksum validation has succeeded or not. First, you have to put files in place, which is usually the job of the DCM. You should substitute "X1METO" for the "identifier" of the dataset you create. You must also use the proper path for where you store files in your dev environment. - ``mkdir /usr/local/payara5/glassfish/domains/domain1/files/10.5072/FK2/X1METO`` - ``mkdir /usr/local/payara5/glassfish/domains/domain1/files/10.5072/FK2/X1METO/X1METO`` @@ -130,7 +130,7 @@ At this point you should be able to download a placeholder rsync script. Dataver -Now the files are in place and you need to send JSON to Dataverse with a success or failure message as described above. Make a copy of ``doc/sphinx-guides/source/_static/installation/files/root/big-data-support/checksumValidationSuccess.json`` and put the identifier in place such as "X1METO" under "uploadFolder"). Then use curl as described above to send the JSON. +Now the files are in place and you need to send JSON to your Dataverse installation with a success or failure message as described above. Make a copy of ``doc/sphinx-guides/source/_static/installation/files/root/big-data-support/checksumValidationSuccess.json`` and put the identifier in place such as "X1METO" under "uploadFolder"). Then use curl as described above to send the JSON. Troubleshooting ~~~~~~~~~~~~~~~ @@ -155,7 +155,7 @@ Optional steps for setting up the S3 Docker DCM Variant ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Before: the default bucket for DCM to hold files in S3 is named test-dcm. It is coded into `post_upload_s3.bash` (line 30). Change to a different bucket if needed. -- Also Note: With the new support for multiple file store in Dataverse, DCM requires a store with id="s3" and DCM will only work with this store. +- Also Note: With the new support for multiple file store in the Dataverse Software, DCM requires a store with id="s3" and DCM will only work with this store. - Add AWS bucket info to dcmsrv - Add AWS credentials to ``~/.aws/credentials`` @@ -164,7 +164,7 @@ Optional steps for setting up the S3 Docker DCM Variant - ``aws_access_key_id =`` - ``aws_secret_access_key =`` -- Dataverse configuration (on dvsrv): +- Dataverse installation configuration (on dvsrv): - Set S3 as the storage driver @@ -175,7 +175,7 @@ Optional steps for setting up the S3 Docker DCM Variant - ``./asadmin create-jvm-options "\-Ddataverse.files.s3.label=s3"`` - - Add AWS bucket info to Dataverse + - Add AWS bucket info to your Dataverse installation - Add AWS credentials to ``~/.aws/credentials`` - ``[default]`` @@ -187,13 +187,13 @@ Optional steps for setting up the S3 Docker DCM Variant - ``[default]`` - ``region = us-east-1`` - - Add the S3 bucket names to Dataverse + - Add the S3 bucket names to your Dataverse installation - - S3 bucket for Dataverse + - S3 bucket for your Dataverse installation - ``/usr/local/payara5/glassfish/bin/asadmin create-jvm-options "-Ddataverse.files.s3.bucket-name=iqsstestdcmbucket"`` - - S3 bucket for DCM (as Dataverse needs to do the copy over) + - S3 bucket for DCM (as your Dataverse installation needs to do the copy over) - ``/usr/local/payara5/glassfish/bin/asadmin create-jvm-options "-Ddataverse.files.dcm-s3-bucket-name=test-dcm"`` @@ -255,7 +255,7 @@ Configuring the RSAL Mock Info for configuring the RSAL Mock: https://github.com/sbgrid/rsal/tree/master/mocks -Also, to configure Dataverse to use the new workflow you must do the following (see also the :doc:`workflows` section): +Also, to configure your Dataverse installation to use the new workflow you must do the following (see also the :doc:`workflows` section): 1. Configure the RSAL URL: diff --git a/doc/sphinx-guides/source/developers/coding-style.rst b/doc/sphinx-guides/source/developers/coding-style.rst index f0a05991fc0..0c00f611a7f 100755 --- a/doc/sphinx-guides/source/developers/coding-style.rst +++ b/doc/sphinx-guides/source/developers/coding-style.rst @@ -2,7 +2,7 @@ Coding Style ============ -Like all development teams, the `Dataverse developers at IQSS `_ have their habits and styles when it comes to writing code. Let's attempt to get on the same page. :) +Like all development teams, the `Dataverse Project developers at IQSS `_ have their habits and styles when it comes to writing code. Let's attempt to get on the same page. :) .. contents:: |toctitle| :local: @@ -62,7 +62,7 @@ As you probably gathered from the :doc:`dev-environment` section, IQSS has stand Checking Your Formatting With Checkstyle ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The easiest way to adopt Dataverse coding style is to use Netbeans as your IDE, avoid change the default Netbeans formatting settings, and only reformat code you've changed, as described above. +The easiest way to adopt the Dataverse Project coding style is to use Netbeans as your IDE, avoid change the default Netbeans formatting settings, and only reformat code you've changed, as described above. If you do not use Netbeans, you are encouraged to check the formatting of your code using Checkstyle. @@ -97,7 +97,7 @@ Special strings should be defined as public constants. For example, ``DatasetFie Avoid Hard-Coding User-Facing Messaging in English ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -There is an ongoing effort to translate Dataverse into various languages. Look for "lang" or "languages" in the :doc:`/installation/config` section of the Installation Guide for details if you'd like to help or play around with this feature. +There is an ongoing effort to translate the Dataverse Software into various languages. Look for "lang" or "languages" in the :doc:`/installation/config` section of the Installation Guide for details if you'd like to help or play around with this feature. The translation effort is hampered if you hard code user-facing messages in English in the Java code. Put English strings in ``Bundle.properties`` and use ``BundleUtil`` to pull them out. This is especially important for messages that appear in the UI. We are aware that the API has many, many hard coded English strings in it. If you touch a method in the API and notice English strings, you are strongly encouraged to use that opportunity to move the English to ``Bundle.properties``. From 09b9f608ed51b16c075958ec7e807f7eb6c62b18 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Mon, 11 Jan 2021 13:44:25 -0500 Subject: [PATCH 084/576] more updates --- .../source/developers/configuration.rst | 8 ++++---- doc/sphinx-guides/source/developers/containers.rst | 2 +- .../source/developers/dependencies.rst | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/sphinx-guides/source/developers/configuration.rst b/doc/sphinx-guides/source/developers/configuration.rst index d058a1e1705..9c971668201 100644 --- a/doc/sphinx-guides/source/developers/configuration.rst +++ b/doc/sphinx-guides/source/developers/configuration.rst @@ -4,7 +4,7 @@ Consuming Configuration .. contents:: |toctitle| :local: -Dataverse uses different types of configuration: +The Dataverse Software uses different types of configuration: 1. JVM system properties 2. Simple database value settings @@ -25,11 +25,11 @@ Developers have accessed the simple properties via 3. ``SystemConfig.xxx()`` for specially treated settings, maybe mixed from 1 and 2 and other sources. 4. ``SettingsWrapper``, reading from 2 and 3 for use in frontend pages based on JSF -As of Dataverse 5.3, we start to streamline our efforts into using a more consistent approach, also bringing joy and +As of Dataverse Software version 5.3, we start to streamline our efforts into using a more consistent approach, also bringing joy and happiness to all the system administrators out there. This will be done by adopting the use of `MicroProfile Config `_ over time. -So far we streamlined configuration of these Dataverse parts: +So far we streamlined configuration of these Dataverse Software parts: - ✅ Database Connection @@ -54,7 +54,7 @@ System administrators benefit from: - Lots of database settings have been introduced in the past, but should be more easily configurable and not rely on a database connection. -- Running Dataverse in containers gets much easier when configuration can be provisioned in a +- Running a Dataverse installation in containers gets much easier when configuration can be provisioned in a streamlined fashion, mitigating the need for scripting glue and distinguishing between setting types. - Classic installations have a profit, too: we can enable using a single config file, e.g. living in ``/etc/dataverse/config.properties``. diff --git a/doc/sphinx-guides/source/developers/containers.rst b/doc/sphinx-guides/source/developers/containers.rst index ecf82533160..64c7710f0f5 100755 --- a/doc/sphinx-guides/source/developers/containers.rst +++ b/doc/sphinx-guides/source/developers/containers.rst @@ -2,7 +2,7 @@ Docker, Kubernetes, and Containers ================================== -The Dataverse community is exploring the use of Docker, Kubernetes, and other container-related technologies. The primary community-lead projects to watch are: +The Dataverse Community is exploring the use of Docker, Kubernetes, and other container-related technologies. The primary community-lead projects to watch are: - https://github.com/IQSS/dataverse-kubernetes - https://github.com/IQSS/dataverse-docker diff --git a/doc/sphinx-guides/source/developers/dependencies.rst b/doc/sphinx-guides/source/developers/dependencies.rst index 599606e7714..09d29324d2c 100644 --- a/doc/sphinx-guides/source/developers/dependencies.rst +++ b/doc/sphinx-guides/source/developers/dependencies.rst @@ -5,13 +5,13 @@ Dependency Management .. contents:: |toctitle| :local: -Dataverse is (currently) a Jakarta EE 8 based application, that uses a lot of additional libraries for special purposes. +The Dataverse Software is (currently) a Jakarta EE 8 based application, that uses a lot of additional libraries for special purposes. This includes features like support for SWORD-API, S3 storage and many others. Besides the code that glues together the single pieces, any developer needs to describe used dependencies for the Maven-based build system. As is familiar to any Maven user, this happens inside the "Project Object Model" (POM) living in ``pom.xml`` at the root of the project repository. Recursive and convergent dependency resolution makes dependency -management with Maven very easy. But sometimes, in projects with many complex dependencies like Dataverse, you have +management with Maven very easy. But sometimes, in projects with many complex dependencies like the Dataverse software, you have to help Maven make the right choices. Terms @@ -19,7 +19,7 @@ Terms As a developer, you should familiarize yourself with the following terms: -- **Direct dependencies**: things *you use* yourself in your own code for Dataverse. +- **Direct dependencies**: things *you use* yourself in your own code for the Dataverse Software. - **Transitive dependencies**: things *others use* for things you use, pulled in recursively. See also: `Maven docs `_. @@ -167,7 +167,7 @@ A reduced example, only showing bits relevant to the above cases and usage of an import + Jackson is used by AWS SDK and others, but we also use it in the Dataverse Software. --> com.fasterxml.jackson jackson-bom @@ -175,7 +175,7 @@ A reduced example, only showing bits relevant to the above cases and usage of an import pom - + joda-time joda-time @@ -230,9 +230,9 @@ Repositories Maven receives all dependencies from *repositories*. Those can be public like `Maven Central `_ and others, but you can also use a private repository on premises or in the cloud. Last but not least, you can use -local repositories, which can live next to your application code (see ``local_lib`` dir within Dataverse codebase). +local repositories, which can live next to your application code (see ``local_lib`` dir within the Dataverse Software codebase). -Repositories are defined within the Dataverse POM like this: +Repositories are defined within the Dataverse Software POM like this: .. code:: xml From 1810940deefc7d6eb7bb8c299304c0e20684613f Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Mon, 11 Jan 2021 14:18:44 -0500 Subject: [PATCH 085/576] Update deployment.rst --- .../source/developers/deployment.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/sphinx-guides/source/developers/deployment.rst b/doc/sphinx-guides/source/developers/deployment.rst index 3bb4b99913c..c65ad265add 100755 --- a/doc/sphinx-guides/source/developers/deployment.rst +++ b/doc/sphinx-guides/source/developers/deployment.rst @@ -2,15 +2,15 @@ Deployment ========== -Developers often only deploy Dataverse to their :doc:`dev-environment` but it can be useful to deploy Dataverse to cloud services such as Amazon Web Services (AWS). +Developers often only deploy the Dataverse software to their :doc:`dev-environment` but it can be useful to deploy the Dataverse software to cloud services such as Amazon Web Services (AWS). .. contents:: |toctitle| :local: -Deploying Dataverse to Amazon Web Services (AWS) ------------------------------------------------- +Deploying the Dataverse Software to Amazon Web Services (AWS) +------------------------------------------------------------- -We have written scripts to deploy Dataverse to Amazon Web Services (AWS) but they require some setup. +We have written scripts to deploy the Dataverse Software to Amazon Web Services (AWS) but they require some setup. Install AWS CLI ~~~~~~~~~~~~~~~ @@ -73,14 +73,14 @@ If you are having trouble configuring the files manually as described above, see Configure Ansible File (Optional) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In order to configure Dataverse settings such as the password of the dataverseAdmin user, download https://raw.githubusercontent.com/GlobalDataverseCommunityConsortium/dataverse-ansible/master/defaults/main.yml and edit the file to your liking. +In order to configure Dataverse installation settings such as the password of the dataverseAdmin user, download https://raw.githubusercontent.com/GlobalDataverseCommunityConsortium/dataverse-ansible/master/defaults/main.yml and edit the file to your liking. You can skip this step if you're fine with the values in the "main.yml" file in the link above. Download and Run the "Create Instance" Script ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Once you have done the configuration above, you are ready to try running the "ec2-create-instance.sh" script to spin up Dataverse in AWS. +Once you have done the configuration above, you are ready to try running the "ec2-create-instance.sh" script to spin up a Dataverse installation in AWS. Download `ec2-create-instance.sh`_ and put it somewhere reasonable. For the purpose of these instructions we'll assume it's in the "Downloads" directory in your home directory. @@ -98,7 +98,7 @@ ec2-create-instance accepts a number of command-line switches, including: ``bash ~/Downloads/ec2-create-instance.sh -b develop -r https://github.com/scholarsportal/dataverse.git -g main.yml`` -You will need to wait for 15 minutes or so until the deployment is finished, longer if you've enabled sample data and/or the API test suite. Eventually, the output should tell you how to access the installation of Dataverse in a web browser or via SSH. It will also provide instructions on how to delete the instance when you are finished with it. Please be aware that AWS charges per minute for a running instance. You may also delete your instance from https://console.aws.amazon.com/console/home?region=us-east-1 . +You will need to wait for 15 minutes or so until the deployment is finished, longer if you've enabled sample data and/or the API test suite. Eventually, the output should tell you how to access the Dataverse installation in a web browser or via SSH. It will also provide instructions on how to delete the instance when you are finished with it. Please be aware that AWS charges per minute for a running instance. You may also delete your instance from https://console.aws.amazon.com/console/home?region=us-east-1 . Caveat Recipiens ~~~~~~~~~~~~~~~~ @@ -111,7 +111,7 @@ Migrating Datafiles from Local Storage to S3 A number of pilot Dataverse installations start on local storage, then administrators are tasked with migrating datafiles into S3 or similar object stores. The files may be copied with a command-line utility such as `s3cmd`. You will want to retain the local file hierarchy, keeping the authority (for example: 10.5072) at the bucket "root." -The below example queries may assist with updating dataset and datafile locations in the Dataverse PostgresQL database. Depending on the initial version of Dataverse and subsequent upgrade path, Datafile storage identifiers may or may not include a ``file://`` prefix, so you'll want to catch both cases. +The below example queries may assist with updating dataset and datafile locations in the Dataverse installation's PostgresQL database. Depending on the initial version of the Dataverse software and subsequent upgrade path, Datafile storage identifiers may or may not include a ``file://`` prefix, so you'll want to catch both cases. To Update Dataset Location to S3, Assuming a ``file://`` Prefix ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From a1a4c9c9289054c577ec8998e2d6f2f40a57a239 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Mon, 11 Jan 2021 14:50:37 -0500 Subject: [PATCH 086/576] adding release note similar to note in 5.1 --- doc/release-notes/7188-utf-8-filenames.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/release-notes/7188-utf-8-filenames.md diff --git a/doc/release-notes/7188-utf-8-filenames.md b/doc/release-notes/7188-utf-8-filenames.md new file mode 100644 index 00000000000..660f7bbd54f --- /dev/null +++ b/doc/release-notes/7188-utf-8-filenames.md @@ -0,0 +1,7 @@ +## Notes for Tool Developers and Integrators + +### Spaces in File Names + +Dataverse Installations will no longer replace spaces in file names of downloaded files with the + character. If your tool or integration has any special handling around this, you may need to make further adjustments to maintain backwards compatibility while also supporting Dataverse installations on 5.4+. + +Note that this follows a change from 5.1 that only corrected this for installations running with S3 storage. This makes the behavior consistent across installations running all types of file storage. \ No newline at end of file From 074ac6ae7977982002703ee3eafee26e36eb0705 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 11 Jan 2021 14:55:30 -0500 Subject: [PATCH 087/576] Mention UTF-8 filenames in release note #7188 --- doc/release-notes/7188-utf-8-filenames.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/7188-utf-8-filenames.md b/doc/release-notes/7188-utf-8-filenames.md index 660f7bbd54f..014ea4b4566 100644 --- a/doc/release-notes/7188-utf-8-filenames.md +++ b/doc/release-notes/7188-utf-8-filenames.md @@ -1,7 +1,9 @@ ## Notes for Tool Developers and Integrators -### Spaces in File Names +### UTF-8 Characters and Spaces in File Names + +UTF-8 characters in filenames are now preserved when downloaded. Dataverse Installations will no longer replace spaces in file names of downloaded files with the + character. If your tool or integration has any special handling around this, you may need to make further adjustments to maintain backwards compatibility while also supporting Dataverse installations on 5.4+. -Note that this follows a change from 5.1 that only corrected this for installations running with S3 storage. This makes the behavior consistent across installations running all types of file storage. \ No newline at end of file +Note that this follows a change from 5.1 that only corrected this for installations running with S3 storage. This makes the behavior consistent across installations running all types of file storage. From 1b27ec324fa45968e329b6ae39411b7d9f58b064 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Mon, 11 Jan 2021 15:08:28 -0500 Subject: [PATCH 088/576] Separate unf display from variables + observations on dataverse pg result cards [ref #5210 #6685] --- .../search/SearchIncludeFragment.java | 27 ++++++++++++++----- src/main/webapp/search-include-fragment.xhtml | 12 ++++++--- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java index 61859700077..73aacbdce2d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java @@ -1201,7 +1201,7 @@ public SearchException getSearchException() { } public String tabularDataDisplayInfo(DataFile datafile) { - String ret = ""; + String tabInfo = ""; if (datafile == null) { return null; @@ -1209,22 +1209,35 @@ public String tabularDataDisplayInfo(DataFile datafile) { if (datafile.isTabularData() && datafile.getDataTable() != null) { DataTable datatable = datafile.getDataTable(); - String unf = datatable.getUnf(); Long varNumber = datatable.getVarQuantity(); Long obsNumber = datatable.getCaseQuantity(); if (varNumber != null && varNumber.intValue() != 0) { - ret = ret.concat(varNumber + " " + BundleUtil.getStringFromBundle("file.metaData.dataFile.dataTab.variables")); + tabInfo = tabInfo.concat(varNumber + " " + BundleUtil.getStringFromBundle("file.metaData.dataFile.dataTab.variables")); if (obsNumber != null && obsNumber.intValue() != 0) { - ret = ret.concat(", " + obsNumber + " " + BundleUtil.getStringFromBundle("file.metaData.dataFile.dataTab.observations")); + tabInfo = tabInfo.concat(", " + obsNumber + " " + BundleUtil.getStringFromBundle("file.metaData.dataFile.dataTab.observations")); } - ret = ret.concat(" - "); } + } + + return tabInfo; + } + + public String tabularDataUnfDisplay(DataFile datafile) { + String tabUnf = ""; + + if (datafile == null) { + return null; + } + + if (datafile.isTabularData() && datafile.getDataTable() != null) { + DataTable datatable = datafile.getDataTable(); + String unf = datatable.getUnf(); if (unf != null && !unf.equals("")) { - ret = ret.concat(BundleUtil.getStringFromBundle("file.metaData.dataFile.dataTab.unf") + ": " + unf); + tabUnf = tabUnf.concat(unf); } } - return ret; + return tabUnf; } public String dataFileSizeDisplay(DataFile datafile) { diff --git a/src/main/webapp/search-include-fragment.xhtml b/src/main/webapp/search-include-fragment.xhtml index 956e57242af..d233ee59078 100644 --- a/src/main/webapp/search-include-fragment.xhtml +++ b/src/main/webapp/search-include-fragment.xhtml @@ -619,9 +619,15 @@ title="#{bundle['file.metaData.checksum.copy']} #{result.fileChecksumValue}" jsf:rendered="#{!result.harvested and !SearchIncludeFragment.isTabular(result.entity) and !(empty result.fileChecksumValue)}">#{result.fileChecksumValue} - - + + + + + #{SearchIncludeFragment.tabularDataUnfDisplay(result.entity)}
    From 16a3efad80f995f95a199bf9eb59740a69146f85 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Mon, 11 Jan 2021 15:20:24 -0500 Subject: [PATCH 089/576] Overwrite css from primefaces component.css for overflow property to be visible for datatables [ref #5210 #6685] --- src/main/webapp/resources/css/structure.css | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index dff45133c9f..1b94bcb29bb 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -862,6 +862,9 @@ div.manageTable.ui-datatable div.ui-datatable-tablewrapper, div.manageTable.ui-d div[id$="filesTable"].ui-datatable thead th.col-file-action, div[id$="filesTable"].ui-datatable tbody td.col-file-action {overflow:visible !important;} thead.ui-datatable-scrollable-theadclone {display:none} +/* DATATABE + OVERFLOW VISIBLE */ +.ui-datatable thead th, .ui-datatable tbody td, .ui-datatable tfoot td, .ui-datatable tfoot th {overflow:visible;} + /* FILE */ #file-preview-icon-block > #border-block {/*border:1px solid #DDD;border-radius:4px; padding: 10px;*/} #file-preview-icon-block > #border-block > #preview-square {/*position: relative; overflow:hidden; padding-bottom:100%;*/} From aae992507223dcec08dbe58f99090195e9bc6a18 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Mon, 11 Jan 2021 16:33:48 -0500 Subject: [PATCH 090/576] Fixed float UI issue in file table on dataset pg [ref #5210, #6685] --- src/main/webapp/file-info-fragment.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index 4a96327c736..045128fc166 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -29,7 +29,7 @@
    -
    +
    From 3a76134dc0de2bc855014b699d9e14d3ee165eb0 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Mon, 11 Jan 2021 17:57:48 -0500 Subject: [PATCH 091/576] more updates --- .../source/developers/dev-environment.rst | 30 +++++++++---------- .../source/developers/documentation.rst | 16 +++++----- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 9abfd385c28..192aac2576c 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -10,7 +10,7 @@ These instructions are purposefully opinionated and terse to help you get your d Quick Start ----------- -The quickest way to get Dataverse running is to use Vagrant as described in the :doc:`tools` section, but for day to day development work, we recommended the following setup. +The quickest way to get the Dataverse software running is to use Vagrant as described in the :doc:`tools` section, but for day to day development work, we recommended the following setup. Set Up Dependencies ------------------- @@ -25,7 +25,7 @@ Windows is not well supported, unfortunately, but Vagrant and Minishift environm Install Java ~~~~~~~~~~~~ -Dataverse requires Java 8. +The Dataverse software requires Java 8. We suggest downloading OpenJDK from https://adoptopenjdk.net @@ -36,29 +36,29 @@ Install Netbeans or Maven NetBeans IDE is recommended, and can be downloaded from http://netbeans.org . Developers may use any editor or IDE. We recommend NetBeans because it is free, works cross platform, has good support for Jakarta EE projects, and includes a required build tool, Maven. -Below we describe how to build the Dataverse war file with Netbeans but if you prefer to use only Maven, you can find installation instructions in the :doc:`tools` section. +Below we describe how to build the Dataverse software war file with Netbeans but if you prefer to use only Maven, you can find installation instructions in the :doc:`tools` section. Install Homebrew (Mac Only) ~~~~~~~~~~~~~~~~~~~~~~~~~~~ On Mac, install Homebrew to simplify the steps below: https://brew.sh -Clone the Dataverse Git Repo -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Clone the Dataverse Software Git Repo +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Fork https://github.com/IQSS/dataverse and then clone your fork like this: ``git clone git@github.com:[YOUR GITHUB USERNAME]/dataverse.git`` -Build the Dataverse War File +Build the Dataverse Software War File ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If you installed Netbeans, follow these steps: -- Launch Netbeans and click "File" and then "Open Project". Navigate to where you put the Dataverse code and double-click "dataverse" to open the project. +- Launch Netbeans and click "File" and then "Open Project". Navigate to where you put the Dataverse software code and double-click "Dataverse" to open the project. - If you see "resolve project problems," go ahead and let Netbeans try to resolve them. This will probably including downloading dependencies, which can take a while. - Allow Netbeans to install nb-javac (required for Java 8 and below). -- Select "dataverse" under Projects and click "Run" in the menu and then "Build Project (dataverse)". Check back for "BUILD SUCCESS" at the end. +- Select "Dataverse" under Projects and click "Run" in the menu and then "Build Project (Dataverse)". Check back for "BUILD SUCCESS" at the end. If you installed Maven instead of Netbeans, run ``mvn package``. Check for "BUILD SUCCESS" at the end. @@ -155,10 +155,10 @@ To install Solr, execute the following commands: ``bin/solr create_core -c collection1 -d server/solr/collection1/conf`` -Run the Dataverse Installer Script ----------------------------------- +Run the Dataverse Software Installer Script +------------------------------------------- -Navigate to the directory where you cloned the Dataverse git repo change directories to the ``scripts/installer`` directory like this: +Navigate to the directory where you cloned the Dataverse software git repo change directories to the ``scripts/installer`` directory like this: ``cd scripts/installer`` @@ -176,10 +176,10 @@ Finally, run the installer (see also :download:`README_python.txt <../../../../s ``python3 install.py`` -Verify Dataverse is Running -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Verify the Dataverse Software is Running +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -After the script has finished, you should be able to log into Dataverse with the following credentials: +After the script has finished, you should be able to log into your Dataverse installation with the following credentials: - http://localhost:8080 - username: dataverseAdmin @@ -197,7 +197,7 @@ This will disable DOI registration by using a fake (in-code) DOI provider. Pleas Next Steps ---------- -If you can log in to Dataverse, great! If not, please see the :doc:`troubleshooting` section. For further assitance, please see "Getting Help" in the :doc:`intro` section. +If you can log in to the Dataverse installation, great! If not, please see the :doc:`troubleshooting` section. For further assitance, please see "Getting Help" in the :doc:`intro` section. You're almost ready to start hacking on code. Now that the installer script has you up and running, you need to continue on to the :doc:`tips` section to get set up to deploy code from your IDE or the command line. diff --git a/doc/sphinx-guides/source/developers/documentation.rst b/doc/sphinx-guides/source/developers/documentation.rst index 01f7cad1ecc..d2101f37587 100755 --- a/doc/sphinx-guides/source/developers/documentation.rst +++ b/doc/sphinx-guides/source/developers/documentation.rst @@ -19,17 +19,17 @@ If you find a typo or a small error in the documentation you can fix it using Gi - `user`_ - Find the file you want to edit under one of the folders above. -- Click the pencil icon in the upper-right corner. If this is your first contribution to Dataverse, the hover text over the pencil icon will say "Fork this project and edit this file". +- Click the pencil icon in the upper-right corner. If this is your first contribution to the Dataverse Project, the hover text over the pencil icon will say "Fork this project and edit this file". - Make changes to the file and preview them. - In the **Commit changes** box, enter a description of the changes you have made and click **Propose file change**. - Under the **Write** tab, delete the long welcome message and write a few words about what you fixed. - Click **Create Pull Request**. -That's it! Thank you for your contribution! Your pull request will be added manually to the main Dataverse project board at https://github.com/orgs/IQSS/projects/2 and will go through code review and QA before it is merged into the "develop" branch. Along the way, developers might suggest changes or make them on your behalf. Once your pull request has been merged you will be listed as a contributor at https://github.com/IQSS/dataverse/graphs/contributors +That's it! Thank you for your contribution! Your pull request will be added manually to the main Dataverse Project board at https://github.com/orgs/IQSS/projects/2 and will go through code review and QA before it is merged into the "develop" branch. Along the way, developers might suggest changes or make them on your behalf. Once your pull request has been merged you will be listed as a contributor at https://github.com/IQSS/dataverse/graphs/contributors Please see https://github.com/IQSS/dataverse/pull/5857 for an example of a quick fix that was merged (the "Files changed" tab shows how a typo was fixed). -If you would like to read more about the Dataverse project's use of GitHub, please see the :doc:`version-control` section. For bug fixes and features we request that you create an issue before making a pull request but this is not at all necessary for quick fixes to the documentation. +If you would like to read more about the Dataverse Project's use of GitHub, please see the :doc:`version-control` section. For bug fixes and features we request that you create an issue before making a pull request but this is not at all necessary for quick fixes to the documentation. .. _admin: https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source/admin .. _api: https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source/api @@ -40,7 +40,7 @@ If you would like to read more about the Dataverse project's use of GitHub, plea Other Changes (Sphinx) ---------------------- -The documentation for Dataverse was written using Sphinx (http://sphinx-doc.org/). +The documentation for the Dataverse Project was written using Sphinx (http://sphinx-doc.org/). If you are interested in suggesting changes or updates we recommend that you create the html files using Sphinx locally and then submit a pull request through GitHub. Here are the instructions on how to proceed: @@ -71,12 +71,12 @@ This is all you need. You should now be able to build HTML/pdf documentation fro Using Sphinx ~~~~~~~~~~~~ -First, you will need to make a fork of the dataverse repository in GitHub. Then, you will need to make a clone of your fork so you can manipulate the files outside GitHub. +First, you will need to make a fork of the Dataverse software repository in GitHub. Then, you will need to make a clone of your fork so you can manipulate the files outside GitHub. To edit the existing documentation: - Create a branch (refer to http://guides.dataverse.org/en/latest/developers/version-control.html > *Create a New Branch off the develop Branch*) to record the changes you are about to perform. -- Go to ~/dataverse/doc/sphinx-guides/source directory inside your clone. There, you will find the .rst files that correspond to the guides in the dataverse page (http://guides.dataverse.org/en/latest/). +- Go to ~/dataverse/doc/sphinx-guides/source directory inside your clone. There, you will find the .rst files that correspond to the guides in the Dataverse Software Guides page (http://guides.dataverse.org/en/latest/). - Using your preferred text editor, open and edit the necessary files, or create new ones. **NOTE:** When adding ReStructured Text (RST) `cross references `_, use the hyphen character (``-``) as the word separator for the cross reference label. For example, ``my-reference-label`` would be the preferred label for a cross reference as opposed to, for example, ``my_reference_label``. @@ -90,7 +90,7 @@ Once you are done, open a terminal and change directories to ~/dataverse/doc/sph After sphinx is done processing the files you should notice that the html folder in ~/dataverse/doc/sphinx-guides/build directory has been updated. You can click on the files in the html folder to preview the changes. -Now you can make a commit with the changes to your own fork in GitHub and submit a pull request to the original (upstream) dataverse repository. +Now you can make a commit with the changes to your own fork in GitHub and submit a pull request to the original (upstream) Dataverse Software repository. Table of Contents ----------------- @@ -126,7 +126,7 @@ This has been tested and works on Mac, Linux, and Windows. Versions -------- -For installations hosting their own copies of the guides, note that as each version of Dataverse is released, there is an updated version of the guides released with it. Google and other search engines index all versions, which may confuse users who discover your guides in the search results as to which version they should be looking at. When learning about your installation from the search results, it is best to be viewing the *latest* version. +For installations hosting their own copies of the guides, note that as each version of the Dataverse Software is released, there is an updated version of the guides released with it. Google and other search engines index all versions, which may confuse users who discover your guides in the search results as to which version they should be looking at. When learning about your installation from the search results, it is best to be viewing the *latest* version. In order to make it clear to the crawlers that we only want the latest version discoverable in their search results, we suggest adding this to your ``robots.txt`` file:: From 93e6476ac7e5dbe479268d376e2ca121ca41c7e7 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 12 Jan 2021 10:00:51 -0500 Subject: [PATCH 092/576] #3438 update bundle --- src/main/java/propertyFiles/Bundle.properties | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 0d6804efc37..816d46570e9 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -160,6 +160,8 @@ institution.name=A Dataverse Instance account.info=Account Information account.edit=Edit Account account.apiToken=API Token +account.emailvalidation.header=Check Inbox +account.emailvalidation.token.exists=You have already received an email for validation. Please check your inbox. user.isShibUser=Account information cannot be edited when logged in through an institutional account. user.helpShibUserMigrateOffShibBeforeLink=Leaving your institution? Please contact user.helpShibUserMigrateOffShibAfterLink=for assistance. From 60c1fd76a78b0a06a760d04857c484fc2c58b428 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 12 Jan 2021 10:26:36 -0500 Subject: [PATCH 093/576] #3438 update user page logic --- .../providers/builtin/DataverseUserPage.java | 20 +++++++------- .../confirmemail/ConfirmEmailServiceBean.java | 8 ++++++ .../builtin/DataverseUserPageTest.java | 27 ++++++++++++------- 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java index e50d38131c9..f841f2fca74 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java @@ -545,17 +545,17 @@ public void sendConfirmEmail() { } - /** - * Determines whether the button to send a verification email appears on user page - * TODO: cant this be refactored to use confirmEmailService.hasVerifiedEmail(currentUser) ? - * @return - */ public boolean showVerifyEmailButton() { - final Timestamp emailConfirmed = currentUser.getEmailConfirmed(); - final ConfirmEmailData confirmedDate = confirmEmailService.findSingleConfirmEmailDataByUser(currentUser); - return (!getUserAuthProvider().isEmailVerified()) - && confirmedDate == null - && emailConfirmed == null; + return !confirmEmailService.hasVerifiedEmail(currentUser); + } + + public boolean getHasActiveVerificationToken(){ + //for user page to determine how to handle Confirm Email click + return confirmEmailService.hasActiveVerificationToken(currentUser); + } + + public void setHasActiveVerificationToken(boolean value){ + //dummy method } public boolean isEmailIsVerified() { diff --git a/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailServiceBean.java index f96a53b5836..bc3b70326f1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailServiceBean.java @@ -205,6 +205,14 @@ public ConfirmEmailData findSingleConfirmEmailDataByUser(AuthenticatedUser user) } return confirmEmailData; } + + public boolean hasActiveVerificationToken(AuthenticatedUser au) { + if (findSingleConfirmEmailDataByUser(au) == null){ + return false; + } + return !findSingleConfirmEmailDataByUser(au).isExpired(); + } + public List findAllConfirmEmailData() { TypedQuery typedQuery = em.createNamedQuery("ConfirmEmailData.findAll", ConfirmEmailData.class); diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPageTest.java b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPageTest.java index e468ab55201..defbc4416b5 100644 --- a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPageTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPageTest.java @@ -48,21 +48,22 @@ public void tearDown() { // provides inputs to testShowVerifyEmailButton() private static Stream provider_testShowVerifyEmailButton() { return Stream.of( - Arguments.of(true, null, false, false), - Arguments.of(false, null, false, true), - Arguments.of(false, null, true, false), - Arguments.of(false, null, true, true), - Arguments.of(false, new Timestamp(0), false, false), - Arguments.of(false, new Timestamp(0), false, true), - Arguments.of(false, new Timestamp(0), true, false), - Arguments.of(false, new Timestamp(0), true, true) + Arguments.of(true, null, false, false, 1), + Arguments.of(true, null, false, true, 2), + Arguments.of(true, null, true, false, 3), + Arguments.of(true, null, true, true, 4), + Arguments.of(false, new Timestamp(0), false, false, 5), + Arguments.of(false, new Timestamp(0), false, true, 6), + Arguments.of(false, new Timestamp(0), true, false, 7), + Arguments.of(false, new Timestamp(0), true, true, 8) ); } @ParameterizedTest @MethodSource("provider_testShowVerifyEmailButton") - public void testShowVerifyEmailButton(boolean expected, Timestamp emailConfirmed, boolean confirmEmailData, boolean emailVerified) { + public void testShowVerifyEmailButton(boolean expected, Timestamp emailConfirmed, boolean confirmEmailData, boolean emailVerified, int counter) { user.setEmailConfirmed(emailConfirmed); + dataverseUserPage.setCurrentUser(user); if (confirmEmailData) { @@ -71,6 +72,14 @@ public void testShowVerifyEmailButton(boolean expected, Timestamp emailConfirmed } else { Mockito.when(mockConfirmEmailService.findSingleConfirmEmailDataByUser(user)).thenReturn(null); } + + if(emailConfirmed != null){ + + Mockito.when(mockConfirmEmailService.hasVerifiedEmail(user)) + .thenReturn(true); + } else { + Mockito.when(mockConfirmEmailService.hasVerifiedEmail(user)).thenReturn(false); + } Mockito.when(mockAuthProvider.isEmailVerified()).thenReturn(emailVerified); Mockito.when(mockAuthService.lookupProvider(user)).thenReturn(mockAuthProvider); From 0ad2266a6d91feb78b55d576739e50bf8dbb307a Mon Sep 17 00:00:00 2001 From: Michael Heppler <687227+mheppler@users.noreply.github.com> Date: Tue, 12 Jan 2021 10:53:28 -0500 Subject: [PATCH 094/576] Added new file types based on new mime types --- .../propertyFiles/MimeTypeDisplay.properties | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/src/main/java/propertyFiles/MimeTypeDisplay.properties b/src/main/java/propertyFiles/MimeTypeDisplay.properties index 29407ccda40..6899f9b4518 100644 --- a/src/main/java/propertyFiles/MimeTypeDisplay.properties +++ b/src/main/java/propertyFiles/MimeTypeDisplay.properties @@ -5,6 +5,7 @@ application/pdf=Adobe PDF image/pdf=Adobe PDF text/pdf=Adobe PDF application/x-pdf=Adobe PDF +application/cnt=CNT application/msword=MS Word application/vnd.ms-excel=MS Excel Spreadsheet application/vnd.openxmlformats-officedocument.spreadsheetml.sheet=MS Excel Spreadsheet @@ -12,8 +13,10 @@ application/vnd.ms-powerpoint=MS Powerpoint application/vnd.openxmlformats-officedocument.presentationml.presentation=MS Powerpoint application/vnd.openxmlformats-officedocument.wordprocessingml.document=MS Word application/vnd.oasis.opendocument.spreadsheet=OpenOffice Spreadsheet +application/vnd.ms-excel.sheet.macroenabled.12=MS Excel Spreadsheet # Text text/plain=Plain Text +text/x-log=Application Log text/html=HTML application/x-tex=LaTeX text/x-tex=LaTeX @@ -21,6 +24,7 @@ text/markdown=Markdown Text text/x-markdown=Markdown Text text/x-r-markdown=R Markdown Text application/rtf=Rich Text Format +text/x-rst=reStructuredText text/rtf=Rich Text Format text/richtext=Rich Text Format text/turtle=Turtle RDF @@ -28,16 +32,26 @@ application/xml=XML text/xml=XML # Code text/x-c=C++ Source +text/x-c++src=C++ Source text/css=Cascading Style Sheet +text/x-fortran=Fortran Source Code +application/java-vm=Java Class +text/x-java-source=Java Source Code text/javascript=Javascript Code application/javascript=Javascript Code application/x-javascript=Javascript Code text/x-matlab=MATLAB Source Code text/x-mathematica=Mathematica Input +text/x-objcsrc=Objective-C Source Code +text/x-pascal=Pascal Source Code +text/x-perl=Perl Script +text/x-perl-script=Perl Script text/php=PHP Source Code +application/postscript=Postscript text/x-python=Python Source Code text/x-python-script=Python Source Code text/x-r-source=R Source Code +text/x-sh=Shell Script application/x-sh=Shell Script application/x-shellscript=Shell Script application/x-sql=SQL Code @@ -57,13 +71,18 @@ application/x-spss-sps=SPSS Script Syntax text/x-sas-syntax=SAS Syntax application/x-sas-syntax=SAS Syntax type/x-r-syntax=R Syntax +application/vnd.wolfram.mathematica.package=Mathematica Package +application/vnd.wolfram.mathematica=Mathematica Package # Ingested Tabular Data text/tab-separated-values=Tab-Delimited # RawData text/tsv=Tab-Separated Values text/comma-separated-values=Comma Separated Values +text/x-comma-separated-values=Comma Separated Values text/csv=Comma Separated Values text/x-fixed-field=Fixed Field Text Data +application/vnd.flographit=FloGraphIt Media +application/x-r-data=R Data application/x-rlang-transport=R Data application/x-R-2=R Binary application/x-stata=Stata Binary @@ -94,6 +113,7 @@ application/matlab-mat=MATLAB Data application/x-matlab-data=MATLAB Data application/x-matlab-figure=MATLAB Figure application/x-matlab-workspace=MATLAB Workspace +text/x-vcard=Virtual Contact File application/x-xfig=MATLAB Figure application/x-msaccess=MS Access application/netcdf=Network Common Data Form @@ -101,6 +121,10 @@ application/x-netcdf=Network Common Data Form application/vnd.lotus-notes=Notes Storage Facility application/x-nsdstat=NSDstat application/vnd.realvnc.bed=PLINK Binary +application/vnd.ms-pki.stl=Data +application/vnd.isac.fcs=Data +application/java-serialized-object=Java Serialized Object +chemical/x-xyz=Co-Ordinate Animation # FITS image/fits=FITS application/fits=FITS @@ -112,7 +136,9 @@ application/sbn=ESRI Spatial Index application/sbx=ESRI Spatial Index application/shp=Shape application/shx=Shape -application/zipped-shapefile=Shape +application/x-esri-shape=ESRI Shapefile +application/vnd.google-earth.kml+xml=Keyhole Markup Language +application/zipped-shapefile=Zipped Shapefiles # Archive application/zip=ZIP Archive application/x-zip-compressed=ZIP Archive @@ -122,15 +148,18 @@ application/x-bzip2=Bzip Archive application/vnd.google-earth.kmz=Google Earth Archive application/gzip=Gzip Archive application/x-gzip=Gzip Archive +application/x-gzip-compressed=Gzip Archive application/rar=RAR Archive application/x-rar=RAR Archive application/x-rar-compressed=RAR Archive application/tar=TAR Archive application/x-tar=TAR Archive +application/x-compressed=Compressed Archive application/x-compressed-tar=TAR Archive application/x-7z-compressed=7Z Archive application/x-xz=XZ Archive application/warc=Web Archive +application/x-iso9660-image=Optical Disc Image # Image image/gif=GIF Image image/jpeg=JPEG Image @@ -151,8 +180,13 @@ image/tiff=TIFF Image image/bmp=Bitmap Image image/x-xbitmap=Bitmap Image image/RAW=Bitmap Image +image/raw=Bitmap Image +application/x-tgif=TGIF File image/x-xpixmap=Pixmap Image image/x-xwindowdump=X Windows Dump +application/photoshop=Photoshop Image +image/vnd.adobe.photoshop=Photoshop Image +application/x-photoshop=Photoshop Image # Audio audio/x-aiff=AIFF Audio audio/mp3=MP3 Audio @@ -177,4 +211,4 @@ text/xml-graphml=GraphML Network Data # Other application/octet-stream=Unknown # Dataverse-specific -application/vnd.dataverse.file-package=Dataverse Package \ No newline at end of file +application/vnd.dataverse.file-package=Dataverse Package From 2e20826f16f712af1a550dcc47effd0e599a41f0 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 12 Jan 2021 10:57:07 -0500 Subject: [PATCH 095/576] #3438 update user ui --- src/main/webapp/dataverseuser.xhtml | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/dataverseuser.xhtml b/src/main/webapp/dataverseuser.xhtml index 76bfd9bf2b1..c7a4d974922 100644 --- a/src/main/webapp/dataverseuser.xhtml +++ b/src/main/webapp/dataverseuser.xhtml @@ -445,9 +445,11 @@

    - + #{bundle['confirmEmail.submitRequest']} - +
    @@ -716,7 +718,27 @@
    + +

    #{bundle['account.emailvalidation.token.exists']}

    +
    + +
    +     + From 944776c04ddbde0134b41f680d3d89550ce2a96b Mon Sep 17 00:00:00 2001 From: Michael Heppler <687227+mheppler@users.noreply.github.com> Date: Tue, 12 Jan 2021 11:04:38 -0500 Subject: [PATCH 096/576] Missed a couple of data file types --- src/main/java/propertyFiles/MimeTypeDisplay.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/propertyFiles/MimeTypeDisplay.properties b/src/main/java/propertyFiles/MimeTypeDisplay.properties index 6899f9b4518..8b07a87832d 100644 --- a/src/main/java/propertyFiles/MimeTypeDisplay.properties +++ b/src/main/java/propertyFiles/MimeTypeDisplay.properties @@ -121,8 +121,8 @@ application/x-netcdf=Network Common Data Form application/vnd.lotus-notes=Notes Storage Facility application/x-nsdstat=NSDstat application/vnd.realvnc.bed=PLINK Binary -application/vnd.ms-pki.stl=Data -application/vnd.isac.fcs=Data +application/vnd.ms-pki.stl=STL Format +application/vnd.isac.fcs=FCS Data application/java-serialized-object=Java Serialized Object chemical/x-xyz=Co-Ordinate Animation # FITS From 4339eb8ad1a87efb05b2e58c251ab29807ef0761 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 12 Jan 2021 12:07:55 -0500 Subject: [PATCH 097/576] remove worldmap/geoconnect #7409 --- .../source/admin/geoconnect-worldmap.rst | 52 -- doc/sphinx-guides/source/admin/index.rst | 1 - .../source/admin/integrations.rst | 7 - doc/sphinx-guides/source/api/apps.rst | 7 - doc/sphinx-guides/source/api/intro.rst | 1 - .../source/developers/geospatial.rst | 132 +-- doc/sphinx-guides/source/developers/intro.rst | 1 - .../source/installation/config.rst | 16 +- .../source/installation/geoconnect.rst | 17 - .../source/installation/index.rst | 1 - .../source/installation/installation-main.rst | 1 - .../source/installation/prep.rst | 1 - doc/sphinx-guides/source/style/patterns.rst | 2 +- .../source/user/data-exploration/index.rst | 1 - .../source/user/data-exploration/worldmap.rst | 153 ---- .../source/user/dataset-management.rst | 28 - scripts/api/setup-optional-harvard.sh | 3 - src/main/java/ValidationMessages.properties | 7 - .../edu/harvard/iq/dataverse/DataFile.java | 27 +- .../edu/harvard/iq/dataverse/DatasetPage.java | 118 --- .../harvard/iq/dataverse/DatasetVersion.java | 11 +- ...taverseFieldTypeInputLevelServiceBean.java | 6 +- .../iq/dataverse/EjbDataverseEngine.java | 8 - .../harvard/iq/dataverse/FileDownload.java | 6 +- .../iq/dataverse/FileDownloadHelper.java | 14 - .../iq/dataverse/FileDownloadServiceBean.java | 39 +- .../edu/harvard/iq/dataverse/FilePage.java | 36 +- .../harvard/iq/dataverse/MailServiceBean.java | 16 - .../iq/dataverse/MapLayerMetadata.java | 304 ------- .../MapLayerMetadataServiceBean.java | 406 --------- .../iq/dataverse/UserNotification.java | 2 +- .../iq/dataverse/api/AbstractApiBean.java | 4 - .../edu/harvard/iq/dataverse/api/Access.java | 33 +- .../edu/harvard/iq/dataverse/api/Files.java | 27 +- .../harvard/iq/dataverse/api/Geoconnect.java | 78 -- .../iq/dataverse/api/WorldMapRelatedData.java | 795 ------------------ .../providers/builtin/DataverseUserPage.java | 5 - .../users/AuthenticatedUser.java | 12 - .../dataaccess/ImageThumbConverter.java | 53 +- .../WorldMapPermissionHelper.java | 636 -------------- .../engine/command/CommandContext.java | 3 - .../command/impl/DeleteDataFileCommand.java | 24 - .../impl/DeleteMapLayerMetadataCommand.java | 40 - .../FinalizeDatasetPublicationCommand.java | 40 +- .../command/impl/UningestFileCommand.java | 14 - .../dataverse/externaltools/ExternalTool.java | 11 - .../settings/SettingsServiceBean.java | 14 - .../harvard/iq/dataverse/util/MailUtil.java | 4 - .../worldmapauth/TokenApplicationType.java | 268 ------ .../TokenApplicationTypeServiceBean.java | 153 ---- .../dataverse/worldmapauth/WorldMapToken.java | 349 -------- .../WorldMapTokenServiceBean.java | 314 ------- src/main/java/propertyFiles/Bundle.properties | 13 - ...3.0.1__7409-remove-worldmap-geoconnect.sql | 6 + ...erMigrate__1-7256-upsert-referenceData.sql | 10 - src/main/webapp/dataverseuser.xhtml | 17 - .../file-download-button-fragment.xhtml | 23 +- .../webapp/file-download-popup-fragment.xhtml | 7 - src/main/webapp/file.xhtml | 30 +- .../iq/dataverse/api/GeoconnectIT.java | 53 -- .../edu/harvard/iq/dataverse/api/UtilIT.java | 34 - .../dataverse/engine/TestCommandContext.java | 5 - .../iq/dataverse/util/MailUtilTest.java | 14 +- .../WorldMapTokenServiceBeanTest.java | 76 -- .../worldmapauth/WorldMapTokenTest.java | 152 ---- 65 files changed, 32 insertions(+), 4709 deletions(-) delete mode 100644 doc/sphinx-guides/source/admin/geoconnect-worldmap.rst delete mode 100644 doc/sphinx-guides/source/installation/geoconnect.rst delete mode 100644 doc/sphinx-guides/source/user/data-exploration/worldmap.rst delete mode 100644 src/main/java/edu/harvard/iq/dataverse/MapLayerMetadata.java delete mode 100755 src/main/java/edu/harvard/iq/dataverse/MapLayerMetadataServiceBean.java delete mode 100644 src/main/java/edu/harvard/iq/dataverse/api/Geoconnect.java delete mode 100755 src/main/java/edu/harvard/iq/dataverse/api/WorldMapRelatedData.java delete mode 100644 src/main/java/edu/harvard/iq/dataverse/datasetutility/WorldMapPermissionHelper.java delete mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteMapLayerMetadataCommand.java delete mode 100644 src/main/java/edu/harvard/iq/dataverse/worldmapauth/TokenApplicationType.java delete mode 100644 src/main/java/edu/harvard/iq/dataverse/worldmapauth/TokenApplicationTypeServiceBean.java delete mode 100644 src/main/java/edu/harvard/iq/dataverse/worldmapauth/WorldMapToken.java delete mode 100644 src/main/java/edu/harvard/iq/dataverse/worldmapauth/WorldMapTokenServiceBean.java create mode 100644 src/main/resources/db/migration/V5.3.0.1__7409-remove-worldmap-geoconnect.sql delete mode 100644 src/test/java/edu/harvard/iq/dataverse/api/GeoconnectIT.java delete mode 100644 src/test/java/edu/harvard/iq/dataverse/worldmapauth/WorldMapTokenServiceBeanTest.java delete mode 100644 src/test/java/edu/harvard/iq/dataverse/worldmapauth/WorldMapTokenTest.java diff --git a/doc/sphinx-guides/source/admin/geoconnect-worldmap.rst b/doc/sphinx-guides/source/admin/geoconnect-worldmap.rst deleted file mode 100644 index fb8250c0650..00000000000 --- a/doc/sphinx-guides/source/admin/geoconnect-worldmap.rst +++ /dev/null @@ -1,52 +0,0 @@ -Geoconnect and WorldMap -======================= - -One of the optional components listed under "Architecture and Components" in the :doc:`/installation/prep` section of the Installation Guide is `Geoconnect `_, a piece of middleware that allows Dataverse users to create maps in `WorldMap `_ based on geospatial data stored in Dataverse. For more details on the feature from the user perspective, see the :doc:`/user/data-exploration/worldmap` section of the User Guide. - -.. contents:: |toctitle| - :local: - -Update "mapitlink" ------------------- - -SQL commands to point a Dataverse installation at different Geoconnect servers: - - -**Geoconnect Production** *geoconnect.datascience.iq.harvard.edu* - -.. code-block:: sql - - update worldmapauth_tokentype set mapitlink = 'https://geoconnect.datascience.iq.harvard.edu/shapefile/map-it', hostname='geoconnect.datascience.iq.harvard.edu' where name = 'GEOCONNECT'; - -**Heroku Test** *geoconnect-dev.herokuapp.com* - -.. code-block:: sql - - update worldmapauth_tokentype set mapitlink = 'https://geoconnect-dev.herokuapp.com/shapefile/map-it', hostname='geoconnect-dev.herokuapp.com' where name = 'GEOCONNECT'; - - -**View Current Settings** - -.. code-block:: sql - - SELECT * from worldmapauth_tokentype; - - -Removing Dead Explore Links ---------------------------- - -After a map has been created in WorldMap (assuming all the setup has been done), in Dataverse the file will display WorldMap as an explore option. In rare occasions, the map has been deleted on the WorldMap resulting in a dead link, a 404 page not found error. - -Functionality has been added on the Dataverse side to iterate through all the maps Dataverse knows about (stored in the ``maplayermetadata`` database table) and to check for the existence of each map in WorldMap. The status code returned from WorldMap (200, 404, etc.) is recorded in Dataverse along with a timestamp of when the check was performed. To perform this check, you can execute the following ``curl`` command: - -``curl -X POST http://localhost:8080/api/admin/geoconnect/mapLayerMetadatas/check`` - -The output above will contain the ``layerLink`` being checked as well as the HTTP response status code (200, 404, etc.) in the ``lastVerifiedStatus`` field. 200 means OK and 404 means not found. 500 might indicate that the map is only temporarily unavailable. The ``lastVerifiedStatus`` and ``lastVerifiedTime`` will be persisted to the ``maplayermetadata`` database table. - -Armed with this information about WorldMap returning a 404 for a map, you may want to delete any record of the map on the Dataverse side removing WorldMap as an explore option (and so that thumbnail files are cleaned up). To accomplish this, use the following ``curl`` command, substituting the id of the file: - -``curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE http://localhost:8080/api/files/{file_id}/map`` - -End users can also run the ``DELETE`` command above (if they have permission to edit the dataset) but it's more likely that the sysadmin reading this guide will run it for them. It is recommended that you add the "check" command above to cron so that Dataverse periodically checks on the status of maps in WorldMap. In addition to the "check all maps" command above, you can also check an individual map with the following ``curl`` command, substituting the id of the row from the ``maplayermetadata`` table: - -``curl -X POST http://localhost:8080/api/admin/geoconnect/mapLayerMetadatas/check/{id}`` diff --git a/doc/sphinx-guides/source/admin/index.rst b/doc/sphinx-guides/source/admin/index.rst index 55733ffd99a..b97d9161d50 100755 --- a/doc/sphinx-guides/source/admin/index.rst +++ b/doc/sphinx-guides/source/admin/index.rst @@ -21,7 +21,6 @@ This guide documents the functionality only available to superusers (such as "da timers make-data-count integrations - geoconnect-worldmap user-administration dataverses-datasets solr-search-index diff --git a/doc/sphinx-guides/source/admin/integrations.rst b/doc/sphinx-guides/source/admin/integrations.rst index 9e43081bb45..bea210e587e 100644 --- a/doc/sphinx-guides/source/admin/integrations.rst +++ b/doc/sphinx-guides/source/admin/integrations.rst @@ -85,13 +85,6 @@ TwoRavens is a web application for tabular data exploration and statistical anal For installation instructions, see the :doc:`external-tools` section. -WorldMap -++++++++ - -WorldMap helps researchers visualize and explore geospatial data by creating maps. - -For installation instructions, see :doc:`geoconnect-worldmap`. - Compute Button ++++++++++++++ diff --git a/doc/sphinx-guides/source/api/apps.rst b/doc/sphinx-guides/source/api/apps.rst index dca641b1feb..4429da9e306 100755 --- a/doc/sphinx-guides/source/api/apps.rst +++ b/doc/sphinx-guides/source/api/apps.rst @@ -65,13 +65,6 @@ OSF allows you to view, download, and upload files to and from a Dataverse datas https://github.com/CenterForOpenScience/osf.io/tree/develop/addons/dataverse -Geoconnect -~~~~~~~~~~ - -Geoconnect allows Dataverse files to be visualized and explored on `WorldMap `_. Read more about it in the :doc:`/user/data-exploration/worldmap` section of the User Guide. - -https://github.com/IQSS/geoconnect - dataverse-metrics ~~~~~~~~~~~~~~~~~ diff --git a/doc/sphinx-guides/source/api/intro.rst b/doc/sphinx-guides/source/api/intro.rst index 6ce77f5fe7d..3bbd117f68e 100755 --- a/doc/sphinx-guides/source/api/intro.rst +++ b/doc/sphinx-guides/source/api/intro.rst @@ -198,7 +198,6 @@ Please note that some APIs are only documented in other guides that are more sui - :doc:`/admin/metadatacustomization` - :doc:`/admin/metadataexport` - :doc:`/admin/make-data-count` - - :doc:`/admin/geoconnect-worldmap` - :doc:`/admin/solr-search-index` - Installation Guide diff --git a/doc/sphinx-guides/source/developers/geospatial.rst b/doc/sphinx-guides/source/developers/geospatial.rst index 13680a52358..63ed6fd8a30 100644 --- a/doc/sphinx-guides/source/developers/geospatial.rst +++ b/doc/sphinx-guides/source/developers/geospatial.rst @@ -5,19 +5,12 @@ Geospatial Data .. contents:: |toctitle| :local: -Geoconnect ----------- - -Geoconnect works as a middle layer, allowing geospatial data files in Dataverse to be visualized with Harvard WorldMap. To set up a Geoconnect development environment, you can follow the steps outlined in the `local_setup.md `_ guide. You will need Python and a few other prerequisites. - -As mentioned under the :ref:`architecture` section of Preparation in the Installation Guide, Geoconnect is an optional component of Dataverse, so this section is only necessary to follow it you are working on an issue related to this feature. - How Dataverse Ingests Shapefiles -------------------------------- -A shapefile is a set of files, often uploaded/transferred in ``.zip`` format. This set may contain up to fifteen files. A minimum of three specific files (``.shp``, ``.shx``, ``.dbf``) are needed to be a valid shapefile and a fourth file (``.prj``) is required for WorldMap -- or any type of meaningful visualization. +A shapefile is a set of files, often uploaded/transferred in ``.zip`` format. This set may contain up to fifteen files. A minimum of three specific files (``.shp``, ``.shx``, ``.dbf``) are needed to be a valid shapefile and a fourth file (``.prj``) is required for some applications -- or any type of meaningful visualization. -For ingest and connecting to WorldMap, four files are the minimum required: +For ingest, four files are the minimum required: - ``.shp`` - shape format; the feature geometry itself - ``.shx`` - shape index format; a positional index of the feature geometry to allow seeking forwards and backwards quickly @@ -89,127 +82,6 @@ For two "final" shapefile sets, ``bicycles.zip`` and ``subway_line.zip``, a new - Mimetype: ``application/zipped-shapefile`` - Mimetype Label: "Shapefile as ZIP Archive" -WorldMap JoinTargets + API Endpoint ------------------------------------ - -WorldMap supplies target layers -- or JoinTargets -- that a tabular file may be mapped against. A JSON description of these `CGA `_-curated JoinTargets may be retrieved via API at ``http://worldmap.harvard.edu/datatables/api/jointargets/``. Please note: login is required. You may use any WorldMap account credentials via HTTP Basic Auth. - -Example of JoinTarget information returned via the API: - -.. code-block:: json - - { - "data":[ - { - "layer":"geonode:census_tracts_2010_boston_6f6", - "name":"Census Tracts, Boston (GEOID10: State+County+Tract)", - "geocode_type_slug":"us-census-tract", - "geocode_type":"US Census Tract", - "attribute":{ - "attribute":"CT_ID_10", - "type":"xsd:string" - }, - "abstract":"As of the 2010 census, Boston, MA contains 7,288 city blocks [truncated for example]", - "title":"Census Tracts 2010, Boston (BARI)", - "expected_format":{ - "expected_zero_padded_length":-1, - "is_zero_padded":false, - "description":"Concatenation of state, county and tract for 2010 Census Tracts. Reference: https://www.census.gov/geo/maps-data/data/tract_rel_layout.html\r\n\r\nNote: Across the US, this can be a zero-padded \"string\" but the original Boston layer has this column as \"numeric\" ", - "name":"2010 Census Boston GEOID10 (State+County+Tract)" - }, - "year":2010, - "id":28 - }, - { - "layer":"geonode:addresses_2014_boston_1wr", - "name":"Addresses, Boston", - "geocode_type_slug":"boston-administrative-geography", - "geocode_type":"Boston, Administrative Geography", - "attribute":{ - "attribute":"LocationID", - "type":"xsd:int" - }, - "abstract":"Unique addresses present in the parcels data set, which itself is derived from [truncated for example]", - "title":"Addresses 2015, Boston (BARI)", - "expected_format":{ - "expected_zero_padded_length":-1, - "is_zero_padded":false, - "description":"Boston, Administrative Geography, Boston Address Location ID. Example: 1, 2, 3...nearly 120000", - "name":"Boston Address Location ID (integer)" - }, - "year":2015, - "id":18 - }, - { - "layer":"geonode:bra_neighborhood_statistical_areas_2012__ug9", - "name":"BRA Neighborhood Statistical Areas, Boston", - "geocode_type_slug":"boston-administrative-geography", - "geocode_type":"Boston, Administrative Geography", - "attribute":{ - "attribute":"BOSNA_R_ID", - "type":"xsd:double" - }, - "abstract":"BRA Neighborhood Statistical Areas 2015, Boston. Provided by [truncated for example]", - "title":"BRA Neighborhood Statistical Areas 2015, Boston (BARI)", - "expected_format":{ - "expected_zero_padded_length":-1, - "is_zero_padded":false, - "description":"Boston, Administrative Geography, Boston BRA Neighborhood Statistical Area ID (integer). Examples: 1, 2, 3, ... 68, 69", - "name":"Boston BRA Neighborhood Statistical Area ID (integer)" - }, - "year":2015, - "id":17 - } - ], - "success":true - } - -How Geoconnect Uses Join Target Information -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -When a user attempts to map a tabular file, the application looks in the Geoconnect database for ``JoinTargetInformation``. If this information is more than 10 minutes* old, the application will retrieve fresh information and save it to the db. - -(* Change the timing via the Django settings variable ``JOIN_TARGET_UPDATE_TIME``.) - -This JoinTarget info is used to populate HTML forms used to match a tabular file column to a JoinTarget column. Once a JoinTarget is chosen, the JoinTarget ID is an essential piece of information used to make an API call to the WorldMap and attempt to map the file. - -Retrieving Join Target Information from WorldMap API -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The ``get_join_targets()`` function in ``dataverse_layer_services.py`` uses the WorldMap API, retrieves a list of available tabular file JointTargets. (See the `dataverse_layer_services code in GitHub `_.) - -Saving Join Target Information to Geoconnect Database -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The ``get_latest_jointarget_information()`` in ``utils.py`` retrieves recent JoinTarget Information from the database. (See the `utils code in GitHub `_.) - -Setting Up WorldMap Sample Data -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -For the dataset page, this script gives a query to add test WorldMap map data. After the query is run, the explore in WorldMap option should display for tabular files or shapefiles. In the example SQL queries below, substitute ``$DATASET_ID`` and ``$DATAFILE_ID`` with the appropriate ID's. - -To add sample map data for a tabular file: - -.. code:: - - INSERT INTO maplayermetadata (id, isjoinlayer, joindescription, embedmaplink, layerlink, layername, mapimagelink, worldmapusername, dataset_id, datafile_id) - VALUES (DEFAULT, true, 'This file was joined with WorldMap layer x, y, z', - 'https://worldmap.harvard.edu/maps/embed/?layer=geonode:zip_codes_2015_zip_s9i','https://worldmap.harvard.edu/data/geonode:zip_codes_2015_zip_s9i', - 'geonode:zip_codes_2015_zip_s9i', - 'http://worldmap.harvard.edu/download/wms/27289/png?layers=geonode%3Azip_codes_2015_zip_s9i&width=865&bbox=-71.1911091251%2C42.2270382738%2C-70.9228275369%2C42.3976144794&service=WMS&format=image%2Fpng&srs=EPSG%3A4326&request=GetMap&height=550', - 'admin',$DATASET_ID,$DATAFILE_ID}); - -To add sample map data for a tabular shapefile: - -.. code:: - - INSERT INTO maplayermetadata (id, isjoinlayer, embedmaplink, layerlink, layername, mapimagelink, worldmapusername, dataset_id, datafile_id) - VALUES (DEFAULT, false, - 'https://worldmap.harvard.edu/maps/embed/?layer=geonode:zip_codes_2015_zip_s9i','https://worldmap.harvard.edu/data/geonode:zip_codes_2015_zip_s9i', - 'geonode:zip_codes_2015_zip_s9i', - 'http://worldmap.harvard.edu/download/wms/27289/png?layers=geonode%3Azip_codes_2015_zip_s9i&width=865&bbox=-71.1911091251%2C42.2270382738%2C-70.9228275369%2C42.3976144794&service=WMS&format=image%2Fpng&srs=EPSG%3A4326&request=GetMap&height=550', - 'admin',$DATASET_ID,$DATAFILE_ID); - ---- Previous: :doc:`unf/index` | Next: :doc:`remote-users` diff --git a/doc/sphinx-guides/source/developers/intro.rst b/doc/sphinx-guides/source/developers/intro.rst index e7ed6a12a37..1cb2edf74d0 100755 --- a/doc/sphinx-guides/source/developers/intro.rst +++ b/doc/sphinx-guides/source/developers/intro.rst @@ -64,7 +64,6 @@ As a developer, you also may be interested in these projects related to Datavers - dataverse-metrics - aggregate and visualize metrics for installations of Dataverse around the world: https://github.com/IQSS/dataverse-metrics - Configuration management scripts - Ansible, Puppet, etc.: See :ref:`advanced` section in the Installation Guide. - :doc:`/developers/unf/index` (Java) - a Universal Numerical Fingerprint: https://github.com/IQSS/UNF -- GeoConnect (Python) - create a map by uploading files to Dataverse: https://github.com/IQSS/geoconnect - `DataTags `_ (Java and Scala) - tag datasets with privacy levels: https://github.com/IQSS/DataTags - `TwoRavens `_ (Javascript) - a `d3.js `_ interface for exploring data and running Zelig models: https://github.com/IQSS/TwoRavens - `Zelig `_ (R) - run statistical models on files uploaded to Dataverse: https://github.com/IQSS/Zelig diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index caa63b1b1cd..6c2f35c0633 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -774,7 +774,7 @@ Once you have created the analytics file, run this curl command to add it to you Tracking Button Clicks ++++++++++++++++++++++ -The basic analytics configuration above tracks page navigation. However, it does not capture potentially interesting events, such as those from users clicking buttons on pages, that do not result in a new page opening. In Dataverse, these events include file downloads, requesting access to restricted data, exporting metadata, social media sharing, requesting citation text, launching external tools or WorldMap, contacting authors, and launching computations. +The basic analytics configuration above tracks page navigation. However, it does not capture potentially interesting events, such as those from users clicking buttons on pages, that do not result in a new page opening. In Dataverse, these events include file downloads, requesting access to restricted data, exporting metadata, social media sharing, requesting citation text, launching external tools, contacting authors, and launching computations. Both Google and Matomo provide the optional capability to track such events and Dataverse has added CSS style classes (btn-compute, btn-contact, btn-download, btn-explore, btn-export, btn-preview, btn-request, btn-share, and downloadCitation) to it's HTML to facilitate it. @@ -1758,20 +1758,6 @@ The ``:TwoRavensUrl`` option is no longer valid. See :doc:`r-rapache-tworavens` The ``:TwoRavensTabularView`` option is no longer valid. See :doc:`r-rapache-tworavens` and the :doc:`/admin/external-tools` section of the Admin Guide. -:GeoconnectCreateEditMaps -+++++++++++++++++++++++++ - -Set ``GeoconnectCreateEditMaps`` to true to allow the user to create maps using Geoconnect. This boolean enables the map configure tool option for a data file and the ingest to create a shape file. - -``curl -X PUT -d true http://localhost:8080/api/admin/settings/:GeoconnectCreateEditMaps`` - -:GeoconnectViewMaps -+++++++++++++++++++ - -Set ``GeoconnectViewMaps`` to true to allow a user to view existing maps. This boolean enables the map explore tool option for a data file. - -``curl -X PUT -d true http://localhost:8080/api/admin/settings/:GeoconnectViewMaps`` - .. _:DatasetPublishPopupCustomText: :DatasetPublishPopupCustomText diff --git a/doc/sphinx-guides/source/installation/geoconnect.rst b/doc/sphinx-guides/source/installation/geoconnect.rst deleted file mode 100644 index 14a12b50a3f..00000000000 --- a/doc/sphinx-guides/source/installation/geoconnect.rst +++ /dev/null @@ -1,17 +0,0 @@ -Geoconnect -========== - -.. contents:: |toctitle| - :local: - -Geoconnect works as a middle layer, allowing geospatial data files in Dataverse to be visualized with Harvard WorldMap. - -To understand the feature from the user perspective, see the :doc:`/user/data-exploration/worldmap` section of the User Guide. - -As of this writing, the README at https://github.com/IQSS/geoconnect recommends not installing Geoconnect at this time due to an ongoing rewrite of the WorldMap code. If you are not deterred by this, read on! - -To set up a Geoconnect development environment, you can follow the steps outlined in the `local_setup.md `_ guide. Although those instructions are for a local development setup, they may assist in installing Geoconnect in your production environment. - -Harvard Dataverse runs Geoconnect on Heroku. To make use of Heroku, you will need a Heroku account, as well as a few other prerequisites. Follow the instructions outlined in the `heroku_setup.md `_ guide. The `heroku.py `_ settings file may also be adapted for other environments. Please note, for the production environment, remember to set ``DEBUG=False``. - -See also the :doc:`/admin/geoconnect-worldmap` section of the Admin Guide. diff --git a/doc/sphinx-guides/source/installation/index.rst b/doc/sphinx-guides/source/installation/index.rst index 8ed0571b466..931c4016394 100755 --- a/doc/sphinx-guides/source/installation/index.rst +++ b/doc/sphinx-guides/source/installation/index.rst @@ -17,7 +17,6 @@ Installation Guide config upgrading r-rapache-tworavens - geoconnect shibboleth oauth2 oidc diff --git a/doc/sphinx-guides/source/installation/installation-main.rst b/doc/sphinx-guides/source/installation/installation-main.rst index 8d289af716a..29600ff1e44 100755 --- a/doc/sphinx-guides/source/installation/installation-main.rst +++ b/doc/sphinx-guides/source/installation/installation-main.rst @@ -90,7 +90,6 @@ For more information, please see https://docs.payara.fish/documentation/payara-s - exporting to Schema.org format (and showing JSON-LD in HTML's tag) - exporting to DDI format - which Dataverse installation an "external tool" should return to -- which Dataverse installation Geoconnect should return to - URLs embedded in SWORD API responses The supplied site URL will be saved under the JVM option :ref:`dataverse.siteUrl`. diff --git a/doc/sphinx-guides/source/installation/prep.rst b/doc/sphinx-guides/source/installation/prep.rst index a32ab4ebeb1..4dea77755b6 100644 --- a/doc/sphinx-guides/source/installation/prep.rst +++ b/doc/sphinx-guides/source/installation/prep.rst @@ -69,7 +69,6 @@ There are a number of optional components you may choose to install or configure - Apache: a web server that can "reverse proxy" Jakarta EE applications (like Dataverse) and rewrite HTTP traffic. - Shibboleth: an authentication system described in :doc:`shibboleth`. Its use with Dataverse requires Apache. - OAuth2: an authentication system described in :doc:`oauth2`. -- Geoconnect: a system that allows users to create maps from geospatial files, described in :doc:`geoconnect`. See also the :doc:`/admin/integrations` section of the Admin Guide. diff --git a/doc/sphinx-guides/source/style/patterns.rst b/doc/sphinx-guides/source/style/patterns.rst index 92397910c14..aabc2cfd736 100644 --- a/doc/sphinx-guides/source/style/patterns.rst +++ b/doc/sphinx-guides/source/style/patterns.rst @@ -587,7 +587,7 @@ Another variation of icon-only buttons uses the ``.btn-link`` style class from B
  • - WorldMap + Data Explorer
    • diff --git a/doc/sphinx-guides/source/user/data-exploration/index.rst b/doc/sphinx-guides/source/user/data-exploration/index.rst index 52833e3ae50..02c386a3d9b 100755 --- a/doc/sphinx-guides/source/user/data-exploration/index.rst +++ b/doc/sphinx-guides/source/user/data-exploration/index.rst @@ -13,4 +13,3 @@ Contents: .. toctree:: tworavens - worldmap diff --git a/doc/sphinx-guides/source/user/data-exploration/worldmap.rst b/doc/sphinx-guides/source/user/data-exploration/worldmap.rst deleted file mode 100644 index 3f884560cff..00000000000 --- a/doc/sphinx-guides/source/user/data-exploration/worldmap.rst +++ /dev/null @@ -1,153 +0,0 @@ -.. _world-map: - -WorldMap: Geospatial Data Exploration -+++++++++++++++++++++++++++++++++++++ - -.. contents:: |toctitle| - :local: - -Dataverse and WorldMap -====================== - -`WorldMap `_ is developed by the Center for Geographic Analysis (CGA) at Harvard and is open source software that helps researchers visualize and explore their data in maps. The WorldMap and Dataverse collaboration allows researchers to upload shapefiles or tabular files to Dataverse for long term storage and receive a persistent identifier (through DOI), then easily navigate into WorldMap to interact with the data. - -Note: WorldMap hosts their own `user guide `_ that covers some of the same material as this page. - -What is Geoconnect? -=================== - -Geoconnect is a platform that integrates Dataverse and WorldMap, allowing researchers to visualize their geospatial data. Geoconnect can be used to create maps of shapefiles or of tabular files containing geospatial information. Geoconnect is an optional component of Dataverse, so if you are interested in this feature but don't see it in the installation of Dataverse you are using, contact support for that installation to request they enable Geoconnect. - -Once the map has been created using Geoconnect, can be viewed by choosing the WorldMap explore tool option for that data file. - -Mapping shapefiles with Geoconnect -================================== - -Geoconnect is capable of mapping shapefiles which are uploaded to Dataverse in .zip format. Specifically, Dataverse recognizes a zipped shapefile by: - -1. Examining the contents of the .zip file - -2. Checking for the existence of four similarly named files with the following extensions: .dbf, .prj, .shp, .shx - -Once you have uploaded your .zip shapefile, a Map Data button will appear next to the file in the dataset. In order to use this button, you'll need to publish your dataset. Once your dataset has been published, you can click on the Map Data button to be brought to Geoconnect, the portal between Dataverse and WorldMap that will allow you to create your map. - -To get started with visualizing your shapefile, click on the blue "Visualize on WorldMap" button in Geoconnect. It may take up to 45 seconds for the data to be sent to WorldMap and then back to Geoconnect. - -Once this process has finished, you will be taken to a new page where you can style your map through Attribute, Classification Method, Number of Intervals, and Colors. Clicking "Apply Changes" will send your map to both Dataverse and WorldMap, creating a preview of your map that will be visible on your file page and your dataset page. - -Clicking "View on WorldMap" will open WorldMap in a new tab, allowing you to see how your map will be displayed there. - -You can delete your map with the "Delete" button. If you decide to delete the map, it will no longer appear on WorldMap, and your dataset in Dataverse will no longer display the map preview. - -When you're satisfied with your map, you may click "Return to the Dataverse" to go back to Dataverse. - -In the future, to replace your shapefile's map with a new one, simply click the Map Data button on the dataset or file page to return to the Geoconnect edit map page. - -Mapping tabular files with Geoconnect -===================================== - -Geoconnect can map tabular files that contain geospatial information such as latitude/longitude coordinates, census tracts, zip codes, Boston election wards, etc. - - -Preparing a tabular file to be mapped -------------------------------------- - -**1. Ingest** - -Geospatial tabular files need a bit of preparation in Dataverse before they can be mapped in Geoconnect. When you upload your file, Dataverse will take about ten seconds to ingest it. During the ingest process it will identify the file as tabular data. - -|image1| - - -**2.Tag as Geospatial** - -Next, you'll need to let Dataverse know that your tabular file contains geospatial data. Select your file, click the "Edit Files" button, and select "Tags" from the dropdown menu. This will take you to the Edit Tags menu (pictured below). Under the "Tabular Data Tags" dropdown, select "Geospatial". Then click "Save Changes". - -|image2| - - -**3. Publish & Map Data** - -At this point, the configure tool option for the file will be available for your file. Publish this new version of your dataset to map your data. - - -Creating the map ----------------- - -If your tabular file contains **latitude and longitude** columns, then the process is simple: those columns may be directly mapped. Otherwise, you will need to use a *spatial join*. Spatial joins tell WorldMap how to read your tabular data file in order to create a map that accurately represents it. - -To carry out a spatial join, you'll manually connect - -- Geospatial column(s) from your Dataverse tabular file - - e.g., a census tract column from your table - -with - -- A WorldMap "target layer" that contains the same geospatial information - - e.g., WorldMap's "target layer" containing census tract parameters - -The following screenshots illustrate the mapping process: - -**1. Once you've pressed the "Map Data" button, you're brought to this page:** - -|image4| - -**2. Choose a Geospatial Data Type** - -|image5| - -**3. Choose a column from your file to match the WorldMap Layer you selected** - -|image6| - -**4. Choose from the list of WorldMap Layers available for the Geospatial Data Type you selected** - -|image7| - -**5.Submit the data for mapping!** - -|image8| - -**6. View Results** - -At this point you will be presented with a basic map that can be styled to your specifications. The example pictured below includes an error message - some of the rows weren't able to be matched properly. In this case, you could still go forward with your map, but without the information from the unmatched rows. - -|image9| - -Finalizing your map -=================== - -Now that you have created your map: - -- It exists on the WorldMap platform and may be viewed there -- with all of WorldMap's capabilities. - -- Dataverse will contain a preview of the map and links to the larger version on WorldMap. - -The map editor (pictured above) provides a set of options you can use to style your map. Clicking "Apply Changes" saves the current version of your map to Dataverse and Worldmap. The "Return to the Dataverse" button brings you back to Dataverse. "View on WorldMap" takes you to the map's page on WorldMap, which offers additional views and options. - -Return to the editor for further changes to your map in the future by choosing the map configure tool option for your data file in Dataverse. - -Removing your map -================= - -You can delete your map at any time. Select the map configure tool option for the data file in Dataverse, then in Geoconnect select "Delete Map". This completely removes the map and underlying data from the WorldMap platform. - - -.. |image1| image:: ./img/geoconnect-tabular-1.png - :class: img-responsive -.. |image2| image:: ./img/geoconnect-tabular-2.png - :class: img-responsive -.. |image3| image:: ./img/geoconnect-tabular-3.png - :class: img-responsive -.. |image4| image:: ./img/geoconnect-tabular-4.png - :class: img-responsive -.. |image5| image:: ./img/geoconnect-tabular-5.png - :class: img-responsive -.. |image6| image:: ./img/geoconnect-tabular-6.png - :class: img-responsive -.. |image7| image:: ./img/geoconnect-tabular-7.png - :class: img-responsive -.. |image8| image:: ./img/geoconnect-tabular-8.png - :class: img-responsive -.. |image9| image:: ./img/geoconnect-tabular-9.png - :class: img-responsive diff --git a/doc/sphinx-guides/source/user/dataset-management.rst b/doc/sphinx-guides/source/user/dataset-management.rst index e1ca2ad4c41..a618bb93793 100755 --- a/doc/sphinx-guides/source/user/dataset-management.rst +++ b/doc/sphinx-guides/source/user/dataset-management.rst @@ -181,34 +181,6 @@ Additional download options available for tabular data (found in the same drop-d See also :ref:`restricted-tabular-files`. -Geospatial ----------- - -Geospatial `shapefiles `_ can be further explored and manipulated through an optional integration with `WorldMap <../user/data-exploration/worldmap.html>`_, a geospatial data visualization and analysis tool developed by the `Center for Geographic Analysis `_ at Harvard University. A shapefile is a set of files, often uploaded/transferred in .zip format. This set may contain up to 15 files. A minimum of 3 specific files (.shp, .shx, .dbf) are needed to be a valid shapefile and a 4th file (.prj) is required for WorldMap--or any type of meaningful visualization. - -For ingest into Dataverse and connecting to WorldMap, these 4 files are the minimum required: - -* .shp - shape format; the feature geometry itself -* .shx - shape index format; a positional index of the feature geometry to allow seeking forwards and backwards quickly -* .dbf - attribute format; columnar attributes for each shape, in dBase IV format -* .prj - projection format; the coordinate system and projection information, a plain text file describing the projection using well-known text format - -For a zipped shapefile, we require 4 files with these extensions. Other files may be included within the zipped shapefile, but they are not required: - -* .shp -* .shx -* .prj -* .dbf - -For example, if these files were included within a .zip, the Geoconnect configure tool option is displayed: - -* subway_line.shp -* subway_line.shx -* subway_line.prj -* subway_line.dbf - -Once you publish your dataset with your shape files, you will be able to use the configure tool `Geoconnect `_ to visualize and manipulate these files for users to explore this geospatial data on `WorldMap `__. Please note: In order to map your data file, a copy will be sent to Harvard's `WorldMap `__ platform. You have the ability to delete any maps, and associated data, from the Harvard WorldMap platform, at any time. - Astronomy (FITS) ---------------- diff --git a/scripts/api/setup-optional-harvard.sh b/scripts/api/setup-optional-harvard.sh index 39ccbb310b6..e51846bd9e0 100755 --- a/scripts/api/setup-optional-harvard.sh +++ b/scripts/api/setup-optional-harvard.sh @@ -35,9 +35,6 @@ echo "- Enabling Shibboleth" curl -X POST -H "Content-type: application/json" http://localhost:8080/api/admin/authenticationProviders --upload-file ../../doc/sphinx-guides/source/_static/installation/files/etc/shibboleth/shibAuthProvider.json echo "- Enabling TwoRavens" curl -X POST -H 'Content-type: application/json' --upload-file ../../doc/sphinx-guides/source/_static/installation/files/root/external-tools/twoRavens.json http://localhost:8080/api/admin/externalTools -echo "- Enabling Geoconnect" -curl -s -X PUT -d true "$SERVER/admin/settings/:GeoconnectCreateEditMaps" -curl -s -X PUT -d true "$SERVER/admin/settings/:GeoconnectViewMaps" echo "- Setting system email" curl -X PUT -d "Harvard Dataverse Support " http://localhost:8080/api/admin/settings/:SystemEmail curl -X PUT -d ", The President & Fellows of Harvard College" http://localhost:8080/api/admin/settings/:FooterCopyright diff --git a/src/main/java/ValidationMessages.properties b/src/main/java/ValidationMessages.properties index da5c336b54e..877f76b0f9b 100644 --- a/src/main/java/ValidationMessages.properties +++ b/src/main/java/ValidationMessages.properties @@ -34,13 +34,6 @@ filename.illegalCharacters=File Name cannot contain any of the following charact directoryname.illegalCharacters=Directory Name cannot contain invalid characters. Valid characters are a-Z, 0-9, '_', '-', '.', '\\', '/' and ' ' (white space). filename.blank=Please specify a file name. - -map.layername=Please specify a layer name. -map.layerlink=Please specify a layer link. -map.link=Please specify am embedded map link. -map.imagelink=Please specify a map image link. -map.username=Please specify a WorldMap username. - oauth.username=Please enter your username. password.retype=The new password is blank: re-type it again. diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java index 2f0981c80af..cb2b6e221e8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java @@ -7,7 +7,6 @@ import com.google.gson.annotations.Expose; import com.google.gson.annotations.SerializedName; import edu.harvard.iq.dataverse.DatasetVersion.VersionState; -import edu.harvard.iq.dataverse.api.WorldMapRelatedData; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataaccess.StorageIO; @@ -19,7 +18,6 @@ import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.ShapefileHandler; import edu.harvard.iq.dataverse.util.StringUtil; -import edu.harvard.iq.dataverse.worldmapauth.WorldMapToken; import java.io.IOException; import java.util.List; import java.util.ArrayList; @@ -217,14 +215,7 @@ public List getGuestbookResponses() { public void setGuestbookResponses(List guestbookResponses) { this.guestbookResponses = guestbookResponses; } - - // The WorldMap LayerMetadata and AuthToken are here to facilitate a - // clean cascade delete when the DataFile is deleted: - @OneToOne(mappedBy="dataFile", orphanRemoval = true, cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST}) - private MapLayerMetadata mapLayerMetadata; - @OneToMany(mappedBy="dataFile", orphanRemoval = true, cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST}) - private List worldMapTokens; - + private char ingestStatus = INGEST_STATUS_NONE; @OneToOne(mappedBy = "thumbnailFile") @@ -725,21 +716,7 @@ public Dataset getThumbnailForDataset() { public void setAsThumbnailForDataset(Dataset dataset) { thumbnailForDataset = dataset; } - - /** - * URL to use with the WorldMapRelatedData API - * Used within dataset.xhtml - * - * @param dataverseUserID - * @return URL for "Map It" functionality - */ - public String getMapItURL(Long dataverseUserID){ - if (dataverseUserID==null){ - return null; - } - return WorldMapRelatedData.getMapItURL(this.getId(), dataverseUserID); - } - + /* 8/10/2014 - Using the current "open access" url */ diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 19ce99435b2..b6c353090fe 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -89,7 +89,6 @@ import java.util.HashSet; import javax.faces.model.SelectItem; import java.util.logging.Level; -import edu.harvard.iq.dataverse.datasetutility.WorldMapPermissionHelper; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; import edu.harvard.iq.dataverse.engine.command.impl.AbstractSubmitToArchiveCommand; import edu.harvard.iq.dataverse.engine.command.impl.CreateNewDatasetCommand; @@ -185,8 +184,6 @@ public enum DisplayMode { @EJB UserNotificationServiceBean userNotificationService; @EJB - MapLayerMetadataServiceBean mapLayerMetadataService; - @EJB BuiltinUserServiceBean builtinUserService; @EJB DataverseFieldTypeInputLevelServiceBean dataverseFieldTypeInputLevelService; @@ -225,8 +222,6 @@ public enum DisplayMode { @Inject FileDownloadHelper fileDownloadHelper; @Inject - WorldMapPermissionHelper worldMapPermissionHelper; - @Inject ThumbnailServiceWrapper thumbnailServiceWrapper; @Inject SettingsWrapper settingsWrapper; @@ -1359,8 +1354,6 @@ public void setNoDVsRemaining(boolean noDVsRemaining) { this.noDVsRemaining = noDVsRemaining; } - private final Map mapLayerMetadataLookup = new HashMap<>(); - private GuestbookResponse guestbookResponse; private Guestbook selectedGuestbook; @@ -1716,78 +1709,10 @@ public boolean isShapefileType(FileMetadata fm) { return fm.getDataFile().isShapefileType(); } - /* - Check if the FileMetadata.dataFile has an associated MapLayerMetadata object - - The MapLayerMetadata objects have been fetched at page inception by "loadMapLayerMetadataLookup()" - */ - public boolean hasMapLayerMetadata(FileMetadata fm) { - if (fm == null) { - return false; - } - if (fm.getDataFile() == null) { - return false; - } - return doesDataFileHaveMapLayerMetadata(fm.getDataFile()); - } - - /** - * Check if a DataFile has an associated MapLayerMetadata object - * - * The MapLayerMetadata objects have been fetched at page inception by - * "loadMapLayerMetadataLookup()" - */ - private boolean doesDataFileHaveMapLayerMetadata(DataFile df) { - if (df == null) { - return false; - } - if (df.getId() == null) { - return false; - } - return this.mapLayerMetadataLookup.containsKey(df.getId()); - } - - /** - * Using a DataFile id, retrieve an associated MapLayerMetadata object - * - * The MapLayerMetadata objects have been fetched at page inception by - * "loadMapLayerMetadataLookup()" - */ - public MapLayerMetadata getMapLayerMetadata(DataFile df) { - if (df == null) { - return null; - } - return this.mapLayerMetadataLookup.get(df.getId()); - } - private void msg(String s){ // System.out.println(s); } - /** - * Create a hashmap consisting of { DataFile.id : MapLayerMetadata object} - * - * Very few DataFiles will have associated MapLayerMetadata objects so only - * use 1 query to get them - */ - private void loadMapLayerMetadataLookup() { - if (this.dataset == null) { - } - if (this.dataset.getId() == null) { - return; - } - List mapLayerMetadataList = mapLayerMetadataService.getMapLayerMetadataForDataset(this.dataset); - if (mapLayerMetadataList == null) { - return; - } - for (MapLayerMetadata layer_metadata : mapLayerMetadataList) { - mapLayerMetadataLookup.put(layer_metadata.getDataFile().getId(), layer_metadata); - } - - }// A DataFile may have a related MapLayerMetadata object - - - private List displayFileMetadata; public List getDisplayFileMetadata() { @@ -1967,8 +1892,6 @@ private String init(boolean initFull) { //setReleasedVersionTabList(resetReleasedVersionTabList()); //SEK - lazymodel may be needed for datascroller in future release // lazyModel = new LazyFileMetadataDataModel(workingVersion.getId(), datafileService ); - // populate MapLayerMetadata - this.loadMapLayerMetadataLookup(); // A DataFile may have a related MapLayerMetadata object this.guestbookResponse = guestbookResponseService.initGuestbookResponseForFragment(workingVersion, null, session); logger.fine("Checking if rsync support is enabled."); if (DataCaptureModuleUtil.rsyncSupportEnabled(settingsWrapper.getValueForKey(SettingsServiceBean.Key.UploadMethods)) @@ -2400,27 +2323,7 @@ private void refreshSelectedFiles(List filesToRefresh){ } readOnly = false; } - - public void testSelectedFilesForMapData(){ - setSelectedFilesHasMapLayer(false); - for (FileMetadata fmd : selectedFiles){ - if(worldMapPermissionHelper.hasMapLayerMetadata(fmd)){ - setSelectedFilesHasMapLayer(true); - return; //only need one for warning message - } - } - } - - private boolean selectedFilesHasMapLayer; - - public boolean isSelectedFilesHasMapLayer() { - return selectedFilesHasMapLayer; - } - public void setSelectedFilesHasMapLayer(boolean selectedFilesHasMapLayer) { - this.selectedFilesHasMapLayer = selectedFilesHasMapLayer; - } - private Integer chunkSize = 25; public Integer getChunkSize() { @@ -2634,18 +2537,6 @@ private DatasetVersion setDatasetVersionDeaccessionReasonAndURL(DatasetVersion d dvIn.setArchiveNote(getDeaccessionForwardURLFor()); return dvIn; } - - public boolean isMapLayerToBeDeletedOnPublish(){ - - for (FileMetadata fmd : workingVersion.getFileMetadatas()){ - if (worldMapPermissionHelper.hasMapLayerMetadata(fmd)){ - if (fmd.isRestricted() || fmd.isRestrictedUI()){ - return true; - } - } - } - return false; - } private String releaseDataset(boolean minor) { if (session.getUser() instanceof AuthenticatedUser) { @@ -5241,15 +5132,6 @@ public GuestbookResponseServiceBean getGuestbookResponseService() { public void setGuestbookResponseService(GuestbookResponseServiceBean guestbookResponseService) { this.guestbookResponseService = guestbookResponseService; } - - - public WorldMapPermissionHelper getWorldMapPermissionHelper() { - return worldMapPermissionHelper; - } - - public void setWorldMapPermissionHelper(WorldMapPermissionHelper worldMapPermissionHelper) { - this.worldMapPermissionHelper = worldMapPermissionHelper; - } /** * dataset title diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java index 83152d1a75f..50681ad4fa2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java @@ -1510,16 +1510,7 @@ public String getReturnToDatasetURL(String serverName, Dataset dset) { } return serverName + "/dataset.xhtml?id=" + dset.getId() + "&versionId=" + this.getId(); } - - /* - Per #3511 we are returning all users to the File Landing page - If we in the future we are going to return them to the referring page we will need the - getReturnToDatasetURL method and add something to the call to the api to - pass the referring page and some kind of decision point in the getWorldMapDatafileInfo method in - WorldMapRelatedData - SEK 3/24/2017 - */ - + public String getReturnToFilePageURL (String serverName, Dataset dset, DataFile dataFile){ if (serverName == null || dataFile == null) { return null; diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFieldTypeInputLevelServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFieldTypeInputLevelServiceBean.java index ce9b44f734b..42a1290fdbd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFieldTypeInputLevelServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFieldTypeInputLevelServiceBean.java @@ -77,11 +77,7 @@ public List findByDataverseIdAndDatasetFieldTypeId return null; } } - // - - // Query query = em.createQuery("select object(o) from MapLayerMetadata as o where o.dataset=:dataset");// order by o.name"); - // query.setParameter("dataset", dataset); - + public DataverseFieldTypeInputLevel findByDataverseIdDatasetFieldTypeId(Long dataverseId, Long datasetFieldTypeId) { Query query = em.createNamedQuery("DataverseFieldTypeInputLevel.findByDataverseIdDatasetFieldTypeId", DataverseFieldTypeInputLevel.class); query.setParameter("dataverseId", dataverseId); diff --git a/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java b/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java index 9606b24ff14..da9aba43498 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java +++ b/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java @@ -161,9 +161,6 @@ public class EjbDataverseEngine { @EJB DatasetVersionServiceBean datasetVersionService; - @EJB - MapLayerMetadataServiceBean mapLayerMetadata; - @EJB DataCaptureModuleServiceBean dataCaptureModule; @@ -565,11 +562,6 @@ public WorkflowServiceBean workflows() { return workflowService; } - @Override - public MapLayerMetadataServiceBean mapLayerMetadata() { - return mapLayerMetadata; - } - @Override public DataCaptureModuleServiceBean dataCaptureModule() { return dataCaptureModule; diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownload.java b/src/main/java/edu/harvard/iq/dataverse/FileDownload.java index 8463a8278c0..fad03d2a0a1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileDownload.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileDownload.java @@ -54,13 +54,11 @@ public class FileDownload implements Serializable { /** * Possible values for downloadType include "Download", "Subset", - * "WorldMap", or the displayName of an ExternalTool. + * or the displayName of an ExternalTool. * - * TODO: Types like "Download" and "Subset" and probably "WorldMap" should + * TODO: Types like "Download" and "Subset" should * be defined once as constants (likely an enum) rather than having these * strings duplicated in various places when setDownloadtype() is called. - * (Some day it would be nice to convert WorldMap into an ExternalTool but - * it's not worth the effort at this time.) */ private String downloadtype; private String sessionId; diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java index 80625d9cc8e..95ceaec5f67 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java @@ -143,14 +143,6 @@ public void writeGuestbookAndLaunchPackagePopup(GuestbookResponse guestbookRespo fileDownloadService.writeGuestbookResponseRecord(guestbookResponse); } - public String startWorldMapDownloadLink(GuestbookResponse guestbookResponse, FileMetadata fmd){ - - guestbookResponse.setDownloadtype("WorldMap"); - String retVal = fileDownloadService.startWorldMapDownloadLink(guestbookResponse, fmd); - PrimeFaces.current().executeScript("PF('downloadPopup').hide()"); - return retVal; - } - /** * Writes a guestbook entry for either popup scenario: guestbook or terms. */ @@ -198,12 +190,6 @@ public void setSelectedFileId(String selectedFileId) { * WARNING: Before calling this, make sure the user has download * permission for the file!! (See DatasetPage.canDownloadFile()) * - * Should there be a Explore WorldMap Button for this file? - * See table in: https://github.com/IQSS/dataverse/issues/1618 - * - * (1) Does the file have MapLayerMetadata? - * (2) Are the proper settings in place - * * @param fileMetadata * @return boolean */ diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java index a91a28d3d96..4b0272d4a36 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java @@ -8,7 +8,6 @@ import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataaccess.StorageIO; -import edu.harvard.iq.dataverse.datasetutility.WorldMapPermissionHelper; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.impl.CreateGuestbookResponseCommand; import edu.harvard.iq.dataverse.engine.command.impl.RequestAccessCommand; @@ -86,7 +85,6 @@ public class FileDownloadServiceBean implements java.io.Serializable { @Inject DataverseRequestServiceBean dvRequestService; - @Inject WorldMapPermissionHelper worldMapPermissionHelper; @Inject FileDownloadHelper fileDownloadHelper; @Inject MakeDataCountLoggingServiceBean mdcLogService; @@ -334,45 +332,10 @@ public void explore(GuestbookResponse guestbookResponse, FileMetadata fmd, Exter } } - public String startWorldMapDownloadLink(GuestbookResponse guestbookResponse, FileMetadata fmd){ - - if (guestbookResponse != null && guestbookResponse.isWriteResponse() && ((fmd != null && fmd.getDataFile() != null) || guestbookResponse.getDataFile() != null)){ - if(guestbookResponse.getDataFile() == null && fmd != null){ - guestbookResponse.setDataFile(fmd.getDataFile()); - } - if (fmd == null || !fmd.getDatasetVersion().isDraft()){ - writeGuestbookResponseRecord(guestbookResponse); - } - } - DataFile file = null; - if (fmd != null){ - file = fmd.getDataFile(); - } - if (guestbookResponse != null && guestbookResponse.getDataFile() != null && file == null){ - file = guestbookResponse.getDataFile(); - } - - - String retVal = worldMapPermissionHelper.getMapLayerMetadata(file).getLayerLink(); - - try { - FacesContext.getCurrentInstance().getExternalContext().redirect(retVal); - return retVal; - } catch (IOException ex) { - logger.info("Failed to issue a redirect to file download url."); - } - return retVal; - } - public Boolean canSeeTwoRavensExploreButton(){ return false; } - - - public Boolean canUserSeeExploreWorldMapButton(){ - return false; - } - + public void downloadDatasetCitationXML(Dataset dataset) { downloadCitationXML(null, dataset, false); } diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index 17a9904d2b1..da315424220 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -14,7 +14,6 @@ import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser; import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.dataaccess.StorageIO; -import edu.harvard.iq.dataverse.datasetutility.WorldMapPermissionHelper; import edu.harvard.iq.dataverse.engine.command.Command; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; @@ -123,18 +122,9 @@ public class FilePage implements java.io.Serializable { PermissionsWrapper permissionsWrapper; @Inject FileDownloadHelper fileDownloadHelper; - @Inject WorldMapPermissionHelper worldMapPermissionHelper; @Inject MakeDataCountLoggingServiceBean mdcLogService; - public WorldMapPermissionHelper getWorldMapPermissionHelper() { - return worldMapPermissionHelper; - } - - public void setWorldMapPermissionHelper(WorldMapPermissionHelper worldMapPermissionHelper) { - this.worldMapPermissionHelper = worldMapPermissionHelper; - } - private static final Logger logger = Logger.getLogger(FilePage.class.getCanonicalName()); private boolean fileDeleteInProgress = false; @@ -233,7 +223,7 @@ public String init() { configureTools = externalToolService.findFileToolsByTypeAndContentType(ExternalTool.Type.CONFIGURE, contentType); exploreTools = externalToolService.findFileToolsByTypeAndContentType(ExternalTool.Type.EXPLORE, contentType); Collections.sort(exploreTools, CompareExternalToolName); - toolsWithPreviews = addMapLayerAndSortExternalTools(); + toolsWithPreviews = sortExternalTools(); if(!toolsWithPreviews.isEmpty()){ setSelectedTool(toolsWithPreviews.get(0)); } @@ -261,32 +251,12 @@ public Long getDatasetVersionId() { public void setDatasetVersionId(Long datasetVersionId) { this.datasetVersionId = datasetVersionId; } - - private List addMapLayerAndSortExternalTools(){ - List retList = externalToolService.findFileToolsByTypeAndContentType(ExternalTool.Type.PREVIEW, file.getContentType()); - if(!retList.isEmpty()){ - retList.forEach((et) -> { - et.setWorldMapTool(false); - }); - } - if (file != null && worldMapPermissionHelper.getMapLayerMetadata(file) != null && worldMapPermissionHelper.getMapLayerMetadata(file).getEmbedMapLink() != null) { - ExternalTool wpTool = new ExternalTool(); - wpTool.setDisplayName("World Map"); - wpTool.setToolParameters("{}"); - wpTool.setToolUrl(worldMapPermissionHelper.getMapLayerMetadata(file).getEmbedMapLink()); - wpTool.setWorldMapTool(true); - retList.add(wpTool); - } + private List sortExternalTools(){ + List retList = externalToolService.findFileToolsByTypeAndContentType(ExternalTool.Type.PREVIEW, file.getContentType()); Collections.sort(retList, CompareExternalToolName); - return retList; } - - /* - worldMapPermissionHelper.getMapLayerMetadata(FilePage.fileMetadata.dataFile).getEmbedMapLink() - */ - public boolean isDownloadPopupRequired() { if(fileMetadata.getId() == null || fileMetadata.getDatasetVersion().getId() == null ){ diff --git a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java index 0ea36dfdc7a..177efd9ed4d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java @@ -440,19 +440,6 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio )); logger.fine(datasetCreatedMessage); return messageText += datasetCreatedMessage; - case MAPLAYERUPDATED: - version = (DatasetVersion) targetObject; - pattern = BundleUtil.getStringFromBundle("notification.email.worldMap.added"); - String[] paramArrayMapLayer = {version.getDataset().getDisplayName(), getDatasetLink(version.getDataset())}; - messageText += MessageFormat.format(pattern, paramArrayMapLayer); - return messageText; - case MAPLAYERDELETEFAILED: - FileMetadata targetFileMetadata = (FileMetadata) targetObject; - version = targetFileMetadata.getDatasetVersion(); - pattern = BundleUtil.getStringFromBundle("notification.email.maplayer.deletefailed.text"); - String[] paramArrayMapLayerDelete = {targetFileMetadata.getLabel(), getDatasetLink(version.getDataset())}; - messageText += MessageFormat.format(pattern, paramArrayMapLayerDelete); - return messageText; case SUBMITTEDDS: version = (DatasetVersion) targetObject; String mightHaveSubmissionComment = ""; @@ -598,9 +585,6 @@ private Object getObjectOfNotification (UserNotification userNotification){ case GRANTFILEACCESS: case REJECTFILEACCESS: return datasetService.find(userNotification.getObjectId()); - case MAPLAYERDELETEFAILED: - return dataFileService.findFileMetadata(userNotification.getObjectId()); - case MAPLAYERUPDATED: case CREATEDS: case SUBMITTEDDS: case PUBLISHEDDS: diff --git a/src/main/java/edu/harvard/iq/dataverse/MapLayerMetadata.java b/src/main/java/edu/harvard/iq/dataverse/MapLayerMetadata.java deleted file mode 100644 index a7dd4d9eef1..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/MapLayerMetadata.java +++ /dev/null @@ -1,304 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ - -package edu.harvard.iq.dataverse; - -import java.io.Serializable; -import java.sql.Timestamp; -import java.util.Arrays; -import java.util.List; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Index; -import javax.persistence.JoinColumn; -import javax.persistence.ManyToOne; -import javax.persistence.NamedQueries; -import javax.persistence.NamedQuery; -import javax.persistence.Table; -import javax.persistence.Transient; -import org.hibernate.validator.constraints.NotBlank; - -/** - * File metadata: specifically WorldMap layer information for a specific DataFile - * - * @author raprasad - */ -@NamedQueries({ - @NamedQuery(name = "MapLayerMetadata.findAll", - query = "SELECT mlm FROM MapLayerMetadata mlm"),}) -@Entity -@Table(indexes = {@Index(columnList="dataset_id")}) -public class MapLayerMetadata implements Serializable { - - - @Transient - public final static String dataType = "MapLayerMetadata"; - - @Transient - public final static List MANDATORY_JSON_FIELDS = Arrays.asList("layerName", "layerLink", "embedMapLink", "worldmapUsername"); - - - private static final long serialVersionUID = 1L; - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private Long id; - - // ForeignKey to DataFile - //x@ManyToOne - // For now, make this unique: Each DataFile may only have one map - @JoinColumn(nullable=false, unique=true) - private DataFile dataFile; - - // ForeignKey to Dataset. - // This is always reachable via the datafile.getOwner(); - // However, save the Dataset itself to potentially save an extra step - @ManyToOne - @JoinColumn(nullable=false) - private Dataset dataset; - - @Column(nullable=false) - @NotBlank(message = "{map.layername}") - private String layerName; - - @Column(nullable=false) - @NotBlank(message = "{map.layerlink}") - private String layerLink; - - @Column(nullable=false) - @NotBlank(message = "{map.link}") - private String embedMapLink; - - @Column(nullable=true) - @NotBlank(message = "{map.imagelink}") - private String mapImageLink; - - @Column(nullable=false) - @NotBlank(message = "{map.username}") - private String worldmapUsername; - - - /** - * Was this layer created by joining a tabular file - * to an existing file? - */ - private boolean isJoinLayer; - - /** - * Description if this was created via a tabular join, - */ - @Column(columnDefinition = "TEXT") - private String joinDescription; - - /** - * Links to alternative representations of the map - * in JSON format - */ - @Column(columnDefinition = "TEXT") - private String mapLayerLinks; - - /** - * The HTTP Status code (200, 404, etc.) returned when you check to see if - * the map/layer exists on the WorldMap side. - */ - @Column(nullable=true) - private int lastVerifiedStatus; - - /** - * The time that lastVerifiedStatus was last recorded. - */ - @Column(nullable=true) - private Timestamp lastVerifiedTime; - - /** - * Get property layerName. - * @return value of property layerName. - */ - public String getLayerName() { - return this.layerName; - } - - /** - * Set property layerName. - * @param layerName new value of property layerName. - */ - public void setLayerName(String layerName) { - this.layerName = layerName; - } - - /** - * Get property layerLink. - * @return value of property layerLink. - */ - public String getLayerLink() { - return this.layerLink; - } - - /** - * Set property layerLink. - * @param layerLink new value of property layerLink. - */ - public void setLayerLink(String layerLink) { - this.layerLink = layerLink; - } - - /** - * Get property mapImageLink. - * @return value of property mapImageLink. - */ - public String getMapImageLink() { - return this.mapImageLink; - } - - /** - * Set property mapImageLink. - * @param mapImageLink new value of property layerLink. - */ - public void setMapImageLink(String mapImageLink) { - this.mapImageLink = mapImageLink; - } - - - /** - * Get property embedMapLink. - * @return value of property embedMapLink. - */ - public String getEmbedMapLink() { - return this.embedMapLink; - } - - /** - * Set property embedMapLink. - * @param embedMapLink new value of property embedMapLink. - */ - public void setEmbedMapLink(String embedMapLink) { - this.embedMapLink = embedMapLink; - } - - /** - * Get property worldmapUsername. - * @return value of property worldmapUsername. - */ - public String getWorldmapUsername() { - return this.worldmapUsername; - } - - /** - * Set property worldmapUsername. - * @param worldmapUsername new value of property worldmapUsername. - */ - public void setWorldmapUsername(String worldmapUsername) { - this.worldmapUsername = worldmapUsername; - } - - - /** - * Get property isJoinLayer. - * @return value of property isJoinLayer. - */ - public boolean isJoinLayer(){ - return this.isJoinLayer; - } - - /** - * Set property isJoinLayer. - * @param bool new value of property isJoinLayer. - */ - public void setIsJoinLayer(boolean bool) { - this.isJoinLayer = bool; - } - - /** - * Get property joinDescription. - * @return value of property joinDescription. - */ - public String getJoinDescription() { - return this.joinDescription; - } - - /** - * Set property joinDescription. - * @param joinDescription new value of property joinDescription. - */ - public void setJoinDescription(String joinDescription) { - this.joinDescription = joinDescription; - } - - /** - * Get property mapLayerLinks - * @return value of property joinDescription. - */ - public String getMapLayerLinks() { - return this.mapLayerLinks; - } - - /** - * Set property joinDescription. - * @param joinDescription new value of property joinDescription. - */ - public void setMapLayerLinks(String mapLayerLinks) { - this.mapLayerLinks = mapLayerLinks; - } - - public Dataset getDataset() { - return dataset; - } - - public void setDataset(Dataset dataset) { - this.dataset = dataset; - } - - public DataFile getDataFile() { - return dataFile; - } - - public void setDataFile(DataFile dataFile) { - this.dataFile = dataFile; - } - - /** - * Getter for property id. - * @return Value of property id. - */ - public Long getId() { - return this.id; - } - - /** - * Setter for property id. - * @param id New value of property id. - */ - public void setId(Long id) { - this.id = id; - } - - public Timestamp getLastVerifiedTime() { - return lastVerifiedTime; - } - - public void setLastVerifiedTime(Timestamp lastVerifiedTime) { - this.lastVerifiedTime = lastVerifiedTime; - } - - public int getLastVerifiedStatus() { - return lastVerifiedStatus; - } - - public void setLastVerifiedStatus(int lastVerifiedStatus) { - this.lastVerifiedStatus = lastVerifiedStatus; - } - - @Override - public String toString() { - return "edu.harvard.iq.dataverse.MaplayerMetadata[id=" + this.id + "]"; - - //return "WorldMap Layer: " + this.layerName + " for DataFile: " + this.dataFile.toString(); - } - -} diff --git a/src/main/java/edu/harvard/iq/dataverse/MapLayerMetadataServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MapLayerMetadataServiceBean.java deleted file mode 100755 index dbbe8b3d8bb..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/MapLayerMetadataServiceBean.java +++ /dev/null @@ -1,406 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ - -package edu.harvard.iq.dataverse; - -import edu.harvard.iq.dataverse.api.WorldMapRelatedData; -import edu.harvard.iq.dataverse.authorization.Permission; -import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; -import edu.harvard.iq.dataverse.authorization.users.User; -import edu.harvard.iq.dataverse.dataaccess.StorageIO; -import edu.harvard.iq.dataverse.util.SystemConfig; -import edu.harvard.iq.dataverse.worldmapauth.WorldMapToken; -import edu.harvard.iq.dataverse.worldmapauth.WorldMapTokenServiceBean; -import java.io.IOException; -import java.io.InputStream; -import java.net.HttpURLConnection; -import java.net.URL; -import java.net.URLEncoder; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; -import javax.ejb.EJB; -import javax.ejb.Stateless; -import javax.inject.Named; -import javax.persistence.EntityManager; -import javax.persistence.NoResultException; -import javax.persistence.PersistenceContext; -import javax.persistence.Query; -import javax.persistence.TypedQuery; - -/** - * - * @author raprasad - */ -@Stateless -@Named -public class MapLayerMetadataServiceBean { - - - @PersistenceContext(unitName = "VDCNet-ejbPU") - private EntityManager em; - - @EJB - DataFileServiceBean dataFileService; - - @EJB - PermissionServiceBean permissionService; - - @EJB - SystemConfig systemConfig; - - @EJB - WorldMapTokenServiceBean tokenServiceBean; - - private static final Logger logger = Logger.getLogger(MapLayerMetadataServiceBean.class.getCanonicalName()); - - private static final String GEOCONNECT_MAP_DELETE_API = "/tabular/delete-map-no-ui/"; - - public MapLayerMetadata find(Object pk) { - if (pk==null){ - return null; - } - return em.find(MapLayerMetadata.class, pk); - } - - public MapLayerMetadata save( MapLayerMetadata layer_metadata) { - if (layer_metadata==null){ - return null; - } - if ( layer_metadata.getId() == null ) { - em.persist(layer_metadata); - return layer_metadata; - } else { - return em.merge( layer_metadata ); - } - } - - - - /* - Given a datafile id, return the associated MapLayerMetadata object - - */ - public MapLayerMetadata findMetadataByDatafile(DataFile datafile){ - - if (datafile == null){ - return null; - } - - try{ - // String sqlStatement = - Query query = em.createQuery("select m from MapLayerMetadata m WHERE m.dataFile=:datafile", MapLayerMetadata.class); - query.setParameter("datafile", datafile); - query.setMaxResults(1); - //entityManager.createQuery(SQL_QUERY).setParameter(arg0,arg1).setMaxResults(10).getResultList(); - return (MapLayerMetadata) query.getSingleResult(); - } catch ( NoResultException nre ) { - return null; - } - } - - - /* - Delete a mapLayerMetadata object. - - First check if the given user has permission to edit this data. - - */ - public boolean deleteMapLayerMetadataObject(MapLayerMetadata mapLayerMetadata, User user){ - logger.info("deleteMapLayerMetadataObject"); - - if ((mapLayerMetadata == null)||(user==null)){ - return false; - } - - if (permissionService.userOn(user, mapLayerMetadata.getDataFile().getOwner()).has(Permission.EditDataset)) { - - // Remove thumbnails associated with the map metadata - // (this also sets theto set the "preview image" flag to false) - // - boolean success = this.deleteOlderMapThumbnails(mapLayerMetadata.getDataFile()); - - // Remove the actual map metadata - // - em.remove(em.merge(mapLayerMetadata)); - - return true; - } - return false; - } - - - public MapLayerMetadata findMetadataByLayerNameAndDatafile(String layer_name){//, DataFile datafile) { - if ((layer_name == null)){//||(datafile==null)){ - return null; - } - //Query query = em.createQuery("select o.id from MapLayerMetadta as o where o.layer_name =:layerName and o.datafile_id =:datafileID;"); - //Query query = em.createQuery("select m from MapLayerMetadata m where m.layer_name =:layerName ;"); - try{ - return em.createQuery("select m from MapLayerMetadata m WHERE m.layerName=:layerName", MapLayerMetadata.class) - .setParameter("layerName", layer_name) - .getSingleResult(); - } catch ( NoResultException nre ) { - return null; - } - } - - - - public List getMapLayerMetadataForDataset(Dataset dataset){ - if (dataset == null){ - return null; - } - TypedQuery query = em.createQuery("select object(o) from MapLayerMetadata as o where o.dataset=:dataset", MapLayerMetadata.class);// order by o.name"); - query.setParameter("dataset", dataset); - return query.getResultList(); - } - - - /** - * Before downloading a file for map icons (see "retrieveMapImageForIcon" below), - * first remove any existing .img and .img.* files - * - * e.g. delete all that start with (DataFile name) + ".img" - * - * @param mapLayerMetadata - * @return - * @throws IOException - */ - private boolean deleteOlderMapThumbnails(DataFile dataFile) { - if (dataFile==null){ - logger.warning("dataFile is null"); - return false; - } - - // Set the preview image available flag to false - dataFile.setPreviewImageAvailable(false); - dataFile = dataFileService.save(dataFile); - - - try { - StorageIO storageIO = dataFile.getStorageIO(); - - if (storageIO == null) { - logger.warning("Null storageIO in deleteOlderMapThumbnails()"); - return false; - } - storageIO.open(); - List cachedObjectsTags = storageIO.listAuxObjects(); - - if (cachedObjectsTags != null) { - String iconBaseTag = "img"; - String iconThumbTagPrefix = "thumb"; - for (String cachedFileTag : cachedObjectsTags) { - logger.info("found AUX tag: "+cachedFileTag); - if (iconBaseTag.equals(cachedFileTag) || cachedFileTag.startsWith(iconThumbTagPrefix)) { - logger.info("deleting cached AUX object "+cachedFileTag); - storageIO.deleteAuxObject(cachedFileTag); - } - } - } - /* - * Below is the old-style code that was assuming that all the files are - * stored on a local filesystem. The StorageIO code, above, should - * be used instead for all the operations on the physical files associated - * with DataFiles. - // Get the parent directory - // - Path fileDirname = dataAccess.getFileSystemPath().getParent(); - if (fileDirname == null){ - logger.warning("DataFile directory has null path. Directory path: " + dataAccess.getFileSystemPath().toString()); - return false; - } - - // Verify that the directory exists - // - File fileDirectory = new File(fileDirname.normalize().toString()); - if (!(fileDirectory.isDirectory())){ - logger.warning("DataFile directory is not actually a directory. Directory path: " + fileDirectory.toString()); - return false; - } - - /* Iterate through directory and delete any ".img" files for this DataFile - - Example: - Datafile name: 14a5e4abf7d-e7eebfb6474d - Types of files that would be deleted (if they exist): - 14a5e4abf7d-e7eebfb6474d.img - 14a5e4abf7d-e7eebfb6474d.img.thumb64 - 14a5e4abf7d-e7eebfb6474d.img.thumb400 - *//* - String iconBaseFilename = dataAccess.getFileSystemPath().toString() + ".img"; - String iconThumbFilename = dataAccess.getFileSystemPath().toString() + ".thumb"; - - for (File singleFile : fileDirectory.listFiles()) { - if (singleFile.toString().startsWith(iconBaseFilename) || singleFile.toString().startsWith(iconThumbFilename)) { - //logger.info("file found: " + singleFile.toString()); - singleFile.delete(); - //results.add(file.getName()); - } - }*/ - } catch (IOException ioEx) { - logger.warning("IOException in deleteOlderMapThumbnails(): "+ioEx.getMessage()); - return false; - } - - return true; - } - - /** - * Use the mapLayerMetadata.mapImageLink to retrieve a PNG file directly from WorldMap - * - * Next step: Save this image as the default icon - * - * Example mapImageLink: http://worldmap.harvard.edu/download/wms/14708/png?layers=geonode:power_plants_enipedia_jan_2014_kvg&width=948&bbox=76.04800165,18.31860358,132.0322222,50.78441&service=WMS&format=image/png&srs=EPSG:4326&request=GetMap&height=550 - * - * Parameter by parameter (note width/height): - * http://worldmap.harvard.edu/download/wms/14708/png? - * layers=geonode:power_plants_enipedia_jan_2014_kvg - * width=948 - * bbox=76.04800165,18.31860358,132.0322222,50.78441 - * service=WMS - * format=image/png - * srs=EPSG:4326 - * request=GetMap - * height=550 - * - * @param mapLayerMetadata - * @return boolean - * @throws IOException - */ - public boolean retrieveMapImageForIcon(MapLayerMetadata mapLayerMetadata) throws IOException { - if (mapLayerMetadata==null){ - logger.warning("mapLayerMetadata is null"); - return false; - } - - this.deleteOlderMapThumbnails(mapLayerMetadata.getDataFile()); - - if ((mapLayerMetadata.getMapImageLink()==null)||mapLayerMetadata.getMapImageLink().isEmpty()){ - logger.warning("mapLayerMetadata does not have a 'map_image_link' attribute"); - return false; - } - - String imageUrl = mapLayerMetadata.getMapImageLink(); - imageUrl = imageUrl.replace("https:", "http:"); - logger.info("Attempt to retrieve map image: " + imageUrl); - - StorageIO dataAccess = null; - try { - dataAccess = mapLayerMetadata.getDataFile().getStorageIO(); - } catch (IOException ioEx) { - dataAccess = null; - } - - if (dataAccess == null) { - logger.warning("Failed to open Access IO on DataFile "+mapLayerMetadata.getDataFile().getId()); - return false; - } - - URL url = new URL(imageUrl); - logger.info("retrieve url : " + imageUrl); - - logger.info("try to open InputStream"); - InputStream is = null; - - try{ - is = url.openStream(); - }catch(IOException exio){ - logger.warning("Error when retrieving map icon image. Exception: " + exio.getMessage()); - if (is!=null){ - try { is.close(); } catch (IOException ignore) {} - } - return false; - } - - try { - dataAccess.saveInputStreamAsAux(is, "img"); - } catch (IOException ioex) { - logger.warning("Failed to save WorldMap-generated image; "+ioex.getMessage()); - return false; - } - - logger.info("Done"); - return true; - } - - /* - * This method calls GeoConnect and requests that a map layer for this - * DataFile is deleted, from WorldMap and GeoConnect itself. - * The use case here is when a user restricts a tabular file for which - * a geospatial map has already been made. - * This process is initiated on the Dataverse side, without any GeoConnect - * UI in the picture. (The way a user normally deletes a layer map is by - * clicking 'Delete' on the GeoConnect map page). - * Otherwise this call follows the scheme used when accessing the - * /shapefile/map-it on GeoConnect - we send along a WorldMap token and a - * callback url for GC to download the file metadata.metadata - * Depending on how it goes we receive a yes or no response from the server. - */ - public void deleteMapLayerFromWorldMap(DataFile dataFile, AuthenticatedUser user) throws IOException { - - - if (dataFile == null){ - logger.severe("dataFile cannot be null"); - return; - } - - if (user == null){ - logger.severe("user cannot be null"); - return; - } - - // Worldmap token: - WorldMapToken token = tokenServiceBean.getNewToken(dataFile, user); - if (token == null){ - logger.severe("token should NOT be null"); - return; - } - - logger.info("-- new token id: " + token.getId()); - // Callback url for geoConnect: - String callback_url = URLEncoder.encode(systemConfig.getDataverseSiteUrl() + WorldMapRelatedData.GET_WORLDMAP_DATAFILE_API_PATH); - - String geoConnectAddress = token.getApplication().getMapitLink(); - /* - * this is a bit of a hack - there should be a cleaner way to get the base - * geoconnect URL from the token! - */ - geoConnectAddress = geoConnectAddress.replace("/shapefile/map-it", ""); - - logger.log(Level.INFO, "callback_url: {0}", callback_url); - - //String geoConnectCommand = geoConnectAddress + GEOCONNECT_MAP_DELETE_API + token.getApplication().getMapitLink() + "/" + token.getToken() + "/?cb=" + callback_url; - String geoConnectCommand = geoConnectAddress + GEOCONNECT_MAP_DELETE_API + token.getToken() + "/?cb=" + callback_url; - logger.info("-- new token id 2: " + token.getId()); - - - logger.info("Attempting to call GeoConnect to request that the WorldMap layer for DataFile "+dataFile.getId()+":\n"+geoConnectCommand); - URL geoConnectUrl = new URL(geoConnectCommand); - - HttpURLConnection geoConnectConnection = (HttpURLConnection)geoConnectUrl.openConnection(); - - geoConnectConnection.setRequestMethod("GET"); - geoConnectConnection.connect(); - - int code = geoConnectConnection.getResponseCode(); - logger.info("response code: " + code); - if (code != 200) { - throw new IOException ("Failed to delete Map Layer via GeoConnect. /tabular/delete-map HTTP code response: "+code+""); - } - logger.info("response :" + geoConnectConnection.getContent()); - - } - - public List findAll() { - TypedQuery typedQuery = em.createNamedQuery("MapLayerMetadata.findAll", MapLayerMetadata.class); - List mapLayerMetadatas = typedQuery.getResultList(); - return mapLayerMetadatas; - } - -} - diff --git a/src/main/java/edu/harvard/iq/dataverse/UserNotification.java b/src/main/java/edu/harvard/iq/dataverse/UserNotification.java index d404d82c950..bac1df0f2ed 100644 --- a/src/main/java/edu/harvard/iq/dataverse/UserNotification.java +++ b/src/main/java/edu/harvard/iq/dataverse/UserNotification.java @@ -27,7 +27,7 @@ public class UserNotification implements Serializable { public enum Type { - ASSIGNROLE, REVOKEROLE, CREATEDV, CREATEDS, CREATEACC, MAPLAYERUPDATED, MAPLAYERDELETEFAILED, SUBMITTEDDS, RETURNEDDS, PUBLISHEDDS, REQUESTFILEACCESS, GRANTFILEACCESS, REJECTFILEACCESS, FILESYSTEMIMPORT, CHECKSUMIMPORT, CHECKSUMFAIL, CONFIRMEMAIL, APIGENERATED, INGESTCOMPLETED, INGESTCOMPLETEDWITHERRORS, PUBLISHFAILED_PIDREG + ASSIGNROLE, REVOKEROLE, CREATEDV, CREATEDS, CREATEACC, SUBMITTEDDS, RETURNEDDS, PUBLISHEDDS, REQUESTFILEACCESS, GRANTFILEACCESS, REJECTFILEACCESS, FILESYSTEMIMPORT, CHECKSUMIMPORT, CHECKSUMFAIL, CONFIRMEMAIL, APIGENERATED, INGESTCOMPLETED, INGESTCOMPLETEDWITHERRORS, PUBLISHFAILED_PIDREG }; private static final long serialVersionUID = 1L; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 2a82acc7622..551c0a797fc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -18,7 +18,6 @@ import edu.harvard.iq.dataverse.DvObjectServiceBean; import edu.harvard.iq.dataverse.EjbDataverseEngine; import edu.harvard.iq.dataverse.GuestbookResponseServiceBean; -import edu.harvard.iq.dataverse.MapLayerMetadataServiceBean; import edu.harvard.iq.dataverse.MetadataBlock; import edu.harvard.iq.dataverse.MetadataBlockServiceBean; import edu.harvard.iq.dataverse.PermissionServiceBean; @@ -211,9 +210,6 @@ String getWrappedMessageWhenJson() { @EJB protected DatasetVersionServiceBean datasetVersionSvc; - @EJB - protected MapLayerMetadataServiceBean mapLayerMetadataSrv; - @EJB protected SystemConfig systemConfig; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 0c7a4224648..aba894e2478 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -68,7 +68,6 @@ import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.json.JsonPrinter; -import edu.harvard.iq.dataverse.worldmapauth.WorldMapTokenServiceBean; import java.util.logging.Logger; import javax.ejb.EJB; @@ -166,8 +165,6 @@ public class Access extends AbstractApiBean { PermissionServiceBean permissionService; @Inject DataverseSession session; - @EJB - WorldMapTokenServiceBean worldMapTokenServiceBean; @Inject DataverseRequestServiceBean dvRequestService; @EJB @@ -1697,11 +1694,6 @@ private boolean isAccessAuthorized(DataFile df, String apiToken) { return true; } - - // We don't want to return false just yet. - // If all else fails, we'll want to use the special WorldMapAuth - // token authentication before we give up. - //return false; } else { // OK, this is a restricted file. @@ -1755,30 +1747,9 @@ private boolean isAccessAuthorized(DataFile df, String apiToken) { } } } - - - // And if all that failed, we'll still check if the download can be authorized based - // on the special WorldMap token: - if ((apiToken != null)&&(apiToken.length()==64)){ - /* - WorldMap token check - - WorldMap tokens are 64 chars in length - - - Use the worldMapTokenServiceBean to verify token - and check permissions against the requested DataFile - */ - if (!(this.worldMapTokenServiceBean.isWorldMapTokenAuthorizedForDataFileDownload(apiToken, df))){ - return false; - } - - // Yes! User may access file - // - logger.fine("WorldMap token-based auth: Token is valid for the requested datafile"); - return true; - - } else if ((apiToken != null)&&(apiToken.length()!=64)) { + if ((apiToken != null)) { // Will try to obtain the user information from the API token, // if supplied: @@ -1794,7 +1765,7 @@ private boolean isAccessAuthorized(DataFile df, String apiToken) { return false; } - if (permissionService.requestOn(createDataverseRequest(user), df).has(Permission.DownloadFile)) { + if (permissionService.requestOn(createDataverseRequest(user), df).has(Permission.DownloadFile)) { // FIXME: Indent this line more. if (published) { logger.log(Level.FINE, "API token-based auth: User {0} has rights to access the datafile.", user.getIdentifier()); return true; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 81db7f9dec1..defc2c4d9ab 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -22,7 +22,6 @@ import edu.harvard.iq.dataverse.datasetutility.OptionalFileParams; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; -import edu.harvard.iq.dataverse.engine.command.impl.DeleteMapLayerMetadataCommand; import edu.harvard.iq.dataverse.engine.command.impl.GetDataFileCommand; import edu.harvard.iq.dataverse.engine.command.impl.GetDraftFileMetadataIfAvailableCommand; import edu.harvard.iq.dataverse.engine.command.impl.RedetectFileTypeCommand; @@ -468,31 +467,7 @@ public Response getFileMetadata(@PathParam("id") String fileIdOrPersistentId, @P public Response getFileMetadataDraft(@PathParam("id") String fileIdOrPersistentId, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response, Boolean getDraft) throws WrappedResponse, Exception { return getFileMetadata(fileIdOrPersistentId, versionId, uriInfo, headers, response, true); } - - // TODO: Rather than only supporting looking up files by their database IDs, consider supporting persistent identifiers. - // TODO: Rename this start with "delete" rather than "get". - @DELETE - @Path("{id}/map") - public Response getMapLayerMetadatas(@PathParam("id") Long idSupplied) { - DataverseRequest dataverseRequest = null; - try { - dataverseRequest = createDataverseRequest(findUserOrDie()); - } catch (WrappedResponse wr) { - return error(BAD_REQUEST, "Couldn't find user to execute command: " + wr.getLocalizedMessage()); - } - DataFile dataFile = fileService.find(idSupplied); - try { - boolean deleted = engineSvc.submit(new DeleteMapLayerMetadataCommand(dataverseRequest, dataFile)); - if (deleted) { - return ok("Map deleted from file id " + dataFile.getId()); - } else { - return error(BAD_REQUEST, "Could not delete map from file id " + dataFile.getId()); - } - } catch (CommandException ex) { - return error(BAD_REQUEST, "Problem trying to delete map from file id " + dataFile.getId() + ": " + ex.getLocalizedMessage()); - } - } - + @Path("{id}/uningest") @POST public Response uningestDatafile(@PathParam("id") String id) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Geoconnect.java b/src/main/java/edu/harvard/iq/dataverse/api/Geoconnect.java deleted file mode 100644 index 72991e41fef..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/api/Geoconnect.java +++ /dev/null @@ -1,78 +0,0 @@ -package edu.harvard.iq.dataverse.api; - -import com.mashape.unirest.http.Unirest; -import com.mashape.unirest.http.exceptions.UnirestException; -import com.mashape.unirest.request.GetRequest; -import edu.harvard.iq.dataverse.MapLayerMetadata; -import java.sql.Timestamp; -import java.util.Date; -import java.util.logging.Logger; -import javax.json.Json; -import javax.json.JsonArrayBuilder; -import javax.json.JsonObjectBuilder; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.core.Response; -import static javax.ws.rs.core.Response.Status.NOT_FOUND; - -/** - * @todo consolidate with WorldMapRelatedData (/api/worldmap)? - */ -@Path("admin/geoconnect") -public class Geoconnect extends AbstractApiBean { - - private static final Logger logger = Logger.getLogger(Geoconnect.class.getCanonicalName()); - - @POST - @Path("mapLayerMetadatas/check") - public Response checkMapLayerMetadatas() { - JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder(); - mapLayerMetadataSrv.findAll().stream().map((unmodified) -> checkStatus(unmodified)).forEach((jsonObjectBuilder) -> { - jsonArrayBuilder.add(jsonObjectBuilder); - }); - return ok(jsonArrayBuilder); - } - - @POST - @Path("mapLayerMetadatas/check/{id}") - public Response checkMapLayerMetadatas(@PathParam("id") Long idSupplied) { - MapLayerMetadata mapLayerMetadata = mapLayerMetadataSrv.find(idSupplied); - if (mapLayerMetadata == null) { - return error(NOT_FOUND, "Could not find MapLayerMetadata based on id " + idSupplied); - } - return ok(checkStatus(mapLayerMetadata)); - } - - private JsonObjectBuilder checkStatus(MapLayerMetadata unmodified) { - JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder(); - jsonObjectBuilder.add("fileId", unmodified.getDataFile().getId()); - jsonObjectBuilder.add("mapLayerMetadataId", unmodified.getId()); - jsonObjectBuilder.add("layerLink", unmodified.getLayerLink()); - jsonObjectBuilder.add("fileLandingPage", systemConfig.getDataverseSiteUrl() + "/file.xhtml?fileId=" + unmodified.getDataFile().getId()); - MapLayerMetadata modified = updateLastVerifiedTimeAndStatusCode(unmodified); - if (modified != null) { - jsonObjectBuilder.add("lastVerifiedStatus", modified.getLastVerifiedStatus()); - } else { - jsonObjectBuilder.add("error", "Could not check status of map associate with file id " + unmodified.getDataFile().getId()); - } - return jsonObjectBuilder; - } - - private MapLayerMetadata updateLastVerifiedTimeAndStatusCode(MapLayerMetadata mapLayerMetadata) { - String layerLink = mapLayerMetadata.getLayerLink(); - GetRequest getRequest = Unirest.get(layerLink); - try { - int statusCode = getRequest.asBinary().getStatus(); - mapLayerMetadata.setLastVerifiedStatus(statusCode); - Timestamp now = new Timestamp(new Date().getTime()); - mapLayerMetadata.setLastVerifiedTime(now); - logger.fine("Setting status code to " + statusCode + " and timestamp to " + now + " for MapLayerMetadata id " + mapLayerMetadata.getId() + "."); - return mapLayerMetadataSrv.save(mapLayerMetadata); - } catch (UnirestException ex) { - logger.info("Couldn't update last verfied status code or timestamp: " + ex.getLocalizedMessage()); - return null; - } - } - -} diff --git a/src/main/java/edu/harvard/iq/dataverse/api/WorldMapRelatedData.java b/src/main/java/edu/harvard/iq/dataverse/api/WorldMapRelatedData.java deleted file mode 100755 index e9868f91cee..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/api/WorldMapRelatedData.java +++ /dev/null @@ -1,795 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package edu.harvard.iq.dataverse.api; - -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.DataFileServiceBean; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetServiceBean; -import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.Dataverse; -import edu.harvard.iq.dataverse.DataverseSession; -import edu.harvard.iq.dataverse.FileMetadata; -import edu.harvard.iq.dataverse.MapLayerMetadata; -import edu.harvard.iq.dataverse.MapLayerMetadataServiceBean; -import edu.harvard.iq.dataverse.PermissionServiceBean; -import edu.harvard.iq.dataverse.UserNotification; -import edu.harvard.iq.dataverse.worldmapauth.TokenApplicationTypeServiceBean; -import edu.harvard.iq.dataverse.UserNotificationServiceBean; -import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; -import edu.harvard.iq.dataverse.authorization.Permission; -import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; -import edu.harvard.iq.dataverse.worldmapauth.WorldMapToken; -import edu.harvard.iq.dataverse.worldmapauth.WorldMapTokenServiceBean; -import edu.harvard.iq.dataverse.util.SystemConfig; -import java.io.IOException; -import java.io.StringReader; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URLEncoder; -import java.util.logging.Level; -import java.util.logging.Logger; -import javax.ejb.EJB; -import javax.inject.Inject; -import javax.json.Json; -import javax.json.JsonObject; -import javax.json.JsonObjectBuilder; -import javax.json.stream.JsonParsingException; -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.Response; - -/** - * - * @author raprasad - * - * Bean used to communicate information to the WorldMap (currently a Django project) - * - * Within geoconnect, the information sent is validated by the DataverseInfoModel in this project: - * - * https://github.com/IQSS/shared-dataverse-information - * - * @todo Audit these methods to ensure they don't pose a security risk. Consider - * changing the installer so that like "admin" this "worldmap" endpoint is - * blocked out of the box. See - * http://guides.dataverse.org/en/4.6.1/installation/config.html#blocking-api-endpoints - */ -@Path("worldmap") -public class WorldMapRelatedData extends AbstractApiBean { - - private static final Logger logger = Logger.getLogger(WorldMapRelatedData.class.getCanonicalName()); - - - private static final String BASE_PATH = "/api/worldmap/"; - - public static final String MAP_IT_API_PATH_FRAGMENT = "map-it/"; - public static final String MAP_IT_API_PATH = BASE_PATH + MAP_IT_API_PATH_FRAGMENT; - - public static final String MAP_IT_API_TOKEN_ONLY_PATH_FRAGMENT = "map-it-token-only/"; - public static final String MAP_IT_API_TOKEN_ONLY_PATH = BASE_PATH + MAP_IT_API_TOKEN_ONLY_PATH_FRAGMENT; - - public static final String GET_WORLDMAP_DATAFILE_API_PATH_FRAGMENT = "datafile/"; - public static final String GET_WORLDMAP_DATAFILE_API_PATH = BASE_PATH + GET_WORLDMAP_DATAFILE_API_PATH_FRAGMENT; - - - public static final String UPDATE_MAP_LAYER_DATA_API_PATH_FRAGMENT = "update-layer-metadata/"; - public static final String UPDATE_MAP_LAYER_DATA_API_PATH = BASE_PATH + UPDATE_MAP_LAYER_DATA_API_PATH_FRAGMENT; - - public static final String DELETE_MAP_LAYER_DATA_API_PATH_FRAGMENT = "delete-layer-metadata/"; - public static final String DELETE_MAP_LAYER_DATA_API_PATH = BASE_PATH + DELETE_MAP_LAYER_DATA_API_PATH_FRAGMENT; - - public static final String DELETE_WORLDMAP_TOKEN_PATH_FRAGMENT = "delete-token/"; - public static final String DELETE_WORLDMAP_TOKEN_PATH = BASE_PATH + DELETE_WORLDMAP_TOKEN_PATH_FRAGMENT; - - @EJB - MapLayerMetadataServiceBean mapLayerMetadataService; - - @EJB - DatasetServiceBean datasetService; - - @EJB - DataFileServiceBean dataFileService; - - @EJB - UserNotificationServiceBean userNotificationService; - - @EJB - WorldMapTokenServiceBean tokenServiceBean; - - @EJB - TokenApplicationTypeServiceBean tokenAppServiceBean; - - @EJB - AuthenticationServiceBean dataverseUserService; - - @EJB - PermissionServiceBean permissionService; - - @EJB - SystemConfig systemConfig; - - @Inject - DataverseSession session; - - /** - * Create URL for API call to WorldMapRelatedData.mapDataFile(...) - * - * @param dataFileID - * @param dataverseUserID - * @return - */ - public static String getMapItURL(Long dataFileID, Long dataverseUserID){ - if ((dataverseUserID==null)||(dataFileID==null)){ - return null; - } - //test, see if it gets created - return WorldMapRelatedData.MAP_IT_API_PATH + dataFileID + "/" + dataverseUserID; - } - - - // test call to track down problems - // http://127.0.0.1:8080/api/worldmap/t/ - @GET - @Path("t/{identifier}") - public Response checkWorldMapAPI(@Context HttpServletRequest request - , @PathParam("identifier") int identifier) { - - MapLayerMetadata mapLayerMetadata = this.mapLayerMetadataService.find(new Long(identifier)); - logger.info("mapLayerMetadata retrieved. Try to retrieve image (after 1st time):
    " + mapLayerMetadata.getMapImageLink()); - - try { - this.mapLayerMetadataService.retrieveMapImageForIcon(mapLayerMetadata); - } catch (IOException ex) { - logger.info("IOException. Failed to retrieve image. Error:" + ex); - // Logger.getLogger(WorldMapRelatedData.class.getName()).log(Level.SEVERE, null, ex); - }catch(Exception e2){ - logger.info("Failed to retrieve image. Error:" + e2); - } - - return ok( "Looks good " + identifier + " " + mapLayerMetadata.getLayerName()); - } - - - @GET - @Path( MAP_IT_API_PATH_FRAGMENT + "{datafile_id}/{dvuser_id}") - public Response mapDataFile(@Context HttpServletRequest request - , @PathParam("datafile_id") Long datafile_id - , @PathParam("dvuser_id") Long dvuser_id){ - - return this.mapDataFileTokenOnlyOption(request, datafile_id, dvuser_id, false); - } - - @GET - @Path(MAP_IT_API_TOKEN_ONLY_PATH_FRAGMENT + "{datafile_id}/{dvuser_id}") - public Response getMapDataFileToken(@Context HttpServletRequest request - , @PathParam("datafile_id") Long datafile_id - , @PathParam("dvuser_id") Long dvuser_id){ - - //request. - return this.mapDataFileTokenOnlyOption(request, datafile_id, dvuser_id, true); - } - - - /* - Link used within Dataverse for MapIt button - Sends file link to GeoConnect using a Redirect - - */ - //@GET - //@Path( MAP_IT_API_PATH_FRAGMENT + "token-option/{datafile_id}/{dvuser_id}/{token_only}") - private Response mapDataFileTokenOnlyOption(@Context HttpServletRequest request - , Long datafile_id - , Long dvuser_id - , boolean tokenOnly - ){ - - logger.log(Level.INFO, "mapDataFile datafile_id: {0}", datafile_id); - logger.log(Level.INFO, "mapDataFile dvuser_id: {0}", dvuser_id); - - AuthenticatedUser user = null; - - if (session != null) { - if (session.getUser() != null) { - if (session.getUser().isAuthenticated()) { - user = (AuthenticatedUser) session.getUser(); - } - } - } - if (user==null){ - return error(Response.Status.FORBIDDEN, "Not logged in"); - } - - - if (true){ - //return okResponse( "Looks good " + datafile_id); - //tokenAppServiceBean.getGeoConnectApplication(); - //return okResponse("Currently deactivated (mapDataFile)"); - } - - // Check if the user exists - AuthenticatedUser dvUser = dataverseUserService.findByID(dvuser_id); - if ( dvUser == null ){ - return error(Response.Status.FORBIDDEN, "Invalid user"); - } - - // Check if this file exists - DataFile dfile = dataFileService.find(datafile_id); - if (dfile==null){ - return error(Response.Status.NOT_FOUND, "DataFile not found for id: " + datafile_id); - } - - /* - Is the dataset public? - */ - if (!dfile.getOwner().isReleased()){ - return error(Response.Status.FORBIDDEN, "Mapping is only permitted for public datasets/files"); - - } - - // Does this user have permission to edit metadata for this file? - if (!permissionService.request(createDataverseRequest(dvUser)).on(dfile.getOwner()).has(Permission.EditDataset)){ - String errMsg = "The user does not have permission to edit metadata for this file."; - return error(Response.Status.FORBIDDEN, errMsg); - } - - WorldMapToken token = tokenServiceBean.getNewToken(dfile, dvUser); - - if (tokenOnly){ - // Return only the token in a JSON object - final JsonObjectBuilder jsonInfo = Json.createObjectBuilder(); - jsonInfo.add(WorldMapToken.GEOCONNECT_TOKEN_KEY, token.getToken()); - return ok(jsonInfo); - } - - // Redirect to geoconnect url - String callback_url = systemConfig.getDataverseSiteUrl() + GET_WORLDMAP_DATAFILE_API_PATH; - String redirect_url_str = token.getApplication().getMapitLink() + "/" + token.getToken() + "/?cb=" + URLEncoder.encode(callback_url); - - URI redirect_uri; - - try { - redirect_uri = new URI(redirect_url_str); - } catch (URISyntaxException ex) { - return error(Response.Status.NOT_FOUND, "Faile to create URI from: " + redirect_url_str); - } -// Response. - return Response.seeOther(redirect_uri).build(); - - } - - @Deprecated - private String getServerNamePort(HttpServletRequest request){ - if (request == null){ - return ""; - } - String serverName = request.getServerName(); - if (serverName==null){ - return ""; - } - int portNumber = request.getServerPort(); - - String http_prefix = "https://"; - if (serverName.contains("localhost")){ - http_prefix = "http://"; - }else if(serverName.contains("127.0.0.1")){ - http_prefix = "http://"; - } - if (portNumber==80){ - return http_prefix + serverName; - } - return http_prefix + serverName + ":" + portNumber; - - } - - /** - * Parse json looking for the GEOCONNECT_TOKEN_KEY. - * Make sure that the string itself is not null and 64 chars - * - * @param jsonTokenInfo - * @return - */ - private String retrieveTokenValueFromJson(JsonObject jsonTokenInfo){ - if (jsonTokenInfo==null){ - return null; - } - logger.info("retrieveTokenValueFromJson.jsonTokenInfo:"+ jsonTokenInfo); - logger.info("token keys: " + jsonTokenInfo.keySet()); - logger.info("key looked for: " + WorldMapToken.GEOCONNECT_TOKEN_KEY); - if (!jsonTokenInfo.containsKey(WorldMapToken.GEOCONNECT_TOKEN_KEY)){ - logger.warning("Token not found. Permission denied."); - return null; - //return errorResponse( Response.Status.BAD_REQUEST, "Permission denied"); - } - - - //String worldmapTokenParam = worldmapTokenObject.toString(); - String worldmapTokenParam = jsonTokenInfo.getString(WorldMapToken.GEOCONNECT_TOKEN_KEY); - if (worldmapTokenParam==null){ // shouldn't happen - logger.warning("worldmapTokenParam is null when .toString() called. Permission denied."); - return null; - //return errorResponse(Response.Status.UNAUTHORIZED, "No access."); - } - //logger.info("worldmapTokenParam:"+ worldmapTokenParam); - - //logger.info("worldmapTokenParam length:"+ worldmapTokenParam.length()); - if (!(worldmapTokenParam.length()==64)){ - logger.warning("worldmapTokenParam not length 64. Permission denied."); - return null; - // return errorResponse(Response.Status.UNAUTHORIZED, "No access."); - } - return worldmapTokenParam; - } - - - /** - * Retrieve FileMetadata for Use by WorldMap. - * This includes information about the DataFile, Dataset, DatasetVersion, and Dataverse - * - * @param jsonTokenData - * @param request - * @return - */ - @POST - @Path(GET_WORLDMAP_DATAFILE_API_PATH_FRAGMENT)// + "{worldmap_token}") - public Response getWorldMapDatafileInfo(String jsonTokenData, @Context HttpServletRequest request){//, @PathParam("worldmap_token") String worldmapTokenParam) { - if (true){ - //return okResponse("Currently deactivated"); - - // return okResponse("remote server: " + request.getRemoteAddr()); - } - logger.info("API call: getWorldMapDatafileInfo"); - //---------------------------------- - // Auth check: Parse the json message and check for a valid GEOCONNECT_TOKEN_KEY and GEOCONNECT_TOKEN_VALUE - // -- For testing, the GEOCONNECT_TOKEN_VALUE will be dynamic, found in the db - //---------------------------------- - logger.info("(1) jsonTokenData: " + jsonTokenData); - // Parse JSON - JsonObject jsonTokenInfo; - try ( StringReader rdr = new StringReader(jsonTokenData) ) { - jsonTokenInfo = Json.createReader(rdr).readObject(); - } catch ( JsonParsingException jpe ) { - logger.log(Level.SEVERE, "Json: " + jsonTokenData); - return error( Response.Status.BAD_REQUEST, "Error parsing Json: " + jpe.getMessage() ); - } - logger.info("(1a) jsonTokenInfo: " + jsonTokenInfo); - - // Retrieve token string - String worldmapTokenParam = this.retrieveTokenValueFromJson(jsonTokenInfo); - logger.info("(1b) token from JSON: " + worldmapTokenParam); - if (worldmapTokenParam==null){ - return error(Response.Status.BAD_REQUEST, "Token not found in JSON request."); - } - - // Retrieve WorldMapToken and make sure it is valid - // - WorldMapToken wmToken = tokenServiceBean.retrieveAndRefreshValidToken(worldmapTokenParam); - logger.info("(2) token retrieved from db: " + wmToken); - - if (wmToken==null){ - return error(Response.Status.UNAUTHORIZED, "No access. Invalid token."); - } - - // Make sure the token's User still has permissions to access the file - // - logger.info("(3) check permissions"); - if (!(tokenServiceBean.canTokenUserEditFile(wmToken))){ - tokenServiceBean.expireToken(wmToken); - return error(Response.Status.UNAUTHORIZED, "No access. Invalid token."); - } - - - // (1) Retrieve token connected data: DataverseUser, DataFile - // - // Make sure token user and file are still available - // - AuthenticatedUser dvUser = wmToken.getDataverseUser(); - if (dvUser == null) { - return error(Response.Status.NOT_FOUND, "DataverseUser not found for token"); - } - DataFile dfile = wmToken.getDatafile(); - if (dfile == null) { - return error(Response.Status.NOT_FOUND, "DataFile not found for token"); - } - - - // (1a) Retrieve FileMetadata - FileMetadata dfile_meta = dfile.getFileMetadata(); - if (dfile_meta==null){ - return error(Response.Status.NOT_FOUND, "FileMetadata not found"); - } - - // (2) Now get the dataset and the latest DatasetVersion - Dataset dset = dfile.getOwner(); - if (dset==null){ - return error(Response.Status.NOT_FOUND, "Owning Dataset for this DataFile not found"); - } - - // (2a) latest DatasetVersion - // !! How do you check if the lastest version has this specific file? - // - DatasetVersion dset_version = dset.getLatestVersion(); - if (dset_version==null){ - return error(Response.Status.NOT_FOUND, "Latest DatasetVersion for this DataFile not found"); - } - - // (3) get Dataverse - Dataverse dverse = dset.getOwner(); - if (dverse==null){ - return error(Response.Status.NOT_FOUND, "Dataverse for this DataFile's Dataset not found"); - } - - // (4) Roll it all up in a JSON response - final JsonObjectBuilder jsonData = Json.createObjectBuilder(); - - //------------------------------------ - // Type of file, currently: - // - shapefile or - // - tabular file (.tab) with geospatial tag - //------------------------------------ - if (dfile.isShapefileType()){ - jsonData.add("mapping_type", "shapefile"); - }else if (dfile.isTabularData()){ - jsonData.add("mapping_type", "tabular"); - }else{ - logger.log(Level.SEVERE, "This was neither a Shapefile nor a Tabular data file. DataFile id: " + dfile.getId()); - return error( Response.Status.BAD_REQUEST, "Sorry! This file does not have mapping data. Please contact the Dataverse administrator. DataFile id: " + dfile.getId()); - } - - - //------------------------------------ - // DataverseUser Info - //------------------------------------ - jsonData.add("dv_user_id", dvUser.getId()); - jsonData.add("dv_username", dvUser.getUserIdentifier()); - jsonData.add("dv_user_email", dvUser.getEmail()); - - //------------------------------------ - // Dataverse URLs to this server - //------------------------------------ - String serverName = systemConfig.getDataverseSiteUrl(); - jsonData.add("return_to_dataverse_url", dset_version.getReturnToFilePageURL(serverName, dset, dfile)); - jsonData.add("datafile_download_url", dfile.getMapItFileDownloadURL(serverName)); - - //------------------------------------ - // Dataverse - //------------------------------------ - //jsonData.add("dataverse_installation_name", "Harvard Dataverse"); // todo / fix - jsonData.add("dataverse_installation_name", systemConfig.getDataverseSiteUrl()); // is this enough to distinguish a dataverse installation? - - jsonData.add("dataverse_id", dverse.getId()); - jsonData.add("dataverse_name", dverse.getName()); - - String dataverseDesc = dverse.getDescription(); - if (dataverseDesc == null || dataverseDesc.equalsIgnoreCase("")){ - dataverseDesc = ""; - } - jsonData.add("dataverse_description", dataverseDesc); - - //------------------------------------ - // Dataset Info - //------------------------------------ - jsonData.add("dataset_id", dset.getId()); - - //------------------------------------ - // DatasetVersion Info - //------------------------------------ - jsonData.add("dataset_version_id", dset_version.getId()); // database id - jsonData.add("dataset_semantic_version", dset_version.getSemanticVersion()); // major/minor version number, e.g. 3.1 - - jsonData.add("dataset_name", dset_version.getTitle()); - jsonData.add("dataset_citation", dset_version.getCitation(true)); - - jsonData.add("dataset_description", ""); // Need to fix to/do - - jsonData.add("dataset_is_public", dset_version.isReleased()); - - //------------------------------------ - // DataFile/FileMetaData Info - //------------------------------------ - jsonData.add("datafile_id", dfile.getId()); - jsonData.add("datafile_label", dfile_meta.getLabel()); - //jsonData.add("filename", dfile_meta.getLabel()); - jsonData.add("datafile_expected_md5_checksum", dfile.getChecksumValue()); - Long fsize = dfile.getFilesize(); - if (fsize == null){ - fsize= new Long(-1); - } - - jsonData.add("datafile_filesize", fsize); - jsonData.add("datafile_content_type", dfile.getContentType()); - jsonData.add("datafile_create_datetime", dfile.getCreateDate().toString()); - - // restriction status of the DataFile - jsonData.add("datafile_is_restricted", dfile.isRestricted()); - - return ok(jsonData); - - } - - - - /* - For WorldMap/GeoConnect Usage - Create/Updated a MapLayerMetadata object for a given Datafile id - - Example of jsonLayerData String: - { - "layerName": "geonode:boston_census_blocks_zip_cr9" - , "layerLink": "http://localhost:8000/data/geonode:boston_census_blocks_zip_cr9" - , "embedMapLink": "http://localhost:8000/maps/embed/?layer=geonode:boston_census_blocks_zip_cr9" - , "worldmapUsername": "dv_pete" - } - */ - @POST - @Path(UPDATE_MAP_LAYER_DATA_API_PATH_FRAGMENT) - public Response updateWorldMapLayerData(String jsonLayerData){ - - //---------------------------------- - // Auth check: Parse the json message and check for a valid GEOCONNECT_TOKEN_KEY and GEOCONNECT_TOKEN_VALUE - // -- For testing, the GEOCONNECT_TOKEN_VALUE will be dynamic, found in the db - //---------------------------------- - if (jsonLayerData==null){ - logger.log(Level.SEVERE, "jsonLayerData is null"); - return error( Response.Status.BAD_REQUEST, "No JSON data"); - } - - // (1) Parse JSON - // - JsonObject jsonInfo; - try ( StringReader rdr = new StringReader(jsonLayerData) ) { - jsonInfo = Json.createReader(rdr).readObject(); - } catch ( JsonParsingException jpe ) { - logger.log(Level.SEVERE, "Json: " + jsonLayerData); - return error( Response.Status.BAD_REQUEST, "Error parsing Json: " + jpe.getMessage() ); - } - - // Retrieve token string - String worldmapTokenParam = this.retrieveTokenValueFromJson(jsonInfo); - if (worldmapTokenParam==null){ - return error(Response.Status.BAD_REQUEST, "Token not found in JSON request."); - } - - // Retrieve WorldMapToken and make sure it is valid - // - WorldMapToken wmToken = this.tokenServiceBean.retrieveAndRefreshValidToken(worldmapTokenParam); - if (wmToken==null){ - return error(Response.Status.UNAUTHORIZED, "No access. Invalid token."); - } - - // Make sure the token's User still has permissions to access the file - // - if (!(tokenServiceBean.canTokenUserEditFile(wmToken))){ - tokenServiceBean.expireToken(wmToken); - return error(Response.Status.UNAUTHORIZED, "No access. Invalid token."); - } - - - // (2) Make sure the json message has all of the required attributes - // - for (String attr : MapLayerMetadata.MANDATORY_JSON_FIELDS ){ - if (!jsonInfo.containsKey(attr)){ - return error( Response.Status.BAD_REQUEST, "Error parsing Json. Key not found [" + attr + "]\nRequired keys are: " + MapLayerMetadata.MANDATORY_JSON_FIELDS ); - } - } - - // (3) Attempt to retrieve DataverseUser - AuthenticatedUser dvUser = wmToken.getDataverseUser(); - if (dvUser == null) { - return error(Response.Status.NOT_FOUND, "DataverseUser not found for token"); - } - - // (4) Attempt to retrieve DataFile - DataFile dfile = wmToken.getDatafile(); - if (dfile==null){ - return error(Response.Status.NOT_FOUND, "DataFile not found for token"); - } - - // check permissions! - if (!permissionService.request( createDataverseRequest(dvUser) ).on(dfile.getOwner()).has(Permission.EditDataset)){ - String errMsg = "The user does not have permission to edit metadata for this file. (MapLayerMetadata)"; - return error(Response.Status.FORBIDDEN, errMsg); - } - - - // (5) See if a MapLayerMetadata already exists - // - MapLayerMetadata mapLayerMetadata = this.mapLayerMetadataService.findMetadataByDatafile(dfile); - if (mapLayerMetadata==null){ - // Create a new mapLayerMetadata object - mapLayerMetadata = new MapLayerMetadata(); - } - - // Create/Update new MapLayerMetadata object and save it - mapLayerMetadata.setDataFile(dfile); - mapLayerMetadata.setDataset(dfile.getOwner()); - mapLayerMetadata.setLayerName(jsonInfo.getString("layerName")); - mapLayerMetadata.setLayerLink(jsonInfo.getString("layerLink")); - mapLayerMetadata.setEmbedMapLink(jsonInfo.getString("embedMapLink")); - mapLayerMetadata.setWorldmapUsername(jsonInfo.getString("worldmapUsername")); - if (jsonInfo.containsKey("mapImageLink")){ - mapLayerMetadata.setMapImageLink(jsonInfo.getString("mapImageLink")); - } - - // If this was a tabular join set the attributes: - // - isJoinLayer - // - joinDescription - // - String joinDescription = jsonInfo.getString("joinDescription", null); - if ((joinDescription == null) || (joinDescription.equals(""))){ - mapLayerMetadata.setIsJoinLayer(true); - mapLayerMetadata.setJoinDescription(joinDescription); - }else{ - mapLayerMetadata.setIsJoinLayer(false); - mapLayerMetadata.setJoinDescription(null); - } - - // Set the mapLayerLinks - // - String mapLayerLinks = jsonInfo.getString("mapLayerLinks", null); - if ((mapLayerLinks == null) || (mapLayerLinks.equals(""))){ - mapLayerMetadata.setMapLayerLinks(null); - }else{ - mapLayerMetadata.setMapLayerLinks(mapLayerLinks); - } - - - //mapLayer.save(); - MapLayerMetadata savedMapLayerMetadata = mapLayerMetadataService.save(mapLayerMetadata); - if (savedMapLayerMetadata==null){ - logger.log(Level.SEVERE, "Json: " + jsonLayerData); - return error( Response.Status.BAD_REQUEST, "Failed to save map layer! Original JSON: "); - } - - - - // notify user - userNotificationService.sendNotification(dvUser, wmToken.getCurrentTimestamp(), UserNotification.Type.MAPLAYERUPDATED, dfile.getOwner().getLatestVersion().getId()); - - // ------------------------------------------ - // Retrieve a PNG representation from WorldMap - // ------------------------------------------ - try { - logger.info("retrieveMapImageForIcon"); - this.mapLayerMetadataService.retrieveMapImageForIcon(savedMapLayerMetadata); - } catch (IOException ex) { - logger.severe("Failed to retrieve image from WorldMap server"); - Logger.getLogger(WorldMapRelatedData.class.getName()).log(Level.SEVERE, null, ex); - } - - return ok("map layer object saved!"); - - } // end updateWorldMapLayerData - - - - - /* - For WorldMap/GeoConnect Usage - Delete MayLayerMetadata object for a given Datafile - - POST params - { - "GEOCONNECT_TOKEN": "-- some 64 char token which contains a link to the DataFile --" - } - */ - @POST - @Path(DELETE_MAP_LAYER_DATA_API_PATH_FRAGMENT) - public Response deleteWorldMapLayerData(String jsonData){ - - /*---------------------------------- - Parse the json message. - - Auth check: GEOCONNECT_TOKEN - //----------------------------------*/ - if (jsonData==null){ - logger.log(Level.SEVERE, "jsonData is null"); - return error( Response.Status.BAD_REQUEST, "No JSON data"); - } - // (1) Parse JSON - // - JsonObject jsonInfo; - try ( StringReader rdr = new StringReader(jsonData) ) { - jsonInfo = Json.createReader(rdr).readObject(); - } catch ( JsonParsingException jpe ) { - logger.log(Level.SEVERE, "Json: " + jsonData); - return error( Response.Status.BAD_REQUEST, "Error parsing Json: " + jpe.getMessage() ); - } - - // (2) Retrieve token string - String worldmapTokenParam = this.retrieveTokenValueFromJson(jsonInfo); - if (worldmapTokenParam==null){ - return error(Response.Status.BAD_REQUEST, "Token not found in JSON request."); - } - - // (3) Retrieve WorldMapToken and make sure it is valid - // - WorldMapToken wmToken = this.tokenServiceBean.retrieveAndRefreshValidToken(worldmapTokenParam); - if (wmToken==null){ - return error(Response.Status.UNAUTHORIZED, "No access. Invalid token."); - } - - // (4) Make sure the token's User still has permissions to access the file - // - if (!(tokenServiceBean.canTokenUserEditFile(wmToken))){ - tokenServiceBean.expireToken(wmToken); - return error(Response.Status.UNAUTHORIZED, "No access. Invalid token."); - } - - // (5) Attempt to retrieve DataFile and mapLayerMetadata - DataFile dfile = wmToken.getDatafile(); - MapLayerMetadata mapLayerMetadata = this.mapLayerMetadataService.findMetadataByDatafile(dfile); - if (mapLayerMetadata==null){ - return error(Response.Status.EXPECTATION_FAILED, "No map layer metadata found."); - } - - - // (6) Delete the mapLayerMetadata - // (note: permissions checked here for a second time by the mapLayerMetadataService call) - // - if (!(this.mapLayerMetadataService.deleteMapLayerMetadataObject(mapLayerMetadata, wmToken.getDataverseUser()))){ - return error(Response.Status.PRECONDITION_FAILED, "Failed to delete layer"); - }; - - - return ok("Map layer metadata deleted."); - - } // end deleteWorldMapLayerData - - /* - For WorldMap/GeoConnect Usage - Explicitly expire a WorldMap token, removing token from the database - - POST params - { - "GEOCONNECT_TOKEN": "-- some 64 char token which contains a link to the DataFile --" - } - */ - @POST - @Path(DELETE_WORLDMAP_TOKEN_PATH_FRAGMENT) - public Response deleteWorldMapToken(String jsonData){ - - /*---------------------------------- - Parse the json message. - - Auth check: GEOCONNECT_TOKEN - //----------------------------------*/ - if (jsonData==null){ - logger.log(Level.SEVERE, "jsonData is null"); - return error( Response.Status.BAD_REQUEST, "No JSON data"); - } - // (1) Parse JSON - // - JsonObject jsonInfo; - try ( StringReader rdr = new StringReader(jsonData) ) { - jsonInfo = Json.createReader(rdr).readObject(); - } catch ( JsonParsingException jpe ) { - logger.log(Level.SEVERE, "Json: " + jsonData); - return error( Response.Status.BAD_REQUEST, "Error parsing Json: " + jpe.getMessage() ); - } - - // (2) Retrieve token string - String worldmapTokenParam = this.retrieveTokenValueFromJson(jsonInfo); - if (worldmapTokenParam==null){ - return error(Response.Status.BAD_REQUEST, "Token not found in JSON request."); - } - - // (3) Retrieve WorldMapToken - // - WorldMapToken wmToken = this.tokenServiceBean.findByName(worldmapTokenParam); - if (wmToken==null){ - return error(Response.Status.NOT_FOUND, "Token not found."); - } - - // (4) Delete the token - // - tokenServiceBean.deleteToken(wmToken); - return ok("Token has been deleted."); - - } // end deleteWorldMapLayerData - -} // end class diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java index e50d38131c9..c7ffc3f75d9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java @@ -482,7 +482,6 @@ public void displayNotification() { userNotification.setTheObject(datasetService.find(userNotification.getObjectId())); break; - case MAPLAYERUPDATED: case CREATEDS: case SUBMITTEDDS: case PUBLISHEDDS: @@ -491,10 +490,6 @@ public void displayNotification() { userNotification.setTheObject(datasetVersionService.find(userNotification.getObjectId())); break; - case MAPLAYERDELETEFAILED: - userNotification.setTheObject(fileService.findFileMetadata(userNotification.getObjectId())); - break; - case CREATEACC: userNotification.setTheObject(userNotification.getUser()); break; diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java index 175e5743e93..12161eb1a59 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java @@ -12,7 +12,6 @@ import edu.harvard.iq.dataverse.util.BundleUtil; import static edu.harvard.iq.dataverse.util.StringUtil.nonEmpty; import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; -import edu.harvard.iq.dataverse.worldmapauth.WorldMapToken; import java.io.Serializable; import java.sql.Timestamp; import java.util.List; @@ -126,18 +125,7 @@ public class AuthenticatedUser implements User, Serializable { public String getIdentifier() { return IDENTIFIER_PREFIX + userIdentifier; } - - @OneToMany(mappedBy = "dataverseUser", cascade={CascadeType.REMOVE}) - private List worldMapTokens; - - public List getWorldMapTokens() { - return worldMapTokens; - } - public void setWorldMapTokens(List worldMapTokens) { - this.worldMapTokens = worldMapTokens; - } - @OneToMany(mappedBy = "user", cascade={CascadeType.REMOVE}) private List notifications; diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java index ec18f23a5a0..6a69c1f9d77 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java @@ -58,7 +58,6 @@ */ public class ImageThumbConverter { public static String THUMBNAIL_SUFFIX = "thumb"; - public static String WORLDMAP_IMAGE_SUFFIX = "img"; public static String THUMBNAIL_MIME_TYPE = "image/png"; public static int DEFAULT_CARDIMAGE_SIZE = 48; @@ -118,8 +117,6 @@ private static boolean isThumbnailAvailable(StorageIO storageIO, int s return generateImageThumbnail(storageIO, size); } else if (file.getContentType().equalsIgnoreCase("application/pdf")) { return generatePDFThumbnail(storageIO, size); - } else if (file.getContentType().equalsIgnoreCase("application/zipped-shapefile") || (file.isTabularData() && file.hasGeospatialTag())) { - return generateWorldMapThumbnail(storageIO, size); } return false; @@ -127,7 +124,7 @@ private static boolean isThumbnailAvailable(StorageIO storageIO, int s } // Note that this method works on ALL file types for which thumbnail - // generation is supported - image/*, pdf, worldmap and geo-tagged tabular; + // generation is supported - image/*, pdf; // not just on images! The type differentiation is handled inside // isThumbnailAvailable(); if the thumbnail is not yet cached, that // method will attempt to generate and cache it. And once it's cached, @@ -283,52 +280,6 @@ private static boolean generateImageThumbnail(StorageIO storageIO, int } - /* - * Note that the "WorldMapThumbnail" generator does the exact same thing as the - * "regular image" thumbnail generator. - * The only difference is that the image generator uses the main file as - * as the source; and the one for the worldmap uses an auxiliary file - * with the ".img" extension (or the swift, etc. equivalent). This file is - * produced and dropped into the Dataset directory (Swift container, etc.) - * the first time the user actually runs WorldMap on the main file. - * Also note that it works the exact same way for tabular-mapped-as-worldmap - * files as well. - */ - private static boolean generateWorldMapThumbnail(StorageIO storageIO, int size) { - - InputStream worldMapImageInputStream = null; - - try { - storageIO.open(); - - Channel worldMapImageChannel = storageIO.openAuxChannel(WORLDMAP_IMAGE_SUFFIX); - if (worldMapImageChannel == null) { - logger.warning("Could not open channel for aux ." + WORLDMAP_IMAGE_SUFFIX + " object; (" + size + ")"); - return false; - } - worldMapImageInputStream = Channels.newInputStream((ReadableByteChannel) worldMapImageChannel); - - long worldMapImageSize = storageIO.getAuxObjectSize(WORLDMAP_IMAGE_SUFFIX); - - if (isImageOverSizeLimit(worldMapImageSize)) { - logger.fine("WorldMap image too large - skipping"); - worldMapImageInputStream.close(); - return false; - } - return generateImageThumbnailFromInputStream(storageIO, size, worldMapImageInputStream); - } catch (FileNotFoundException fnfe) { - logger.fine("No .img file for this worldmap file yet; giving up. Original Error: " + fnfe); - return false; - - } catch (IOException ioex) { - logger.warning("caught IOException trying to open an input stream for worldmap .img file (" + storageIO.getDataFile().getStorageIdentifier() + "). Original Error: " + ioex); - return false; - } finally { - IOUtils.closeQuietly(worldMapImageInputStream); - } - - } - /* * This is the actual workhorse method that does the rescaling of the full * size image: @@ -489,8 +440,6 @@ public static String getImageThumbnailAsBase64(DataFile file, int size) { generated = generateImageThumbnail(storageIO, size); } else if (file.getContentType().equalsIgnoreCase("application/pdf")) { generated = generatePDFThumbnail(storageIO, size); - } else if (file.getContentType().equalsIgnoreCase("application/zipped-shapefile") || (file.isTabularData() && file.hasGeospatialTag())) { - generated = generateWorldMapThumbnail(storageIO, size); } if (generated) { diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/WorldMapPermissionHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/WorldMapPermissionHelper.java deleted file mode 100644 index 73c0af1d2db..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/WorldMapPermissionHelper.java +++ /dev/null @@ -1,636 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package edu.harvard.iq.dataverse.datasetutility; - -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DataverseSession; -import edu.harvard.iq.dataverse.DvObject; -import edu.harvard.iq.dataverse.FileMetadata; -import edu.harvard.iq.dataverse.MapLayerMetadata; -import edu.harvard.iq.dataverse.MapLayerMetadataServiceBean; -import edu.harvard.iq.dataverse.PermissionServiceBean; -import edu.harvard.iq.dataverse.SettingsWrapper; -import edu.harvard.iq.dataverse.authorization.Permission; -import edu.harvard.iq.dataverse.authorization.users.GuestUser; -import edu.harvard.iq.dataverse.authorization.users.User; -import edu.harvard.iq.dataverse.settings.SettingsServiceBean; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import javax.faces.view.ViewScoped; -import javax.inject.Inject; -import javax.inject.Named; - -/** - * This class originally encapsulated display logic for the DatasetPage - * - * It allows the following checks without redundantly querying the db to - * check permissions or if MapLayerMetadata exists - * - * - canUserSeeMapDataButton (private) - * - canUserSeeMapDataButtonFromPage (public) - * - canUserSeeMapDataButtonFromAPI (public) - * - * - canSeeMapButtonReminderToPublish (private) - * - canSeeMapButtonReminderToPublishFromPage (public) - * - canSeeMapButtonReminderToPublishFromAPI (public) - * - * - canUserSeeExploreWorldMapButton (private) - * - canUserSeeExploreWorldMapButtonFromPage (public) - * - canUserSeeExploreWorldMapButtonFromAPI (public) - * - * @author rmp553 - */ -@ViewScoped -@Named -public class WorldMapPermissionHelper implements java.io.Serializable { - - @Inject SettingsWrapper settingsWrapper; - @Inject MapLayerMetadataServiceBean mapLayerMetadataService; - @Inject PermissionServiceBean permissionService; - @Inject DataverseSession session; - - - private final Map fileMetadataWorldMapExplore = new HashMap<>(); // { FileMetadata.id : Boolean } - private Map mapLayerMetadataLookup = null; - private final Map datasetPermissionMap = new HashMap<>(); // { Permission human_name : Boolean } - - - public WorldMapPermissionHelper( ){ - - } - - - /** - * Using a DataFile id, retrieve an associated MapLayerMetadata object - * - * The MapLayerMetadata objects have been fetched at page inception by - * "loadMapLayerMetadataLookup()" - */ - public MapLayerMetadata getMapLayerMetadata(DataFile df) { - if (df == null) { - return null; - } - if (mapLayerMetadataLookup == null){ - loadMapLayerMetadataLookup(df.getOwner()); - } - return this.mapLayerMetadataLookup.get(df.getId()); - } - - - /* - * Call this when using the API - * - calls private method canUserSeeExploreWorldMapButton - */ - public boolean canUserSeeExploreWorldMapButtonFromAPI(FileMetadata fm, User user){ - - if (fm == null){ - return false; - } - if (user==null){ - return false; - } - if (!this.permissionService.userOn(user, fm.getDataFile()).has(Permission.DownloadFile)){ - return false; - } - - return this.canUserSeeExploreWorldMapButton(fm, true); - } - - /** - * Call this from a Dataset or File page - * - calls private method canUserSeeExploreWorldMapButton - * - * WARNING: Before calling this, make sure the user has download - * permission for the file!! (See DatasetPage.canDownloadFile()) - * - * @param FileMetadata fm - * @return boolean - */ - public boolean canUserSeeExploreWorldMapButtonFromPage(FileMetadata fm){ - - if (fm==null){ - return false; - } - return this.canUserSeeExploreWorldMapButton(fm, true); - } - - /** - * WARNING: Before calling this, make sure the user has download - * permission for the file!! (See DatasetPage.canDownloadFile()) - * - * Should there be a Explore WorldMap Button for this file? - * See table in: https://github.com/IQSS/dataverse/issues/1618 - * - * (1) Does the file have MapLayerMetadata? - * (2) Are the proper settings in place - * - * @param fm FileMetadata - * @return boolean - */ - private boolean canUserSeeExploreWorldMapButton(FileMetadata fm, boolean permissionsChecked){ - if (fm==null){ - return false; - } - // This is only here to make the public method users think... - if (!permissionsChecked){ - return false; - } - if (this.fileMetadataWorldMapExplore.containsKey(fm.getId())){ - // Yes, return previous answer - //logger.info("using cached result for candownloadfile on filemetadata "+fid); - return this.fileMetadataWorldMapExplore.get(fm.getId()); - } - - /* ----------------------------------------------------- - Does a Map Exist? - ----------------------------------------------------- */ - if (!(this.hasMapLayerMetadata(fm))) { - //See if it does - MapLayerMetadata layer_metadata = mapLayerMetadataService.findMetadataByDatafile(fm.getDataFile()); - if (layer_metadata != null) { - if (mapLayerMetadataLookup == null) { - loadMapLayerMetadataLookup(fm.getDataFile().getOwner()); - } - // yes: keep going... - mapLayerMetadataLookup.put(layer_metadata.getDataFile().getId(), layer_metadata); - } else { - // Nope: no button - this.fileMetadataWorldMapExplore.put(fm.getId(), false); - return false; - } - } - - /* - Is setting for GeoconnectViewMaps true? - Nope? no button - */ - if (!settingsWrapper.isTrueForKey(SettingsServiceBean.Key.GeoconnectViewMaps, false)){ - this.fileMetadataWorldMapExplore.put(fm.getId(), false); - return false; - } - //---------------------------------------------------------------------- - //(0) Before we give it to you - if version is deaccessioned and user - // does not have edit dataset permission then may download - //---------------------------------------------------------------------- - - if (fm.getDatasetVersion().isDeaccessioned()) { - if (this.doesSessionUserHavePermission( Permission.EditDataset, fm)) { - // Yes, save answer and return true - this.fileMetadataWorldMapExplore.put(fm.getId(), true); - return true; - } else { - this.fileMetadataWorldMapExplore.put(fm.getId(), false); - return false; - } - } - //Check for restrictions - - boolean isRestrictedFile = fm.isRestricted(); - - // -------------------------------------------------------------------- - // Is the file Unrestricted ? - // -------------------------------------------------------------------- - if (!isRestrictedFile){ - // Yes, save answer and return true - this.fileMetadataWorldMapExplore.put(fm.getId(), true); - return true; - } - - // -------------------------------------------------------------------- - // Conditions (2) through (4) are for Restricted files - // -------------------------------------------------------------------- - - - if (session.getUser() instanceof GuestUser){ - this.fileMetadataWorldMapExplore.put(fm.getId(), false); - return false; - } - - - // -------------------------------------------------------------------- - // (3) Does the User have DownloadFile Permission at the **Dataset** level - // -------------------------------------------------------------------- - - - if (!this.doesSessionUserHavePermission(Permission.DownloadFile, fm)){ - // Yes, save answer and return true - this.fileMetadataWorldMapExplore.put(fm.getId(), false); - return false; - } - - /* ----------------------------------------------------- - Yes: User can view button! - ----------------------------------------------------- */ - this.fileMetadataWorldMapExplore.put(fm.getId(), true); - return true; - } - - - /* - Check if the FileMetadata.dataFile has an associated MapLayerMetadata object - - The MapLayerMetadata objects have been fetched at page inception by "loadMapLayerMetadataLookup()" - */ - public boolean hasMapLayerMetadata(FileMetadata fm) { - if (fm == null) { - return false; - } - if (fm.getDataFile() == null) { - return false; - } - if (mapLayerMetadataLookup == null) { - loadMapLayerMetadataLookup(fm.getDataFile().getOwner()); - } - return doesDataFileHaveMapLayerMetadata(fm.getDataFile()); - } - - /** - * Check if a DataFile has an associated MapLayerMetadata object - * - * The MapLayerMetadata objects have been fetched at page inception by - * "loadMapLayerMetadataLookup()" - */ - private boolean doesDataFileHaveMapLayerMetadata(DataFile df) { - if (df == null) { - return false; - } - if (df.getId() == null) { - return false; - } - return this.mapLayerMetadataLookup.containsKey(df.getId()); - } - - - /** - * Create a hashmap consisting of { DataFile.id : MapLayerMetadata object} - * - * Very few DataFiles will have associated MapLayerMetadata objects so only - * use 1 query to get them - */ - private void loadMapLayerMetadataLookup(Dataset dataset) { - mapLayerMetadataLookup = new HashMap<>(); - if (dataset == null) { - } - if (dataset.getId() == null) { - return; - } - List mapLayerMetadataList = mapLayerMetadataService.getMapLayerMetadataForDataset(dataset); - if (mapLayerMetadataList == null) { - return; - } - for (MapLayerMetadata layer_metadata : mapLayerMetadataList) { - mapLayerMetadataLookup.put(layer_metadata.getDataFile().getId(), layer_metadata); - } - - }// A DataFile may have a related MapLayerMetadata object - - - /** - * Check if this is a mappable file type. - * - * Currently (2/2016) - * - Shapefile (zipped shapefile) - * - Tabular file with Geospatial Data tag - * - * @param fm - * @return - */ - private boolean isPotentiallyMappableFileType(FileMetadata fm){ - if (fm==null){ - return false; - } - - // Yes, it's a shapefile - // - if (this.isShapefileType(fm)){ - return true; - } - - // Yes, it's tabular with a geospatial tag - // - if (fm.getDataFile().isTabularData()){ - if (fm.getDataFile().hasGeospatialTag()){ - return true; - } - } - return false; - } - - - - public boolean isShapefileType(FileMetadata fm) { - if (fm == null) { - return false; - } - if (fm.getDataFile() == null) { - return false; - } - - return fm.getDataFile().isShapefileType(); - } - - - /** - * Call this from a Dataset or File page - * - calls private method canSeeMapButtonReminderToPublish - * - * WARNING: Assumes user isAuthenicated AND has Permission.EditDataset - * - These checks should be made on the DatasetPage or FilePage which calls this method - * - * - * @param FileMetadata fm - * @return boolean - */ - public boolean canSeeMapButtonReminderToPublishFromPage(FileMetadata fm){ - if (fm == null){ - return false; - } - - if (mapLayerMetadataLookup == null){ - loadMapLayerMetadataLookup(fm.getDatasetVersion().getDataset()); - } - - return this.canSeeMapButtonReminderToPublish(fm, true); - - } - - - /** - * Call this when using the API - * - calls private method canSeeMapButtonReminderToPublish - * - * @param fm - * @param user - * @return - */ - public boolean canSeeMapButtonReminderToPublishFromAPI(FileMetadata fm, User user){ - if (fm == null){ - return false; - } - if (user==null){ - return false; - } - - if (!this.permissionService.userOn(user, fm.getDataFile().getOwner()).has(Permission.EditDataset)){ - return false; - } - - return this.canSeeMapButtonReminderToPublish(fm, true); - - } - - - - /** - * Assumes permissions have been checked!! - * - * See table in: https://github.com/IQSS/dataverse/issues/1618 - * - * Can the user see a reminder to publish button? - * (1) Is the view GeoconnectViewMaps - * (2) Is this file a Shapefile or a Tabular file tagged as Geospatial? - * (3) Is this DataFile released? Yes, don't need reminder - * (4) Does a map already exist? Yes, don't need reminder - */ - private boolean canSeeMapButtonReminderToPublish(FileMetadata fm, boolean permissionsChecked){ - if (fm==null){ - return false; - } - - // Is this user authenticated with EditDataset permission? - // - if (!(isUserAuthenticatedWithEditDatasetPermission(fm))){ - return false; - } - - // This is only here as a reminder to the public method users - if (!permissionsChecked){ - return false; - } - - // (1) Is the view GeoconnectViewMaps - if (!settingsWrapper.isTrueForKey(SettingsServiceBean.Key.GeoconnectCreateEditMaps, false)){ - return false; - } - - - // (2) Is this file a Shapefile or a Tabular file tagged as Geospatial? - // - if (!(this.isPotentiallyMappableFileType(fm))){ - return false; - } - - // (3) Is this DataFile released? Yes, don't need reminder - // - if (fm.getDataFile().isReleased()){ - return false; - } - - // (4) Does a map already exist? Yes, don't need reminder - // - if (this.hasMapLayerMetadata(fm)){ - return false; - } - - // Looks good - // - return true; - } - - /** - * - * WARNING: Assumes user isAuthenicated AND has Permission.EditDataset - * - These checks are made on the DatasetPage which calls this method - * - */ - public boolean canUserSeeMapDataButtonFromPage(FileMetadata fm){ - - if (fm==null){ - return false; - } - - // Is this user authenticated with EditDataset permission? - // - if (!(isUserAuthenticatedWithEditDatasetPermission(fm))){ - return false; - } - if (mapLayerMetadataLookup == null){ - loadMapLayerMetadataLookup(fm.getDatasetVersion().getDataset()); - } - return this.canUserSeeMapDataButton(fm, true); - } - - - - /** - * Call this when using the API - * - calls private method canUserSeeMapDataButton - * - * @param fm - * @param user - * @return - */ - public boolean canUserSeeMapDataButtonFromAPI(FileMetadata fm, User user){ - if (fm == null){ - return false; - } - if (user==null){ - return false; - } - - if (!this.permissionService.userOn(user, fm.getDataFile().getOwner()).has(Permission.EditDataset)){ - return false; - } - - return this.canUserSeeMapDataButton(fm, true); - - } - - /** - * - * WARNING: Assumes user isAuthenicated AND has Permission.EditDataset - * - These checks are made on the DatasetPage which calls this method - * - * Should there be a Map Data Button for this file? - * see table in: https://github.com/IQSS/dataverse/issues/1618 - * (1) Is the user logged in? - * (2) Is this file a Shapefile or a Tabular file tagged as Geospatial? - * (3) Does the logged in user have permission to edit the Dataset to which this FileMetadata belongs? - * (4) Is the create Edit Maps flag set to true? - * (5) Any of these conditions: - * 9a) File Published - * (b) Draft: File Previously published - * @param fm FileMetadata - * @return boolean - */ - private boolean canUserSeeMapDataButton(FileMetadata fm, boolean permissionsChecked){ - if (fm==null){ - return false; - } - - // This is only here as a reminder to the public method users - if (!permissionsChecked){ - - return false; - } - - // (1) Is this file a Shapefile or a Tabular file tagged as Geospatial? - // TO DO: EXPAND FOR TABULAR FILES TAGGED AS GEOSPATIAL! - // - if (!(this.isPotentiallyMappableFileType(fm))){ - - return false; - } - - - // (2) Is the view GeoconnectViewMaps - if (!settingsWrapper.isTrueForKey(SettingsServiceBean.Key.GeoconnectCreateEditMaps, false)){ - - return false; - } - - - // (3) Is File restricted? if Yes - no button. - if (fm.isRestricted() || fm.getDataFile().isRestricted()){ - - return false; - } - - - - // (4) Is File released? - // - if (fm.getDataFile().isReleased()){ - - return true; - } - - // Nope - return false; - } - - private boolean isUserAuthenticatedWithEditDatasetPermission( FileMetadata fm){ - - // Is the user authenticated? - // - if (!(isSessionUserAuthenticated())){ - return false; - } - - // If so, can the logged in user edit the Dataset to which this FileMetadata belongs? - // - if (!this.doesSessionUserHavePermission(Permission.EditDataset, fm)){ - return false; - } - - return true; - } - - public boolean isSessionUserAuthenticated() { - - - if (session == null) { - return false; - } - - if (session.getUser() == null) { - return false; - } - - return session.getUser().isAuthenticated(); - - } - - private boolean doesSessionUserHavePermission(Permission permissionToCheck, FileMetadata fileMetadata){ - if (permissionToCheck == null){ - return false; - } - - DvObject objectToCheck = null; - - if (permissionToCheck.equals(Permission.EditDataset)){ - objectToCheck = fileMetadata.getDatasetVersion().getDataset(); - } else if (permissionToCheck.equals(Permission.DownloadFile)){ - objectToCheck = fileMetadata.getDataFile(); - } - - if (objectToCheck == null){ - return false; - } - - if (this.session.getUser() == null){ - return false; - } - - if (this.permissionService == null){ - return false; - } - - String permName = permissionToCheck.getHumanName(); - - // Has this check already been done? - // - if (this.datasetPermissionMap.containsKey(permName)){ - // Yes, return previous answer - return this.datasetPermissionMap.get(permName); - } - - // Check the permission - // - - boolean hasPermission = this.permissionService.userOn(this.session.getUser(), objectToCheck).has(permissionToCheck); - - // Save the permission - this.datasetPermissionMap.put(permName, hasPermission); - - // return true/false - return hasPermission; - } - - -} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java index cf0d6e781b0..8e555d5f7a2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java @@ -18,7 +18,6 @@ import edu.harvard.iq.dataverse.FileDownloadServiceBean; import edu.harvard.iq.dataverse.GuestbookResponseServiceBean; import edu.harvard.iq.dataverse.GuestbookServiceBean; -import edu.harvard.iq.dataverse.MapLayerMetadataServiceBean; import edu.harvard.iq.dataverse.search.IndexServiceBean; import edu.harvard.iq.dataverse.PermissionServiceBean; import edu.harvard.iq.dataverse.RoleAssigneeServiceBean; @@ -134,8 +133,6 @@ public interface CommandContext { public WorkflowServiceBean workflows(); - public MapLayerMetadataServiceBean mapLayerMetadata(); - public DataCaptureModuleServiceBean dataCaptureModule(); public FileDownloadServiceBean fileDownload(); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataFileCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataFileCommand.java index bc2ce4286f7..3a7bfb984d3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataFileCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataFileCommand.java @@ -213,31 +213,7 @@ public FileVisitResult postVisitDirectory(final Path dir, final IOException e) } catch (Exception e) { logger.log(Level.WARNING, "Identifier deletion was not successfull:", e.getMessage()); } - - // If there is a Map Layer associated with this file, we may need to - // try and remove the layer data on the WorldMap side. - if (ctxt.mapLayerMetadata().findMetadataByDatafile(doomed) != null) { - // (We need an AuthenticatedUser in order to produce a WorldMap token!) - String id = getUser().getIdentifier(); - id = id.startsWith("@") ? id.substring(1) : id; - AuthenticatedUser authenticatedUser = ctxt.authentication().getAuthenticatedUser(id); - try { - ctxt.mapLayerMetadata().deleteMapLayerFromWorldMap(doomed, authenticatedUser); - // We have the dedicatd command DeleteMapLayerMetadataCommand, but - // there's no need to use it explicitly, since the Dataverse-side - // MapLayerMetadata entity will be deleted by the database - // cascade on the DataFile. -- L.A. Apr. 2020 - - } catch (Exception ex) { - // We are not going to treat it as a fatal condition and bail out, - // but we will send a notification to the user, warning them - // there may still be some data associated with the mapped layer, - // on the WorldMap side, un-deleted: - ctxt.notifications().sendNotification(authenticatedUser, new Timestamp(new Date().getTime()), UserNotification.Type.MAPLAYERDELETEFAILED, doomed.getFileMetadata().getId()); - } - - } DataFile doomedAndMerged = ctxt.em().merge(doomed); ctxt.em().remove(doomedAndMerged); /** diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteMapLayerMetadataCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteMapLayerMetadataCommand.java deleted file mode 100644 index e4fd4c18031..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteMapLayerMetadataCommand.java +++ /dev/null @@ -1,40 +0,0 @@ -package edu.harvard.iq.dataverse.engine.command.impl; - -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.MapLayerMetadata; -import edu.harvard.iq.dataverse.authorization.Permission; -import edu.harvard.iq.dataverse.engine.command.AbstractCommand; -import edu.harvard.iq.dataverse.engine.command.CommandContext; -import edu.harvard.iq.dataverse.engine.command.DataverseRequest; -import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; -import edu.harvard.iq.dataverse.engine.command.exception.CommandException; -import java.util.logging.Logger; - -@RequiredPermissions(Permission.EditDataset) -public class DeleteMapLayerMetadataCommand extends AbstractCommand { - - private static final Logger logger = Logger.getLogger(DeleteMapLayerMetadataCommand.class.getCanonicalName()); - - private final DataFile dataFile; - - public DeleteMapLayerMetadataCommand(DataverseRequest aRequest, DataFile datafile) { - super(aRequest, datafile); - this.dataFile = datafile; - } - - @Override - public Boolean execute(CommandContext ctxt) throws CommandException { - if (dataFile == null) { - return false; - } - MapLayerMetadata mapLayerMetadata = ctxt.mapLayerMetadata().findMetadataByDatafile(dataFile); - if (mapLayerMetadata == null) { - return false; - } - boolean mapDeleted = ctxt.mapLayerMetadata().deleteMapLayerMetadataObject(mapLayerMetadata, getUser()); - logger.info("Boolean returned from deleteMapLayerMetadataObject: " + mapDeleted); - return mapDeleted; - - } - -} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java index c062070f566..7a32f515d36 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java @@ -408,45 +408,11 @@ private void updateFiles(Timestamp updateTime, CommandContext ctxt) throws Comma if (dataFile.isRestricted()) { - // A couple things need to happen if the file has been restricted: - // 1. If there's a map layer associated with this shape file, or - // tabular-with-geo-tag file, all that map layer data (that - // includes most of the actual data in the file!) need to be - // removed from WorldMap and GeoConnect, since anyone can get - // download the data from there; - // 2. If this (image) file has been assigned as the dedicated + // If the file has been restricted: + // If this (image) file has been assigned as the dedicated // thumbnail for the dataset, we need to remove that assignment, // now that the file is restricted. - - // Map layer: - - if (ctxt.mapLayerMetadata().findMetadataByDatafile(dataFile) != null) { - // (We need an AuthenticatedUser in order to produce a WorldMap token!) - String id = getUser().getIdentifier(); - id = id.startsWith("@") ? id.substring(1) : id; - AuthenticatedUser authenticatedUser = ctxt.authentication().getAuthenticatedUser(id); - try { - ctxt.mapLayerMetadata().deleteMapLayerFromWorldMap(dataFile, authenticatedUser); - - // If that was successful, delete the layer on the Dataverse side as well: - //SEK 4/20/2017 - //Command to delete from Dataverse side - ctxt.engine().submit(new DeleteMapLayerMetadataCommand(this.getRequest(), dataFile)); - - // RP - Bit of hack, update the datafile here b/c the reference to the datafile - // is not being passed all the way up/down the chain. - // - dataFile.setPreviewImageAvailable(false); - - } catch (IOException ioex) { - // We are not going to treat it as a fatal condition and bail out, - // but we will send a notification to the user, warning them about - // the layer still being out there, un-deleted: - ctxt.notifications().sendNotification(authenticatedUser, getTimestamp(), UserNotification.Type.MAPLAYERDELETEFAILED, dataFile.getFileMetadata().getId()); - } - - } - + // Dataset thumbnail assignment: if (dataFile.equals(getDataset().getThumbnailFile())) { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UningestFileCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UningestFileCommand.java index 5690e58c3cc..29180f65e36 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UningestFileCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UningestFileCommand.java @@ -10,7 +10,6 @@ import edu.harvard.iq.dataverse.DataTable; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.FileMetadata; -import edu.harvard.iq.dataverse.MapLayerMetadata; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.dataaccess.DataAccess; @@ -165,19 +164,6 @@ protected void executeImpl(CommandContext ctxt) throws CommandException { ctxt.datasetVersion().fixMissingUnf(dv.getId().toString(), true); } - MapLayerMetadata mapLayerMetadata = ctxt.mapLayerMetadata().findMetadataByDatafile(uningest); - if (mapLayerMetadata != null) { - try { - String id = getUser().getIdentifier(); - id = id.startsWith("@") ? id.substring(1) : id; - AuthenticatedUser authenticatedUser = ctxt.authentication().getAuthenticatedUser(id); - ctxt.mapLayerMetadata().deleteMapLayerFromWorldMap(uningest, authenticatedUser); - } catch (Exception e) { - logger.warning("Unable to delete WorldMap file - may not have existed. Data File id: " + uningest.getId()); - } - ctxt.mapLayerMetadata().deleteMapLayerMetadataObject(mapLayerMetadata, getUser()); - } - try{ dataAccess.deleteAllAuxObjects(); } catch (IOException e){ diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java index 0ae8c9c496d..c996e332bdb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java @@ -97,17 +97,6 @@ public class ExternalTool implements Serializable { @Column(nullable = true, columnDefinition = "TEXT") private String contentType; - @Transient - private boolean worldMapTool; - - public boolean isWorldMapTool() { - return worldMapTool; - } - - public void setWorldMapTool(boolean worldMapTool) { - this.worldMapTool = worldMapTool; - } - /** * This default constructor is only here to prevent this error at * deployment: diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index e292ee39722..1f53281b2fa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -228,20 +228,6 @@ public enum Key { SPSS/sav format, "RData" for R, etc. for example: :TabularIngestSizeLimit:RData */ TabularIngestSizeLimit, - /** - Whether to allow user to create GeoConnect Maps - This boolean effects whether the user sees the map button on - the dataset page and if the ingest will create a shape file - Default is false - */ - GeoconnectCreateEditMaps, - /** - Whether to allow a user to view existing maps - This boolean effects whether a user may see the - Explore World Map Button - Default is false; - */ - GeoconnectViewMaps, /** The message added to a popup upon dataset publish * diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java index 88f0b9a61c5..37667d16b55 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java @@ -44,10 +44,6 @@ public static String getSubjectTextBasedOnNotification(UserNotification userNoti return BundleUtil.getStringFromBundle("notification.email.grant.file.access.subject", rootDvNameAsList); case REJECTFILEACCESS: return BundleUtil.getStringFromBundle("notification.email.rejected.file.access.subject", rootDvNameAsList); - case MAPLAYERUPDATED: - return BundleUtil.getStringFromBundle("notification.email.update.maplayer", rootDvNameAsList); - case MAPLAYERDELETEFAILED: - return BundleUtil.getStringFromBundle("notification.email.maplayer.deletefailed.subject", rootDvNameAsList); case CREATEDS: return BundleUtil.getStringFromBundle("notification.email.create.dataset.subject", rootDvNameAsList); case SUBMITTEDDS: diff --git a/src/main/java/edu/harvard/iq/dataverse/worldmapauth/TokenApplicationType.java b/src/main/java/edu/harvard/iq/dataverse/worldmapauth/TokenApplicationType.java deleted file mode 100644 index 2530f9b1285..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/worldmapauth/TokenApplicationType.java +++ /dev/null @@ -1,268 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ - -package edu.harvard.iq.dataverse.worldmapauth; - -import java.sql.Timestamp; -import java.util.Date; -import java.util.logging.Logger; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Index; -import javax.persistence.Table; - -/** - * - * @author raprasad - */ -@Entity -@Table(name="worldmapauth_tokentype", indexes = {@Index(name = "application_name", columnList="name", unique = true)}) -public class TokenApplicationType implements java.io.Serializable { - - private static final Logger logger = Logger.getLogger(TokenApplicationType.class.getCanonicalName()); - public static final String DEFAULT_GEOCONNECT_APPLICATION_NAME = "GEOCONNECT"; - public static final int DEFAULT_TOKEN_TIME_LIMIT_MINUTES = 30; - public static final String LOCAL_DEV_MAPIT_LINK = "http://127.0.0.1:8070/shapefile/map-it"; // local GeoConnect - public static final String DEV_MAPIT_LINK = "http://geoconnect.datascience.iq.harvard.edu/shapefile/map-it"; // dev GeoConnect - - - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private Long id; - - @Column(nullable=false) - private String name; - - private String contactEmail; - - @Column(nullable=true) - private String hostname; - - @Column(nullable=true) - private String ipAddress; - - - @Column(nullable=false) - private String mapitLink; - - //api_permissions = models.ManyToManyField(APIPermission, blank=True, null=True) - @Column(nullable=false, name="timeLimitMinutes", columnDefinition="int default 30") - private int timeLimitMinutes;// = models.IntegerField(default=30, help_text='in minutes') - - @Column(nullable=false, name="timeLimitSeconds", columnDefinition="bigint default 1800") - private long timeLimitSeconds; - - @Column(nullable=false) - private String md5; - - @Column(nullable=false) - private Timestamp created; - - @Column(nullable=false) - private Timestamp modified; - - public TokenApplicationType(){ - - this.name = DEFAULT_GEOCONNECT_APPLICATION_NAME; - this.timeLimitMinutes = DEFAULT_TOKEN_TIME_LIMIT_MINUTES; - this.setCreated(); - this.setModified(); - } - - /** - * Get property id. - * @return Long, value of property id. - */ - public Long getId() { - return this.id; - } - - /** - * Set property id. - * @param id new value of property id. - */ - public void setId(Long id) { - this.id = id; - } - - - /** - * Get property name. - * @return String, value of property name. - */ - public String getName() { - return this.name; - } - - /** - * Set property name. - * @param name new value of property name. - */ - public void setName(String name) { - this.name = name; - } - - - /** - * Get property contactEmail. - * @return String, value of property contactEmail. - */ - public String getContactEmail() { - return this.contactEmail; - } - - /** - * Set property contactEmail. - * @param contactEmail new value of property contactEmail. - */ - public void setContactEmail(String contactEmail) { - this.contactEmail = contactEmail; - } - - - /** - * Get property hostname. - * @return String, value of property hostname. - */ - public String getHostname() { - return this.hostname; - } - - /** - * Set property hostname. - * @param hostname new value of property hostname. - */ - public void setHostname(String hostname) { - this.hostname = hostname; - } - - - /** - * Get property ipAddress. - * @return String, value of property ipAddress. - */ - public String getIpAddress() { - return this.ipAddress; - } - - /** - * Set property ipAddress. - * @param ipAddress new value of property ipAddress. - */ - public void setIpAddress(String ipAddress) { - this.ipAddress = ipAddress; - } - - - /** - * Get property mapitLink. - * @return String, value of property mapitLink. - */ - public String getMapitLink() { - return this.mapitLink; - } - - /** - * Set property mapitLink. - * @param mapitLink new value of property mapitLink. - */ - public void setMapitLink(String mapitLink) { - this.mapitLink = mapitLink; - } - - - /** - * Get property timeLimitMinutes. - * @return int, value of property timeLimitMinutes. - */ - public int getTimeLimitMinutes() { - return this.timeLimitMinutes; - } - - /** - * Set property timeLimitMinutes. - * @param timeLimitMinutes new value of property timeLimitMinutes. Also sets timeLimitSeconds - */ - public void setTimeLimitMinutes(int timeLimitMinutes) { - if (timeLimitMinutes < 1){ - logger.warning("Cannot set Token time limit to less than 1 minute"); - return; - } - this.timeLimitMinutes = timeLimitMinutes; - this.timeLimitSeconds = timeLimitMinutes * 60; - } - - - /** - * Get property timeLimitSeconds. - * @return long, value of property timeLimitSeconds. - */ - public long getTimeLimitSeconds() { - return this.timeLimitSeconds; - } - - - /** - * Get property md5. - * @return String, value of property md5. - */ - public String getMd5() { - return this.md5; - } - - /** - * Set property md5. - * @param md5 new value of property md5. - */ - public void setMd5(String md5) { - this.md5 = md5; - } - - - /** - * Get property modified. - * @return Timestamp, value of property modified. - */ - public Timestamp getModified() { - return this.modified; - } - - /** - * Set property modified. - */ - public void setModified() { - this.modified = new Timestamp(new Date().getTime()); - } - - - /** - * Get property created. - * @return Timestamp, value of property created. - */ - public Timestamp getCreated() { - return this.created; - } - - /** - * Set property created. - */ - public void setCreated() { - this.created = new Timestamp(new Date().getTime()); - } - - - @Override - public String toString() { - return "edu.harvard.iq.dataverse.worldmapauth.TokenApplicationType[id=" + this.id + "]"; - - //return "WorldMap Layer: " + this.layerName + " for DataFile: " + this.dataFile.toString(); - } - -} diff --git a/src/main/java/edu/harvard/iq/dataverse/worldmapauth/TokenApplicationTypeServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/worldmapauth/TokenApplicationTypeServiceBean.java deleted file mode 100644 index 24c2f0c1c1e..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/worldmapauth/TokenApplicationTypeServiceBean.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ - -package edu.harvard.iq.dataverse.worldmapauth; - -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.List; -import java.util.logging.Logger; -import javax.ejb.Stateless; -import javax.inject.Named; -import javax.persistence.EntityManager; -import javax.persistence.NoResultException; -import javax.persistence.PersistenceContext; -import javax.persistence.TypedQuery; - -/** - * - * @author raprasad - */ -@Stateless -@Named -public class TokenApplicationTypeServiceBean { - - private static final Logger logger = Logger.getLogger(TokenApplicationTypeServiceBean.class.getCanonicalName()); - - @PersistenceContext(unitName = "VDCNet-ejbPU") - private EntityManager em; - - public TokenApplicationType getGeoConnectApplication(){ - logger.info("--getGeoConnectApplication--"); - TokenApplicationType tat = this.findByName(TokenApplicationType.DEFAULT_GEOCONNECT_APPLICATION_NAME); - if (tat != null){ - logger.info("-- Got it!!"); - return tat; - } - // Make a default application for GeoConnect - tat = new TokenApplicationType(); - tat.setName(TokenApplicationType.DEFAULT_GEOCONNECT_APPLICATION_NAME); - tat.setContactEmail("info@iq.harvard.edu"); - tat.setHostname("geoconnect.datascience.iq.harvard.edu"); - tat.setIpAddress("140.247.115.127"); - tat.setTimeLimitMinutes(TokenApplicationType.DEFAULT_TOKEN_TIME_LIMIT_MINUTES); - //tat.setMapitLink(TokenApplicationType.LOCAL_DEV_MAPIT_LINK); - tat.setMapitLink(TokenApplicationType.DEV_MAPIT_LINK); - - return this.save(tat); - - //return null; - } - public TokenApplicationType find(Object pk) { - if (pk==null){ - return null; - } - return em.find(TokenApplicationType.class, pk); - } - - /** - * - * Convert string to md5 hash - * - import hashlib - m = hashlib.md5() - m.update("Give me python or give me...more time, more time -- c.mena") - m.hexdigest() #'266cf94160a22fe1ef118c907379cd60' - * @param stringToHash - * @return - */ - public String getMD5Hash(String stringToHash){ - if (stringToHash==null){ - return null; - } - MessageDigest md; - try { - md = MessageDigest.getInstance("MD5"); - } catch (NoSuchAlgorithmException ex) { - logger.severe("Failed to set TokenApplicationType for 'Map It' request!!!"); - return null; - } - md.update(stringToHash.getBytes()); - - byte[] mdbytes = md.digest(); - StringBuilder sb = new StringBuilder(""); - for (int i = 0; i < mdbytes.length; i++) { - sb.append(Integer.toString((mdbytes[i] & 0xff) + 0x100, 16).substring(1)); - } - return sb.toString(); - - - } - - public TokenApplicationType save( TokenApplicationType tokenApp ) { - - if (tokenApp==null){ - return null; - } - - if (tokenApp.getName()==null){ - tokenApp.setName(TokenApplicationType.DEFAULT_GEOCONNECT_APPLICATION_NAME); - } - - if (tokenApp.getMapitLink()==null){ - logger.warning("mapitLink is missing for tokenApp"); - return null; - } - - // Set time limit minutes - Integer time_limit_minutes = tokenApp.getTimeLimitMinutes(); - if (time_limit_minutes == null){ - tokenApp.setTimeLimitMinutes(TokenApplicationType.DEFAULT_TOKEN_TIME_LIMIT_MINUTES); - // (also sets the time limit seconds) - } - - // set md5 - tokenApp.setMd5(this.getMD5Hash(tokenApp.getName())); - - if ( tokenApp.getId() == null ) { - tokenApp.setCreated(); - em.persist(tokenApp); - logger.fine("New tokenApp saved"); - return tokenApp; - } else { - tokenApp.setModified(); - logger.fine("Existing tokenApp saved"); - return em.merge( tokenApp ); - } - } - - - public TokenApplicationType findByName(String name){ - if (name == null){ - return null; - } - try{ - return em.createQuery("select m from TokenApplicationType m WHERE m.name=:name", TokenApplicationType.class) - .setParameter("name", name) - .getSingleResult(); - } catch ( NoResultException nre ) { - return null; - } - } - - - public List getAllTokenApplicationTypes(){ - String qr = "select object(o) from TokenApplicationType order by o.modified desc"; - TypedQuery query = em.createQuery(qr, TokenApplicationType.class); - return query.getResultList(); - } - -} \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/worldmapauth/WorldMapToken.java b/src/main/java/edu/harvard/iq/dataverse/worldmapauth/WorldMapToken.java deleted file mode 100644 index ccae1d1a920..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/worldmapauth/WorldMapToken.java +++ /dev/null @@ -1,349 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ - -package edu.harvard.iq.dataverse.worldmapauth; - -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.sql.Timestamp; -import java.util.Date; -import java.util.Random; -import java.util.logging.Logger; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Index; -import javax.persistence.JoinColumn; -import javax.persistence.ManyToOne; -import javax.persistence.Table; -import javax.persistence.Transient; - -/** - * - * @author raprasad - */ -@Entity -@Table(name="worldmapauth_token" - , indexes = {@Index(name = "token_value", columnList="token", unique = true) - , @Index(columnList="application_id") - , @Index(columnList="datafile_id") - , @Index(columnList="dataverseuser_id") - }) -public class WorldMapToken implements java.io.Serializable { - - @Transient - public static final String GEOCONNECT_TOKEN_KEY = "GEOCONNECT_TOKEN"; - @Transient - public static final long MAX_HOURS_TOKEN_CAN_BE_USED = 10; - - private static final Logger logger = Logger.getLogger(WorldMapToken.class.getCanonicalName()); - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private Long id; - - @Column(nullable=true) - private String token; // save it to get an id, then set token - - @ManyToOne - @JoinColumn(nullable=false) - private TokenApplicationType application; - - @ManyToOne - @JoinColumn(nullable=false) - private AuthenticatedUser dataverseUser; - - @ManyToOne - @JoinColumn(nullable=false) - private DataFile dataFile; - - - @Column(nullable=false) - private boolean hasExpired; - - @Column(nullable=false) - private Timestamp lastRefreshTime; - - @Column(nullable=false) - private Timestamp modified; - - @Column(nullable=false) - private Timestamp created; - - public WorldMapToken(){ - this.setLastRefreshTime(this.getCurrentTimestamp()); - this.setCreated(); - this.setModified(); - } - /** - * Get property token. - * @return String, value of property token. - */ - public String getToken() { - return this.token; - } - - public Long getId(){ - return this.id; - } - /** - * Set property token. - * @param token new value of property token. - */ - public boolean setToken(){ - if (this.token != null){ - return false; - } - if ((this.dataFile==null)||(this.dataverseUser==null)){ - return false; - } - MessageDigest md; - try { - md = MessageDigest.getInstance("SHA-256"); - } catch (NoSuchAlgorithmException ex) { - logger.severe("Failed to set token for 'Map It' request!!!"); - return false; - } - md.update(this.getCurrentTimestamp().toString().getBytes()); - md.update(this.dataFile.toString().getBytes()); - md.update(this.dataverseUser.toString().getBytes()); - Random rand = new Random(); - Integer rnd_int = rand.nextInt(); - md.update(rnd_int.byteValue()); - - byte[] mdbytes = md.digest(); - StringBuilder sb = new StringBuilder(""); - for (int i = 0; i < mdbytes.length; i++) { - sb.append(Integer.toString((mdbytes[i] & 0xff) + 0x100, 16).substring(1)); - } - this.token = sb.toString(); - //this.token = md.toString(); - return true; - } - - - /** - * Get property application. - * @return TokenApplicationType, value of property application. - */ - public TokenApplicationType getApplication() { - return this.application; - } - - /** - * Set property application. - * @param application new value of property application. - */ - public void setApplication(TokenApplicationType application) { - this.application = application; - } - - - /** - * Get property dataverseUser. - * @return DataverseUser, value of property dataverseUser. - */ - public AuthenticatedUser getDataverseUser() { - return this.dataverseUser; - } - - /** - * Set property dataverseUser. - * @param dataverseUser new value of property dataverseUser. - */ - public void setDataverseUser(AuthenticatedUser dataverseUser) { - this.dataverseUser = dataverseUser; - } - - - /** - * Get property datafile. - * @return DataFile, value of property datafile. - */ - public DataFile getDatafile() { - return this.dataFile; - } - - /** - * Set property datafile. - * @param datafile new value of property datafile. - */ - public void setDatafile(DataFile dataFile) { - this.dataFile = dataFile; - } - - - /** - * Get property hasExpired. - * @return boolean, value of property hasExpired. - */ - public boolean getHasExpired() { - return this.hasExpired; - } - - /** - * Set property hasExpired. - * @param hasExpired new value of property hasExpired. - */ - public void setHasExpired(boolean hasExpired) { - this.hasExpired = hasExpired; - } - - - /** - * Get property lastRefreshTime. - * @return Timestamp, value of property lastRefreshTime. - */ - public Timestamp getLastRefreshTime() { - return this.lastRefreshTime; - } - - /** - * Set property lastRefreshTime. - * @param lastRefreshTime new value of property lastRefreshTime. - */ - public void setLastRefreshTime(Timestamp lastRefreshTime) { - this.lastRefreshTime = this.getCurrentTimestamp(); - } - - - /** - * Get property modified. - * @return Timestamp, value of property modified. - */ - public Timestamp getModified() { - return this.modified; - } - - /** - * Set property modified. - */ - public void setModified() { - this.modified = this.getCurrentTimestamp(); - } - - - /** - * Get property created. - * @return Timestamp, value of property created. - */ - public Timestamp getCreated() { - return this.created; - } - - - /** - * Set property created. - */ - public void setCreated() { - this.created = this.getCurrentTimestamp(); - } - - public boolean hasTokenExpired(){ - logger.fine("hasTokenExpired"); - // long currentTime = this.getCurrentTimestamp();//new Date().getTime(); - return this.hasTokenExpired(this.getCurrentTimestamp()); - } - - - private long getElapsedHours(Timestamp currentTime, Timestamp oldTime){ - - // If values are null, send back an elapsed time - if ((currentTime==null)||(oldTime==null)){ - return this.application.getTimeLimitSeconds() + 100000; - } - - long milliseconds1 = oldTime.getTime(); - long milliseconds2 = currentTime.getTime(); - - long diff = milliseconds2 - milliseconds1; - //long diffSeconds = diff / 1000; - //long diffMinutes = diff / (60 * 1000); - long diffHours = diff / (60 * 60 * 1000); - //long diffDays = diff / (24 * 60 * 60 * 1000); - - return diffHours; - } - - private long getElapsedSeconds(Timestamp currentTime, Timestamp oldTime){ - - // If values are null, send back an elapsed time - if ((currentTime==null)||(oldTime==null)){ - return this.application.getTimeLimitSeconds() + 100000; - } - - long milliseconds1 = oldTime.getTime(); - long milliseconds2 = currentTime.getTime(); - - long diff = milliseconds2 - milliseconds1; - long diffSeconds = diff / 1000; - //long diffMinutes = diff / (60 * 1000); - //long diffHours = diff / (60 * 60 * 1000); - //long diffDays = diff / (24 * 60 * 60 * 1000); - - return diffSeconds; - } - - public boolean hasTokenExpired(Timestamp currentTimestamp){ - logger.fine("hasTokenExpired (w/timestamp)"); - - if (this.getHasExpired()){ - return true; - } - - if ((currentTimestamp==null)||(this.lastRefreshTime==null)){ - this.expireToken(); - return true; - } - logger.fine("currentTimestamp: " + currentTimestamp); - logger.fine("lastRefreshTime: " + lastRefreshTime); - - //System.out.println(" ..pre diff: "+ currentTimestamp); - long hours = this.getElapsedHours(currentTimestamp, this.created); - logger.fine("elapsed hours: " + hours); - - if (hours > MAX_HOURS_TOKEN_CAN_BE_USED){ - this.expireToken(); - return true; - } - - long diffSeconds = this.getElapsedSeconds(currentTimestamp, this.lastRefreshTime); - //System.out.println(" ..diffSeconds: "+ diffSeconds); - //logger.fine("this.application.getTimeLimitSeconds: "+ this.application.getTimeLimitSeconds()); - if (diffSeconds > this.application.getTimeLimitSeconds()){ - this.expireToken(); - return true; - } - return false; - - } - - public void expireToken(){ - this.setHasExpired(true); - } - - - public Timestamp getCurrentTimestamp(){ - return new Timestamp(new Date().getTime()); - } - - public boolean refreshToken(){ - - if (this.getHasExpired()){ - return false; - } - Timestamp currentTime = this.getCurrentTimestamp(); - if (this.hasTokenExpired(currentTime)){ - return false; - } - this.setLastRefreshTime(currentTime); - return true; - } -} diff --git a/src/main/java/edu/harvard/iq/dataverse/worldmapauth/WorldMapTokenServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/worldmapauth/WorldMapTokenServiceBean.java deleted file mode 100644 index 7ca5c62b1f3..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/worldmapauth/WorldMapTokenServiceBean.java +++ /dev/null @@ -1,314 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ - -package edu.harvard.iq.dataverse.worldmapauth; - -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.PermissionServiceBean; -import edu.harvard.iq.dataverse.authorization.Permission; -import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; -import javax.ejb.EJB; -import javax.ejb.Stateless; -import javax.ejb.TransactionAttribute; -import static javax.ejb.TransactionAttributeType.REQUIRES_NEW; -import javax.inject.Named; -import javax.persistence.EntityManager; -import javax.persistence.NoResultException; -import javax.persistence.PersistenceContext; -import javax.persistence.TypedQuery; -import javax.ws.rs.WebApplicationException; -import javax.ws.rs.core.Response; - -/** - * - * @author raprasad - */ -@Stateless -@Named -public class WorldMapTokenServiceBean { - - @EJB - PermissionServiceBean permissionService; - - @EJB - TokenApplicationTypeServiceBean tokenApplicationService; - - private static final Logger logger = Logger.getLogger(TokenApplicationTypeServiceBean.class.getCanonicalName()); - - @PersistenceContext(unitName = "VDCNet-ejbPU") - private EntityManager em; - - /** - * - * Retrieve a fresh token to map a Dataverse file on GeoConnect - * - * @param dataFile - * @param dvUser - * @return WorldMapToken - */ - @TransactionAttribute(REQUIRES_NEW) - public WorldMapToken getNewToken(DataFile dataFile, AuthenticatedUser dvUser){ - if ((dataFile==null)||(dvUser==null)){ - logger.severe("dataFile or dvUser is null"); - return null; - } - WorldMapToken token = new WorldMapToken(); - token.setApplication(tokenApplicationService.getGeoConnectApplication()); - token.setDatafile(dataFile); - token.setDataverseUser(dvUser); - token.setModified(); - token.setToken(); - return this.save(token); - } - - public WorldMapToken find(Object pk) { - if (pk==null){ - return null; - } - return em.find(WorldMapToken.class, pk); - } - - - /** - * Expire the token, set the "hasExpired" flag to True - * - * @param wmToken - */ - public void expireToken(WorldMapToken wmToken){ - if (wmToken==null){ - return; - } - wmToken.expireToken(); - em.merge(wmToken); - } - - /** - * Expire and then Delete the token - * (The expire is a bit extraneous) - * - * @param wmToken - */ - public void deleteToken(WorldMapToken wmToken){ - - if (wmToken==null){ - return; - } - em.remove(em.merge(wmToken)); - } - - /* - Remove expired tokens from the database - */ - public void deleteExpiredTokens(){ - - TypedQuery query = em.createQuery("select object(w) from WorldMapToken as w where w.hasExpired IS TRUE", WorldMapToken.class);// order by o.name"); - List tokenList = query.getResultList(); - for (WorldMapToken wmToken : tokenList) { - // em.remove(token); - em.remove(em.merge(wmToken)); - } - } - - public WorldMapToken save( WorldMapToken dvToken ) { - - if (dvToken==null){ - return null; - } - - if (dvToken.getToken()==null){ - dvToken.setToken(); - } - - if (dvToken.getApplication().getMapitLink()==null){ - logger.warning("mapitLink is missing for token_application"); - return null; - } - - TokenApplicationType dat = dvToken.getApplication(); - if (dat==null){ - logger.warning("TokenApplicationType is null for WorldMapToken"); - return null; - } - - if ( dvToken.getId()== null ) { - em.persist(dvToken); - logger.fine("New token_application saved"); - return dvToken; - } else { - logger.fine("Existing token_application saved"); - return em.merge( dvToken ); - } - } - - - public WorldMapToken findByName(String token){ - if (token == null){ - return null; - } - - try{ - return em.createQuery("select m from WorldMapToken m WHERE m.token=:token", WorldMapToken.class) - .setParameter("token", token) - .getSingleResult(); - } catch ( NoResultException nre ) { - return null; - } - } - - - - /** - * Given a string token, retrieve the related WorldMapToken object - * - * @param worldmapTokenParam - * @return WorldMapToken object (if it hasn't expired) - */ - public WorldMapToken retrieveAndRefreshValidToken(String worldmapTokenParam){ - if (worldmapTokenParam==null){ - logger.warning("worldmapTokenParam is null. Permission denied."); - return null; - } - WorldMapToken wmToken = this.findByName(worldmapTokenParam); - if (wmToken==null){ - logger.warning("WorldMapToken not found for '" + worldmapTokenParam + "'. Permission denied."); - return null; - } - if (wmToken.hasTokenExpired()){ - em.remove(em.merge(wmToken)); // Delete expired token from the database. - logger.warning("WorldMapToken has expired. Permission denied."); - return null; - } - wmToken.refreshToken(); - - logger.info("WorldMapToken refreshed."); - this.save(wmToken); - - return wmToken; - } - - - /* - Can the user connected to the WorldMapToken still - edit the dataset for the file connected to the token? - */ - public boolean canTokenUserEditFile(WorldMapToken wmToken){ - if (wmToken==null){ - return false; - } - if (permissionService.userOn(wmToken.getDataverseUser(), wmToken.getDatafile()).has(Permission.EditDataset)) { - logger.info("WorldMap token-based auth: Token's User is still authorized to edit the dataset for the datafile."); - return true; - } - return false; - - } - - /* - Can the user connected to the WorldMapToken still - download the file connected to the token? - */ - private boolean canTokenUserDownloadFile(WorldMapToken wmToken){ - if (wmToken==null){ - return false; - } - if (permissionService.userOn(wmToken.getDataverseUser(), wmToken.getDatafile()).has(Permission.DownloadFile)) { - logger.info("WorldMap token-based auth: Token's User is still authorized to download the datafile."); - return true; - } - return false; - - } - - - /* - Given a string for a WorldMapToken and DataFile, check: - - (1) Is this token valid? - (2) Does the token correspond to this DataFile? - - @return boolean if token is valid and corresponds to the given DataFile - - */ - public boolean isWorldMapTokenAuthorizedForDataFileDownload(String worldmapTokenParam, DataFile df){ - logger.info("-- isWorldMapTokenAuthorizedForDataFileworldmapTokenParam " + worldmapTokenParam); - - if ((worldmapTokenParam == null)||(df == null)){ - logger.info("nope: worldmapTokenParam or data file is null"); - return false; - } - - // Check 1: Is this a valid WorldMap token? - // - WorldMapToken token = this.retrieveAndRefreshValidToken(worldmapTokenParam); - if (token==null){ - logger.info("WorldMap token-based auth: Token is not invalid."); - return false; - } - - - // Check 2: Does this WorldMap token's datafile match the requested datafile? - // - if (!(token.getDatafile().getId().equals(df.getId()))){ - logger.info("WorldMap token-based auth: Token's datafile does not match the requested datafile."); - return false; - } - - // Check 3: Does this WorldMap token's user have permission for the requested datafile? - // - if (!(this.canTokenUserDownloadFile(token))){ - logger.info("WorldMap token-based auth: Token's User is not authorized for the requested datafile."); - return false; - } - - - logger.info("WorldMap token-based auth: Token is valid for the requested datafile"); - return true; - - } - - - /* - Given a string for a WorldMapToken, check: - - (1) Is this token valid? - (2) Re-verify that can edit the Dataset connected to the token's DataFile - - @return boolean if token is valid and corresponds to the given DataFile - - */ - public boolean isWorldMapTokenAuthorizedForMetadataRetrievalAndUpdates(String worldmapTokenParam){ - logger.info("-- isWorldMapTokenAuthorizedForDataFile?"); - if (worldmapTokenParam == null){ - logger.info("nope: worldmapTokenParam or data file is null"); - return false; - } - - // Check 1: Is this a valid WorldMap token? - // - WorldMapToken token = this.retrieveAndRefreshValidToken(worldmapTokenParam); - if (token==null){ - logger.info("WorldMap token-based auth: Token is not invalid."); - return false; - } - - - - // Check 2: Does this WorldMap token's user still have edit permission for the requested datafile? - // - if (!(this.canTokenUserEditFile(token))){ - logger.info("WorldMap token-based auth: Token's User is not authorized for the requested datafile."); - return false; - } - - - logger.info("WorldMap token-based auth: Token is valid for the requested datafile"); - return true; - - } -} // end of class diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 0d6804efc37..057c6827a56 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -174,7 +174,6 @@ wasPublished=, was published in wasReturnedByReviewer=, was returned by the curator of # TODO: Confirm that "toReview" can be deleted. toReview=Don't forget to publish it or send it back to the contributor! -worldMap.added=dataset had a WorldMap layer data added to it. # Bundle file editors, please note that "notification.welcome" is used in a unit test. notification.welcome=Welcome to {0}! Get started by adding or finding data. Have questions? Check out the {1}. Want to test out Dataverse features? Use our {2}. Also, check for your welcome email to verify your address. notification.demoSite=Demo Site @@ -191,8 +190,6 @@ notification.wasPublished={0} was published in {1}. notification.publishFailedPidReg={0} in {1} could not be published due to a failure to register, or update the Global Identifier for the dataset or one of the files in it. Contact support if this continues to happen. notification.ingestCompleted=Dataset {1} ingest has successfully finished. notification.ingestCompletedWithErrors=Dataset {1} ingest has finished with errors. -notification.worldMap.added={0}, dataset had WorldMap layer data added to it. -notification.maplayer.deletefailed=Failed to delete the map layer associated with the restricted file {0} from WorldMap. Please try again, or contact WorldMap and/or Dataverse support. (Dataset: {1}) notification.generic.objectDeleted=The dataverse, dataset, or file for this notification has been deleted. notification.access.granted.dataverse=You have been granted the {0} role for {1}. notification.access.granted.dataset=You have been granted the {0} role for {1}. @@ -641,9 +638,6 @@ notification.email.create.dataset.subject={0}: Your dataset has been created notification.email.request.file.access.subject={0}: Access has been requested for a restricted file notification.email.grant.file.access.subject={0}: You have been granted access to a restricted file notification.email.rejected.file.access.subject={0}: Your request for access to a restricted file has been rejected -notification.email.update.maplayer={0}: WorldMap layer added to dataset -notification.email.maplayer.deletefailed.subject={0}: Failed to delete WorldMap layer -notification.email.maplayer.deletefailed.text=We failed to delete the WorldMap layer associated with the restricted file {0}, and any related data that may still be publicly available on the WorldMap site. Please try again, or contact WorldMap and/or Dataverse support. (Dataset: {1}) notification.email.submit.dataset.subject={0}: Your dataset has been submitted for review notification.email.publish.dataset.subject={0}: Your dataset has been published notification.email.publishFailure.dataset.subject={0}: Failed to publish your dataset @@ -669,7 +663,6 @@ notification.email.wasSubmittedForReview={0} (view at {1}) was submitted for rev notification.email.wasReturnedByReviewer={0} (view at {1}) was returned by the curator of {2} (view at {3}). notification.email.wasPublished={0} (view at {1}) was published in {2} (view at {3}). notification.email.publishFailedPidReg={0} (view at {1}) in {2} (view at {3}) could not be published due to a failure to register, or update the Global Identifier for the dataset or one of the files in it. Contact support if this continues to happen. -notification.email.worldMap.added={0} (view at {1}) had WorldMap layer data added to it. notification.email.closing=\n\nYou may contact us for support at {0}.\n\nThank you,\n{1} notification.email.assignRole=You are now {0} for the {1} "{2}" (view at {3}). notification.email.revokeRole=One of your roles for the {0} "{1}" has been revoked (view at {2}). @@ -1353,7 +1346,6 @@ dataset.compute.computeBtn=Compute dataset.compute.computeBatchListHeader=Compute Batch dataset.compute.computeBatchRestricted=This dataset contains restricted files you may not compute on because you have not been granted access. dataset.delete.error=Could not deaccession the dataset because the {0} update failed. -dataset.publish.worldMap.deleteConfirm=Please note that your data and map on WorldMap will be removed due to restricted file access changes in this dataset version which you are publishing. Do you want to continue? dataset.publish.workflow.message=Publish in Progress dataset.publish.workflow.inprogress=This dataset is locked until publication. dataset.pidRegister.workflow.inprogress=The dataset is locked while the persistent identifiers are being registered or updated, and/or the physical files are being validated. @@ -1593,11 +1585,6 @@ file.spss-porExtraLabels.title=Upload an additional text file with extra variabl file.spss-porExtraLabels.selectToAddBtn=Select File to Add file.ingestFailed.header=Upload Completed with Errors file.ingestFailed.message=Tabular data ingest failed. -file.map=Map -file.mapData=Map Data -file.mapData.worldMap=WorldMap -file.mapData.unpublished.header=Data Not Published -file.mapData.unpublished.message=In order to map your data with WorldMap, your data must be published. Please publish this dataset, then retry the Map Data feature. file.downloadBtn.format.all=All File Formats + Information file.downloadBtn.format.tab=Tab-Delimited file.downloadBtn.format.original=Original File Format ({0}) diff --git a/src/main/resources/db/migration/V5.3.0.1__7409-remove-worldmap-geoconnect.sql b/src/main/resources/db/migration/V5.3.0.1__7409-remove-worldmap-geoconnect.sql new file mode 100644 index 00000000000..c42a9e3c081 --- /dev/null +++ b/src/main/resources/db/migration/V5.3.0.1__7409-remove-worldmap-geoconnect.sql @@ -0,0 +1,6 @@ +-- psql:src/main/resources/db/migration/V5.3.0.1__7409-remove-worldmap-geoconnect.sql:5: ERROR: cannot drop table worldmapauth_tokentype because other objects depend on it +-- DETAIL: constraint fk_worldmapauth_token_application_id on table worldmapauth_token depends on table worldmapauth_tokentype +-- HINT: Use DROP ... CASCADE to drop the dependent objects too. +drop table if exists worldmapauth_tokentype cascade; +drop table if exists worldmapauth_token; +drop table if exists maplayermetadata; diff --git a/src/main/resources/db/migration/afterMigrate__1-7256-upsert-referenceData.sql b/src/main/resources/db/migration/afterMigrate__1-7256-upsert-referenceData.sql index d45c03bc03b..07e9b2c6266 100644 --- a/src/main/resources/db/migration/afterMigrate__1-7256-upsert-referenceData.sql +++ b/src/main/resources/db/migration/afterMigrate__1-7256-upsert-referenceData.sql @@ -39,13 +39,3 @@ INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname INSERT INTO guestbook (emailrequired, enabled, institutionrequired, createtime, name, namerequired, positionrequired, dataverse_id) SELECT false, true, false, now(), 'Default', false, false, null WHERE NOT EXISTS (SELECT id FROM guestbook); - --- Simple trick: WHERE NOT EXISTS (SELECT id FROM table) is only true if the table is empty. -INSERT INTO worldmapauth_tokentype - (name, created, contactemail, hostname, - ipaddress, mapitlink, - md5, modified, timelimitminutes) - SELECT 'GEOCONNECT', current_timestamp, 'support@dataverse.org', 'geoconnect.datascience.iq.harvard.edu', - '140.247.115.127', 'http://geoconnect.datascience.iq.harvard.edu/shapefile/map-it', - '38c0a931b2d582a5c43fc79405b30c22', current_timestamp, 30 - WHERE NOT EXISTS (SELECT id from worldmapauth_tokentype); diff --git a/src/main/webapp/dataverseuser.xhtml b/src/main/webapp/dataverseuser.xhtml index 76bfd9bf2b1..82e528f1124 100644 --- a/src/main/webapp/dataverseuser.xhtml +++ b/src/main/webapp/dataverseuser.xhtml @@ -189,23 +189,6 @@ - - - - - #{item.theObject.getDataset().getDisplayName()} - - - - - - - - - #{item.theObject.getDatasetVersion().getDataset().getDisplayName()} - - - diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml index da88d1ae64d..85fe60863b4 100644 --- a/src/main/webapp/file-download-button-fragment.xhtml +++ b/src/main/webapp/file-download-button-fragment.xhtml @@ -220,9 +220,8 @@ - + -
  • #{bundle['dataset.accessBtn.header.explore']}
    • @@ -246,26 +245,6 @@ - -
  • - - #{bundle['file.mapData.worldMap']} - - - #{bundle['file.mapData.worldMap']} - -
    • -     diff --git a/src/main/webapp/file-download-popup-fragment.xhtml b/src/main/webapp/file-download-popup-fragment.xhtml index 1283c3a6e3c..b9a8c6e0425 100644 --- a/src/main/webapp/file-download-popup-fragment.xhtml +++ b/src/main/webapp/file-download-popup-fragment.xhtml @@ -164,7 +164,6 @@ - - - -
    - - - - - - - - - - - - - - - - - - -
    diff --git a/src/test/java/edu/harvard/iq/dataverse/api/GeoconnectIT.java b/src/test/java/edu/harvard/iq/dataverse/api/GeoconnectIT.java deleted file mode 100644 index 2b18e3283d7..00000000000 --- a/src/test/java/edu/harvard/iq/dataverse/api/GeoconnectIT.java +++ /dev/null @@ -1,53 +0,0 @@ -package edu.harvard.iq.dataverse.api; - -import com.jayway.restassured.RestAssured; -import com.jayway.restassured.response.Response; -import static javax.ws.rs.core.Response.Status.OK; -import org.junit.BeforeClass; -import org.junit.Test; - -public class GeoconnectIT { - - @BeforeClass - public static void setUp() { - RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); - } - -// @Ignore - @Test - public void checkMapLayerMetadatas() { - Response createUser = UtilIT.createRandomUser(); - String username = UtilIT.getUsernameFromResponse(createUser); - String apiToken = UtilIT.getApiTokenFromResponse(createUser); - - Response checkMapLayerMetadatas = UtilIT.checkMapLayerMetadatas(apiToken); - checkMapLayerMetadatas.prettyPrint(); - checkMapLayerMetadatas.then().assertThat() - .statusCode(OK.getStatusCode()); - } - -// @Ignore - @Test - public void checkSingleMapLayerMetadata() { - Response createUser = UtilIT.createRandomUser(); - String username = UtilIT.getUsernameFromResponse(createUser); - String apiToken = UtilIT.getApiTokenFromResponse(createUser); - - Response checkMapLayerMetadatas = UtilIT.checkMapLayerMetadatas(1l, apiToken); - checkMapLayerMetadatas.prettyPrint(); - checkMapLayerMetadatas.then().assertThat() - .statusCode(OK.getStatusCode()); - } - -// @Ignore - @Test - public void deleteMapFromFile() { - String apiToken = "b5a144d8-b3a3-452b-9543-131360884281"; - long fileId = 2661; - Response deleteMapFromFile = UtilIT.deleteMapFromFile(fileId, apiToken); - deleteMapFromFile.prettyPrint(); - deleteMapFromFile.then().assertThat() - .statusCode(OK.getStatusCode()); - } - -} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 97afd7e060e..27b34aba5e1 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1845,40 +1845,6 @@ private static byte[] inputStreamToBytes(InputStream inputStream) { } } - static Response listMapLayerMetadatas() { - return given().get("/api/admin/geoconnect/mapLayerMetadatas"); - } - - static Response checkMapLayerMetadatas(String apiToken) { - return given() - .header(API_TOKEN_HTTP_HEADER, apiToken) - .post("/api/admin/geoconnect/mapLayerMetadatas/check"); - } - - static Response checkMapLayerMetadatas(Long mapLayerMetadataId, String apiToken) { - return given() - .header(API_TOKEN_HTTP_HEADER, apiToken) - .post("/api/admin/geoconnect/mapLayerMetadatas/check/" + mapLayerMetadataId); - } - - static Response getMapFromFile(long fileId, String apiToken) { - return given() - .header(API_TOKEN_HTTP_HEADER, apiToken) - .get("/api/files/" + fileId + "/map"); - } - - static Response checkMapFromFile(long fileId, String apiToken) { - return given() - .header(API_TOKEN_HTTP_HEADER, apiToken) - .get("/api/files/" + fileId + "/map/check"); - } - - static Response deleteMapFromFile(long fileId, String apiToken) { - return given() - .header(API_TOKEN_HTTP_HEADER, apiToken) - .delete("/api/files/" + fileId + "/map?key=" + apiToken); - } - static Response getRsyncScript(String datasetPersistentId, String apiToken) { RequestSpecification requestSpecification = given(); if (apiToken != null) { diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java b/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java index 329f8509ec6..f0ddcf4c81c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java @@ -220,11 +220,6 @@ public WorkflowServiceBean workflows() { return null; } - @Override - public MapLayerMetadataServiceBean mapLayerMetadata() { - return null; - } - @Override public DataCaptureModuleServiceBean dataCaptureModule() { return null; diff --git a/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java index 03a9f1f3a20..af0c414d356 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java @@ -71,19 +71,7 @@ public void testSubjectRejectFileAccess() { userNotification.setType(UserNotification.Type.REJECTFILEACCESS); assertEquals("LibraScholar: Your request for access to a restricted file has been rejected", MailUtil.getSubjectTextBasedOnNotification(userNotification, rootDataverseName, null)); } - - @Test - public void testSubjectMapLayerUpdated() { - userNotification.setType(UserNotification.Type.MAPLAYERUPDATED); - assertEquals("LibraScholar: WorldMap layer added to dataset", MailUtil.getSubjectTextBasedOnNotification(userNotification, rootDataverseName, null)); - } - - @Test - public void testSubjectMapLayerDeleteFailed() { - userNotification.setType(UserNotification.Type.MAPLAYERDELETEFAILED); - assertEquals("LibraScholar: Failed to delete WorldMap layer", MailUtil.getSubjectTextBasedOnNotification(userNotification, rootDataverseName, null)); - } - + @Test public void testSubjectCreateDataset() { userNotification.setType(UserNotification.Type.CREATEDS); diff --git a/src/test/java/edu/harvard/iq/dataverse/worldmapauth/WorldMapTokenServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/worldmapauth/WorldMapTokenServiceBeanTest.java deleted file mode 100644 index e2d392249d7..00000000000 --- a/src/test/java/edu/harvard/iq/dataverse/worldmapauth/WorldMapTokenServiceBeanTest.java +++ /dev/null @@ -1,76 +0,0 @@ -package edu.harvard.iq.dataverse.worldmapauth; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; - -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.PermissionServiceBean; -import edu.harvard.iq.dataverse.PermissionServiceBean.StaticPermissionQuery; -import edu.harvard.iq.dataverse.authorization.Permission; -import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; - -public class WorldMapTokenServiceBeanTest { - - private WorldMapTokenServiceBean worldMapTokenServiceBean; - - @Before - public void setUp() { - this.worldMapTokenServiceBean = new WorldMapTokenServiceBean(); - this.worldMapTokenServiceBean.permissionService = mock(PermissionServiceBean.class); - } - - @After - public void tearDown() { - this.worldMapTokenServiceBean = null; - } - - private WorldMapToken createWorldMapToken(AuthenticatedUser user, DataFile file) { - WorldMapToken worldMapToken = new WorldMapToken(); - worldMapToken.setDataverseUser(user); - worldMapToken.setDatafile(file); - - return worldMapToken; - } - - private void mockPermissionServiceCallForEditDataset(AuthenticatedUser user, DataFile file, - boolean isUserPermitted) { - StaticPermissionQuery query = mock(StaticPermissionQuery.class); - Mockito.when(this.worldMapTokenServiceBean.permissionService.userOn(user, file)).thenReturn(query); - Mockito.when(query.has(Permission.EditDataset)).thenReturn(isUserPermitted); - } - - @Test - public void testCanTokenUserEditFileWithUndefinedWorldMapToken() { - WorldMapToken worldMapToken = null; - - assertFalse(this.worldMapTokenServiceBean.canTokenUserEditFile(worldMapToken)); - } - - @Test - public void testCanTokenUserEditFileWithPermission() { - AuthenticatedUser user = new AuthenticatedUser(); - DataFile file = new DataFile(); - WorldMapToken worldMapToken = this.createWorldMapToken(user, file); - - this.mockPermissionServiceCallForEditDataset(user, file, true); - - assertTrue(this.worldMapTokenServiceBean.canTokenUserEditFile(worldMapToken)); - } - - @Test - public void testCanTokenUserEditFileWithoutPermission() { - AuthenticatedUser user = new AuthenticatedUser(); - DataFile file = new DataFile(); - WorldMapToken worldMapToken = this.createWorldMapToken(user, file); - - this.mockPermissionServiceCallForEditDataset(user, file, false); - - assertFalse(this.worldMapTokenServiceBean.canTokenUserEditFile(worldMapToken)); - } -} \ No newline at end of file diff --git a/src/test/java/edu/harvard/iq/dataverse/worldmapauth/WorldMapTokenTest.java b/src/test/java/edu/harvard/iq/dataverse/worldmapauth/WorldMapTokenTest.java deleted file mode 100644 index 94380d95651..00000000000 --- a/src/test/java/edu/harvard/iq/dataverse/worldmapauth/WorldMapTokenTest.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ - -package edu.harvard.iq.dataverse.worldmapauth; - -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.NonEssentialTests; -import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; -import java.sql.Timestamp; -import java.util.Date; -import javax.ejb.embeddable.EJBContainer; -import static org.junit.Assert.assertEquals; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -/** - * - * @author raprasad - */ -public class WorldMapTokenTest { - - private static EJBContainer c; - - - public void msg(String s){ - System.out.println(s); - } - - public void msgt(String s){ - msg("------------------------------------------------------------"); - msg(s); - msg("------------------------------------------------------------"); - } - - - private TokenApplicationType makeTokenApplicationType(int timeLimitMinutes){ - TokenApplicationType tat = new TokenApplicationType(); - tat.setName("GeoConnect"); - tat.setContactEmail("info@iq.harvard.edu"); - tat.setHostname("geoconnect.datascience.iq.harvard.edu"); - tat.setIpAddress("127.0.0.1"); - tat.setTimeLimitMinutes(timeLimitMinutes); - return tat; - } - private WorldMapToken makeNewToken(TokenApplicationType tat){ - WorldMapToken token; - token = new WorldMapToken(); - token.setApplication(tat); - token.setDatafile(new DataFile()); - token.setDataverseUser(new AuthenticatedUser()); - token.refreshToken(); - token.setToken(); - return token; - } - - @Category(NonEssentialTests.class) - @Test - public void testTokenValues(){ - msgt("WorldMapTokenTest!"); - TokenApplicationType tat = this.makeTokenApplicationType(30); - - WorldMapToken token = this.makeNewToken(tat); - String token_str1 = token.getToken(); - - // Should only be able to set token value once--it doesn't "reset" - token.setToken(); - assertEquals(token.getToken().equalsIgnoreCase(token_str1), true); - - - WorldMapToken token2 = this.makeNewToken(tat); - WorldMapToken token3 = this.makeNewToken(tat); - assertEquals(token2.getToken().equalsIgnoreCase(token.getToken()), false); - assertEquals(token2.getToken().equalsIgnoreCase(token3.getToken()), false); - } - - @Category(NonEssentialTests.class) - @Test - public void testTokenTimes(){ - msgt("testTokenTimes"); - - TokenApplicationType tat = this.makeTokenApplicationType(30); - - assertEquals(30*60, tat.getTimeLimitSeconds()); - msg("time limit seconds: " + tat.getTimeLimitSeconds()); - tat.setTimeLimitMinutes(1); - msg("time limit seconds (2): " + tat.getTimeLimitSeconds()); - assertEquals(1*60, tat.getTimeLimitSeconds()); - - tat.setTimeLimitMinutes(30); - WorldMapToken token = this.makeNewToken(tat); - assertEquals(token.hasTokenExpired(), false); - - //msg("Future token time (31 min): " + getFutureTimeStamp(31)); - msg("token time limit (minutes): " + token.getApplication().getTimeLimitMinutes()); - msg("Did token expire in 10 minutes? (should be no)"); - //msg("expired? " + token.hasTokenExpired(getFutureTimeStamp(10))); - - assertEquals(token.hasTokenExpired(getFutureTimeStamp(10)), false); - - msg("Did token expire? (automatically check current time)"); - assertEquals(token.hasTokenExpired(), false); - - msg("Did token expire at 30 minutes? (should be no)"); - assertEquals(token.hasTokenExpired(getFutureTimeStamp(30)), false); - - msg("Did token expire in 31 minutes? (should be yes)"); - assertEquals(token.hasTokenExpired(getFutureTimeStamp(31)), true); - - msg("Did token expire in 45 minutes? (should be yes)"); - assertEquals(token.hasTokenExpired(getFutureTimeStamp(45)), true); - - msg("token time limit (minutes): 10 minutes"); - token.getApplication().setTimeLimitMinutes(10); - assertEquals(token.getApplication().getTimeLimitMinutes(), 10); - - msg("Did token expire if null sent? (should be yes)"); - assertEquals(token.hasTokenExpired(null), true); - - msg("Refresh token (but fails b/c sending null automatically expires it"); - token.refreshToken(); - msg("Did token expire in 5 minutes? (should be no)"); - assertEquals(token.hasTokenExpired(getFutureTimeStamp(5)), true); - msg("Did token expire? (auto-check current time)"); - assertEquals(token.hasTokenExpired(), true); - - msgt("Get a new Token"); - WorldMapToken token2 = this.makeNewToken(tat); - - - msg("Did token expire? (automatically check current time)"); - assertEquals(token2.hasTokenExpired(), false); - - msg("Manually expire token: setHasExpired(true)"); - token2.setHasExpired(true); - msg("Did token expire in 1 minute? (should be yes--b/c manually expired)"); - assertEquals(token2.hasTokenExpired(getFutureTimeStamp(1)), true); - - - } - - private Timestamp getFutureTimeStamp(int futuremMinutes){ - - long ONE_MINUTE_IN_MILLIS=60000;//millisecs - long currentTimeMillisec = new Date().getTime(); - long inFutureMinutesMillisec = currentTimeMillisec + (futuremMinutes * ONE_MINUTE_IN_MILLIS); - return new Timestamp(new Date(inFutureMinutesMillisec).getTime()); - - } -} From 0e21c22c0374919f5e87c430abd59b6e3239a1d0 Mon Sep 17 00:00:00 2001 From: Kevin Condon Date: Tue, 12 Jan 2021 15:11:29 -0500 Subject: [PATCH 098/576] Update 7502-more-mime-types.md Updated release notes to specify reindex rather than reidentify. --- doc/release-notes/7502-more-mime-types.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/7502-more-mime-types.md b/doc/release-notes/7502-more-mime-types.md index 8fc3f33f92f..e5a7f7c95ed 100644 --- a/doc/release-notes/7502-more-mime-types.md +++ b/doc/release-notes/7502-more-mime-types.md @@ -1,4 +1,7 @@ ## Upgrade Steps -- Redetect file types using the Redetect File Types API: -https://guides.dataverse.org/en/latest/api/native-api.html#redetect-file-type \ No newline at end of file +In addition to mapping friendly names to these file types, the types are further mapped to aggregate file types facets on the homepage. This requires them to be reindexed to appear in the facet count. + +Kick off full reindex + +http://guides.dataverse.org/en/4.20/admin/solr-search-index.html From f3ab76c02597b1847971a7c48c83a2f8e13e4324 Mon Sep 17 00:00:00 2001 From: Michael Heppler <687227+mheppler@users.noreply.github.com> Date: Tue, 12 Jan 2021 15:17:14 -0500 Subject: [PATCH 099/576] Revisions from Ana - thanks! --- src/main/java/propertyFiles/MimeTypeDisplay.properties | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/propertyFiles/MimeTypeDisplay.properties b/src/main/java/propertyFiles/MimeTypeDisplay.properties index 8b07a87832d..614ecb9b12a 100644 --- a/src/main/java/propertyFiles/MimeTypeDisplay.properties +++ b/src/main/java/propertyFiles/MimeTypeDisplay.properties @@ -5,7 +5,7 @@ application/pdf=Adobe PDF image/pdf=Adobe PDF text/pdf=Adobe PDF application/x-pdf=Adobe PDF -application/cnt=CNT +application/cnt=Windows Help Contents File application/msword=MS Word application/vnd.ms-excel=MS Excel Spreadsheet application/vnd.openxmlformats-officedocument.spreadsheetml.sheet=MS Excel Spreadsheet @@ -71,8 +71,8 @@ application/x-spss-sps=SPSS Script Syntax text/x-sas-syntax=SAS Syntax application/x-sas-syntax=SAS Syntax type/x-r-syntax=R Syntax -application/vnd.wolfram.mathematica.package=Mathematica Package -application/vnd.wolfram.mathematica=Mathematica Package +application/vnd.wolfram.mathematica.package=Wolfram Mathematica Code +application/vnd.wolfram.mathematica=Wolfram Mathematica Code # Ingested Tabular Data text/tab-separated-values=Tab-Delimited # RawData From 7cc0ea642403e9971863167ca033736140abb20a Mon Sep 17 00:00:00 2001 From: Kevin Condon Date: Tue, 12 Jan 2021 15:19:42 -0500 Subject: [PATCH 100/576] Update 7502-more-mime-types.md Tweak based on Leonid's input --- doc/release-notes/7502-more-mime-types.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/7502-more-mime-types.md b/doc/release-notes/7502-more-mime-types.md index e5a7f7c95ed..0c57dc3e389 100644 --- a/doc/release-notes/7502-more-mime-types.md +++ b/doc/release-notes/7502-more-mime-types.md @@ -1,6 +1,6 @@ ## Upgrade Steps -In addition to mapping friendly names to these file types, the types are further mapped to aggregate file types facets on the homepage. This requires them to be reindexed to appear in the facet count. +In addition to mapping friendly names to these file types, the types are further mapped to aggregate file types facets on the homepage. A full reindex is required for the facets to be refreshed. Kick off full reindex From 6b4fb249a3a4a5bf84ab9fb5c67eec3b7e8f4415 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Tue, 12 Jan 2021 16:33:32 -0500 Subject: [PATCH 101/576] Undo on bug testing revision mistakenly left in java function [ref #5210, #6685] --- .../harvard/iq/dataverse/search/SearchIncludeFragment.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java index 73aacbdce2d..f7a3b874c24 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java @@ -1204,7 +1204,7 @@ public String tabularDataDisplayInfo(DataFile datafile) { String tabInfo = ""; if (datafile == null) { - return null; + return ""; } if (datafile.isTabularData() && datafile.getDataTable() != null) { @@ -1226,7 +1226,7 @@ public String tabularDataUnfDisplay(DataFile datafile) { String tabUnf = ""; if (datafile == null) { - return null; + return ""; } if (datafile.isTabularData() && datafile.getDataTable() != null) { From 8364149b8fa9415c6d8066e5db04c4e3f5fb83f3 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 12 Jan 2021 17:29:27 -0500 Subject: [PATCH 102/576] Create 7409-remove-worldmap.md --- doc/release-notes/7409-remove-worldmap.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/release-notes/7409-remove-worldmap.md diff --git a/doc/release-notes/7409-remove-worldmap.md b/doc/release-notes/7409-remove-worldmap.md new file mode 100644 index 00000000000..3e6ea3ed06e --- /dev/null +++ b/doc/release-notes/7409-remove-worldmap.md @@ -0,0 +1,9 @@ +## Release Highlights + +### Worldmap/Geoconnect Integration Now Obsolete + +As of this release, the Geoconnect/Worldmap integration is no longer available. The Harvard University Worldmap is going through a migration process, and instead of updating this code to work with the new infrastructure, the decision was made to pursue future Geospatial exploration/analysis through other tools, following the External Tools Framework in the Dataverse Software. + +## Notes to Admins + +Take a backup of the Worldmap links, if any. \ No newline at end of file From f45d0353627590723052b7c6dae293741b048679 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 12 Jan 2021 22:25:12 -0500 Subject: [PATCH 103/576] more updates --- .../source/developers/geospatial.rst | 22 ++++++++-------- doc/sphinx-guides/source/developers/intro.rst | 26 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/doc/sphinx-guides/source/developers/geospatial.rst b/doc/sphinx-guides/source/developers/geospatial.rst index 13680a52358..1cbe85cd12e 100644 --- a/doc/sphinx-guides/source/developers/geospatial.rst +++ b/doc/sphinx-guides/source/developers/geospatial.rst @@ -8,12 +8,12 @@ Geospatial Data Geoconnect ---------- -Geoconnect works as a middle layer, allowing geospatial data files in Dataverse to be visualized with Harvard WorldMap. To set up a Geoconnect development environment, you can follow the steps outlined in the `local_setup.md `_ guide. You will need Python and a few other prerequisites. +Geoconnect works as a middle layer, allowing geospatial data files in your Dataverse installation to be visualized with Harvard WorldMap. To set up a Geoconnect development environment, you can follow the steps outlined in the `local_setup.md `_ guide. You will need Python and a few other prerequisites. -As mentioned under the :ref:`architecture` section of Preparation in the Installation Guide, Geoconnect is an optional component of Dataverse, so this section is only necessary to follow it you are working on an issue related to this feature. +As mentioned under the :ref:`architecture` section of Preparation in the Installation Guide, Geoconnect is an optional component of the Dataverse Platform, so this section is only necessary to follow it you are working on an issue related to this feature. -How Dataverse Ingests Shapefiles --------------------------------- +How Your Dataverse Installation Ingests Shapefiles +-------------------------------------------------- A shapefile is a set of files, often uploaded/transferred in ``.zip`` format. This set may contain up to fifteen files. A minimum of three specific files (``.shp``, ``.shx``, ``.dbf``) are needed to be a valid shapefile and a fourth file (``.prj``) is required for WorldMap -- or any type of meaningful visualization. @@ -27,7 +27,7 @@ For ingest and connecting to WorldMap, four files are the minimum required: Ingest ~~~~~~ -When uploaded to Dataverse, the ``.zip`` is unpacked (same as all ``.zip`` files). Shapefile sets are recognized by the same base name and specific extensions. These individual files constitute a shapefile set. The first four are the minimum required (``.shp``, ``.shx``, ``.dbf``, ``.prj``) +When uploaded to a Dataverse installation, the ``.zip`` is unpacked (same as all ``.zip`` files). Shapefile sets are recognized by the same base name and specific extensions. These individual files constitute a shapefile set. The first four are the minimum required (``.shp``, ``.shx``, ``.dbf``, ``.prj``) For example: @@ -38,19 +38,19 @@ For example: - bicycles.sbx (NOT required extension) - bicycles.sbn (NOT required extension) -Upon recognition of the four required files, Dataverse will group them as well as any other relevant files into a shapefile set. Files with these extensions will be included in the shapefile set: +Upon recognition of the four required files, the Dataverse installation will group them as well as any other relevant files into a shapefile set. Files with these extensions will be included in the shapefile set: - Required: ``.shp``, ``.shx``, ``.dbf``, ``.prj`` - Optional: ``.sbn``, ``.sbx``, ``.fbn``, ``.fbx``, ``.ain``, ``.aih``, ``.ixs``, ``.mxs``, ``.atx``, ``.cpg``, ``shp.xml`` -Then Dataverse creates a new ``.zip`` with mimetype as a shapefile. The shapefile set will persist as this new ``.zip``. +Then the Dataverse installation creates a new ``.zip`` with mimetype as a shapefile. The shapefile set will persist as this new ``.zip``. Example ~~~~~~~ **1a.** Original ``.zip`` contents: -A file named ``bikes_and_subways.zip`` is uploaded to the Dataverse. This ``.zip`` contains the following files. +A file named ``bikes_and_subways.zip`` is uploaded to the Dataverse installation. This ``.zip`` contains the following files. - ``bicycles.shp`` (shapefile set #1) - ``bicycles.shx`` (shapefile set #1) @@ -66,9 +66,9 @@ A file named ``bikes_and_subways.zip`` is uploaded to the Dataverse. This ``.zip - ``subway_line.prj`` (shapefile set #2) - ``subway_line.dbf`` (shapefile set #2) -**1b.** Dataverse unzips and re-zips files: +**1b.** The Dataverse installation unzips and re-zips files: -Upon ingest, Dataverse unpacks the file ``bikes_and_subways.zip``. Upon recognizing the shapefile sets, it groups those files together into new ``.zip`` files: +Upon ingest, the Dataverse installation unpacks the file ``bikes_and_subways.zip``. Upon recognizing the shapefile sets, it groups those files together into new ``.zip`` files: - files making up the "bicycles" shapefile become a new ``.zip`` - files making up the "subway_line" shapefile become a new ``.zip`` @@ -76,7 +76,7 @@ Upon ingest, Dataverse unpacks the file ``bikes_and_subways.zip``. Upon recogniz To ensure that a shapefile set remains intact, individual files such as ``bicycles.sbn`` are kept in the set -- even though they are not used for mapping. -**1c.** Dataverse final file listing: +**1c.** The Dataverse installation final file listing: - ``bicycles.zip`` (contains shapefile set #1: ``bicycles.shp``, ``bicycles.shx``, ``bicycles.prj``, ``bicycles.dbf``, ``bicycles.sbx``, ``bicycles.sbn``) - ``bicycles.txt`` (separate, not part of a shapefile set) diff --git a/doc/sphinx-guides/source/developers/intro.rst b/doc/sphinx-guides/source/developers/intro.rst index e7ed6a12a37..a4e7b312bc9 100755 --- a/doc/sphinx-guides/source/developers/intro.rst +++ b/doc/sphinx-guides/source/developers/intro.rst @@ -2,7 +2,7 @@ Introduction ============ -Welcome! `Dataverse `_ is an `open source `_ project that loves `contributors `_! +Welcome! `The Dataverse Project `_ is an `open source `_ project that loves `contributors `_! .. contents:: |toctitle| :local: @@ -10,7 +10,7 @@ Welcome! `Dataverse `_ is an `open source `_ application that is compiled into a WAR file and deployed to an application server (app server) which is configured to work with a relational database (PostgreSQL) and a search engine (Solr). +The Dataverse software is a `Jakarta EE `_ application that is compiled into a WAR file and deployed to an application server (app server) which is configured to work with a relational database (PostgreSQL) and a search engine (Solr). We make use of a variety of Jakarta EE technologies such as JPA, JAX-RS, JMS, and JSF. The front end is built using PrimeFaces and Bootstrap. @@ -33,7 +33,7 @@ In addition, we start to adopt parts of Eclipse MicroProfile, namely `MicroProfi Roadmap ------- -For the Dataverse development roadmap, please see https://www.iq.harvard.edu/roadmap-dataverse-project +For the Dataverse software development roadmap, please see https://www.iq.harvard.edu/roadmap-dataverse-project Kanban Board ------------ @@ -48,7 +48,7 @@ We use GitHub Issues as our issue tracker: https://github.com/IQSS/dataverse/iss Related Guides -------------- -If you are a developer who wants to make use of Dataverse APIs, please see the :doc:`/api/index`. If you have front-end UI questions, please see the :doc:`/style/index`. +If you are a developer who wants to make use of the Dataverse Software APIs, please see the :doc:`/api/index`. If you have front-end UI questions, please see the :doc:`/style/index`. If you are a sysadmin who likes to code, you may be interested in hacking on installation scripts mentioned in the :doc:`/installation/index`. We validate the installation scripts with :doc:`/developers/tools` such as `Vagrant `_ and Docker (see the :doc:`containers` section). @@ -57,20 +57,20 @@ Related Projects As a developer, you also may be interested in these projects related to Dataverse: -- External Tools - add additional features to Dataverse without modifying the core: :doc:`/api/external-tools` -- Dataverse API client libraries - use Dataverse APIs from various languages: :doc:`/api/client-libraries` -- DVUploader - a stand-alone command-line Java application that uses the Dataverse API to support upload of files from local disk to a Dataset: https://github.com/IQSS/dataverse-uploader +- External Tools - add additional features to the Dataverse Software without modifying the core: :doc:`/api/external-tools` +- Dataverse Software API client libraries - use Dataverse Software APIs from various languages: :doc:`/api/client-libraries` +- DVUploader - a stand-alone command-line Java application that uses the Dataverse Software API to support upload of files from local disk to a Dataset: https://github.com/IQSS/dataverse-uploader - dataverse-sample-data - populate your Dataverse installation with sample data: https://github.com/IQSS/dataverse-sample-data -- dataverse-metrics - aggregate and visualize metrics for installations of Dataverse around the world: https://github.com/IQSS/dataverse-metrics +- dataverse-metrics - aggregate and visualize metrics for Dataverse installations around the world: https://github.com/IQSS/dataverse-metrics - Configuration management scripts - Ansible, Puppet, etc.: See :ref:`advanced` section in the Installation Guide. - :doc:`/developers/unf/index` (Java) - a Universal Numerical Fingerprint: https://github.com/IQSS/UNF -- GeoConnect (Python) - create a map by uploading files to Dataverse: https://github.com/IQSS/geoconnect +- GeoConnect (Python) - create a map by uploading files to a Dataverse installation: https://github.com/IQSS/geoconnect - `DataTags `_ (Java and Scala) - tag datasets with privacy levels: https://github.com/IQSS/DataTags - `TwoRavens `_ (Javascript) - a `d3.js `_ interface for exploring data and running Zelig models: https://github.com/IQSS/TwoRavens -- `Zelig `_ (R) - run statistical models on files uploaded to Dataverse: https://github.com/IQSS/Zelig +- `Zelig `_ (R) - run statistical models on files uploaded to a Dataverse installation: https://github.com/IQSS/Zelig - `Matrix `_ - a visualization showing the connectedness and collaboration between authors and their affiliations. -- Third party apps - make use of Dataverse APIs: :doc:`/api/apps` -- chat.dataverse.org - chat interface for Dataverse users and developers: https://github.com/IQSS/chat.dataverse.org +- Third party apps - make use of Dataverse installation APIs: :doc:`/api/apps` +- chat.dataverse.org - chat interface for Dataverse Project users and developers: https://github.com/IQSS/chat.dataverse.org - [Your project here] :) ---- From c69abed0dfb282b4d04cbeb3bf9fd84f1628b10e Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 12 Jan 2021 22:27:45 -0500 Subject: [PATCH 104/576] Update make-data-count.rst --- .../source/developers/make-data-count.rst | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/sphinx-guides/source/developers/make-data-count.rst b/doc/sphinx-guides/source/developers/make-data-count.rst index dac1b074d68..f604e2c6c62 100644 --- a/doc/sphinx-guides/source/developers/make-data-count.rst +++ b/doc/sphinx-guides/source/developers/make-data-count.rst @@ -1,7 +1,7 @@ Make Data Count =============== -Support for Make Data Count is a feature of Dataverse that is described in the :doc:`/admin/make-data-count` section of the Admin Guide. In order for developers to work on the feature, they must install Counter Processor, a Python 3 application, as described below. Counter Processor can be found at https://github.com/CDLUC3/counter-processor +Support for Make Data Count is a feature of the Dataverse Software that is described in the :doc:`/admin/make-data-count` section of the Admin Guide. In order for developers to work on the feature, they must install Counter Processor, a Python 3 application, as described below. Counter Processor can be found at https://github.com/CDLUC3/counter-processor .. contents:: |toctitle| :local: @@ -9,7 +9,7 @@ Support for Make Data Count is a feature of Dataverse that is described in the : Architecture ------------ -There are many components involved in Dataverse's architecture for Make Data Count as shown in the diagram below. +There are many components involved in the Dataverse software's architecture for Make Data Count as shown in the diagram below. |makedatacount_components| @@ -28,20 +28,20 @@ To insert a citation you could insert a row like below, changing "72" in the exa Full Setup ~~~~~~~~~~ -The recommended way to work on the Make Data Count feature is to spin up an EC2 instance that has both Dataverse and Counter Processor installed. Go to the :doc:`deployment` page for details on how to spin up an EC2 instance and make sure that your Ansible file is configured to install Counter Processor before running the "create" script. +The recommended way to work on the Make Data Count feature is to spin up an EC2 instance that has both the Dataverse software and Counter Processor installed. Go to the :doc:`deployment` page for details on how to spin up an EC2 instance and make sure that your Ansible file is configured to install Counter Processor before running the "create" script. (Alternatively, you can try installing Counter Processor in Vagrant. :download:`setup-counter-processor.sh <../../../../scripts/vagrant/setup-counter-processor.sh>` might help you get it installed.) -After you have spun to your EC2 instance, set ``:MDCLogPath`` so that Dataverse creates a log for Counter Processor to operate on. For more on this database setting, see the :doc:`/installation/config` section of the Installation Guide. +After you have spun to your EC2 instance, set ``:MDCLogPath`` so that the Dataverse installation creates a log for Counter Processor to operate on. For more on this database setting, see the :doc:`/installation/config` section of the Installation Guide. -Next you need to have Dataverse add some entries to the log that Counter Processor will operate on. To do this, click on some published datasets and download some files. +Next you need to have the Dataverse installation add some entries to the log that Counter Processor will operate on. To do this, click on some published datasets and download some files. Next you should run Counter Processor to convert the log into a SUSHI report, which is in JSON format. Before running Counter Processor, you need to put a configuration file into place. As a starting point use :download:`counter-processor-config.yaml <../../../../scripts/vagrant/counter-processor-config.yaml>` and edit the file, paying particular attention to the following settings: - ``log_name_pattern`` You might want something like ``/usr/local/payara5/glassfish/domains/domain1/logs/counter_(yyyy-mm-dd).log`` - ``year_month`` You should probably set this to the current month. - ``output_file`` This needs to be a directory that the "dataverse" Unix user can read but that the "counter" user can write to. In dev, you can probably get away with "/tmp" as the directory. -- ``platform`` Out of the box from Counter Processor this is set to ``Dash`` but this should be changed to match the name of your Dataverse. Examples are "Harvard Dataverse" for Harvard University or "LibraData" for the University of Virginia. +- ``platform`` Out of the box from Counter Processor this is set to ``Dash`` but this should be changed to match the name of your Dataverse installation. Examples are "Harvard Dataverse Repository" for Harvard University or "LibraData" for the University of Virginia. - ``upload_to_hub`` This should be "False" unless you are testing sending SUSHI reports to the DataCite hub. - ``simulate_date`` You should probably set this to tomorrow. @@ -73,10 +73,10 @@ If all this is working and you want to send data to the test instance of the Dat For how to put citations into your dev database and how to get them out again, see :ref:`MDC-updateCitationsForDataset` section in Make Data Count of the Admin Guide. -Testing Make Data Count and Dataverse -------------------------------------- +Testing Make Data Count and Your Dataverse Installation +------------------------------------------------------- -A developer running Counter Processor alongside Dataverse for development or testing purposes will notice that once the raw Dataverse logs have been processed, there is no straightforward way to re-test those same logs. +A developer running Counter Processor alongside the Dataverse installation for development or testing purposes will notice that once the raw Dataverse installation logs have been processed, there is no straightforward way to re-test those same logs. The first thing to fix is to clear two files from Counter Processor ``state`` folder, ``statefile.json`` and ``counter_db_[yyyy-mm].sqlite3`` @@ -88,7 +88,7 @@ To get the ``REPORT_ID``, look at the logs generated in ``/usr/local/counter-pro To read more about the Make Data Count api, see https://github.com/datacite/sashimi -You can compare the MDC metrics display with Dataverse's original by toggling the ``:DisplayMDCMetrics`` setting (true by default to display MDC metrics). +You can compare the MDC metrics display with the Dataverse installation's original by toggling the ``:DisplayMDCMetrics`` setting (true by default to display MDC metrics). Resources --------- From 0ad21ce37510f70e94b344c9b1e546b29d115545 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 12 Jan 2021 22:29:52 -0500 Subject: [PATCH 105/576] YET MORE UPDATES --- doc/sphinx-guides/source/developers/remote-users.rst | 2 +- doc/sphinx-guides/source/developers/selinux.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/developers/remote-users.rst b/doc/sphinx-guides/source/developers/remote-users.rst index 66af0c71eda..6e658c15543 100755 --- a/doc/sphinx-guides/source/developers/remote-users.rst +++ b/doc/sphinx-guides/source/developers/remote-users.rst @@ -8,7 +8,7 @@ Shibboleth and OAuth Shibboleth and OAuth -------------------- -If you are working on anything related to users, please keep in mind that your changes will likely affect Shibboleth and OAuth users. For some background on user accounts in Dataverse, see :ref:`auth-modes` section of Configuration in the Installation Guide. +If you are working on anything related to users, please keep in mind that your changes will likely affect Shibboleth and OAuth users. For some background on user accounts in the Dataverse software, see :ref:`auth-modes` section of Configuration in the Installation Guide. Rather than setting up Shibboleth on your laptop, developers are advised to simply add a value to their database to enable Shibboleth "dev mode" like this: diff --git a/doc/sphinx-guides/source/developers/selinux.rst b/doc/sphinx-guides/source/developers/selinux.rst index d7f5b0d7519..52fd62cc308 100644 --- a/doc/sphinx-guides/source/developers/selinux.rst +++ b/doc/sphinx-guides/source/developers/selinux.rst @@ -44,7 +44,7 @@ Use ``semodule -l | grep shibboleth`` to see if the ``shibboleth.te`` rules are Exercising SELinux denials ~~~~~~~~~~~~~~~~~~~~~~~~~~ -As of this writing, there are two optional components of Dataverse that are known not to work with SELinux out of the box with SELinux: Shibboleth and rApache. +As of this writing, there are two optional components of the Dataverse software that are known not to work with SELinux out of the box with SELinux: Shibboleth and rApache. We will be exercising SELinux denials with Shibboleth, and the SELinux-related issues are expected out the box: From 92bc34bf33e91bba85312d306b4299182a93df66 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 12 Jan 2021 22:43:56 -0500 Subject: [PATCH 106/576] updates, updates everywhere --- .../source/developers/sql-upgrade-scripts.rst | 6 +-- .../source/developers/testing.rst | 38 +++++++++---------- doc/sphinx-guides/source/developers/tips.rst | 30 +++++++-------- doc/sphinx-guides/source/developers/tools.rst | 16 ++++---- 4 files changed, 45 insertions(+), 45 deletions(-) diff --git a/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst b/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst index 6d49f927a9e..bf27242f79e 100644 --- a/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst +++ b/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst @@ -2,7 +2,7 @@ SQL Upgrade Scripts =================== -The database schema for Dataverse is constantly evolving and we have adopted a tool called Flyway to help keep your development environment up to date and in working order. As you make changes to the database schema (changes to ``@Entity`` classes), you must write SQL upgrade scripts when needed and follow Flyway file naming conventions. +The database schema for the Dataverse software is constantly evolving and we have adopted a tool called Flyway to help keep your development environment up to date and in working order. As you make changes to the database schema (changes to ``@Entity`` classes), you must write SQL upgrade scripts when needed and follow Flyway file naming conventions. .. contents:: |toctitle| :local: @@ -30,7 +30,7 @@ Create a new file called something like ``V4.11.0.1__5565-sanitize-directory-lab The SQL migration script you wrote will be part of the war file and executed when the war file is deployed. To see a history of Flyway database migrations that have been applied, look at the ``flyway_schema_history`` table. -As with any task related to Dataverse development, if you need any help writing SQL upgrade scripts, please reach out using any of the channels mentioned under "Getting Help" in the :doc:`intro` section. +As with any task related to the development of the Dataverse software, if you need any help writing SQL upgrade scripts, please reach out using any of the channels mentioned under "Getting Help" in the :doc:`intro` section. Troubleshooting --------------- @@ -38,7 +38,7 @@ Troubleshooting Renaming SQL Upgrade Scripts ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Please note that if you need to rename your script (because a new version of Dataverse was released, for example), you will see the error "FlywayException: Validate failed: Detected applied migration not resolved locally" when you attempt to deploy and deployment will fail. +Please note that if you need to rename your script (because a new version of the Dataverse software was released, for example), you will see the error "FlywayException: Validate failed: Detected applied migration not resolved locally" when you attempt to deploy and deployment will fail. To resolve this problem, delete the old migration from the ``flyway_schema_history`` table and attempt to redeploy. diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst index 4ec1ff5cec0..c2e14749dd8 100755 --- a/doc/sphinx-guides/source/developers/testing.rst +++ b/doc/sphinx-guides/source/developers/testing.rst @@ -2,7 +2,7 @@ Testing ======= -In order to keep our codebase healthy, the Dataverse project encourages developers to write automated tests in the form of unit tests and integration tests. We also welcome ideas for how to improve our automated testing. +In order to keep our codebase healthy, the Dataverse Project encourages developers to write automated tests in the form of unit tests and integration tests. We also welcome ideas for how to improve our automated testing. .. contents:: |toctitle| :local: @@ -113,16 +113,16 @@ Generally speaking, unit tests have been flagged as non-essential because they a Integration Tests ----------------- -Unit tests are fantastic for low level testing of logic but aren't especially real-world-applicable because they do not exercise Dataverse as it runs in production with a database and other runtime dependencies. We test in-depth by also writing integration tests to exercise a running system. +Unit tests are fantastic for low level testing of logic but aren't especially real-world-applicable because they do not exercise the Dataverse software as it runs in production with a database and other runtime dependencies. We test in-depth by also writing integration tests to exercise a running system. Unfortunately, the term "integration tests" can mean different things to -different people. For our purposes, an integration test can have to flavors: +different people. For our purposes, an integration test can have two flavors: 1. Be an API Test: - - Exercise Dataverse APIs. + - Exercise the Dataverse Software APIs. - Running not automatically on developers' laptops. - - Operate on an installation of Dataverse that is running and able to talk to both PostgreSQL and Solr. + - Operate on a Dataverse installation that is running and able to talk to both PostgreSQL and Solr. - Written using REST Assured. 2. Be a `Testcontainers `__ Test: @@ -173,32 +173,32 @@ Alternatively, you can run tests against the app server running on your laptop b Getting Set Up to Run REST Assured Tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Unit tests are run automatically on every build, but dev environments and servers require special setup to run REST Assured tests. In short, Dataverse needs to be placed into an insecure mode that allows arbitrary users and datasets to be created and destroyed. This differs greatly from the out-of-the-box behavior of Dataverse, which we strive to keep secure for sysadmins installing the software for their institutions in a production environment. +Unit tests are run automatically on every build, but dev environments and servers require special setup to run REST Assured tests. In short, the Dataverse software needs to be placed into an insecure mode that allows arbitrary users and datasets to be created and destroyed. This differs greatly from the out-of-the-box behavior of the Dataverse software, which we strive to keep secure for sysadmins installing the software for their institutions in a production environment. -The :doc:`dev-environment` section currently refers developers here for advice on getting set up to run REST Assured tests, but we'd like to add some sort of "dev" flag to the installer to put Dataverse in "insecure" mode, with lots of scary warnings that this dev mode should not be used in production. +The :doc:`dev-environment` section currently refers developers here for advice on getting set up to run REST Assured tests, but we'd like to add some sort of "dev" flag to the installer to put the Dataverse software in "insecure" mode, with lots of scary warnings that this dev mode should not be used in production. The instructions below assume a relatively static dev environment on a Mac. There is a newer "all in one" Docker-based approach documented in the :doc:`/developers/containers` section under "Docker" that you may like to play with as well. The Burrito Key ^^^^^^^^^^^^^^^ -For reasons that have been lost to the mists of time, Dataverse really wants you to to have a burrito. Specifically, if you're trying to run REST Assured tests and see the error "Dataverse config issue: No API key defined for built in user management", you must run the following curl command (or make an equivalent change to your database): +For reasons that have been lost to the mists of time, the Dataverse software really wants you to to have a burrito. Specifically, if you're trying to run REST Assured tests and see the error "Dataverse config issue: No API key defined for built in user management", you must run the following curl command (or make an equivalent change to your database): ``curl -X PUT -d 'burrito' http://localhost:8080/api/admin/settings/BuiltinUsers.KEY`` -Without this "burrito" key in place, REST Assured will not be able to create users. We create users to create objects we want to test, such as dataverses, datasets, and files. +Without this "burrito" key in place, REST Assured will not be able to create users. We create users to create objects we want to test, such as Dataverse collections, datasets, and files. -Root Dataverse Permissions -^^^^^^^^^^^^^^^^^^^^^^^^^^ +Root Dataverse Collection Permissions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -In your browser, log in as dataverseAdmin (password: admin) and click the "Edit" button for your root dataverse. Navigate to Permissions, then the Edit Access button. Under "Who can add to this dataverse?" choose "Anyone with a Dataverse account can add sub dataverses and datasets" if it isn't set to this already. +In your browser, log in as dataverseAdmin (password: admin) and click the "Edit" button for your root Dataverse collection. Navigate to Permissions, then the Edit Access button. Under "Who can add to this Dataverse collection?" choose "Anyone with a Dataverse installation account can add sub Dataverse collections and datasets" if it isn't set to this already. Alternatively, this same step can be done with this script: ``scripts/search/tests/grant-authusers-add-on-root`` -Publish Root Dataverse -^^^^^^^^^^^^^^^^^^^^^^ +Publish Root Dataverse Collection +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The root dataverse must be published for some of the REST Assured tests to run. +The root Dataverse collection must be published for some of the REST Assured tests to run. dataverse.siteUrl ^^^^^^^^^^^^^^^^^ @@ -235,7 +235,7 @@ Remember, it’s only a test (and it's not graded)! Some guidelines to bear in m - Let the code do the labor; automate everything that happens when you run your test file. - Just as with any development, if you’re stuck: ask for help! -To execute existing integration tests on your local Dataverse, a helpful command line tool to use is `Maven `_. You should have Maven installed as per the `Development Environment `_ guide, but if not it’s easily done via Homebrew: ``brew install maven``. +To execute existing integration tests on your local Dataverse installation, a helpful command line tool to use is `Maven `_. You should have Maven installed as per the `Development Environment `_ guide, but if not it’s easily done via Homebrew: ``brew install maven``. Once installed, you may run commands with ``mvn [options] [] []``. @@ -410,7 +410,7 @@ Enhance build time by caching dependencies ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In the future, CI builds in ephemeral build environments and Docker builds can benefit from caching all dependencies and plugins. -As Dataverse is a huge project, build times can be enhanced by avoiding re-downloading everything when the Maven POM is unchanged. +As the Dataverse Project is a huge project, build times can be enhanced by avoiding re-downloading everything when the Maven POM is unchanged. To seed the cache, use the following Maven goal before using Maven in (optional) offline mode in your scripts: .. code:: shell @@ -456,9 +456,9 @@ Accessibility Testing Accessibility Policy ~~~~~~~~~~~~~~~~~~~~ -Dataverse aims to improve the user experience for those with disabilities, and are in the process of following the recommendations of the `Harvard University Digital Accessibility Policy `__, which use the Worldwide Web Consortium’s Web Content Accessibility Guidelines version 2.1, Level AA Conformance (WCAG 2.1 Level AA) as the standard. +The Dataverse Project aims to improve the user experience for those with disabilities, and are in the process of following the recommendations of the `Harvard University Digital Accessibility Policy `__, which use the Worldwide Web Consortium’s Web Content Accessibility Guidelines version 2.1, Level AA Conformance (WCAG 2.1 Level AA) as the standard. -To report an accessibility issue with Dataverse, you can create a new issue in our GitHub repo at: https://github.com/IQSS/dataverse/issues/ +To report an accessibility issue with the Dataverse software, you can create a new issue in our GitHub repo at: https://github.com/IQSS/dataverse/issues/ Accessibility Tools ~~~~~~~~~~~~~~~~~~~ diff --git a/doc/sphinx-guides/source/developers/tips.rst b/doc/sphinx-guides/source/developers/tips.rst index 47265c97eff..5d38c2f8aba 100755 --- a/doc/sphinx-guides/source/developers/tips.rst +++ b/doc/sphinx-guides/source/developers/tips.rst @@ -10,16 +10,16 @@ If you just followed the steps in :doc:`dev-environment` for the first time, you Iterating on Code and Redeploying --------------------------------- -When you followed the steps in the :doc:`dev-environment` section, the war file was deployed to Payara by the Dataverse installation script. That's fine but once you're ready to make a change to the code you will need to get comfortable with undeploying and redeploying code (a war file) to Payara. +When you followed the steps in the :doc:`dev-environment` section, the war file was deployed to Payara by the Dataverse software installation script. That's fine but once you're ready to make a change to the code you will need to get comfortable with undeploying and redeploying code (a war file) to Payara. -It's certainly possible to manage deployment and undeployment of the war file via the command line using the ``asadmin`` command that ships with Payara (that's what the Dataverse installation script uses and the steps are documented below), but we recommend getting set up with an IDE such as Netbeans to manage deployment for you. +It's certainly possible to manage deployment and undeployment of the war file via the command line using the ``asadmin`` command that ships with Payara (that's what the Dataverse software installation script uses and the steps are documented below), but we recommend getting set up with an IDE such as Netbeans to manage deployment for you. -Undeploy the war File from the Dataverse Installation Script -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Undeploy the war File from the Dataverse Software Installation Script +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Because the initial deployment of the war file was done outside of Netbeans by the Dataverse installation script, it's a good idea to undeploy that war file to give Netbeans a clean slate to work with. +Because the initial deployment of the war file was done outside of Netbeans by the Dataverse Software installation script, it's a good idea to undeploy that war file to give Netbeans a clean slate to work with. -Assuming you installed Payara in ``/usr/local/payara5``, run the following ``asadmin`` command to see the version of Dataverse that the Dataverse installation script deployed: +Assuming you installed Payara in ``/usr/local/payara5``, run the following ``asadmin`` command to see the version of the Dataverse Software that the Dataverse Software installation script deployed: ``/usr/local/payara5/bin/asadmin list-applications`` @@ -38,7 +38,7 @@ Please note that if you are on a Mac, Netbeans may be unable to start Payara due At this point you can manage Payara using Netbeans. Click "Window" and then "Services". Expand "Servers" and right-click Payara to stop and then start it so that it appears in the Output window. Note that you can expand "Payara" and "Applications" to see if any applications are deployed. -Ensure that Dataverse Will Be Deployed to Payara +Ensure that the Dataverse Software Will Be Deployed to Payara ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Click "Window" and then "Projects". Click "File" and then "Project Properties (dataverse)". Click "Run" and change "Server" from "No Server Selected" to your installation of Payara. Click OK. @@ -81,7 +81,7 @@ For faster iteration while working on JSF pages, it is highly recommended that y Database Schema Exploration --------------------------- -With over 100 tables, the Dataverse PostgreSQL database ("dvndb") can be somewhat daunting for newcomers. Here are some tips for coming up to speed. (See also the :doc:`sql-upgrade-scripts` section.) +With over 100 tables, the Dataverse Software PostgreSQL database ("dvndb") can be somewhat daunting for newcomers. Here are some tips for coming up to speed. (See also the :doc:`sql-upgrade-scripts` section.) pgAdmin ~~~~~~~~ @@ -109,13 +109,13 @@ For the ``asadmin`` commands below, we assume you have already changed directori There are four steps to this process: 1. Build the war file: ``mvn package`` -2. Check which version of Dataverse is deployed: ``./asadmin list-applications`` -3. Undeploy the Dataverse application (if necessary): ``./asadmin undeploy dataverse-VERSION`` +2. Check which version of the Dataverse software is deployed: ``./asadmin list-applications`` +3. Undeploy the Dataverse software application (if necessary): ``./asadmin undeploy dataverse-VERSION`` 4. Copy the war file to the server (if necessary) 5. Deploy the new code: ``./asadmin deploy /path/to/dataverse-VERSION.war`` -Running the Dataverse Installation Script in Non-Interactive Mode ------------------------------------------------------------------ +Running the Dataverse Software Installation Script in Non-Interactive Mode +-------------------------------------------------------------------------- Rather than running the installer in "interactive" mode, it's possible to put the values in a file. See "non-interactive mode" in the :doc:`/installation/installation-main` section of the Installation Guide. @@ -128,9 +128,9 @@ Solr ---- .. TODO: This section should be moved into a dedicated guide about Solr for developers. It should be extended with - information about the way Solr is used within Dataverse, ideally explaining concepts and links to upstream docs. + information about the way Solr is used within the Dataverse software, ideally explaining concepts and links to upstream docs. -Once some dataverses, datasets, and files have been created and indexed, you can experiment with searches directly from Solr at http://localhost:8983/solr/#/collection1/query and look at the JSON output of searches, such as this wildcard search: http://localhost:8983/solr/collection1/select?q=*%3A*&wt=json&indent=true . You can also get JSON output of static fields Solr knows about: http://localhost:8983/solr/collection1/schema/fields +Once some Dataverse collections, datasets, and files have been created and indexed, you can experiment with searches directly from Solr at http://localhost:8983/solr/#/collection1/query and look at the JSON output of searches, such as this wildcard search: http://localhost:8983/solr/collection1/select?q=*%3A*&wt=json&indent=true . You can also get JSON output of static fields Solr knows about: http://localhost:8983/solr/collection1/schema/fields You can simply double-click "start.jar" rather that running ``java -jar start.jar`` from the command line. Figuring out how to stop Solr after double-clicking it is an exercise for the reader. @@ -165,7 +165,7 @@ When you prefer manual updates, there is another script, see above: :ref:`custom Sample Data ----------- -You may want to populate your **non-production** installation(s) of Dataverse with sample data. You have a couple options: +You may want to populate your **non-production** Dataverse installations with sample data. You have a couple options: - Code in https://github.com/IQSS/dataverse-sample-data (recommended). This set of sample data includes several common data types, data subsetted from production datasets in dataverse.harvard.edu, datasets with file hierarchy, and more. - Scripts called from ``scripts/deploy/phoenix.dataverse.org/post``. diff --git a/doc/sphinx-guides/source/developers/tools.rst b/doc/sphinx-guides/source/developers/tools.rst index 8fc86796742..49e92c1b5cf 100755 --- a/doc/sphinx-guides/source/developers/tools.rst +++ b/doc/sphinx-guides/source/developers/tools.rst @@ -30,11 +30,11 @@ With Maven installed you can run ``mvn package`` and ``mvn test`` from the comma Vagrant +++++++ -Vagrant allows you to spin up a virtual machine running Dataverse on your development workstation. You'll need to install Vagrant from https://www.vagrantup.com and VirtualBox from https://www.virtualbox.org. +Vagrant allows you to spin up a virtual machine running the Dataverse software on your development workstation. You'll need to install Vagrant from https://www.vagrantup.com and VirtualBox from https://www.virtualbox.org. We assume you have already cloned the repo from https://github.com/IQSS/dataverse as explained in the :doc:`/developers/dev-environment` section. -From the root of the git repo (where the ``Vagrantfile`` is), run ``vagrant up`` and eventually you should be able to reach an installation of Dataverse at http://localhost:8888 (the ``forwarded_port`` indicated in the ``Vagrantfile``). +From the root of the git repo (where the ``Vagrantfile`` is), run ``vagrant up`` and eventually you should be able to reach a Dataverse installation at http://localhost:8888 (the ``forwarded_port`` indicated in the ``Vagrantfile``). Please note that running ``vagrant up`` for the first time should run the ``downloads/download.sh`` script for you to download required software such as an app server, Solr, etc. However, these dependencies change over time so it's a place to look if ``vagrant up`` was working but later fails. @@ -73,7 +73,7 @@ The first time you run ``./pagekite.py`` a file at ``~/.pagekite.rc`` will be created. You can edit this file to configure PageKite to serve up port 8080 (the default app server HTTP port) or the port of your choosing. -According to https://pagekite.net/support/free-for-foss/ PageKite (very generously!) offers free accounts to developers writing software the meets http://opensource.org/docs/definition.php such as Dataverse. +According to https://pagekite.net/support/free-for-foss/ PageKite (very generously!) offers free accounts to developers writing software the meets http://opensource.org/docs/definition.php such as the Dataverse Project. MSV +++ @@ -160,7 +160,7 @@ jmap allows you to look at the contents of the java heap. It can be used to crea will output a list of all classes, sorted by the number of instances of each individual class, with the size in bytes. This can be very useful when looking for memory leaks in the application. Another useful tool is ``jstat``, that can be used in combination with ``jmap`` to monitor the effectiveness of garbage collection in reclaiming allocated memory. -In the example script below we stress running Dataverse applicatione with GET requests to a specific dataverse page, use ``jmap`` to see how many Dataverse, Dataset and DataFile class object get allocated, then run ``jstat`` to see how the numbers are affected by both "Young Generation" and "Full" garbage collection runs (``YGC`` and ``FGC`` respectively): +In the example script below we stress running Dataverse software applicatione with GET requests to a specific page in a Dataverse installation, use ``jmap`` to see how many Dataverse collection, Dataset and DataFile class object get allocated, then run ``jstat`` to see how the numbers are affected by both "Young Generation" and "Full" garbage collection runs (``YGC`` and ``FGC`` respectively): (This is script is provided **as an example only**! You will have to experiment and expand it to suit any specific needs and any specific problem you may be trying to diagnose, and this is just to give an idea of how to go about it) @@ -173,7 +173,7 @@ In the example script below we stress running Dataverse applicatione with GET re while : do - # Access the dataverse xxx 10 times in a row: + # Access the Dataverse collection xxx 10 times in a row: for ((i = 0; i < 10; i++)) do # hide the output, standard and stderr: @@ -186,7 +186,7 @@ In the example script below we stress running Dataverse applicatione with GET re jmap -histo $id > /tmp/jmap.histo.out - # grep the output for Dataverse, Dataset and DataFile classes: + # grep the output for Dataverse Collection, Dataset and DataFile classes: grep '\.Dataverse$' /tmp/jmap.histo.out grep '\.Dataset$' /tmp/jmap.histo.out grep '\.DataFile$' /tmp/jmap.histo.out @@ -237,7 +237,7 @@ The script above will run until you stop it, and will output something like: How to analyze the output, what to look for: -First, look at the numbers in the jmap output. In the example above, you can immediately see, after the first three iterations, that every 10 dataverse page loads results in the increase of the number of Dataset classes by 160. I.e., each page load leaves 16 of these on the heap. We can also see that each of the 10 page load cycles increased the heap by roughly 3GB; that each cycle resulted in a couple of YG (young generation) garbage collections, and in the old generation allocation being almost 70% full. These numbers in the example are clearly quite high and are an indication of some problematic memory allocation by the dataverse page - if this is the result of something you have added to the page, you probably would want to investigate and fix it. However, overly generous memory use **is not the same as a leak** necessarily. What you want to see now is how much of this allocation can be reclaimed by "Full GC". If all of it gets freed by ``FGC``, it is not the end of the world (even though you do not want your system to spend too much time running ``FGC``; it costs CPU cycles, and actually freezes the application while it's in progress!). It is however a **really** serious problem, if you determine that a growing portion of the old. gen. memory (``"O"`` in the ``jmap`` output) is not getting freed, even by ``FGC``. This *is* a real leak now, i.e. something is leaving behind some objects that are still referenced and thus off limits to garbage collector. So look for the lines where the ``FGC`` counter is incremented. For example, the first ``FGC`` in the example output above: +First, look at the numbers in the jmap output. In the example above, you can immediately see, after the first three iterations, that every 10 Dataverse installation page loads results in the increase of the number of Dataset classes by 160. I.e., each page load leaves 16 of these on the heap. We can also see that each of the 10 page load cycles increased the heap by roughly 3GB; that each cycle resulted in a couple of YG (young generation) garbage collections, and in the old generation allocation being almost 70% full. These numbers in the example are clearly quite high and are an indication of some problematic memory allocation by the Dataverse installation page - if this is the result of something you have added to the page, you probably would want to investigate and fix it. However, overly generous memory use **is not the same as a leak** necessarily. What you want to see now is how much of this allocation can be reclaimed by "Full GC". If all of it gets freed by ``FGC``, it is not the end of the world (even though you do not want your system to spend too much time running ``FGC``; it costs CPU cycles, and actually freezes the application while it's in progress!). It is however a **really** serious problem, if you determine that a growing portion of the old. gen. memory (``"O"`` in the ``jmap`` output) is not getting freed, even by ``FGC``. This *is* a real leak now, i.e. something is leaving behind some objects that are still referenced and thus off limits to garbage collector. So look for the lines where the ``FGC`` counter is incremented. For example, the first ``FGC`` in the example output above: .. code-block:: none @@ -257,7 +257,7 @@ First, look at the numbers in the jmap output. In the example above, you can imm 0.00 100.00 71.95 20.12 ? 22 25.034 1 4.455 29.489 Wed Aug 14 23:21:40 EDT 2019 -We can see that the first ``FGC`` resulted in reducing the ``"O"`` by almost 7GB, from 15GB down to 8GB (from 88% to 20% full). The number of Dataset classes has not budged at all - it has grown by the same 160 objects as before (very suspicious!). To complicate matters, ``FGC`` does not **guarantee** to free everything that can be freed - it will balance how much the system needs memory vs. how much it is willing to spend in terms of CPU cycles performing GC (remember, the application freezes while ``FGC`` is running!). So you should not assume that the "20% full" number above means that you have 20% of your stack already wasted and unrecoverable. Instead, look for the next **minium** value of ``"O"``; then for the next, etc. Now compare these consecutive miniums. With the above test (this is an output of a real experiment, a particularly memory-hungry feature added to the dataverse page), the minimums sequence (of old. gen. usage, in %) was looking as follows: +We can see that the first ``FGC`` resulted in reducing the ``"O"`` by almost 7GB, from 15GB down to 8GB (from 88% to 20% full). The number of Dataset classes has not budged at all - it has grown by the same 160 objects as before (very suspicious!). To complicate matters, ``FGC`` does not **guarantee** to free everything that can be freed - it will balance how much the system needs memory vs. how much it is willing to spend in terms of CPU cycles performing GC (remember, the application freezes while ``FGC`` is running!). So you should not assume that the "20% full" number above means that you have 20% of your stack already wasted and unrecoverable. Instead, look for the next **minium** value of ``"O"``; then for the next, etc. Now compare these consecutive miniums. With the above test (this is an output of a real experiment, a particularly memory-hungry feature added to the Dataverse installation page), the minimums sequence (of old. gen. usage, in %) was looking as follows: .. code-block:: none From c8e4404af36c45f787c41d721d0fd4bce0516462 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 12 Jan 2021 22:48:43 -0500 Subject: [PATCH 107/576] more updates --- .../source/developers/troubleshooting.rst | 16 +++++------ .../source/developers/version-control.rst | 28 +++++++++---------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/doc/sphinx-guides/source/developers/troubleshooting.rst b/doc/sphinx-guides/source/developers/troubleshooting.rst index 2964684f7a9..a53a08ce77f 100755 --- a/doc/sphinx-guides/source/developers/troubleshooting.rst +++ b/doc/sphinx-guides/source/developers/troubleshooting.rst @@ -2,7 +2,7 @@ Troubleshooting =============== -Over in the :doc:`dev-environment` section we described the "happy path" of when everything goes right as you set up your Dataverse development environment. Here are some common problems and solutions for when things go wrong. +Over in the :doc:`dev-environment` section we described the "happy path" of when everything goes right as you set up your Dataverse software development environment. Here are some common problems and solutions for when things go wrong. .. contents:: |toctitle| :local: @@ -14,7 +14,7 @@ For unknown reasons, Netbeans will sometimes change the following line under ``s ``/`` -Sometimes Netbeans will change ``/`` to ``/dataverse``. Sometimes it will delete the line entirely. Either way, you will see very strange behavior when attempting to click around Dataverse in a browser. The homepage will load but icons will be missing. Any other page will fail to load entirely and you'll see an app server error. +Sometimes Netbeans will change ``/`` to ``/dataverse``. Sometimes it will delete the line entirely. Either way, you will see very strange behavior when attempting to click around the Dataverse installation in a browser. The homepage will load but icons will be missing. Any other page will fail to load entirely and you'll see an app server error. The solution is to put the file back to how it was before Netbeans touched it. If anyone knows of an open Netbeans bug about this, please let us know. @@ -41,7 +41,7 @@ This command helps verify what host your domain is using to send mail. Even if i 2. From the left-side panel, select **JavaMail Sessions** 3. You should see one session named **mail/notifyMailSession** -- click on that. -From this window you can modify certain fields of your Dataverse's notifyMailSession, which is the JavaMail session for outgoing system email (such as on user signup or data publication). Two of the most important fields we need are: +From this window you can modify certain fields of your Dataverse installation's notifyMailSession, which is the JavaMail session for outgoing system email (such as on user signup or data publication). Two of the most important fields we need are: - **Mail Host:** The DNS name of the default mail server (e.g. smtp.gmail.com) - **Default User:** The username provided to your Mail Host when you connect to it (e.g. johndoe@gmail.com) @@ -50,7 +50,7 @@ Most of the other defaults can safely be left as is. **Default Sender Address** If your user credentials for the SMTP server require a password, you'll need to configure some **Additional Properties** at the bottom. -**IMPORTANT:** Before continuing, it's highly recommended that your Default User account does NOT use a password you share with other accounts, as one of the additional properties includes entering the Default User's password (without concealing it on screen). For smtp.gmail.com you can safely use an `app password `_ or create an extra Gmail account for use with your Dataverse dev environment. +**IMPORTANT:** Before continuing, it's highly recommended that your Default User account does NOT use a password you share with other accounts, as one of the additional properties includes entering the Default User's password (without concealing it on screen). For smtp.gmail.com you can safely use an `app password `_ or create an extra Gmail account for use with your Dataverse software development environment. Authenticating yourself to a Mail Host can be tricky. As an example, we'll walk through setting up our JavaMail Session to use smtp.gmail.com as a host by way of SSL on port 465. Use the Add Property button to generate a blank property for each name/value pair. @@ -74,9 +74,9 @@ The mail session can also be set from command line. To use this method, you will - Delete: ``./asadmin delete-javamail-resource mail/MyMailSession`` - Create (remove brackets and replace the variables inside): ``./asadmin create-javamail-resource --mailhost [smtp.gmail.com] --mailuser [test\@test\.com] --fromaddress [test\@test\.com] --property mail.smtp.auth=[true]:mail.smtp.password=[password]:mail.smtp.port=[465]:mail.smtp.socketFactory.port=[465]:mail.smtp.socketFactory.fallback=[false]:mail.smtp.socketFactory.class=[javax.net.ssl.SSLSocketFactory] mail/notifyMailSession`` -These properties can be tailored to your own preferred mail service, but if all else fails these settings work fine with Dataverse development environments for your localhost. +These properties can be tailored to your own preferred mail service, but if all else fails these settings work fine with Dataverse software development environments for your localhost. -+ If you're seeing a "Relay access denied" error in your app server logs when Dataverse attempts to send an email, double check your user/password credentials for the Mail Host you're using. ++ If you're seeing a "Relay access denied" error in your app server logs when the Dataverse installation attempts to send an email, double check your user/password credentials for the Mail Host you're using. + If you're seeing a "Connection refused" / similar error upon email sending, try another port. As another example, here is how to create a Mail Host via command line for Amazon SES: @@ -97,8 +97,8 @@ A script called :download:`dev-rebuild.sh <../../../../scripts/dev/dev-rebuild.s - Deploys the war file located in the ``target`` directory. - Runs ``setup-all.sh`` in insecure mode so tests will pass. - Runs post-install SQL statements. -- Publishes the root dataverse. -- Adjusts permissions on on the root dataverse so tests will pass. +- Publishes the root Dataverse collection. +- Adjusts permissions on on the root Dataverse collection so tests will pass. To execute the script, make sure you have built a war file already and then ``cd`` to the root of the source tree and run ``scripts/dev/dev-rebuild.sh``. Feedback on this script is welcome! diff --git a/doc/sphinx-guides/source/developers/version-control.rst b/doc/sphinx-guides/source/developers/version-control.rst index eaaf0fa1911..e6161966563 100644 --- a/doc/sphinx-guides/source/developers/version-control.rst +++ b/doc/sphinx-guides/source/developers/version-control.rst @@ -2,16 +2,16 @@ Version Control ================== -The Dataverse Project uses git for version control and GitHub for hosting. On this page we'll explain where to find the code, our branching strategey, advice on how to make a pull request, and other git tips. +The Dataverse Project uses git for version control and GitHub for hosting. On this page we'll explain where to find the code, our branching strategy, advice on how to make a pull request, and other git tips. .. contents:: |toctitle| :local: -Where to Find the Dataverse Code --------------------------------- +Where to Find the Dataverse Software Code +----------------------------------------- -The main Dataverse code at https://github.com/IQSS/dataverse but as explained in the :doc:`intro` section under "Related Projects", there are many other code bases you can hack on if you wish! +The main Dataverse software code at https://github.com/IQSS/dataverse but as explained in the :doc:`intro` section under "Related Projects", there are many other code bases you can hack on if you wish! Branching Strategy ------------------ @@ -19,7 +19,7 @@ Branching Strategy Goals ~~~~~ -The goals of the Dataverse branching strategy are: +The goals of the Dataverse software branching strategy are: - allow for concurrent development - only ship stable code @@ -32,7 +32,7 @@ Branches The "master" Branch ******************* -The "`master `_" branch represents released versions of Dataverse. As mentioned in the :doc:`making-releases` section, at release time we update the master branch to include all the code for that release. Commits are never made directly to master. Rather, master is updated only when we merge code into it from the "develop" branch. +The "`master `_" branch represents released versions of the Dataverse software. As mentioned in the :doc:`making-releases` section, at release time we update the master branch to include all the code for that release. Commits are never made directly to master. Rather, master is updated only when we merge code into it from the "develop" branch. The "develop" Branch ******************** @@ -60,7 +60,7 @@ Find or Create a GitHub Issue For guidance on which issue to work on, please ask! Also, see https://github.com/IQSS/dataverse/blob/develop/CONTRIBUTING.md -Let's say you want to tackle https://github.com/IQSS/dataverse/issues/3728 which points out a typo in a page of Dataverse's documentation. +Let's say you want to tackle https://github.com/IQSS/dataverse/issues/3728 which points out a typo in a page of the Dataverse software's documentation. If you tell us your GitHub username we are happy to add you to the "read only" team at https://github.com/orgs/IQSS/teams/dataverse-readonly/members so that we can assign the issue to you while you're working on it. You can also tell us if you'd like to be added to the `Dataverse Community Contributors spreadsheet `_. @@ -77,7 +77,7 @@ Making a commit (or several commits) to that branch. Ideally the first line of y Push Your Branch to GitHub ~~~~~~~~~~~~~~~~~~~~~~~~~~ -Push your feature branch to your fork of Dataverse. Your git command may look something like ``git push origin 3728-doc-apipolicy-fix``. +Push your feature branch to your fork of the Dataverse software. Your git command may look something like ``git push origin 3728-doc-apipolicy-fix``. Make a Pull Request ~~~~~~~~~~~~~~~~~~~ @@ -98,8 +98,8 @@ Summary of Git commands This section provides sequences of Git commands for two scenarios: -* preparing the first request, when the IQSS Dataverse repository and the forked repository are identical -* creating an additional request after some time, when the IQSS Dataverse repository is ahead of the forked repository +* preparing the first request, when the IQSS Dataverse software repository and the forked repository are identical +* creating an additional request after some time, when the IQSS Dataverse software repository is ahead of the forked repository In the examples we use 123-COOL-FEATURE as the name of the feature branch, and https://github.com/YOUR_NAME/dataverse.git as your forked repository's URL. In practice modify both accordingly. @@ -157,7 +157,7 @@ How to Resolve Conflicts in Your Pull Request Unfortunately, pull requests can quickly become "stale" and unmergable as other pull requests are merged into the develop branch ahead of you. This is completely normal, and often occurs because other developers made their pull requests before you did. -The Dataverse team may ping you to ask you to merge the latest from the develop branch into your branch and resolve merge conflicts. If this sounds daunting, please just say so and we will assist you. +The Dataverse Project team may ping you to ask you to merge the latest from the develop branch into your branch and resolve merge conflicts. If this sounds daunting, please just say so and we will assist you. If you'd like to resolve the merge conflicts yourself, here are some steps to do so that make use of GitHub Desktop and Netbeans. @@ -169,7 +169,7 @@ If you'd like to resolve the merge conflicts yourself, here are some steps to do **In Netbeans:** -4. Click Window -> Favorites and open your local Dataverse project folder in the Favorites panel. +4. Click Window -> Favorites and open your local Dataverse software project folder in the Favorites panel. 5. In this file browser, you can follow the red cylinder icon to find files with merge conflicts. 6. Double click the red merge conflicted file. 7. Right click on the red tab for that file and select Git -> Resolve Conflicts. @@ -182,14 +182,14 @@ If you'd like to resolve the merge conflicts yourself, here are some steps to do **In GitHub Issues:** -11. Leave a comment for the Dataverse team that you have resolved the merge conflicts. +11. Leave a comment for the Dataverse Project team that you have resolved the merge conflicts. Adding Commits to a Pull Request from a Fork -------------------------------------------- By default, when a pull request is made from a fork, "Allow edits from maintainers" is checked as explained at https://help.github.com/articles/allowing-changes-to-a-pull-request-branch-created-from-a-fork/ -This is a nice feature of GitHub because it means that the core dev team for Dataverse can make small (or even large) changes to a pull request from a contributor to help the pull request along on its way to QA and being merged. +This is a nice feature of GitHub because it means that the core dev team for the Dataverse Project can make small (or even large) changes to a pull request from a contributor to help the pull request along on its way to QA and being merged. GitHub documents how to make changes to a fork at https://help.github.com/articles/committing-changes-to-a-pull-request-branch-created-from-a-fork/ but as of this writing the steps involve making a new clone of the repo. This works but you might find it more convenient to add a "remote" to your existing clone. The example below uses the fork at https://github.com/OdumInstitute/dataverse and the branch ``4709-postgresql_96`` but the technique can be applied to any fork and branch: From 0aa8c5e7e934e514805ddb434c229f2cf9b16aa9 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 12 Jan 2021 22:53:35 -0500 Subject: [PATCH 108/576] more changes --- .../source/developers/windows.rst | 10 +++++----- .../source/developers/workflows.rst | 20 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst index ec5d4cb971f..0cfc329377e 100755 --- a/doc/sphinx-guides/source/developers/windows.rst +++ b/doc/sphinx-guides/source/developers/windows.rst @@ -4,13 +4,13 @@ Windows Development Development on Windows is not well supported, unfortunately. You will have a much easier time if you develop on Mac or Linux as described under :doc:`dev-environment` section. -Vagrant commands appear below and were tested on Windows 10 but the Vagrant environement is currently broken. Please see https://github.com/IQSS/dataverse/issues/6849 +Vagrant commands appear below and were tested on Windows 10 but the Vagrant environment is currently broken. Please see https://github.com/IQSS/dataverse/issues/6849 .. contents:: |toctitle| :local: -Running Dataverse in Vagrant ----------------------------- +Running the Dataverse Software in Vagrant +----------------------------------------- Install Vagrant ~~~~~~~~~~~~~~~ @@ -69,7 +69,7 @@ The ``dataverse`` directory you changed is the one you just cloned. Vagrant will ``vagrant up`` -After a long while you hopefully will have Dataverse installed at http://localhost:8888 +After a long while you hopefully will have a Dataverse installation available at http://localhost:8888 Improving Windows Support ------------------------- @@ -82,4 +82,4 @@ We have been unable to get Windows Subsystem for Linux (WSL) to work. We tried f Discussion and Feedback ~~~~~~~~~~~~~~~~~~~~~~~ -For more discussion of Windows support for Dataverse development see our community list thread `"Do you want to develop on Windows?" `_ We would be happy to incorporate feedback from Windows developers into this page. The :doc:`documentation` section describes how. +For more discussion of Windows support for Dataverse software development see our community list thread `"Do you want to develop on Windows?" `_ We would be happy to incorporate feedback from Windows developers into this page. The :doc:`documentation` section describes how. diff --git a/doc/sphinx-guides/source/developers/workflows.rst b/doc/sphinx-guides/source/developers/workflows.rst index b9090b86be3..f823f8459b1 100644 --- a/doc/sphinx-guides/source/developers/workflows.rst +++ b/doc/sphinx-guides/source/developers/workflows.rst @@ -1,7 +1,7 @@ Workflows ================ -Dataverse has a flexible workflow mechanism that can be used to trigger actions before and after Dataset publication. +The Dataverse software has a flexible workflow mechanism that can be used to trigger actions before and after Dataset publication. .. contents:: |toctitle| :local: @@ -10,35 +10,35 @@ Dataverse has a flexible workflow mechanism that can be used to trigger actions Introduction ------------ -Dataverse can perform two sequences of actions when datasets are published: one prior to publishing (marked by a ``PrePublishDataset`` trigger), and one after the publication has succeeded (``PostPublishDataset``). The pre-publish workflow is useful for having an external system prepare a dataset for being publicly accessed (a possibly lengthy activity that requires moving files around, uploading videos to a streaming server, etc.), or to start an approval process. A post-publish workflow might be used for sending notifications about the newly published dataset. +The Dataverse software can perform two sequences of actions when datasets are published: one prior to publishing (marked by a ``PrePublishDataset`` trigger), and one after the publication has succeeded (``PostPublishDataset``). The pre-publish workflow is useful for having an external system prepare a dataset for being publicly accessed (a possibly lengthy activity that requires moving files around, uploading videos to a streaming server, etc.), or to start an approval process. A post-publish workflow might be used for sending notifications about the newly published dataset. -Workflow steps are created using *step providers*. Dataverse ships with an internal step provider that offers some basic functionality, and with the ability to load 3rd party step providers. This allows installations to implement functionality they need without changing the Dataverse source code. +Workflow steps are created using *step providers*. The Dataverse software ships with an internal step provider that offers some basic functionality, and with the ability to load 3rd party step providers. This allows installations to implement functionality they need without changing the Dataverse software source code. -Steps can be internal (say, writing some data to the log) or external. External steps involve Dataverse sending a request to an external system, and waiting for the system to reply. The wait period is arbitrary, and so allows the external system unbounded operation time. This is useful, e.g., for steps that require human intervension, such as manual approval of a dataset publication. +Steps can be internal (say, writing some data to the log) or external. External steps involve the Dataverse software sending a request to an external system, and waiting for the system to reply. The wait period is arbitrary, and so allows the external system unbounded operation time. This is useful, e.g., for steps that require human intervention, such as manual approval of a dataset publication. -The external system reports the step result back to dataverse, by sending a HTTP ``POST`` command to ``api/workflows/{invocation-id}``. The body of the request is passed to the paused step for further processing. +The external system reports the step result back to the Dataverse installation, by sending a HTTP ``POST`` command to ``api/workflows/{invocation-id}``. The body of the request is passed to the paused step for further processing. -If a step in a workflow fails, Dataverse make an effort to roll back all the steps that preceded it. Some actions, such as writing to the log, cannot be rolled back. If such an action has a public external effect (e.g. send an EMail to a mailing list) it is advisable to put it in the post-release workflow. +If a step in a workflow fails, the Dataverse installation makes an effort to roll back all the steps that preceded it. Some actions, such as writing to the log, cannot be rolled back. If such an action has a public external effect (e.g. send an EMail to a mailing list) it is advisable to put it in the post-release workflow. .. tip:: - For invoking external systems using a REST api, Dataverse's internal step + For invoking external systems using a REST api, the Dataverse software's internal step provider offers a step for sending and receiving customizable HTTP requests. It's called *http/sr*, and is detailed below. Administration ~~~~~~~~~~~~~~ -A Dataverse instance stores a set of workflows in its database. Workflows can be managed using the ``api/admin/workflows/`` endpoints of the :doc:`/api/native-api`. Sample workflow files are available in ``scripts/api/data/workflows``. +A Dataverse installation stores a set of workflows in its database. Workflows can be managed using the ``api/admin/workflows/`` endpoints of the :doc:`/api/native-api`. Sample workflow files are available in ``scripts/api/data/workflows``. At the moment, defining a workflow for each trigger is done for the entire instance, using the endpoint ``api/admin/workflows/default/«trigger type»``. -In order to prevent unauthorized resuming of workflows, Dataverse maintains a "white list" of IP addresses from which resume requests are honored. This list is maintained using the ``/api/admin/workflows/ip-whitelist`` endpoint of the :doc:`/api/native-api`. By default, Dataverse honors resume requests from localhost only (``127.0.0.1;::1``), so set-ups that use a single server work with no additional configuration. +In order to prevent unauthorized resuming of workflows, the Dataverse installation maintains a "white list" of IP addresses from which resume requests are honored. This list is maintained using the ``/api/admin/workflows/ip-whitelist`` endpoint of the :doc:`/api/native-api`. By default, the Dataverse installation honors resume requests from localhost only (``127.0.0.1;::1``), so set-ups that use a single server work with no additional configuration. Available Steps ~~~~~~~~~~~~~~~ -Dataverse has an internal step provider, whose id is ``:internal``. It offers the following steps: +The Dataverse software has an internal step provider, whose id is ``:internal``. It offers the following steps: log +++ From 2f4fadd5ac09ca5afaf9bd65bf316a30fcb6186b Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 12 Jan 2021 23:00:23 -0500 Subject: [PATCH 109/576] onto the installation guide! --- doc/sphinx-guides/source/installation/config.rst | 8 ++++---- doc/sphinx-guides/source/installation/external-tools.rst | 2 +- doc/sphinx-guides/source/installation/geoconnect.rst | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 4a877eabff7..09bc1647d19 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2,9 +2,9 @@ Configuration ============= -Now that you've successfully logged into Dataverse with a superuser account after going through a basic :doc:`installation-main`, you'll need to secure and configure your installation. +Now that you've successfully logged into your Dataverse installation with a superuser account after going through a basic :doc:`installation-main`, you'll need to secure and configure your installation. -Settings within Dataverse itself are managed via JVM options or by manipulating values in the ``setting`` table directly or through API calls. +Settings within your Dataverse installation itself are managed via JVM options or by manipulating values in the ``setting`` table directly or through API calls. Once you have finished securing and configuring your Dataverse installation, you may proceed to the :doc:`/admin/index` for more information on the ongoing administration of a Dataverse installation. Advanced configuration topics are covered in the :doc:`r-rapache-tworavens`, :doc:`shibboleth` and :doc:`oauth2` sections. @@ -45,9 +45,9 @@ To avoid having your users send credentials in the clear, it's strongly recommen Recording User IP Addresses +++++++++++++++++++++++++++ -By default, Dataverse captures the IP address from which requests originate. This is used for multiple purposes including controlling access to the admin API, IP-based user groups and Make Data Count reporting. When Dataverse is configured behind a proxy such as a load balancer, this default setup may not capture the correct IP address. In this case all the incoming requests will be logged in the access logs, MDC logs etc., as if they are all coming from the IP address(es) of the load balancer itself. Proxies usually save the original address in an added HTTP header, from which it can be extracted. For example, AWS LB records the "true" original address in the standard ``X-Forwarded-For`` header. If your Dataverse is running behind an IP-masking proxy, but you would like to use IP groups, or record the true geographical location of the incoming requests with Make Data Count, you may enable the IP address lookup from the proxy header using the JVM option ``dataverse.useripaddresssourceheader``, described further below. +By default, the Dataverse installation captures the IP address from which requests originate. This is used for multiple purposes including controlling access to the admin API, IP-based user groups and Make Data Count reporting. When the Dataverse installation is configured behind a proxy such as a load balancer, this default setup may not capture the correct IP address. In this case all the incoming requests will be logged in the access logs, MDC logs etc., as if they are all coming from the IP address(es) of the load balancer itself. Proxies usually save the original address in an added HTTP header, from which it can be extracted. For example, AWS LB records the "true" original address in the standard ``X-Forwarded-For`` header. If your Dataverse installation is running behind an IP-masking proxy, but you would like to use IP groups, or record the true geographical location of the incoming requests with Make Data Count, you may enable the IP address lookup from the proxy header using the JVM option ``dataverse.useripaddresssourceheader``, described further below. -Before doing so however, you must absolutely **consider the security risks involved**! This option must be enabled **only** on a Dataverse that is in fact fully behind a proxy that properly, and consistently, adds the ``X-Forwarded-For`` (or a similar) header to every request it forwards. Consider the implications of activating this option on a Dataverse that is not running behind a proxy, *or running behind one, but still accessible from the insecure locations bypassing the proxy*: Anyone can now add the header above to an incoming reqest, supplying an arbitrary IP address that Dataverse will trust as the true origin of the call. Thus giving an attacker an easy way to, for example, get in a privileged IP group. The implications could be even more severe if an attacker were able to pretend to be coming from ``localhost``, if Dataverse is configured to trust localhost connections for unrestricted access to the admin API! We have addressed this by making it so that Dataverse should never accept ``localhost``, ``127.0.0.1``, ``0:0:0:0:0:0:0:1`` etc. when supplied in such a header. But if you have reasons to still find this risk unacceptable, you may want to consider turning open localhost access to the API off (See :ref:`Securing Your Installation ` for more information.) +Before doing so however, you must absolutely **consider the security risks involved**! This option must be enabled **only** on a Dataverse installation that is in fact fully behind a proxy that properly, and consistently, adds the ``X-Forwarded-For`` (or a similar) header to every request it forwards. Consider the implications of activating this option on a Dataverse installation that is not running behind a proxy, *or running behind one, but still accessible from the insecure locations bypassing the proxy*: Anyone can now add the header above to an incoming request, supplying an arbitrary IP address that the Dataverse installation will trust as the true origin of the call. Thus giving an attacker an easy way to, for example, get in a privileged IP group. The implications could be even more severe if an attacker were able to pretend to be coming from ``localhost``, if a Dataverse installation is configured to trust localhost connections for unrestricted access to the admin API! We have addressed this by making it so that Dataverse installation should never accept ``localhost``, ``127.0.0.1``, ``0:0:0:0:0:0:0:1`` etc. when supplied in such a header. But if you have reasons to still find this risk unacceptable, you may want to consider turning open localhost access to the API off (See :ref:`Securing Your Installation ` for more information.) This is how to verify that your proxy or load balancer, etc. is handling the originating address headers properly and securely: Make sure access logging is enabled in your application server (Payara) configuration. (```` in the ``domain.xml``). Add the address header to the access log format. For example, on a system behind AWS ELB, you may want to use something like ``%client.name% %datetime% %request% %status% %response.length% %header.referer% %header.x-forwarded-for%``. Once enabled, access the Dataverse from outside the LB. You should now see the real IP address of your remote client in the access log. For example, something like: ``"1.2.3.4" "01/Jun/2020:12:00:00 -0500" "GET /dataverse.xhtml HTTP/1.1" 200 81082 "NULL-REFERER" "128.64.32.16"`` diff --git a/doc/sphinx-guides/source/installation/external-tools.rst b/doc/sphinx-guides/source/installation/external-tools.rst index 028802fd736..2517a2c14c8 100644 --- a/doc/sphinx-guides/source/installation/external-tools.rst +++ b/doc/sphinx-guides/source/installation/external-tools.rst @@ -1,7 +1,7 @@ External Tools ============== -External tools can provide additional features that are not part of Dataverse itself, such as data exploration. +External tools can provide additional features that are not part of the Dataverse software itself, such as data exploration. .. contents:: |toctitle| :local: diff --git a/doc/sphinx-guides/source/installation/geoconnect.rst b/doc/sphinx-guides/source/installation/geoconnect.rst index 14a12b50a3f..430c0eb807c 100644 --- a/doc/sphinx-guides/source/installation/geoconnect.rst +++ b/doc/sphinx-guides/source/installation/geoconnect.rst @@ -4,7 +4,7 @@ Geoconnect .. contents:: |toctitle| :local: -Geoconnect works as a middle layer, allowing geospatial data files in Dataverse to be visualized with Harvard WorldMap. +Geoconnect works as a middle layer, allowing geospatial data files in a Dataverse installation to be visualized with Harvard WorldMap. To understand the feature from the user perspective, see the :doc:`/user/data-exploration/worldmap` section of the User Guide. @@ -12,6 +12,6 @@ As of this writing, the README at https://github.com/IQSS/geoconnect recommends To set up a Geoconnect development environment, you can follow the steps outlined in the `local_setup.md `_ guide. Although those instructions are for a local development setup, they may assist in installing Geoconnect in your production environment. -Harvard Dataverse runs Geoconnect on Heroku. To make use of Heroku, you will need a Heroku account, as well as a few other prerequisites. Follow the instructions outlined in the `heroku_setup.md `_ guide. The `heroku.py `_ settings file may also be adapted for other environments. Please note, for the production environment, remember to set ``DEBUG=False``. +The Harvard Dataverse Repository runs Geoconnect on Heroku. To make use of Heroku, you will need a Heroku account, as well as a few other prerequisites. Follow the instructions outlined in the `heroku_setup.md `_ guide. The `heroku.py `_ settings file may also be adapted for other environments. Please note, for the production environment, remember to set ``DEBUG=False``. See also the :doc:`/admin/geoconnect-worldmap` section of the Admin Guide. From b81bef41da0e4164df4d508fae2b556c55dabbf6 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 12 Jan 2021 23:35:47 -0500 Subject: [PATCH 110/576] more updates --- .../source/installation/config.rst | 257 +++++++++--------- 1 file changed, 128 insertions(+), 129 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 09bc1647d19..37f419f2cbd 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -49,16 +49,16 @@ By default, the Dataverse installation captures the IP address from which reques Before doing so however, you must absolutely **consider the security risks involved**! This option must be enabled **only** on a Dataverse installation that is in fact fully behind a proxy that properly, and consistently, adds the ``X-Forwarded-For`` (or a similar) header to every request it forwards. Consider the implications of activating this option on a Dataverse installation that is not running behind a proxy, *or running behind one, but still accessible from the insecure locations bypassing the proxy*: Anyone can now add the header above to an incoming request, supplying an arbitrary IP address that the Dataverse installation will trust as the true origin of the call. Thus giving an attacker an easy way to, for example, get in a privileged IP group. The implications could be even more severe if an attacker were able to pretend to be coming from ``localhost``, if a Dataverse installation is configured to trust localhost connections for unrestricted access to the admin API! We have addressed this by making it so that Dataverse installation should never accept ``localhost``, ``127.0.0.1``, ``0:0:0:0:0:0:0:1`` etc. when supplied in such a header. But if you have reasons to still find this risk unacceptable, you may want to consider turning open localhost access to the API off (See :ref:`Securing Your Installation ` for more information.) -This is how to verify that your proxy or load balancer, etc. is handling the originating address headers properly and securely: Make sure access logging is enabled in your application server (Payara) configuration. (```` in the ``domain.xml``). Add the address header to the access log format. For example, on a system behind AWS ELB, you may want to use something like ``%client.name% %datetime% %request% %status% %response.length% %header.referer% %header.x-forwarded-for%``. Once enabled, access the Dataverse from outside the LB. You should now see the real IP address of your remote client in the access log. For example, something like: +This is how to verify that your proxy or load balancer, etc. is handling the originating address headers properly and securely: Make sure access logging is enabled in your application server (Payara) configuration. (```` in the ``domain.xml``). Add the address header to the access log format. For example, on a system behind AWS ELB, you may want to use something like ``%client.name% %datetime% %request% %status% %response.length% %header.referer% %header.x-forwarded-for%``. Once enabled, access the Dataverse installation from outside the LB. You should now see the real IP address of your remote client in the access log. For example, something like: ``"1.2.3.4" "01/Jun/2020:12:00:00 -0500" "GET /dataverse.xhtml HTTP/1.1" 200 81082 "NULL-REFERER" "128.64.32.16"`` In this example, ``128.64.32.16`` is your remote address (that you should verify), and ``1.2.3.4`` is the address of your LB. If you're not seeing your remote address in the log, do not activate the JVM option! Also, verify that all the entries in the log have this header populated. The only entries in the access log that you should be seeing without this header (logged as ``"NULL-HEADER-X-FORWARDED-FOR"``) are local requests, made from localhost, etc. In this case, since the request is not coming through the proxy, the local IP address should be logged as the primary one (as the first value in the log entry, ``%client.name%``). If you see any requests coming in from remote, insecure subnets without this header - do not use the JVM option! -Once you are ready, enable the :ref:`JVM option `. Verify that the remote locations are properly tracked in your MDC metrics, and/or your IP groups are working. As a final test, if your Dataverse is allowing unrestricted localhost access to the admin API, imitate an attack in which a malicious request is pretending to be coming from ``127.0.0.1``. Try the following from a remote, insecure location: +Once you are ready, enable the :ref:`JVM option `. Verify that the remote locations are properly tracked in your MDC metrics, and/or your IP groups are working. As a final test, if your Dataverse installation is allowing unrestricted localhost access to the admin API, imitate an attack in which a malicious request is pretending to be coming from ``127.0.0.1``. Try the following from a remote, insecure location: ``curl https://your.dataverse.edu/api/admin/settings --header "X-FORWARDED-FOR: 127.0.0.1"`` -First of all, confirm that access is denied! If you are in fact able to access the settings api from a location outside the proxy, **something is seriously wrong**, so please let us know, and stop using the JVM option. Otherwise check the access log entry for the header value. What you should see is something like ``"127.0.0.1, 128.64.32.16"``. Where the second address should be the real IP of your remote client. The fact that the "fake" ``127.0.0.1`` you sent over is present in the header is perfectly ok. This is the proper proxy behavior - it preserves any incoming values in the ``X-Forwarded-Header``, if supplied, and adds the detected incoming address to it, *on the right*. It is only this rightmost comma-separated value that Dataverse should ever be using. +First of all, confirm that access is denied! If you are in fact able to access the settings api from a location outside the proxy, **something is seriously wrong**, so please let us know, and stop using the JVM option. Otherwise check the access log entry for the header value. What you should see is something like ``"127.0.0.1, 128.64.32.16"``. Where the second address should be the real IP of your remote client. The fact that the "fake" ``127.0.0.1`` you sent over is present in the header is perfectly ok. This is the proper proxy behavior - it preserves any incoming values in the ``X-Forwarded-Header``, if supplied, and adds the detected incoming address to it, *on the right*. It is only this rightmost comma-separated value that Dataverse installation should ever be using. Still feel like activating this option in your configuration? - Have fun and be safe! @@ -71,7 +71,7 @@ Privacy Considerations Email Privacy ^^^^^^^^^^^^^ -Out of the box, Dataverse will list email addresses of the contacts for datasets when users visit a dataset page and click the "Export Metadata" button. Additionally, out of the box, Dataverse will list email addresses of dataverse contacts via API (see :ref:`View a Dataverse ` in the :doc:`/api/native-api` section of the API Guide). If you would like to exclude these email addresses from export, set :ref:`:ExcludeEmailFromExport <:ExcludeEmailFromExport>` to true. +Out of the box, your Dataverse installation will list email addresses of the contacts for datasets when users visit a dataset page and click the "Export Metadata" button. Additionally, out of the box, the Dataverse installation will list email addresses of Dataverse collection contacts via API (see :ref:`View a Dataverse Collection ` in the :doc:`/api/native-api` section of the API Guide). If you would like to exclude these email addresses from export, set :ref:`:ExcludeEmailFromExport <:ExcludeEmailFromExport>` to true. Additional Recommendations ++++++++++++++++++++++++++ @@ -86,9 +86,9 @@ Related to this is that you should remove ``/root/.payara/pass`` to ensure that Enforce Strong Passwords for User Accounts ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Dataverse only stores passwords (as salted hash, and using a strong hashing algorithm) for "builtin" users. You can increase the password complexity rules to meet your security needs. If you have configured your Dataverse installation to allow login from remote authentication providers such as Shibboleth, ORCID, GitHub or Google, you do not have any control over those remote providers' password complexity rules. See the :ref:`auth-modes` section below for more on login options. +Your Dataverse installation only stores passwords (as salted hash, and using a strong hashing algorithm) for "builtin" users. You can increase the password complexity rules to meet your security needs. If you have configured your Dataverse installation to allow login from remote authentication providers such as Shibboleth, ORCID, GitHub or Google, you do not have any control over those remote providers' password complexity rules. See the :ref:`auth-modes` section below for more on login options. -Even if you are satisfied with the out-of-the-box password complexity rules Dataverse ships with, for the "dataverseAdmin" account you should use a strong password so the hash cannot easily be cracked through dictionary attacks. +Even if you are satisfied with the out-of-the-box password complexity rules the Dataverse software ships with, for the "dataverseAdmin" account you should use a strong password so the hash cannot easily be cracked through dictionary attacks. Password complexity rules for "builtin" accounts can be adjusted with a variety of settings documented below. Here's a list: @@ -106,15 +106,15 @@ Password complexity rules for "builtin" accounts can be adjusted with a variety Network Ports ------------- -Remember how under "Decisions to Make" in the :doc:`prep` section we mentioned you'll need to make a decision about whether or not to introduce a proxy in front of Dataverse such as Apache or nginx? The time has come to make that decision. +Remember how under "Decisions to Make" in the :doc:`prep` section we mentioned you'll need to make a decision about whether or not to introduce a proxy in front of the Dataverse software such as Apache or nginx? The time has come to make that decision. -The need to redirect port HTTP (port 80) to HTTPS (port 443) for security has already been mentioned above and the fact that Payara puts these services on 8080 and 8181, respectively, was touched on in the :doc:`installation-main` section. In production, you don't want to tell your users to use Dataverse on ports 8080 and 8181. You should have them use the standard HTTPS port, which is 443. +The need to redirect port HTTP (port 80) to HTTPS (port 443) for security has already been mentioned above and the fact that Payara puts these services on 8080 and 8181, respectively, was touched on in the :doc:`installation-main` section. In production, you don't want to tell your users to use your Dataverse installation on ports 8080 and 8181. You should have them use the standard HTTPS port, which is 443. -Your decision to proxy or not should primarily be driven by which features of Dataverse you'd like to use. If you'd like to use Shibboleth, the decision is easy because proxying or "fronting" Payara with Apache is required. The details are covered in the :doc:`shibboleth` section. +Your decision to proxy or not should primarily be driven by which features of the Dataverse software you'd like to use. If you'd like to use Shibboleth, the decision is easy because proxying or "fronting" Payara with Apache is required. The details are covered in the :doc:`shibboleth` section. If you'd like to use TwoRavens, you should also consider fronting with Apache because you will be required to install an Apache anyway to make use of the rApache module. For details, see the :doc:`r-rapache-tworavens` section. -Even if you have no interest in Shibboleth nor TwoRavens, you may want to front Dataverse with Apache or nginx to simply the process of installing SSL certificates. There are many tutorials on the Internet for adding certs to Apache, including a some `notes used by the Dataverse team `_, but the process of adding a certificate to Payara is arduous and not for the faint of heart. The Dataverse team cannot provide much help with adding certificates to Payara beyond linking to `tips `_ on the web. +Even if you have no interest in Shibboleth nor TwoRavens, you may want to front your Dataverse installation with Apache or nginx to simply the process of installing SSL certificates. There are many tutorials on the Internet for adding certs to Apache, including a some `notes used by the Dataverse Project team `_, but the process of adding a certificate to Payara is arduous and not for the faint of heart. The Dataverse Project team cannot provide much help with adding certificates to Payara beyond linking to `tips `_ on the web. Still not convinced you should put Payara behind another web server? Even if you manage to get your SSL certificate into Payara, how are you going to run Payara on low ports such as 80 and 443? Are you going to run Payara as root? Bad idea. This is a security risk. Under "Additional Recommendations" under "Securing Your Installation" above you are advised to configure Payara to run as a user other than root. @@ -126,7 +126,7 @@ If you really don't want to front Payara with any proxy (not recommended), you c ``./asadmin set server-config.network-config.network-listeners.network-listener.http-listener-2.port=443`` -What about port 80? Even if you don't front Dataverse with Apache, you may want to let Apache run on port 80 just to rewrite HTTP to HTTPS as described above. You can use a similar command as above to change the HTTP port that Payara uses from 8080 to 80 (substitute ``http-listener-1.port=80``). Payara can be used to enforce HTTPS on its own without Apache, but configuring this is an exercise for the reader. Answers here may be helpful: http://stackoverflow.com/questions/25122025/glassfish-v4-java-7-port-unification-error-not-able-to-redirect-http-to +What about port 80? Even if you don't front your Dataverse installation with Apache, you may want to let Apache run on port 80 just to rewrite HTTP to HTTPS as described above. You can use a similar command as above to change the HTTP port that Payara uses from 8080 to 80 (substitute ``http-listener-1.port=80``). Payara can be used to enforce HTTPS on its own without Apache, but configuring this is an exercise for the reader. Answers here may be helpful: http://stackoverflow.com/questions/25122025/glassfish-v4-java-7-port-unification-error-not-able-to-redirect-http-to If you are running an installation with Apache and Payara on the same server, and would like to restrict Payara from responding to any requests to port 8080 from external hosts (in other words, not through Apache), you can restrict the AJP listener to localhost only with: @@ -134,19 +134,19 @@ If you are running an installation with Apache and Payara on the same server, an You should **NOT** use the configuration option above if you are running in a load-balanced environment, or otherwise have the web server on a different host than the application server. -Root Dataverse Permissions --------------------------- +Root Dataverse Collection Permissions +------------------------------------- -The user who creates a dataverse is given the "Admin" role on that dataverse. The root dataverse is created automatically for you by the installer and the "Admin" is the superuser account ("dataverseAdmin") we used in the :doc:`installation-main` section to confirm that we can log in. These next steps of configuring the root dataverse require the "Admin" role on the root dataverse, but not the much more powerful superuser attribute. In short, users with the "Admin" role are subject to the permission system. A superuser, on the other hand, completely bypasses the permission system. You can give non-superusers the "Admin" role on the root dataverse if you'd like them to configure the root dataverse. +The user who creates a Dataverse collection is given the "Admin" role on that Dataverse collection. The root Dataverse collection is created automatically for you by the installer and the "Admin" is the superuser account ("dataverseAdmin") we used in the :doc:`installation-main` section to confirm that we can log in. These next steps of configuring the root Dataverse collection require the "Admin" role on the root Dataverse collection, but not the much more powerful superuser attribute. In short, users with the "Admin" role are subject to the permission system. A superuser, on the other hand, completely bypasses the permission system. You can give non-superusers the "Admin" role on the root Dataverse collection if you'd like them to configure the root Dataverse collection. -In order for non-superusers to start creating dataverses or datasets, you need click "Edit" then "Permissions" and make choices about which users can add dataverses or datasets within the root dataverse. (There is an API endpoint for this operation as well.) Again, the user who creates a dataverse will be granted the "Admin" role on that dataverse. Non-superusers who are not "Admin" on the root dataverse will not be able to do anything useful until the root dataverse has been published. +In order for non-superusers to start creating Dataverse collections or datasets, you need click "Edit" then "Permissions" and make choices about which users can add Dataverse collections or datasets within the root Dataverse collection. (There is an API endpoint for this operation as well.) Again, the user who creates a Dataverse collection will be granted the "Admin" role on that Dataverse collection. Non-superusers who are not "Admin" on the root Dataverse collection will not be able to do anything useful until the root Dataverse collection has been published. -As the person installing Dataverse you may or may not be a local metadata expert. You may want to have others sign up for accounts and grant them the "Admin" role at the root dataverse to configure metadata fields, templates, browse/search facets, guestbooks, etc. For more on these topics, consult the :doc:`/user/dataverse-management` section of the User Guide. +As the person installing the Dataverse software, you may or may not be a local metadata expert. You may want to have others sign up for accounts and grant them the "Admin" role at the root Dataverse collection to configure metadata fields, templates, browse/search facets, guestbooks, etc. For more on these topics, consult the :doc:`/user/dataverse-management` section of the User Guide. Persistent Identifiers and Publishing Datasets ---------------------------------------------- -Persistent identifiers are a required and integral part of the Dataverse platform. They provide a URL that is guaranteed to resolve to the datasets or files they represent. Dataverse currently supports creating identifiers using DOI and Handle. +Persistent identifiers are a required and integral part of the Dataverse Platform. They provide a URL that is guaranteed to resolve to the datasets or files they represent. The Dataverse software currently supports creating identifiers using DOI and Handle. By default, the installer configures a default DOI namespace (10.5072) with DataCite as the registration provider. Please note that as of the release 4.9.3, we can no longer use EZID as the provider. Unlike EZID, DataCite requires that you register for a test account, configured with your own prefix (please contact support@datacite.org). Once you receive the login name, password, and prefix for the account, configure the credentials in your domain.xml, as the following two JVM options:: @@ -157,16 +157,16 @@ and restart Payara. The prefix can be configured via the API (where it is referr ``curl -X PUT -d 10.xxxx http://localhost:8080/api/admin/settings/:Authority`` -Once this is done, you will be able to publish datasets and files, but the persistent identifiers will not be citable, and they will only resolve from the DataCite test environment (and then only if the Dataverse from which you published them is accessible - DOIs minted from your laptop will not resolve). Note that any datasets or files created using the test configuration cannot be directly migrated and would need to be created again once a valid DOI namespace is configured. +Once this is done, you will be able to publish datasets and files, but the persistent identifiers will not be citable, and they will only resolve from the DataCite test environment (and then only if the Dataverse installation from which you published them is accessible - DOIs minted from your laptop will not resolve). Note that any datasets or files created using the test configuration cannot be directly migrated and would need to be created again once a valid DOI namespace is configured. To properly configure persistent identifiers for a production installation, an account and associated namespace must be acquired for a fee from a DOI or HDL provider. **DataCite** (https://www.datacite.org) is the recommended DOI provider (see https://dataverse.org/global-dataverse-community-consortium for more on joining DataCite) but **EZID** (http://ezid.cdlib.org) is an option for the University of California according to https://www.cdlib.org/cdlinfo/2017/08/04/ezid-doi-service-is-evolving/ . **Handle.Net** (https://www.handle.net) is the HDL provider. -Once you have your DOI or Handle account credentials and a namespace, configure Dataverse to use them using the JVM options and database settings below. +Once you have your DOI or Handle account credentials and a namespace, configure your Dataverse installation to use them using the JVM options and database settings below. -Configuring Dataverse for DOIs -++++++++++++++++++++++++++++++ +Configuring Your Dataverse Installation for DOIs +++++++++++++++++++++++++++++++++++++++++++++++++ -By default Dataverse attempts to register DOIs for each dataset and file under a test authority, though you must apply for your own credentials as explained above. +By default, your Dataverse installation attempts to register DOIs for each dataset and file under a test authority, though you must apply for your own credentials as explained above. Here are the configuration options for DOIs: @@ -187,8 +187,8 @@ Here are the configuration options for DOIs: - :ref:`:DataFilePIDFormat <:DataFilePIDFormat>` (optional) - :ref:`:FilePIDsEnabled <:FilePIDsEnabled>` (optional, defaults to true) -Configuring Dataverse for Handles -+++++++++++++++++++++++++++++++++ +Configuring Your Dataverse Installation for Handles ++++++++++++++++++++++++++++++++++++++++++++++++++++ Here are the configuration options for handles: @@ -213,17 +213,17 @@ Note: If you are **minting your own handles** and plan to set up your own handle Auth Modes: Local vs. Remote vs. Both ------------------------------------- -There are three valid configurations or modes for authenticating users to Dataverse: +There are three valid configurations or modes for authenticating users to your Dataverse installation: Local Only Auth +++++++++++++++ -Out of the box, Dataverse is configured in "local only" mode. The "dataverseAdmin" superuser account mentioned in the :doc:`/installation/installation-main` section is an example of a local account. Internally, these accounts are called "builtin" because they are built in to the Dataverse application itself. +Out of the box, your Dataverse installation is configured in "local only" mode. The "dataverseAdmin" superuser account mentioned in the :doc:`/installation/installation-main` section is an example of a local account. Internally, these accounts are called "builtin" because they are built in to the Dataverse software application itself. Both Local and Remote Auth ++++++++++++++++++++++++++ -The ``authenticationproviderrow`` database table controls which "authentication providers" are available within Dataverse. Out of the box, a single row with an id of "builtin" will be present. For each user in Dataverse, the ``authenticateduserlookup`` table will have a value under ``authenticationproviderid`` that matches this id. For example, the default "dataverseAdmin" user will have the value "builtin" under ``authenticationproviderid``. Why is this important? Users are tied to a specific authentication provider but conversion mechanisms are available to switch a user from one authentication provider to the other. As explained in the :doc:`/user/account` section of the User Guide, a graphical workflow is provided for end users to convert from the "builtin" authentication provider to a remote provider. Conversion from a remote authentication provider to the builtin provider can be performed by a sysadmin with access to the "admin" API. See the :doc:`/api/native-api` section of the API Guide for how to list users and authentication providers as JSON. +The ``authenticationproviderrow`` database table controls which "authentication providers" are available within a Dataverse installation. Out of the box, a single row with an id of "builtin" will be present. For each user in a Dataverse installation, the ``authenticateduserlookup`` table will have a value under ``authenticationproviderid`` that matches this id. For example, the default "dataverseAdmin" user will have the value "builtin" under ``authenticationproviderid``. Why is this important? Users are tied to a specific authentication provider but conversion mechanisms are available to switch a user from one authentication provider to the other. As explained in the :doc:`/user/account` section of the User Guide, a graphical workflow is provided for end users to convert from the "builtin" authentication provider to a remote provider. Conversion from a remote authentication provider to the builtin provider can be performed by a sysadmin with access to the "admin" API. See the :doc:`/api/native-api` section of the API Guide for how to list users and authentication providers as JSON. Adding and enabling a second authentication provider (:ref:`native-api-add-auth-provider` and :ref:`api-toggle-auth-provider`) will result in the Log In page showing additional providers for your users to choose from. By default, the Log In page will show the "builtin" provider, but you can adjust this via the :ref:`conf-default-auth-provider` configuration option. Further customization can be achieved by setting :ref:`conf-allow-signup` to "false", thus preventing users from creating local accounts via the web interface. Please note that local accounts can also be created through the API by enabling the ``builtin-users`` endpoint (:ref:`:BlockedApiEndpoints`) and setting the ``BuiltinUsers.KEY`` database setting (:ref:`BuiltinUsers.KEY`). @@ -237,28 +237,28 @@ As for the "Remote only" authentication mode, it means that: - Shibboleth or OAuth has been enabled. - ``:AllowSignUp`` is set to "false" to prevent users from creating local accounts via the web interface. - ``:DefaultAuthProvider`` has been set to use the desired authentication provider -- The "builtin" authentication provider has been disabled (:ref:`api-toggle-auth-provider`). Note that disabling the "builtin" authentication provider means that the API endpoint for converting an account from a remote auth provider will not work. Converting directly from one remote authentication provider to another (i.e. from GitHub to Google) is not supported. Conversion from remote is always to "builtin". Then the user initiates a conversion from "builtin" to remote. Note that longer term, the plan is to permit multiple login options to the same Dataverse account per https://github.com/IQSS/dataverse/issues/3487 (so all this talk of conversion will be moot) but for now users can only use a single login option, as explained in the :doc:`/user/account` section of the User Guide. In short, "remote only" might work for you if you only plan to use a single remote authentication provider such that no conversion between remote authentication providers will be necessary. +- The "builtin" authentication provider has been disabled (:ref:`api-toggle-auth-provider`). Note that disabling the "builtin" authentication provider means that the API endpoint for converting an account from a remote auth provider will not work. Converting directly from one remote authentication provider to another (i.e. from GitHub to Google) is not supported. Conversion from remote is always to "builtin". Then the user initiates a conversion from "builtin" to remote. Note that longer term, the plan is to permit multiple login options to the same Dataverse installation account per https://github.com/IQSS/dataverse/issues/3487 (so all this talk of conversion will be moot) but for now users can only use a single login option, as explained in the :doc:`/user/account` section of the User Guide. In short, "remote only" might work for you if you only plan to use a single remote authentication provider such that no conversion between remote authentication providers will be necessary. File Storage: Using a Local Filesystem and/or Swift and/or S3 object stores --------------------------------------------------------------------------- By default, a Dataverse installation stores all data files (files uploaded by end users) on the filesystem at ``/usr/local/payara5/glassfish/domains/domain1/files``. This path can vary based on answers you gave to the installer (see the :ref:`dataverse-installer` section of the Installation Guide) or afterward by reconfiguring the ``dataverse.files.directory`` JVM option described below. -Dataverse can alternately store files in a Swift or S3-compatible object store, and can now be configured to support multiple stores at once. With a multi-store configuration, the location for new files can be controlled on a per-dataverse basis. +A Dataverse installation can alternately store files in a Swift or S3-compatible object store, and can now be configured to support multiple stores at once. With a multi-store configuration, the location for new files can be controlled on a per-Dataverse collection basis. The following sections describe how to set up various types of stores and how to configure for multiple stores. Multi-store Basics ++++++++++++++++++ -To support multiple stores, Dataverse now requires an id, type, and label for each store (even for a single store configuration). These are configured by defining two required jvm options: +To support multiple stores, a Dataverse installation now requires an id, type, and label for each store (even for a single store configuration). These are configured by defining two required jvm options: .. code-block:: none ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files..type=" ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files..label=
    diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index 15c14e7d8bd..463fdac89ca 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -40,6 +40,16 @@ function bind_bsui_components(){ var dialog_id = this.jq.attr('id').split(/[:]+/).pop(); handleResizeDialog(dialog_id); } + + // clipboard.js copy btn + var clipboard = new Clipboard('button.btn-copy, span.checksum-truncate'); + + clipboard.on('success', function (e) { + console.log(e); + }); + clipboard.on('error', function (e) { + console.log(e); + }); } function bind_tooltip_popover(){ From b2161963b9ea83007adee1a67a075310e63c3284 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Tue, 26 Jan 2021 14:09:08 -0500 Subject: [PATCH 170/576] Updated clipboard.js to latest and greatest [ref #6039] --- src/main/webapp/resources/js/clipboard.min.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) mode change 100755 => 100644 src/main/webapp/resources/js/clipboard.min.js diff --git a/src/main/webapp/resources/js/clipboard.min.js b/src/main/webapp/resources/js/clipboard.min.js old mode 100755 new mode 100644 index 1993676f992..98d4ccba698 --- a/src/main/webapp/resources/js/clipboard.min.js +++ b/src/main/webapp/resources/js/clipboard.min.js @@ -1,7 +1,7 @@ /*! - * clipboard.js v1.6.1 - * https://zenorocha.github.io/clipboard.js + * clipboard.js v2.0.6 + * https://clipboardjs.com/ * * Licensed MIT © Zeno Rocha */ -!function(e){if("object"==typeof exports&&"undefined"!=typeof module)module.exports=e();else if("function"==typeof define&&define.amd)define([],e);else{var t;t="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:this,t.Clipboard=e()}}(function(){var e,t,n;return function e(t,n,o){function i(a,c){if(!n[a]){if(!t[a]){var l="function"==typeof require&&require;if(!c&&l)return l(a,!0);if(r)return r(a,!0);var u=new Error("Cannot find module '"+a+"'");throw u.code="MODULE_NOT_FOUND",u}var s=n[a]={exports:{}};t[a][0].call(s.exports,function(e){var n=t[a][1][e];return i(n?n:e)},s,s.exports,e,t,n,o)}return n[a].exports}for(var r="function"==typeof require&&require,a=0;a0&&void 0!==arguments[0]?arguments[0]:{};this.action=t.action,this.emitter=t.emitter,this.target=t.target,this.text=t.text,this.trigger=t.trigger,this.selectedText=""}},{key:"initSelection",value:function e(){this.text?this.selectFake():this.target&&this.selectTarget()}},{key:"selectFake",value:function e(){var t=this,n="rtl"==document.documentElement.getAttribute("dir");this.removeFake(),this.fakeHandlerCallback=function(){return t.removeFake()},this.fakeHandler=document.body.addEventListener("click",this.fakeHandlerCallback)||!0,this.fakeElem=document.createElement("textarea"),this.fakeElem.style.fontSize="12pt",this.fakeElem.style.border="0",this.fakeElem.style.padding="0",this.fakeElem.style.margin="0",this.fakeElem.style.position="absolute",this.fakeElem.style[n?"right":"left"]="-9999px";var o=window.pageYOffset||document.documentElement.scrollTop;this.fakeElem.style.top=o+"px",this.fakeElem.setAttribute("readonly",""),this.fakeElem.value=this.text,document.body.appendChild(this.fakeElem),this.selectedText=(0,i.default)(this.fakeElem),this.copyText()}},{key:"removeFake",value:function e(){this.fakeHandler&&(document.body.removeEventListener("click",this.fakeHandlerCallback),this.fakeHandler=null,this.fakeHandlerCallback=null),this.fakeElem&&(document.body.removeChild(this.fakeElem),this.fakeElem=null)}},{key:"selectTarget",value:function e(){this.selectedText=(0,i.default)(this.target),this.copyText()}},{key:"copyText",value:function e(){var t=void 0;try{t=document.execCommand(this.action)}catch(e){t=!1}this.handleResult(t)}},{key:"handleResult",value:function e(t){this.emitter.emit(t?"success":"error",{action:this.action,text:this.selectedText,trigger:this.trigger,clearSelection:this.clearSelection.bind(this)})}},{key:"clearSelection",value:function e(){this.target&&this.target.blur(),window.getSelection().removeAllRanges()}},{key:"destroy",value:function e(){this.removeFake()}},{key:"action",set:function e(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:"copy";if(this._action=t,"copy"!==this._action&&"cut"!==this._action)throw new Error('Invalid "action" value, use either "copy" or "cut"')},get:function e(){return this._action}},{key:"target",set:function e(t){if(void 0!==t){if(!t||"object"!==("undefined"==typeof t?"undefined":r(t))||1!==t.nodeType)throw new Error('Invalid "target" value, use a valid Element');if("copy"===this.action&&t.hasAttribute("disabled"))throw new Error('Invalid "target" attribute. Please use "readonly" instead of "disabled" attribute');if("cut"===this.action&&(t.hasAttribute("readonly")||t.hasAttribute("disabled")))throw new Error('Invalid "target" attribute. You can\'t cut text from elements with "readonly" or "disabled" attributes');this._target=t}},get:function e(){return this._target}}]),e}();e.exports=c})},{select:5}],8:[function(t,n,o){!function(i,r){if("function"==typeof e&&e.amd)e(["module","./clipboard-action","tiny-emitter","good-listener"],r);else if("undefined"!=typeof o)r(n,t("./clipboard-action"),t("tiny-emitter"),t("good-listener"));else{var a={exports:{}};r(a,i.clipboardAction,i.tinyEmitter,i.goodListener),i.clipboard=a.exports}}(this,function(e,t,n,o){"use strict";function i(e){return e&&e.__esModule?e:{default:e}}function r(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function a(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!=typeof t&&"function"!=typeof t?e:t}function c(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function, not "+typeof t);e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),t&&(Object.setPrototypeOf?Object.setPrototypeOf(e,t):e.__proto__=t)}function l(e,t){var n="data-clipboard-"+e;if(t.hasAttribute(n))return t.getAttribute(n)}var u=i(t),s=i(n),f=i(o),d=function(){function e(e,t){for(var n=0;n0&&void 0!==arguments[0]?arguments[0]:{};this.action="function"==typeof t.action?t.action:this.defaultAction,this.target="function"==typeof t.target?t.target:this.defaultTarget,this.text="function"==typeof t.text?t.text:this.defaultText}},{key:"listenClick",value:function e(t){var n=this;this.listener=(0,f.default)(t,"click",function(e){return n.onClick(e)})}},{key:"onClick",value:function e(t){var n=t.delegateTarget||t.currentTarget;this.clipboardAction&&(this.clipboardAction=null),this.clipboardAction=new u.default({action:this.action(n),target:this.target(n),text:this.text(n),trigger:n,emitter:this})}},{key:"defaultAction",value:function e(t){return l("action",t)}},{key:"defaultTarget",value:function e(t){var n=l("target",t);if(n)return document.querySelector(n)}},{key:"defaultText",value:function e(t){return l("text",t)}},{key:"destroy",value:function e(){this.listener.destroy(),this.clipboardAction&&(this.clipboardAction.destroy(),this.clipboardAction=null)}}],[{key:"isSupported",value:function e(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:["copy","cut"],n="string"==typeof t?[t]:t,o=!!document.queryCommandSupported;return n.forEach(function(e){o=o&&!!document.queryCommandSupported(e)}),o}}]),t}(s.default);e.exports=h})},{"./clipboard-action":7,"good-listener":4,"tiny-emitter":6}]},{},[8])(8)}); \ No newline at end of file +!function(t,e){"object"==typeof exports&&"object"==typeof module?module.exports=e():"function"==typeof define&&define.amd?define([],e):"object"==typeof exports?exports.ClipboardJS=e():t.ClipboardJS=e()}(this,function(){return n={134:(t,e,n)=>{"use strict";n.d(e,{default:()=>r});var e=n(817),o=n.n(e);function i(t){return(i="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t})(t)}function a(t,e){for(var n=0;n{var e;"undefined"==typeof Element||Element.prototype.matches||((e=Element.prototype).matches=e.matchesSelector||e.mozMatchesSelector||e.msMatchesSelector||e.oMatchesSelector||e.webkitMatchesSelector),t.exports=function(t,e){for(;t&&9!==t.nodeType;){if("function"==typeof t.matches&&t.matches(e))return t;t=t.parentNode}}},438:(t,e,n)=>{var a=n(828);function i(t,e,n,r,o){var i=function(e,n,t,r){return function(t){t.delegateTarget=a(t.target,n),t.delegateTarget&&r.call(e,t)}}.apply(this,arguments);return t.addEventListener(n,i,o),{destroy:function(){t.removeEventListener(n,i,o)}}}t.exports=function(t,e,n,r,o){return"function"==typeof t.addEventListener?i.apply(null,arguments):"function"==typeof n?i.bind(null,document).apply(null,arguments):("string"==typeof t&&(t=document.querySelectorAll(t)),Array.prototype.map.call(t,function(t){return i(t,e,n,r,o)}))}},879:(t,n)=>{n.node=function(t){return void 0!==t&&t instanceof HTMLElement&&1===t.nodeType},n.nodeList=function(t){var e=Object.prototype.toString.call(t);return void 0!==t&&("[object NodeList]"===e||"[object HTMLCollection]"===e)&&"length"in t&&(0===t.length||n.node(t[0]))},n.string=function(t){return"string"==typeof t||t instanceof String},n.fn=function(t){return"[object Function]"===Object.prototype.toString.call(t)}},370:(t,e,n)=>{var u=n(879),s=n(438);t.exports=function(t,e,n){if(!t&&!e&&!n)throw new Error("Missing required arguments");if(!u.string(e))throw new TypeError("Second argument must be a String");if(!u.fn(n))throw new TypeError("Third argument must be a Function");if(u.node(t))return c=e,l=n,(a=t).addEventListener(c,l),{destroy:function(){a.removeEventListener(c,l)}};if(u.nodeList(t))return r=t,o=e,i=n,Array.prototype.forEach.call(r,function(t){t.addEventListener(o,i)}),{destroy:function(){Array.prototype.forEach.call(r,function(t){t.removeEventListener(o,i)})}};if(u.string(t))return t=t,e=e,n=n,s(document.body,t,e,n);throw new TypeError("First argument must be a String, HTMLElement, HTMLCollection, or NodeList");var r,o,i,a,c,l}},817:t=>{t.exports=function(t){var e,n="SELECT"===t.nodeName?(t.focus(),t.value):"INPUT"===t.nodeName||"TEXTAREA"===t.nodeName?((e=t.hasAttribute("readonly"))||t.setAttribute("readonly",""),t.select(),t.setSelectionRange(0,t.value.length),e||t.removeAttribute("readonly"),t.value):(t.hasAttribute("contenteditable")&&t.focus(),n=window.getSelection(),(e=document.createRange()).selectNodeContents(t),n.removeAllRanges(),n.addRange(e),n.toString());return n}},279:t=>{function e(){}e.prototype={on:function(t,e,n){var r=this.e||(this.e={});return(r[t]||(r[t]=[])).push({fn:e,ctx:n}),this},once:function(t,e,n){var r=this;function o(){r.off(t,o),e.apply(n,arguments)}return o._=e,this.on(t,o,n)},emit:function(t){for(var e=[].slice.call(arguments,1),n=((this.e||(this.e={}))[t]||[]).slice(),r=0,o=n.length;r{var e=t&&t.__esModule?()=>t.default:()=>t;return r.d(e,{a:e}),e},r.d=(t,e)=>{for(var n in e)r.o(e,n)&&!r.o(t,n)&&Object.defineProperty(t,n,{enumerable:!0,get:e[n]})},r.o=(t,e)=>Object.prototype.hasOwnProperty.call(t,e),r(134).default;function r(t){if(o[t])return o[t].exports;var e=o[t]={exports:{}};return n[t](e,e.exports,r),e.exports}var n,o}); \ No newline at end of file From 28cd9fb7621a9528337397e9b85111759fc4bf3b Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Tue, 26 Jan 2021 14:10:46 -0500 Subject: [PATCH 171/576] Added copy to clipboard to private url popup, other code clean up on api token account pg [ref #6039] --- src/main/java/propertyFiles/Bundle.properties | 1 + src/main/webapp/dataset.xhtml | 3 +++ src/main/webapp/dataverseuser.xhtml | 4 ++-- src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js | 4 ++-- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 3d3349a0b52..2a73ea97307 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -54,6 +54,7 @@ harvested=Harvested apply=Apply add=Add delete=Delete +copyClipboard=Copy to Clipboard yes=Yes no=No previous=Previous diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 65a30280ceb..53c383381d4 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -974,6 +974,9 @@
    + - -
    -
    -
    - -

    - - - - - - - - - -

    - -

    - - - - - - - - - - -

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - #{bundle['user.username']} - - #{DataverseUserPage.currentUser.identifier.replaceFirst("@", "")} -
    - #{bundle['user.firstName']} - - #{DataverseUserPage.currentUser.firstName} -
    - #{bundle['user.lastName']} - - #{DataverseUserPage.currentUser.lastName} -
    - #{bundle.email} - -

    - #{DataverseUserPage.currentUser.email} -   - - #{bundle['confirmEmail.notVerified']} - - - #{bundle['confirmEmail.verified']} - -

    -   - - #{bundle['confirmEmail.submitRequest']} - -
    - #{DataverseUserPage.userAuthProvider.persistentIdName} - -   - - - -
    - #{bundle.affiliation} - - #{DataverseUserPage.currentUser.affiliation} -
    - #{bundle['user.position']} - - #{DataverseUserPage.currentUser.position} -
    -
    -

    From 4117548fa15dc6d8c7cb380c2c2a1e0a9db86c16 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 29 Jan 2021 10:45:33 -0500 Subject: [PATCH 212/576] fix null value in test --- .../harvard/iq/dataverse/ControlledVocabularyValue.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ControlledVocabularyValue.java b/src/main/java/edu/harvard/iq/dataverse/ControlledVocabularyValue.java index 00c264afeb7..581405ada1f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ControlledVocabularyValue.java +++ b/src/main/java/edu/harvard/iq/dataverse/ControlledVocabularyValue.java @@ -116,7 +116,12 @@ public String getLocaleStrValue() String key = strValue.toLowerCase().replace(" " , "_"); key = StringUtils.stripAccents(key); try { - return BundleUtil.getStringFromPropertyFile("controlledvocabulary." + this.datasetFieldType.getName() + "." + key, getDatasetFieldType().getMetadataBlock().getName()); + String val = BundleUtil.getStringFromPropertyFile("controlledvocabulary." + this.datasetFieldType.getName() + "." + key, getDatasetFieldType().getMetadataBlock().getName()); + if( val== null) { + //Default to raw value + val=strValue; + } + return val; } catch (MissingResourceException e) { return strValue; } From 89023bae820cf46b2481da191e6ddaf626fabbd1 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Fri, 29 Jan 2021 11:53:17 -0500 Subject: [PATCH 213/576] Fix checksum flashing and rebinding issues on download, search [ref #7081] --- src/main/webapp/file-info-fragment.xhtml | 5 ++++- src/main/webapp/filesFragment.xhtml | 8 ++++---- src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js | 7 ++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index a3cd5d14ca9..d797eb0be23 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -71,10 +71,13 @@

    - #{fileMetadata.dataFile.checksumType}: #{fileMetadata.dataFile.checksumValue} +
    diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index b64570eec68..7654cbc3436 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -101,10 +101,10 @@ - + - + #{bundle['dataverse.search.btn.find']} @@ -320,14 +320,14 @@   - +   - +

    diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index bd9b89a9e81..811436e65a5 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -45,7 +45,7 @@ function bind_bsui_components(){ } // clipboard.js copy btn - var clipboard = new ClipboardJS('button.btn-copy, span.checksum-truncate'); + var clipboard = new ClipboardJS('button.btn-copy, span.btn-copy, span.checksum-truncate'); clipboard.on('success', function (e) { // DEV TOOL DEBUG @@ -171,6 +171,7 @@ function sharrre(){ */ function checksumTruncate(){ $('span.checksum-truncate').each(function () { + $(this).toggleClass('sr-only').toggleClass('visisble'); var checksumText = $(this).text(); if (checksumText.length > 25) { // COUNT ":" IN UNF VERSION LABEL @@ -187,10 +188,6 @@ function checksumTruncate(){ $(this).text(checksumText.substr(0, 3) + '...' + checksumText.substr((checksumText.length - 3), checksumText.length)); } } - // ADD CLIPBOARD ICON - $(this).append(' '); - - // COPY TO CLIPBOARD MOVE TO bind_bsui_components() ABOVE }); } From 85755bad3aba0e577d539c447aa650a63e26252f Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Fri, 29 Jan 2021 12:05:32 -0500 Subject: [PATCH 214/576] Fix checksum flashing and rebinding issues on download [ref #7081] --- src/main/webapp/filesFragment.xhtml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index 7654cbc3436..27ed25cdbe2 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -398,7 +398,7 @@ #{bundle.download} @@ -407,7 +407,7 @@ styleClass="btn btn-default btn-download" disabled="#{false and DatasetPage.lockedFromDownload}" action="#{DatasetPage.validateFilesForDownload(true, false)}" - update="@form" oncomplete=""> + update="@form" oncomplete="bind_bsui_components();"> #{bundle.download}
    @@ -422,7 +422,7 @@
  • #{bundle.downloadOriginal} @@ -430,14 +430,14 @@ + update="@form" oncomplete="bind_bsui_components();"> #{bundle.downloadOriginal}
  • #{bundle.downloadArchival} @@ -445,7 +445,7 @@ + update="@form" oncomplete="bind_bsui_components();"> #{bundle.downloadArchival}
    • @@ -454,7 +454,8 @@ #{bundle['file.requestAccess']} @@ -554,7 +555,7 @@
    -     +     From 56122d5646c819793553ecf5fca455cb6c3c1faa Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 29 Jan 2021 12:33:20 -0500 Subject: [PATCH 215/576] updates platform to software, release notes --- doc/release-notes/7501-guides-update | 6 +++++ doc/sphinx-guides/source/admin/dashboard.rst | 4 ++-- .../source/admin/external-tools.rst | 2 +- .../source/admin/harvestclients.rst | 2 +- .../source/admin/harvestserver.rst | 2 +- .../source/admin/integrations.rst | 8 +++---- .../source/admin/make-data-count.rst | 16 ++++++------- .../source/admin/solr-search-index.rst | 2 +- doc/sphinx-guides/source/api/apps.rst | 10 ++++---- doc/sphinx-guides/source/api/auth.rst | 4 ++-- .../source/api/client-libraries.rst | 2 +- .../source/api/external-tools.rst | 8 +++---- doc/sphinx-guides/source/api/faq.rst | 8 +++---- .../source/api/getting-started.rst | 6 ++--- doc/sphinx-guides/source/api/intro.rst | 24 +++++++++---------- doc/sphinx-guides/source/api/metrics.rst | 2 +- doc/sphinx-guides/source/api/native-api.rst | 8 +++---- doc/sphinx-guides/source/api/search.rst | 2 +- doc/sphinx-guides/source/api/sword.rst | 4 ++-- .../source/installation/config.rst | 2 +- 20 files changed, 64 insertions(+), 58 deletions(-) create mode 100644 doc/release-notes/7501-guides-update diff --git a/doc/release-notes/7501-guides-update b/doc/release-notes/7501-guides-update new file mode 100644 index 00000000000..6ad08a34446 --- /dev/null +++ b/doc/release-notes/7501-guides-update @@ -0,0 +1,6 @@ +## Release Highlights + +The Dataverse Guides have been updated following Merce's recent note to the Dataverse Community: + +https://groups.google.com/g/dataverse-community/c/pD-aFrpXMPo + diff --git a/doc/sphinx-guides/source/admin/dashboard.rst b/doc/sphinx-guides/source/admin/dashboard.rst index a3ab90110ea..520126c5299 100644 --- a/doc/sphinx-guides/source/admin/dashboard.rst +++ b/doc/sphinx-guides/source/admin/dashboard.rst @@ -1,7 +1,7 @@ Dashboard ========= -The Dataverse Platform offers a dashboard of administrative tools for superusers only. If you are a logged-in superuser, you can access it by clicking your username in the navbar, and then clicking "Dashboard" from the dropdown. You can verify that you are a superuser by checking the color of your username in the navbar. If it's red, you have the right permissions to use the Dashboard. Superusers can give other users the superuser status via :doc:`user-administration`. +The Dataverse software offers a dashboard of administrative tools for superusers only. If you are a logged-in superuser, you can access it by clicking your username in the navbar, and then clicking "Dashboard" from the dropdown. You can verify that you are a superuser by checking the color of your username in the navbar. If it's red, you have the right permissions to use the Dashboard. Superusers can give other users the superuser status via :doc:`user-administration`. .. contents:: Contents: :local: @@ -22,7 +22,7 @@ This dashboard tool allows you to define sets of local datasets to make availabl Metadata Export --------------- -This part of the Dashboard is simply a reminder message that metadata export happens through the Dataverse Platform API. See the :doc:`metadataexport` section and the :doc:`/api/native-api` section of the API Guide for more details. +This part of the Dashboard is simply a reminder message that metadata export happens through the Dataverse software API. See the :doc:`metadataexport` section and the :doc:`/api/native-api` section of the API Guide for more details. Users ----- diff --git a/doc/sphinx-guides/source/admin/external-tools.rst b/doc/sphinx-guides/source/admin/external-tools.rst index 4426ab2d37f..6e35d5f1d92 100644 --- a/doc/sphinx-guides/source/admin/external-tools.rst +++ b/doc/sphinx-guides/source/admin/external-tools.rst @@ -1,7 +1,7 @@ External Tools ============== -External tools can provide additional features that are not part of the Dataverse Platform itself, such as data file previews, visualization, and curation. +External tools can provide additional features that are not part of the Dataverse software itself, such as data file previews, visualization, and curation. .. contents:: |toctitle| :local: diff --git a/doc/sphinx-guides/source/admin/harvestclients.rst b/doc/sphinx-guides/source/admin/harvestclients.rst index a3de6690b9f..c62c64b80fe 100644 --- a/doc/sphinx-guides/source/admin/harvestclients.rst +++ b/doc/sphinx-guides/source/admin/harvestclients.rst @@ -26,4 +26,4 @@ What if a Run Fails? Each harvesting client run logs a separate file per run to the app server's default logging directory (``/usr/local/payara5/glassfish/domains/domain1/logs/`` unless you've changed it). Look for filenames in the format ``harvest_TARGET_YYYY_MM_DD_timestamp.log`` to get a better idea of what's going wrong. -Note that you'll want to run a minimum Dataverse Platform version of 4.6, optimally 4.18, for the best OAI-PMH interoperability. +Note that you'll want to run a minimum Dataverse software version of 4.6, optimally 4.18, for the best OAI-PMH interoperability. diff --git a/doc/sphinx-guides/source/admin/harvestserver.rst b/doc/sphinx-guides/source/admin/harvestserver.rst index 3550599a67c..71c272d3056 100644 --- a/doc/sphinx-guides/source/admin/harvestserver.rst +++ b/doc/sphinx-guides/source/admin/harvestserver.rst @@ -50,7 +50,7 @@ the terms "OAI Set" and "Harvesting Set" are used interchangeably. Sets are defined by search queries. Any such query that finds any number of published, local (non-harvested) datasets can be used to create an OAI set. Sets can overlap local Dataverse collections, and can include as few or as many of your local datasets as you wish. A -good way to master the Dataverse Platform search query language is to +good way to master the Dataverse software search query language is to experiment with the Advanced Search page. We also recommend that you consult the :doc:`/api/search` section of the API Guide. diff --git a/doc/sphinx-guides/source/admin/integrations.rst b/doc/sphinx-guides/source/admin/integrations.rst index 72e86ba5807..b1683639c9a 100644 --- a/doc/sphinx-guides/source/admin/integrations.rst +++ b/doc/sphinx-guides/source/admin/integrations.rst @@ -111,8 +111,8 @@ Researchers can import datasets from a Dataverse installation into their Renku p command-line interface (CLI) by using the dataset's DOI. See the `renku Dataset documentation `_ -for details. Currently Dataverse Platform ``>=4.8.x`` is required for the import to work. If you need -support for an earlier version of the Dataverse Platform, please get in touch with the Renku team at +for details. Currently Dataverse software ``>=4.8.x`` is required for the import to work. If you need +support for an earlier version of the Dataverse software, please get in touch with the Renku team at `Discourse `_ or `GitHub `_. Avgidea Data Search @@ -123,12 +123,12 @@ Researchers can use a Google Sheets add-on to search for Dataverse installation' Discoverability --------------- -Integration with `DataCite `_ is built in to the Dataverse Platform. When datasets are published, metadata is sent to DataCite. You can further increase the discoverability of your datasets by setting up additional integrations. +Integration with `DataCite `_ is built in to the Dataverse software. When datasets are published, metadata is sent to DataCite. You can further increase the discoverability of your datasets by setting up additional integrations. OAI-PMH (Harvesting) ++++++++++++++++++++ -The Dataverse Platform supports a protocol called OAI-PMH that facilitates harvesting datasets from one system into another. For details on harvesting, see the :doc:`harvestserver` section. +The Dataverse software supports a protocol called OAI-PMH that facilitates harvesting datasets from one system into another. For details on harvesting, see the :doc:`harvestserver` section. SHARE +++++ diff --git a/doc/sphinx-guides/source/admin/make-data-count.rst b/doc/sphinx-guides/source/admin/make-data-count.rst index ac37d9eb11f..e2942d8f24a 100644 --- a/doc/sphinx-guides/source/admin/make-data-count.rst +++ b/doc/sphinx-guides/source/admin/make-data-count.rst @@ -1,7 +1,7 @@ Make Data Count =============== -`Make Data Count`_ is a project to collect and standardize metrics on data use, especially views, downloads, and citations. The Dataverse Platform can integrate Make Data Count to collect and display usage metrics including counts of dataset views, file downloads, and dataset citations. +`Make Data Count`_ is a project to collect and standardize metrics on data use, especially views, downloads, and citations. The Dataverse software can integrate Make Data Count to collect and display usage metrics including counts of dataset views, file downloads, and dataset citations. .. contents:: Contents: :local: @@ -9,7 +9,7 @@ Make Data Count Introduction ------------ -`Make Data Count`_ is part of a broader Research Data Alliance (RDA) `Data Usage Metrics Working Group`_ which helped to produce a specification called the `COUNTER Code of Practice for Research Data`_ (`PDF`_, `HTML`_) that the Dataverse Platform makes every effort to comply with. The Code of Practice (CoP) is built on top of existing standards such as COUNTER and SUSHI that come out of the article publishing world. The Make Data Count project has emphasized that they would like feedback on the code of practice. You can keep up to date on the Make Data Count project by subscribing to their `newsletter`_. +`Make Data Count`_ is part of a broader Research Data Alliance (RDA) `Data Usage Metrics Working Group`_ which helped to produce a specification called the `COUNTER Code of Practice for Research Data`_ (`PDF`_, `HTML`_) that the Dataverse software makes every effort to comply with. The Code of Practice (CoP) is built on top of existing standards such as COUNTER and SUSHI that come out of the article publishing world. The Make Data Count project has emphasized that they would like feedback on the code of practice. You can keep up to date on the Make Data Count project by subscribing to their `newsletter`_. Architecture ------------ @@ -29,7 +29,7 @@ The most important takeaways from the diagram are: - You should run Counter Processor once a day to create reports in SUSHI (JSON) format that are saved to disk for your Dataverse installation to process and that are sent to the DataCite hub. - You should set up a cron job to have your Dataverse installation process the daily SUSHI reports, updating the Dataverse installation database with the latest metrics. - You should set up a cron job to have your Dataverse installation pull the latest list of citations for each dataset on a periodic basis, perhaps weekly or daily. These citations come from Crossref via the DataCite hub. -- APIs are available in the Dataverse Platform to retrieve Make Data Count metrics: views, downloads, and citations. +- APIs are available in the Dataverse software to retrieve Make Data Count metrics: views, downloads, and citations. Limitations for Dataverse Installations Using Handles Rather Than DOIs @@ -49,7 +49,7 @@ Data repositories using Handles and other identifiers are not supported by Make | | MDC citations | | +---------------------+---------------+------------------+ -This being said, the Dataverse Platform usage logging can still generate logs and process those logs with Counter Processor to create json that details usage on a dataset level. Dataverse installations can ingest this locally generated json. +This being said, the Dataverse software usage logging can still generate logs and process those logs with Counter Processor to create json that details usage on a dataset level. Dataverse installations can ingest this locally generated json. When editing the ``counter-processor-config.yaml`` file mentioned below, make sure that the ``upload_to_hub`` boolean is set to ``False``. @@ -127,9 +127,9 @@ Soon we will be setting up a cron job to run nightly but we start with a single Populate Views and Downloads Nightly ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Running ``main.py`` to create the SUSHI JSON file and the subsequent calling of the Dataverse Platform API to process it should be added as a cron job. +Running ``main.py`` to create the SUSHI JSON file and the subsequent calling of the Dataverse software API to process it should be added as a cron job. -The Dataverse Platform provides example scripts that run the steps to process new accesses and uploads and update your Dataverse installation's database (`counter_daily.sh`) and to retrieve citations for all Datasets from DataCite (`counter_weekly.sh`). These scripts should be configured for your environment and can be run manually or as cron jobs. +The Dataverse software provides example scripts that run the steps to process new accesses and uploads and update your Dataverse installation's database (`counter_daily.sh`) and to retrieve citations for all Datasets from DataCite (`counter_weekly.sh`). These scripts should be configured for your environment and can be run manually or as cron jobs. Sending Usage Metrics to the DataCite Hub ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -164,7 +164,7 @@ Citations will be retrieved for each published dataset and recorded in the your For how to get the citations out of your Dataverse installation, see "Retrieving Citations for a Dataset" under :ref:`Dataset Metrics ` in the :doc:`/api/native-api` section of the API Guide. -Please note that while the Dataverse Platform has a metadata field for "Related Dataset" this information is not currently sent as a citation to Crossref. +Please note that while the Dataverse software has a metadata field for "Related Dataset" this information is not currently sent as a citation to Crossref. Retrieving Make Data Count Metrics from the DataCite Hub -------------------------------------------------------- @@ -180,7 +180,7 @@ The following metrics can be downloaded directly from the DataCite hub (see http Retrieving Make Data Count Metrics from a Dataverse Installation ---------------------------------------------------------------- -The Dataverse Platform API endpoints for retrieving Make Data Count metrics are described below under :ref:`Dataset Metrics ` in the :doc:`/api/native-api` section of the API Guide. +The Dataverse software API endpoints for retrieving Make Data Count metrics are described below under :ref:`Dataset Metrics ` in the :doc:`/api/native-api` section of the API Guide. Please note that it is also possible to retrieve metrics from the DataCite hub itself via https://api.datacite.org diff --git a/doc/sphinx-guides/source/admin/solr-search-index.rst b/doc/sphinx-guides/source/admin/solr-search-index.rst index 5dcc866b137..5ce36179e6b 100644 --- a/doc/sphinx-guides/source/admin/solr-search-index.rst +++ b/doc/sphinx-guides/source/admin/solr-search-index.rst @@ -69,7 +69,7 @@ Dataverse collections must be referenced by database object ID. If you have dire ``select id from dataverse where alias='datavarsealias';`` -should work, or you may click the Dataverse Platform's "Edit" menu and look for dataverseId= in the URLs produced by the drop-down. Then, to re-index: +should work, or you may click the Dataverse software's "Edit" menu and look for dataverseId= in the URLs produced by the drop-down. Then, to re-index: ``curl http://localhost:8080/api/admin/index/dataverses/135`` diff --git a/doc/sphinx-guides/source/api/apps.rst b/doc/sphinx-guides/source/api/apps.rst index a0eb8645a41..3ac158f15cb 100755 --- a/doc/sphinx-guides/source/api/apps.rst +++ b/doc/sphinx-guides/source/api/apps.rst @@ -1,9 +1,9 @@ Apps ==== -The introduction of Dataverse Platform APIs has fostered the development of a variety of software applications that are listed in the :doc:`/admin/integrations`, :doc:`/admin/external-tools`, and :doc:`/admin/reporting-tools-and-queries` sections of the Admin Guide. +The introduction of Dataverse software APIs has fostered the development of a variety of software applications that are listed in the :doc:`/admin/integrations`, :doc:`/admin/external-tools`, and :doc:`/admin/reporting-tools-and-queries` sections of the Admin Guide. -The apps below are open source and demonstrate how to use Dataverse Platform APIs. Some of these apps are built on :doc:`/api/client-libraries` that are available for Dataverse Platform APIs in Python, Javascript, R, and Java. +The apps below are open source and demonstrate how to use Dataverse software APIs. Some of these apps are built on :doc:`/api/client-libraries` that are available for Dataverse software APIs in Python, Javascript, R, and Java. .. contents:: |toctitle| :local: @@ -42,7 +42,7 @@ https://github.com/IQSS/TwoRavens Python ------ -Please note that there are multiple Python modules for Dataverse Platform APIs listed in the :doc:`client-libraries` section. +Please note that there are multiple Python modules for Dataverse software APIs listed in the :doc:`client-libraries` section. dataverse-sample-data ~~~~~~~~~~~~~~~~~~~~~ @@ -97,12 +97,12 @@ https://github.com/jupyter/repo2docker/blob/master/repo2docker/contentproviders/ Java ---- -Please note that there is a Java library for Dataverse Platform APIs listed in the :doc:`client-libraries` section. +Please note that there is a Java library for Dataverse software APIs listed in the :doc:`client-libraries` section. DVUploader ~~~~~~~~~~ -The open-source DVUploader tool is a stand-alone command-line Java application that uses the Dataverse Platform API to upload files to a specified Dataset. Files can be specified by name, or the DVUploader can upload all files in a directory or recursively from a directory tree. The DVUploader can also verify that uploaded files match their local sources by comparing the local and remote fixity checksums. Source code, release 1.0.0- jar file, and documentation are available on GitHub. DVUploader's creation was supported by the Texas Digital Library. +The open-source DVUploader tool is a stand-alone command-line Java application that uses the Dataverse software API to upload files to a specified Dataset. Files can be specified by name, or the DVUploader can upload all files in a directory or recursively from a directory tree. The DVUploader can also verify that uploaded files match their local sources by comparing the local and remote fixity checksums. Source code, release 1.0.0- jar file, and documentation are available on GitHub. DVUploader's creation was supported by the Texas Digital Library. https://github.com/IQSS/dataverse-uploader diff --git a/doc/sphinx-guides/source/api/auth.rst b/doc/sphinx-guides/source/api/auth.rst index e21555a07be..9bcbd125719 100644 --- a/doc/sphinx-guides/source/api/auth.rst +++ b/doc/sphinx-guides/source/api/auth.rst @@ -1,7 +1,7 @@ API Tokens and Authentication ============================= -An API token is similar to a password and allows you to authenticate to Dataverse Platform APIs to perform actions as you. Many Dataverse Platform APIs require the use of an API token. +An API token is similar to a password and allows you to authenticate to Dataverse software APIs to perform actions as you. Many Dataverse software APIs require the use of an API token. .. contents:: |toctitle| :local: @@ -25,7 +25,7 @@ Note: The SWORD API uses a different way of passing the API token. Please see :r See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. -There are two ways to pass your API token to Dataverse Platform APIs. The preferred method is to send the token in the ``X-Dataverse-key`` HTTP header, as in the following curl example. +There are two ways to pass your API token to Dataverse software APIs. The preferred method is to send the token in the ``X-Dataverse-key`` HTTP header, as in the following curl example. .. code-block:: bash diff --git a/doc/sphinx-guides/source/api/client-libraries.rst b/doc/sphinx-guides/source/api/client-libraries.rst index 406064424cb..d6a803a4f0b 100755 --- a/doc/sphinx-guides/source/api/client-libraries.rst +++ b/doc/sphinx-guides/source/api/client-libraries.rst @@ -34,6 +34,6 @@ The package is currently maintained by `Will Beasley `_ from `ResearchSpace `_ created and maintains this library. diff --git a/doc/sphinx-guides/source/api/external-tools.rst b/doc/sphinx-guides/source/api/external-tools.rst index 5ea4a1aa83b..f7a0e051803 100644 --- a/doc/sphinx-guides/source/api/external-tools.rst +++ b/doc/sphinx-guides/source/api/external-tools.rst @@ -1,7 +1,7 @@ Building External Tools ======================= -External tools can provide additional features that are not part of the Dataverse Platform itself, such as data exploration. Thank you for your interest in building an external tool for the Dataverse Platform! +External tools can provide additional features that are not part of the Dataverse software itself, such as data exploration. Thank you for your interest in building an external tool for the Dataverse software! .. contents:: |toctitle| :local: @@ -9,7 +9,7 @@ External tools can provide additional features that are not part of the Datavers Introduction ------------ -External tools are additional applications the user can access or open from your Dataverse installation to preview, explore, and manipulate data files and datasets. The term "external" is used to indicate that the tool is not part of the main Dataverse Platform. +External tools are additional applications the user can access or open from your Dataverse installation to preview, explore, and manipulate data files and datasets. The term "external" is used to indicate that the tool is not part of the main Dataverse software. Once you have created the external tool itself (which is most of the work!), you need to teach a Dataverse installation how to construct URLs that your tool needs to operate. For example, if you've deployed your tool to fabulousfiletool.com your tool might want the ID of a file and the siteUrl of the Dataverse installation like this: https://fabulousfiletool.com?fileId=42&siteUrl=http://demo.dataverse.org @@ -164,12 +164,12 @@ Once you've gotten your tool working, please make a pull request to update the l Unless your tool runs entirely in a browser, you may have integrated server-side software with your Dataverse installation. If so, please double check that your software is listed in the :doc:`/admin/integrations` section of the Admin Guide and if not, please open an issue or pull request to add it. Thanks! -If you've thought to yourself that there ought to be an app store for Dataverse Platform external tools, you're not alone. Please see https://github.com/IQSS/dataverse/issues/5688 :) +If you've thought to yourself that there ought to be an app store for Dataverse software external tools, you're not alone. Please see https://github.com/IQSS/dataverse/issues/5688 :) Demoing Your External Tool ++++++++++++++++++++++++++ -https://demo.dataverse.org is the place to play around with the Dataverse Platform and your tool can be included. Please email support@dataverse.org to start the conversation about adding your tool. Additionally, you are welcome to open an issue at https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible which already includes a number of the tools listed above. +https://demo.dataverse.org is the place to play around with the Dataverse software and your tool can be included. Please email support@dataverse.org to start the conversation about adding your tool. Additionally, you are welcome to open an issue at https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible which already includes a number of the tools listed above. Announcing Your External Tool +++++++++++++++++++++++++++++ diff --git a/doc/sphinx-guides/source/api/faq.rst b/doc/sphinx-guides/source/api/faq.rst index e49a6648dd8..b73fbe694a1 100644 --- a/doc/sphinx-guides/source/api/faq.rst +++ b/doc/sphinx-guides/source/api/faq.rst @@ -11,12 +11,12 @@ What is an API? See "What is an API?" in the :doc:`intro` section. -What Are Common Use Cases for Dataverse Platform APIs? +What Are Common Use Cases for Dataverse software APIs? ---------------------------------------------------------- See the :doc:`getting-started` section for common use cases for researchers and curators. Other types of API users should find starting points at :ref:`types-of-api-users`. -Where Can I Find Examples of Using Dataverse Platform APIs? +Where Can I Find Examples of Using Dataverse software APIs? ----------------------------------------------------------- See the :doc:`getting-started` section links to examples using curl. @@ -26,7 +26,7 @@ For examples in Javascript, Python, R, and Java, and PHP, see the :doc:`apps` an When Should I Use the Native API vs. the SWORD API? --------------------------------------------------- -The :doc:`sword` is based on a standard, works fine, and is fully supported, but much more development effort has been going into the :doc:`native-api`, which is not based on a standard. It is specific to the Dataverse Platform. +The :doc:`sword` is based on a standard, works fine, and is fully supported, but much more development effort has been going into the :doc:`native-api`, which is not based on a standard. It is specific to the Dataverse software. SWORD uses XML. The Native API uses JSON. @@ -54,7 +54,7 @@ Note that when multiple query parameters are used (such as ``persistentId`` and Where is the Comprehensive List of All API Functionality? --------------------------------------------------------- -There are so many Dataverse Platform APIs that a single page in this guide would probably be overwhelming. See :ref:`list-of-dataverse-apis` for links to various pages. +There are so many Dataverse software APIs that a single page in this guide would probably be overwhelming. See :ref:`list-of-dataverse-apis` for links to various pages. It is possible to get a complete list of API functionality in Swagger/OpenAPI format if you deploy Dataverse Software version 5+. For details, see https://github.com/IQSS/dataverse/issues/5794 diff --git a/doc/sphinx-guides/source/api/getting-started.rst b/doc/sphinx-guides/source/api/getting-started.rst index 0b4fd42ad4a..27fd2a18c5d 100644 --- a/doc/sphinx-guides/source/api/getting-started.rst +++ b/doc/sphinx-guides/source/api/getting-started.rst @@ -1,7 +1,7 @@ Getting Started with APIs ========================= -If you are a researcher or curator who wants to automate parts of your workflow, this section should help you get started. The :doc:`intro` section lists resources for other groups who may be interested in Dataverse Platform APIs such as developers of integrations and support teams. +If you are a researcher or curator who wants to automate parts of your workflow, this section should help you get started. The :doc:`intro` section lists resources for other groups who may be interested in Dataverse software APIs such as developers of integrations and support teams. .. contents:: |toctitle| :local: @@ -16,7 +16,7 @@ If you would rather have full control over your own test server, deployments to Getting an API Token -------------------- -Many Dataverse Platform APIs require an API token. +Many Dataverse software APIs require an API token. Once you have identified a server to test with, create an account, click on your name, and get your API token. For more details, see the :doc:`auth` section. @@ -147,7 +147,7 @@ In addition to the tasks listed above, your Dataverse installation supports many See :ref:`list-of-dataverse-apis` and :ref:`types-of-api-users` to get oriented. -If you're looking for some inspiration for how you can use the Dataverse Platform APIs, there are open source projects that integrate with the Dataverse Platform listed in the :doc:`apps` section. +If you're looking for some inspiration for how you can use the Dataverse software APIs, there are open source projects that integrate with the Dataverse software listed in the :doc:`apps` section. Getting Help ------------- diff --git a/doc/sphinx-guides/source/api/intro.rst b/doc/sphinx-guides/source/api/intro.rst index e096d658368..03f950f0ec3 100755 --- a/doc/sphinx-guides/source/api/intro.rst +++ b/doc/sphinx-guides/source/api/intro.rst @@ -1,7 +1,7 @@ Introduction ============ -The Dataverse Platform APIs allow users to accomplish many tasks such as... +The Dataverse software APIs allow users to accomplish many tasks such as... - creating datasets - uploading files @@ -10,7 +10,7 @@ The Dataverse Platform APIs allow users to accomplish many tasks such as... ... all without using the Dataverse installation's web interface. -APIs open the door for integrations between the Dataverse Platform and other software. For a list, see the :doc:`/admin/integrations` section of the Admin Guide. +APIs open the door for integrations between the Dataverse software and other software. For a list, see the :doc:`/admin/integrations` section of the Admin Guide. .. contents:: |toctitle| :local: @@ -18,7 +18,7 @@ APIs open the door for integrations between the Dataverse Platform and other sof What is an API? --------------- -API stands for "Application Programming Interface" and an example is the Dataverse Platform's "file upload" API. In the diagram below, we can see that while users can click a button within Dataverse installation's web interface to upload a file, there are many other ways to get files into a Dataverse installation, all using an API that allows for uploading of files. +API stands for "Application Programming Interface" and an example is the Dataverse software's "file upload" API. In the diagram below, we can see that while users can click a button within Dataverse installation's web interface to upload a file, there are many other ways to get files into a Dataverse installation, all using an API that allows for uploading of files. .. graphviz:: @@ -72,7 +72,7 @@ The diagram above shows only a few examples of software using a specific API but .. _types-of-api-users: -Types of Dataverse Platform API Users +Types of Dataverse software API Users ------------------------------------- This guide is intended to serve multiple audiences but pointers various sections of the guide are provided below based on the type of API user you are. @@ -104,7 +104,7 @@ Power users may be researchers or curators who are comfortable with automating p Support Teams and Superusers ++++++++++++++++++++++++++++ -Support teams that answer questions about their Dataverse installation should familiarize themselves with the :doc:`getting-started` section to get a sense of common tasks that researchers and curators might be trying to accomplish by using Dataverse Platform APIs. +Support teams that answer questions about their Dataverse installation should familiarize themselves with the :doc:`getting-started` section to get a sense of common tasks that researchers and curators might be trying to accomplish by using Dataverse software APIs. Superusers of a Dataverse installation have access a superuser dashboard described in the :doc:`/admin/dashboard` section of the Admin Guide but some operations can only be done via API. @@ -113,7 +113,7 @@ Superusers of a Dataverse installation have access a superuser dashboard describ Sysadmins +++++++++ -Sysadmins often write scripts to automate tasks and Dataverse Platform APIs make this possible. Sysadmins have control over the server that the Dataverse installation is running on and may be called upon to execute API commands that are limited to "localhost" (the server itself) for security reasons. +Sysadmins often write scripts to automate tasks and Dataverse software APIs make this possible. Sysadmins have control over the server that the Dataverse installation is running on and may be called upon to execute API commands that are limited to "localhost" (the server itself) for security reasons. |Start| A good starting point for sysadmins is "Blocking API Endpoints" in the :doc:`/installation/config` section of the Installation Guide, followed by the :doc:`getting-started` section of this guide, followed by the :doc:`/admin/troubleshooting` section of the Admin Guide. @@ -132,18 +132,18 @@ The Dataverse Project loves contributors! Depending on your interests and skills Developers of Integrations, External Tools, and Apps ++++++++++++++++++++++++++++++++++++++++++++++++++++ -One of the primary purposes for Dataverse Platform APIs in the first place is to enable integrations with third party software. Integrations are listed in the following places: +One of the primary purposes for Dataverse software APIs in the first place is to enable integrations with third party software. Integrations are listed in the following places: - The :doc:`/admin/integrations` section of the Admin Guide. - The :doc:`/api/external-tools` section this guide. - The :doc:`apps` section of this guide. -|Start| Good starting points are the three sections above to get a sense of third-party software that already integrates with the Dataverse Platform, followed by the :doc:`getting-started` section. +|Start| Good starting points are the three sections above to get a sense of third-party software that already integrates with the Dataverse software, followed by the :doc:`getting-started` section. -Developers of Dataverse Platform API Client Libraries +Developers of Dataverse software API Client Libraries +++++++++++++++++++++++++++++++++++++++++++++++++++++ -A client library helps developers using a specific programming language such as Python, Javascript, R, or Java interact with Dataverse Platform APIs in a manner that is idiomatic for their language. For example, a Python programmer may want to +A client library helps developers using a specific programming language such as Python, Javascript, R, or Java interact with Dataverse software APIs in a manner that is idiomatic for their language. For example, a Python programmer may want to |Start| A good starting point is the :doc:`client-libraries` section, followed by the :doc:`getting-started` section. @@ -207,7 +207,7 @@ Please note that some APIs are only documented in other guides that are more sui Client Libraries ~~~~~~~~~~~~~~~~ -See :doc:`client-libraries` for how to use Dataverse Platform APIs from Python, R, Java, and Javascript. +See :doc:`client-libraries` for how to use Dataverse software APIs from Python, R, Java, and Javascript. Examples ~~~~~~~~ @@ -224,7 +224,7 @@ See :doc:`faq`. Getting Help ------------ -Dataverse Platform API questions are on topic in all the usual places: +Dataverse software API questions are on topic in all the usual places: - The dataverse-community Google Group: https://groups.google.com/forum/#!forum/dataverse-community - Dataverse Project community calls: https://dataverse.org/community-calls diff --git a/doc/sphinx-guides/source/api/metrics.rst b/doc/sphinx-guides/source/api/metrics.rst index e29cb3a2c9e..b273ef9eaff 100755 --- a/doc/sphinx-guides/source/api/metrics.rst +++ b/doc/sphinx-guides/source/api/metrics.rst @@ -1,7 +1,7 @@ Metrics API =========== -The Metrics API provides counts of downloads, datasets created, files uploaded, and more, as described below. The Dataverse Platform also supports Make Data Count, which is described in the :doc:`/admin/make-data-count` section of the Admin Guide. +The Metrics API provides counts of downloads, datasets created, files uploaded, and more, as described below. The Dataverse software also supports Make Data Count, which is described in the :doc:`/admin/make-data-count` section of the Admin Guide. .. contents:: |toctitle| :local: diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 9abe5dea3b4..d4712dd7a86 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1,7 +1,7 @@ Native API ========== -The Dataverse Platform exposes most of its GUI functionality via a REST-based API. This section describes that functionality. Most API endpoints require an API token that can be passed as the ``X-Dataverse-key`` HTTP header or in the URL as the ``key`` query parameter. +The Dataverse software exposes most of its GUI functionality via a REST-based API. This section describes that functionality. Most API endpoints require an API token that can be passed as the ``X-Dataverse-key`` HTTP header or in the URL as the ``key`` query parameter. .. note:: |CORS| Some API endpoint allow CORS_ (cross-origin resource sharing), which makes them usable from scripts running in web browsers. These endpoints are marked with a *CORS* badge. @@ -9,7 +9,7 @@ The Dataverse Platform exposes most of its GUI functionality via a REST-based AP .. _CORS: https://www.w3.org/TR/cors/ -.. warning:: The Dataverse Platform's API is versioned at the URI - all API calls may include the version number like so: ``http://server-address/api/v1/...``. Omitting the ``v1`` part would default to the latest API version (currently 1). When writing scripts/applications that will be used for a long time, make sure to specify the API version, so they don't break when the API is upgraded. +.. warning:: The Dataverse software's API is versioned at the URI - all API calls may include the version number like so: ``http://server-address/api/v1/...``. Omitting the ``v1`` part would default to the latest API version (currently 1). When writing scripts/applications that will be used for a long time, make sure to specify the API version, so they don't break when the API is upgraded. .. contents:: |toctitle| :local: @@ -933,7 +933,7 @@ For these deletes your JSON file must include an exact match of those dataset fi Publish a Dataset ~~~~~~~~~~~~~~~~~ -When publishing a dataset it's good to be aware of the Dataverse Platform's versioning system, which is described in the :doc:`/user/dataset-management` section of the User Guide. +When publishing a dataset it's good to be aware of the Dataverse software's versioning system, which is described in the :doc:`/user/dataset-management` section of the User Guide. If this is the first version of the dataset, its version number will be set to ``1.0``. Otherwise, the new dataset version number is determined by the most recent version number and the ``type`` parameter. Passing ``type=minor`` increases the minor version number (2.3 is updated to 2.4). Passing ``type=major`` increases the major version number (2.3 is updated to 3.0). (Superusers can pass ``type=updatecurrent`` to update metadata without changing the version number.) @@ -1816,7 +1816,7 @@ Note: at present, the API cannot be used on a file that's already successfully i Redetect File Type ~~~~~~~~~~~~~~~~~~ -The Dataverse Platform uses a variety of methods for determining file types (MIME types or content types) and these methods (listed below) are updated periodically. If you have files that have an unknown file type, you can have the Dataverse Platform attempt to redetect the file type. +The Dataverse software uses a variety of methods for determining file types (MIME types or content types) and these methods (listed below) are updated periodically. If you have files that have an unknown file type, you can have the Dataverse software attempt to redetect the file type. When using the curl command below, you can pass ``dryRun=true`` if you don't want any changes to be saved to the database. Change this to ``dryRun=false`` (or omit it) to save the change. diff --git a/doc/sphinx-guides/source/api/search.rst b/doc/sphinx-guides/source/api/search.rst index 60e799f00fc..15b964f867a 100755 --- a/doc/sphinx-guides/source/api/search.rst +++ b/doc/sphinx-guides/source/api/search.rst @@ -4,7 +4,7 @@ Search API .. contents:: |toctitle| :local: -The Search API supports the same searching, sorting, and faceting operations as the Dataverse Platform's web interface. +The Search API supports the same searching, sorting, and faceting operations as the Dataverse software's web interface. To search unpublished content, you must pass in an API token as described in the :doc:`auth` section. diff --git a/doc/sphinx-guides/source/api/sword.rst b/doc/sphinx-guides/source/api/sword.rst index cd49238a432..d58e003e636 100755 --- a/doc/sphinx-guides/source/api/sword.rst +++ b/doc/sphinx-guides/source/api/sword.rst @@ -11,7 +11,7 @@ About Introduced in Dataverse Network (DVN) `3.6 `_, the SWORD API was formerly known as the "Data Deposit API" and ``data-deposit/v1`` appeared in the URLs. For backwards compatibility these URLs continue to work (with deprecation warnings). Due to architectural changes and security improvements (especially the introduction of API tokens) in Dataverse Software 4.0, a few backward incompatible changes were necessarily introduced and for this reason the version has been increased to ``v1.1``. For details, see :ref:`incompatible`. -The Dataverse Platform implements most of SWORDv2_, which is specified at http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html . Please reference the `SWORDv2 specification`_ for expected HTTP status codes (i.e. 201, 204, 404, etc.), headers (i.e. "Location"), etc. +The Dataverse software implements most of SWORDv2_, which is specified at http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html . Please reference the `SWORDv2 specification`_ for expected HTTP status codes (i.e. 201, 204, 404, etc.), headers (i.e. "Location"), etc. As a profile of AtomPub, XML is used throughout SWORD. As of Dataverse Software 4.0 datasets can also be created via JSON using the "native" API. SWORD is limited to the dozen or so fields listed below in the crosswalk, but the native API allows you to populate all metadata fields available in a Dataverse installation. @@ -49,7 +49,7 @@ Backward incompatible changes For better security than in DVN 3.x, usernames and passwords are no longer accepted. The use of an API token is required. -Differences in Dataverse Platform 4 from DVN 3.x lead to a few minor backward incompatible changes in the Dataverse Platform implementation of SWORD, which are listed below. Old ``v1`` URLs should continue to work but the ``Service Document`` will contain a deprecation warning and responses will contain ``v1.1`` URLs. See also :ref:`known-issues`. +Differences in Dataverse software 4 from DVN 3.x lead to a few minor backward incompatible changes in the Dataverse software implementation of SWORD, which are listed below. Old ``v1`` URLs should continue to work but the ``Service Document`` will contain a deprecation warning and responses will contain ``v1.1`` URLs. See also :ref:`known-issues`. - Newly required fields when creating/editing datasets for compliance with the `Joint Declaration for Data Citation principles `_. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 9393233295a..4c5130f524b 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -146,7 +146,7 @@ As the person installing the Dataverse software, you may or may not be a local m Persistent Identifiers and Publishing Datasets ---------------------------------------------- -Persistent identifiers are a required and integral part of the Dataverse Platform. They provide a URL that is guaranteed to resolve to the datasets or files they represent. The Dataverse software currently supports creating identifiers using DOI and Handle. +Persistent identifiers are a required and integral part of the Dataverse software. They provide a URL that is guaranteed to resolve to the datasets or files they represent. The Dataverse software currently supports creating identifiers using DOI and Handle. By default, the installer configures a default DOI namespace (10.5072) with DataCite as the registration provider. Please note that as of the release 4.9.3, we can no longer use EZID as the provider. Unlike EZID, DataCite requires that you register for a test account, configured with your own prefix (please contact support@datacite.org). Once you receive the login name, password, and prefix for the account, configure the credentials in your domain.xml, as the following two JVM options:: From 1e3d8b98b6caff3c9e89e9500d6ec4908a3bc2af Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 29 Jan 2021 13:39:14 -0500 Subject: [PATCH 216/576] Catch NPE from Bundle, use raw value in type checks --- .../edu/harvard/iq/dataverse/ControlledVocabularyValue.java | 4 ++-- src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ControlledVocabularyValue.java b/src/main/java/edu/harvard/iq/dataverse/ControlledVocabularyValue.java index 581405ada1f..0b868c73c42 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ControlledVocabularyValue.java +++ b/src/main/java/edu/harvard/iq/dataverse/ControlledVocabularyValue.java @@ -117,12 +117,12 @@ public String getLocaleStrValue() key = StringUtils.stripAccents(key); try { String val = BundleUtil.getStringFromPropertyFile("controlledvocabulary." + this.datasetFieldType.getName() + "." + key, getDatasetFieldType().getMetadataBlock().getName()); - if( val== null) { + if( val == null) { //Default to raw value val=strValue; } return val; - } catch (MissingResourceException e) { + } catch (MissingResourceException | NullPointerException e) { return strValue; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java index 50681ad4fa2..dba9efac5aa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java @@ -843,7 +843,7 @@ public List getDatasetAuthors() { datasetAuthor.setAffiliation(subField); } if (subField.getDatasetFieldType().getName().equals(DatasetFieldConstant.authorIdType)){ - datasetAuthor.setIdType(subField.getDisplayValue()); + datasetAuthor.setIdType(subField.getRawValue()); } if (subField.getDatasetFieldType().getName().equals(DatasetFieldConstant.authorIdValue)){ datasetAuthor.setIdValue(subField.getDisplayValue()); @@ -871,7 +871,7 @@ public List getFunders() { contributorName = subField.getDisplayValue(); } if (subField.getDatasetFieldType().getName().equals(DatasetFieldConstant.contributorType)) { - contributorType = subField.getDisplayValue(); + contributorType = subField.getRawValue(); } } //SEK 02/12/2019 move outside loop to prevent contrib type to carry over to next contributor From 495ba605ca16a34e8aa98a9a41155d1ac54b8d1f Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 29 Jan 2021 15:12:27 -0500 Subject: [PATCH 217/576] Create 7501-guides-update.md --- doc/release-notes/7501-guides-update.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 doc/release-notes/7501-guides-update.md diff --git a/doc/release-notes/7501-guides-update.md b/doc/release-notes/7501-guides-update.md new file mode 100644 index 00000000000..6ad08a34446 --- /dev/null +++ b/doc/release-notes/7501-guides-update.md @@ -0,0 +1,6 @@ +## Release Highlights + +The Dataverse Guides have been updated following Merce's recent note to the Dataverse Community: + +https://groups.google.com/g/dataverse-community/c/pD-aFrpXMPo + From 43b5c35f78d8ac554c7ee8d936386914bf53e692 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 29 Jan 2021 15:47:47 -0500 Subject: [PATCH 218/576] reverting tracking changes --- conf/docker-aio/run-test-suite.sh | 4 +- .../7398-saved-search-performance.md | 30 +++++++ doc/release-notes/7518-guides-updates.md | 5 ++ .../admin/dataverse-external-tools.tsv | 2 +- doc/sphinx-guides/source/admin/dashboard.rst | 4 + .../source/admin/harvestclients.rst | 4 + doc/sphinx-guides/source/api/apps.rst | 2 +- .../source/installation/prerequisites.rst | 5 +- .../installation/r-rapache-tworavens.rst | 2 +- scripts/issues/7398/delete_dld.txt | 21 +++++ scripts/issues/7398/delete_ss.txt | 18 ++++ scripts/issues/7398/dld_for_deletion.txt | 19 +++++ scripts/issues/7398/ss_for_deletion.txt | 10 +++ .../harvard/iq/dataverse/DataversePage.java | 38 +-------- .../savedsearch/SavedSearchServiceBean.java | 84 +++++++++---------- src/main/java/propertyFiles/Bundle.properties | 1 + ....3.0.2__7512-update-notification-types.sql | 6 ++ src/main/webapp/dataset.xhtml | 3 + src/main/webapp/dataverseuser.xhtml | 9 +- src/main/webapp/filesFragment.xhtml | 15 ---- src/main/webapp/resources/js/clipboard.min.js | 6 +- .../resources/js/dv_rebind_bootstrap_ui.js | 10 +++ .../iq/dataverse/api/ExternalToolsIT.java | 2 +- 23 files changed, 188 insertions(+), 112 deletions(-) create mode 100644 doc/release-notes/7398-saved-search-performance.md create mode 100644 doc/release-notes/7518-guides-updates.md create mode 100644 scripts/issues/7398/delete_dld.txt create mode 100644 scripts/issues/7398/delete_ss.txt create mode 100644 scripts/issues/7398/dld_for_deletion.txt create mode 100644 scripts/issues/7398/ss_for_deletion.txt create mode 100644 src/main/resources/db/migration/V5.3.0.2__7512-update-notification-types.sql mode change 100755 => 100644 src/main/webapp/resources/js/clipboard.min.js diff --git a/conf/docker-aio/run-test-suite.sh b/conf/docker-aio/run-test-suite.sh index d0dd5d5318c..2b24f6c90b2 100755 --- a/conf/docker-aio/run-test-suite.sh +++ b/conf/docker-aio/run-test-suite.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # This is the canonical list of which "IT" tests are expected to pass. dvurl=$1 @@ -8,4 +8,4 @@ fi # Please note the "dataverse.test.baseurl" is set to run for "all-in-one" Docker environment. # TODO: Rather than hard-coding the list of "IT" classes here, add a profile to pom.xml. -source maven/maven.sh mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT -Ddataverse.test.baseurl=$dvurl +source maven/maven.sh && mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT -Ddataverse.test.baseurl=$dvurl diff --git a/doc/release-notes/7398-saved-search-performance.md b/doc/release-notes/7398-saved-search-performance.md new file mode 100644 index 00000000000..4986524ed4f --- /dev/null +++ b/doc/release-notes/7398-saved-search-performance.md @@ -0,0 +1,30 @@ +## Release Highlights + +### Saved Search Performance Improvements + +A refactoring has greatly improved Saved Search performance in the application. If your installation has multiple, potentially long-running Saved Searches in place, this greatly improves the probability that those search jobs will complete without timing out. + +## Notes for Dataverse Installation Administrators + +### DB Cleanup for Saved Searches + +A previous version of dataverse changed the indexing logic so that when a user links a dataverse, its children are also indexed as linked. This means that the children do not need to be separately linked, and in this version we removed the logic that creates a saved search to create those links when a dataverse is linked. + +We recommend cleaning up the db to a) remove these saved searches and b) remove the links for the objects. We can do this via a few queries, which are available in the folder here: + +https://github.com/IQSS/dataverse/raw/develop/scripts/issues/7398/ + +There are four sets of queries available, and they should be run in this order: + +- ss_for_deletion.txt to identify the Saved Searches to be deleted +- delete_ss.txt to delete the Saved Searches identified in the previous query +- dld_for_deletion.txt to identify the linked datasets and dataverses to be deleted +- delete_dld.txt to delete the linked datasets and dataverses identified in the previous queries + +Note: removing these saved searches and links should not affect what users will see as linked due to the aforementioned indexing change. Similarly, not removing these saved searches and links should not affect anything, but is a cleanup of unnecessary rows in the database. + +## Additional Upgrade Instructions + +X\. (Optional, but recommended) DB Cleanup + +Perform the DB Cleanup for Saved Searches and Linked Objects, summarized in the "Notes for Dataverse Installation Administrators" section above. diff --git a/doc/release-notes/7518-guides-updates.md b/doc/release-notes/7518-guides-updates.md new file mode 100644 index 00000000000..b64e0c3a0ee --- /dev/null +++ b/doc/release-notes/7518-guides-updates.md @@ -0,0 +1,5 @@ +## Guides Updates + +The Dataverse Software Guides have been updated to follow recent changes to how different terms are used across the Dataverse Project. For more information, see Mercè's note the community: + + diff --git a/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv b/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv index 881cf448221..521d61735dc 100644 --- a/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv +++ b/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv @@ -1,6 +1,6 @@ Tool Type Scope Description TwoRavens explore file A system of interlocking statistical tools for data exploration, analysis, and meta-analysis: http://2ra.vn. See the :doc:`/user/data-exploration/tworavens` section of the User Guide for more information on TwoRavens from the user perspective and the :doc:`/installation/r-rapache-tworavens` section of the Installation Guide. -Data Explorer explore file A GUI which lists the variables in a tabular data file allowing searching, charting and cross tabulation analysis. See the README.md file at https://github.com/scholarsportal/Dataverse-Data-Explorer for the instructions on adding Data Explorer to your Dataverse; and the :doc:`/installation/prerequisites` section of the Installation Guide for the instructions on how to set up **basic R configuration required** (specifically, Dataverse uses R to generate .prep metadata files that are needed to run Data Explorer). +Data Explorer explore file A GUI which lists the variables in a tabular data file allowing searching, charting and cross tabulation analysis. See the README.md file at https://github.com/scholarsportal/dataverse-data-explorer-v2 for the instructions on adding Data Explorer to your Dataverse. Whole Tale explore dataset A platform for the creation of reproducible research packages that allows users to launch containerized interactive analysis environments based on popular tools such as Jupyter and RStudio. Using this integration, Dataverse users can launch Jupyter and RStudio environments to analyze published datasets. For more information, see the `Whole Tale User Guide `_. File Previewers explore file A set of tools that display the content of files - including audio, html, `Hypothes.is `_ annotations, images, PDF, text, video, tabular data, and spreadsheets - allowing them to be viewed without downloading. The previewers can be run directly from github.io, so the only required step is using the Dataverse API to register the ones you want to use. Documentation, including how to optionally brand the previewers, and an invitation to contribute through github are in the README.md file. Initial development was led by the Qualitative Data Repository and the spreasdheet previewer was added by the Social Sciences and Humanities Open Cloud (SSHOC) project. https://github.com/GlobalDataverseCommunityConsortium/dataverse-previewers Data Curation Tool configure file A GUI for curating data by adding labels, groups, weights and other details to assist with informed reuse. See the README.md file at https://github.com/scholarsportal/Dataverse-Data-Curation-Tool for the installation instructions. diff --git a/doc/sphinx-guides/source/admin/dashboard.rst b/doc/sphinx-guides/source/admin/dashboard.rst index 520126c5299..6472c9e0e0f 100644 --- a/doc/sphinx-guides/source/admin/dashboard.rst +++ b/doc/sphinx-guides/source/admin/dashboard.rst @@ -22,7 +22,11 @@ This dashboard tool allows you to define sets of local datasets to make availabl Metadata Export --------------- +<<<<<<< HEAD This part of the Dashboard is simply a reminder message that metadata export happens through the Dataverse software API. See the :doc:`metadataexport` section and the :doc:`/api/native-api` section of the API Guide for more details. +======= +This part of the Dashboard is simply a reminder message that metadata export happens through the Dataverse Software API. See the :doc:`metadataexport` section and the :doc:`/api/native-api` section of the API Guide for more details. +>>>>>>> parent of f9f1faff9... updating branch from tracking Users ----- diff --git a/doc/sphinx-guides/source/admin/harvestclients.rst b/doc/sphinx-guides/source/admin/harvestclients.rst index c62c64b80fe..78a79178175 100644 --- a/doc/sphinx-guides/source/admin/harvestclients.rst +++ b/doc/sphinx-guides/source/admin/harvestclients.rst @@ -26,4 +26,8 @@ What if a Run Fails? Each harvesting client run logs a separate file per run to the app server's default logging directory (``/usr/local/payara5/glassfish/domains/domain1/logs/`` unless you've changed it). Look for filenames in the format ``harvest_TARGET_YYYY_MM_DD_timestamp.log`` to get a better idea of what's going wrong. +<<<<<<< HEAD Note that you'll want to run a minimum Dataverse software version of 4.6, optimally 4.18, for the best OAI-PMH interoperability. +======= +Note that you'll want to run a minimum Dataverse software version of 4.6, optimally 4.18 or beyond, for the best OAI-PMH interoperability. +>>>>>>> parent of f9f1faff9... updating branch from tracking diff --git a/doc/sphinx-guides/source/api/apps.rst b/doc/sphinx-guides/source/api/apps.rst index 3ac158f15cb..0000527a86e 100755 --- a/doc/sphinx-guides/source/api/apps.rst +++ b/doc/sphinx-guides/source/api/apps.rst @@ -16,7 +16,7 @@ Data Explorer Data Explorer is a GUI which lists the variables in a tabular data file allowing searching, charting and cross tabulation analysis. -https://github.com/scholarsportal/Dataverse-Data-Explorer +https://github.com/scholarsportal/Dataverse-Data-Explorer-v2 Data Curation Tool ~~~~~~~~~~~~~~~~~~ diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index fe439018e40..84fe012efd1 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -303,10 +303,7 @@ R The Dataverse software uses `R `_ to handle tabular data files. The instructions below describe a **minimal** R installation. It will allow you to ingest R (.RData) files as tabular -data; to export tabular data as .RData files; and to run `Data -Explorer `_ -(specifically, R is used to generate .prep metadata files that Data -Explorer uses). R can be considered an optional component, meaning +data and to export tabular data as .RData files. R can be considered an optional component, meaning that if you don't have R installed, you will still be able to run and use the Dataverse software - but the functionality specific to tabular data mentioned above will not be available to your users. **Note** that if diff --git a/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst b/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst index 72e6034d5ad..48dd3a6da95 100644 --- a/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst +++ b/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst @@ -25,7 +25,7 @@ correct versions of the required third party R packages. setup described in the "Prerequisites" portion of the Installation Guide. Meaning that once completed, it installs everything needed to run TwoRavens, PLUS all the libraries and components required to -ingest RData files, export as RData, and use Data Explorer.** +ingest RData files and export as RData.** diff --git a/scripts/issues/7398/delete_dld.txt b/scripts/issues/7398/delete_dld.txt new file mode 100644 index 00000000000..30edf42774b --- /dev/null +++ b/scripts/issues/7398/delete_dld.txt @@ -0,0 +1,21 @@ +-- these queries will delete linked objects identified using the query in dld_for_deletion + +begin; + +delete from datasetlinkingdataverse where id in ( +select dld.id +from datasetlinkingdataverse dld, dvobject dvo, dataverselinkingdataverse dvld +where dld.dataset_id = dvo.id +and dld.linkingdataverse_id = dvld.linkingdataverse_id +and dvo.owner_id = dvld.dataverse_id +); + +delete from dataverselinkingdataverse where id in ( +select dld.id +from dataverselinkingdataverse dld, dvobject dvo, dataverselinkingdataverse dvld +where dld.dataverse_id = dvo.id +and dld.linkingdataverse_id = dvld.linkingdataverse_id +and dvo.owner_id = dvld.dataverse_id +); + +commit; \ No newline at end of file diff --git a/scripts/issues/7398/delete_ss.txt b/scripts/issues/7398/delete_ss.txt new file mode 100644 index 00000000000..3cf053ce5af --- /dev/null +++ b/scripts/issues/7398/delete_ss.txt @@ -0,0 +1,18 @@ +-- these queries will delete the saved searches identified using the ss_for_deletion query + +begin; + +create temporary table delete_ss on commit drop as ( +Select ss.id +from savedsearch ss, savedsearchfilterquery ssfq, dataverselinkingdataverse dld +where ss.id = ssfq.savedsearch_id +and ss.definitionpoint_id = dld.linkingdataverse_id +and dld.dataverse_id = rtrim(reverse(split_part(reverse(ssfq.filterquery),'/',1)),'"')::integer +and ss.query='*' +and ssfq.filterquery like 'subtreePaths%' +); + +delete from savedsearchfilterquery where savedsearch_id in (select id from delete_ss); +delete from savedsearch where id in (select id from delete_ss); + +commit; diff --git a/scripts/issues/7398/dld_for_deletion.txt b/scripts/issues/7398/dld_for_deletion.txt new file mode 100644 index 00000000000..51aa9ce1458 --- /dev/null +++ b/scripts/issues/7398/dld_for_deletion.txt @@ -0,0 +1,19 @@ +-- these queries will show you the linked objects that will get deleted + +-- datasets +select dld.dataset_id, dvo.owner_id, dld.linkingdataverse_id, +dvld.dataverse_id, dvld.linkingdataverse_id +from datasetlinkingdataverse dld, dvobject dvo, dataverselinkingdataverse dvld +where dld.dataset_id = dvo.id +and dld.linkingdataverse_id = dvld.linkingdataverse_id +and dvo.owner_id = dvld.dataverse_id +order by dld.linkingdataverse_id; + +-- dataverses +select dld.dataverse_id, dvo.owner_id, dld.linkingdataverse_id, +dvld.dataverse_id, dvld.linkingdataverse_id +from dataverselinkingdataverse dld, dvobject dvo, dataverselinkingdataverse dvld +where dld.dataverse_id = dvo.id +and dld.linkingdataverse_id = dvld.linkingdataverse_id +and dvo.owner_id = dvld.dataverse_id +order by dld.linkingdataverse_id; diff --git a/scripts/issues/7398/ss_for_deletion.txt b/scripts/issues/7398/ss_for_deletion.txt new file mode 100644 index 00000000000..11ad8bbc49a --- /dev/null +++ b/scripts/issues/7398/ss_for_deletion.txt @@ -0,0 +1,10 @@ +-- this query will show you the saved searches that will get deleted + +select ss.id, ss.definitionpoint_id, dld.dataverse_id, ssfq.filterquery +from savedsearch ss, savedsearchfilterquery ssfq, dataverselinkingdataverse dld +where ss.id = ssfq.savedsearch_id +and ss.definitionpoint_id = dld.linkingdataverse_id +and dld.dataverse_id = rtrim(reverse(split_part(reverse(ssfq.filterquery),'/',1)),'"')::integer +and ss.query='*' +and ssfq.filterquery like 'subtreePaths%' +order by ss.definitionpoint_id; \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java index 165c1759b5e..34efc928ab9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java @@ -787,18 +787,9 @@ public String saveLinkedDataverse() { return ""; } - AuthenticatedUser savedSearchCreator = getAuthenticatedUser(); - if (savedSearchCreator == null) { - String msg = BundleUtil.getStringFromBundle("dataverse.link.user"); - logger.severe(msg); - JsfHelper.addErrorMessage(msg); - return returnRedirect(); - } - linkingDataverse = dataverseService.find(linkingDataverseId); LinkDataverseCommand cmd = new LinkDataverseCommand(dvRequestService.getDataverseRequest(), linkingDataverse, dataverse); - //LinkDvObjectCommand cmd = new LinkDvObjectCommand (session.getUser(), linkingDataverse, dataverse); try { commandEngine.submit(cmd); } catch (CommandException ex) { @@ -808,32 +799,9 @@ public String saveLinkedDataverse() { JsfHelper.addErrorMessage(msg); return returnRedirect(); } - - SavedSearch savedSearchOfChildren = createSavedSearchForChildren(savedSearchCreator); - - boolean createLinksAndIndexRightNow = false; - if (createLinksAndIndexRightNow) { - try { - // create links (does indexing) right now (might be expensive) - boolean debug = false; - DataverseRequest dataverseRequest = new DataverseRequest(savedSearchCreator, SavedSearchServiceBean.getHttpServletRequest()); - savedSearchService.makeLinksForSingleSavedSearch(dataverseRequest, savedSearchOfChildren, debug); - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataverse.linked.success", getSuccessMessageArguments())); - return returnRedirect(); - } catch (SearchException | CommandException ex) { - // error: solr is down, etc. can't link children right now - JsfHelper.addErrorMessage(BundleUtil.getStringFromBundle("dataverse.linked.internalerror", getSuccessMessageArguments())); - String msg = dataverse.getDisplayName() + " has been successfully linked to " + linkingDataverse.getDisplayName() + " but contents will not appear until an internal error has been fixed."; - logger.log(Level.SEVERE, "{0} {1}", new Object[]{msg, ex}); - //JsfHelper.addErrorMessage(msg); - return returnRedirect(); - } - } else { - // defer: please wait for the next timer/cron job - //JsfHelper.addSuccessMessage(dataverse.getDisplayName() + " has been successfully linked to " + linkingDataverse.getDisplayName() + ". Please wait for its contents to appear."); - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataverse.linked.success.wait", getSuccessMessageArguments())); - return returnRedirect(); - } + + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataverse.linked.success.wait", getSuccessMessageArguments())); + return returnRedirect(); } private List getSuccessMessageArguments() { diff --git a/src/main/java/edu/harvard/iq/dataverse/search/savedsearch/SavedSearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/savedsearch/SavedSearchServiceBean.java index 20cb1a8629a..a495842e40d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/savedsearch/SavedSearchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/savedsearch/SavedSearchServiceBean.java @@ -8,6 +8,7 @@ import edu.harvard.iq.dataverse.DvObjectServiceBean; import edu.harvard.iq.dataverse.EjbDataverseEngine; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.authorization.users.GuestUser; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.search.SearchServiceBean; import edu.harvard.iq.dataverse.search.SolrQueryResponse; @@ -20,6 +21,7 @@ import edu.harvard.iq.dataverse.search.SortBy; import edu.harvard.iq.dataverse.util.SystemConfig; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.List; import java.util.logging.Level; @@ -171,35 +173,40 @@ public JsonObjectBuilder makeLinksForSingleSavedSearch(DataverseRequest dvReq, S JsonArrayBuilder savedSearchArrayBuilder = Json.createArrayBuilder(); JsonArrayBuilder infoPerHit = Json.createArrayBuilder(); SolrQueryResponse queryResponse = findHits(savedSearch); + + List skipList = new ArrayList(); // a list for the definition point itself and already linked objects + skipList.add(savedSearch.getDefinitionPoint().getId()); - // get linked objects and add to a list TypedQuery typedQuery = em.createNamedQuery("DataverseLinkingDataverse.findIdsByLinkingDataverseId", Long.class) .setParameter("linkingDataverseId", savedSearch.getDefinitionPoint().getId()); - List alreadyLinkedObjectIds = typedQuery.getResultList(); + skipList.addAll(typedQuery.getResultList()); typedQuery = em.createNamedQuery("DatasetLinkingDataverse.findIdsByLinkingDataverseId", Long.class) .setParameter("linkingDataverseId", savedSearch.getDefinitionPoint().getId()); - alreadyLinkedObjectIds.addAll(typedQuery.getResultList()); - + skipList.addAll(typedQuery.getResultList()); + for (SolrSearchResult solrSearchResult : queryResponse.getSolrSearchResults()) { JsonObjectBuilder hitInfo = Json.createObjectBuilder(); hitInfo.add("name", solrSearchResult.getNameSort()); hitInfo.add("dvObjectId", solrSearchResult.getEntityId()); + + if (skipList.contains(solrSearchResult.getEntityId())) { + hitInfo.add(resultString, "Skipping because would link to itself or an already linked entity."); + infoPerHit.add(hitInfo); + continue; + } DvObject dvObjectThatDefinitionPointWillLinkTo = dvObjectService.findDvObject(solrSearchResult.getEntityId()); if (dvObjectThatDefinitionPointWillLinkTo == null) { hitInfo.add(resultString, "Could not find DvObject with id " + solrSearchResult.getEntityId()); infoPerHit.add(hitInfo); - break; - } + continue; + } + if (dvObjectThatDefinitionPointWillLinkTo.isInstanceofDataverse()) { Dataverse dataverseToLinkTo = (Dataverse) dvObjectThatDefinitionPointWillLinkTo; - if (wouldResultInLinkingToItself(savedSearch.getDefinitionPoint(), dataverseToLinkTo)) { - hitInfo.add(resultString, "Skipping because dataverse id " + dataverseToLinkTo.getId() + " would link to itself."); - } else if (alreadyLinkedToTheDataverse(alreadyLinkedObjectIds, dataverseToLinkTo)) { - hitInfo.add(resultString, "Skipping because dataverse " + savedSearch.getDefinitionPoint().getId() + " already links to dataverse " + dataverseToLinkTo.getId() + "."); - } else if (dataverseToLinkToIsAlreadyPartOfTheSubtree(savedSearch.getDefinitionPoint(), dataverseToLinkTo)) { + if (dataverseToLinkToIsAlreadyPartOfTheSubtree(savedSearch.getDefinitionPoint(), dataverseToLinkTo)) { hitInfo.add(resultString, "Skipping because " + dataverseToLinkTo + " is already part of the subtree for " + savedSearch.getDefinitionPoint()); } else { DataverseLinkingDataverse link = commandEngine.submitInNewTransaction(new LinkDataverseCommand(dvReq, savedSearch.getDefinitionPoint(), dataverseToLinkTo)); @@ -207,29 +214,22 @@ public JsonObjectBuilder makeLinksForSingleSavedSearch(DataverseRequest dvReq, S } } else if (dvObjectThatDefinitionPointWillLinkTo.isInstanceofDataset()) { Dataset datasetToLinkTo = (Dataset) dvObjectThatDefinitionPointWillLinkTo; - if (alreadyLinkedToTheDataset(alreadyLinkedObjectIds, datasetToLinkTo)) { - hitInfo.add(resultString, "Skipping because dataverse " + savedSearch.getDefinitionPoint() + " already links to dataset " + datasetToLinkTo + "."); - } else if (datasetToLinkToIsAlreadyPartOfTheSubtree(savedSearch.getDefinitionPoint(), datasetToLinkTo)) { + if (datasetToLinkToIsAlreadyPartOfTheSubtree(savedSearch.getDefinitionPoint(), datasetToLinkTo)) { // already there from normal search/browse hitInfo.add(resultString, "Skipping because dataset " + datasetToLinkTo.getId() + " is already part of the subtree for " + savedSearch.getDefinitionPoint().getAlias()); } else if (datasetAncestorAlreadyLinked(savedSearch.getDefinitionPoint(), datasetToLinkTo)) { hitInfo.add(resultString, "FIXME: implement this?"); - } else if (!datasetToLinkTo.isReleased()) { - hitInfo.add(resultString, "Skipping because dataset " + datasetToLinkTo.getId() + " is not released " ); } else { DatasetLinkingDataverse link = commandEngine.submitInNewTransaction(new LinkDatasetCommand(dvReq, savedSearch.getDefinitionPoint(), datasetToLinkTo)); - alreadyLinkedObjectIds.add(datasetToLinkTo.getId()); // because search results could produce two hits (published and draft) hitInfo.add(resultString, "Persisted DatasetLinkingDataverse id " + link.getId() + " link of " + link.getDataset() + " to " + link.getLinkingDataverse()); } - } else if (dvObjectThatDefinitionPointWillLinkTo.isInstanceofDataFile()) { - hitInfo.add(resultString, "Skipping because the search matched a file. The matched file id was " + dvObjectThatDefinitionPointWillLinkTo.getId() + "."); } else { hitInfo.add(resultString, "Unexpected DvObject type."); } infoPerHit.add(hitInfo); } - + JsonObjectBuilder info = getInfo(savedSearch, infoPerHit); if (debugFlag) { info.add("debug", getDebugInfo(savedSearch)); @@ -237,13 +237,12 @@ public JsonObjectBuilder makeLinksForSingleSavedSearch(DataverseRequest dvReq, S savedSearchArrayBuilder.add(info); response.add("hits for saved search id " + savedSearch.getId(), savedSearchArrayBuilder); - Date end = new Date(); - logger.info("SAVED SEARCH (" + savedSearch.getId() + ") total time in ms: " + (end.getTime() - start.getTime())); + logger.info("SAVED SEARCH (" + savedSearch.getId() + ") total time in ms: " + (new Date().getTime() - start.getTime())); return response; } private SolrQueryResponse findHits(SavedSearch savedSearch) throws SearchException { - String sortField = SearchFields.RELEVANCE; + String sortField = SearchFields.TYPE; // first return dataverses, then datasets String sortOrder = SortBy.DESCENDING; SortBy sortBy = new SortBy(sortField, sortOrder); int paginationStart = 0; @@ -251,16 +250,23 @@ private SolrQueryResponse findHits(SavedSearch savedSearch) throws SearchExcepti int numResultsPerPage = Integer.MAX_VALUE; List dataverses = new ArrayList<>(); dataverses.add(savedSearch.getDefinitionPoint()); + + // since saved search can only link Dataverses and Datasets, we can limit our search + List searchFilterQueries = savedSearch.getFilterQueriesAsStrings(); + searchFilterQueries.add("dvObjectType:(dataverses OR datasets)"); + + // run the search as GuestUser to only link published objects SolrQueryResponse solrQueryResponse = searchService.search( - new DataverseRequest(savedSearch.getCreator(), getHttpServletRequest()), + new DataverseRequest(GuestUser.get(), getHttpServletRequest()), dataverses, savedSearch.getQuery(), - savedSearch.getFilterQueriesAsStrings(), + searchFilterQueries, sortBy.getField(), sortBy.getOrder(), paginationStart, dataRelatedToMe, - numResultsPerPage + numResultsPerPage, + false // do not retrieve entities ); return solrQueryResponse; } @@ -289,18 +295,6 @@ private JsonArrayBuilder getFilterQueries(SavedSearch savedSearch) { return filterQueriesArrayBuilder; } - private boolean alreadyLinkedToTheDataverse(List alreadyLinkedObjectIds, Dataverse dataverseToLinkTo) { - return alreadyLinkedObjectIds.contains(dataverseToLinkTo.getId()); - } - - private boolean alreadyLinkedToTheDataset(List alreadyLinkedObjectIds, Dataset linkToThisDataset) { - return alreadyLinkedObjectIds.contains(linkToThisDataset.getId()); - } - - private static boolean wouldResultInLinkingToItself(Dataverse savedSearchDefinitionPoint, Dataverse dataverseToLinkTo) { - return savedSearchDefinitionPoint.equals(dataverseToLinkTo); - } - private boolean datasetToLinkToIsAlreadyPartOfTheSubtree(Dataverse definitionPoint, Dataset datasetWeMayLinkTo) { Dataverse ancestor = datasetWeMayLinkTo.getOwner(); while (ancestor != null) { @@ -351,14 +345,12 @@ public static HttpServletRequest getHttpServletRequest() { * Saved Search was created. The default IP address in the * DataverseRequest constructor is used instead, which as of this * writing is 0.0.0.0 to mean "undefined". Is this a feature or a bug? - * What is the expected interplay between Saved Search and IP Groups? - * Users might be surprised to see certain DvObjects in the results of - * their query when creating the Saved Search and later find that those - * DvObjects, which are only visible due to an IP Groups membership, are - * not found by Saved Search when executed by cron, for example. As of - * this writing Saved Search is a superuser-only feature so perhaps IP - * Groups are irrelevant because all DvObjects are discoverable to - * superusers. + * This is not an issue for the search itself, since it is now run as the + * Guest User, but would present a problem if the user does not have + * permission to create links and could only create the saved search due to + * a granted permission from the IP Group. + * As of this writing Saved Search is a superuser-only feature; so IP + * Groups are irrelevant because all superusers can create links. */ return null; } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 3d3349a0b52..2a73ea97307 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -54,6 +54,7 @@ harvested=Harvested apply=Apply add=Add delete=Delete +copyClipboard=Copy to Clipboard yes=Yes no=No previous=Previous diff --git a/src/main/resources/db/migration/V5.3.0.2__7512-update-notification-types.sql b/src/main/resources/db/migration/V5.3.0.2__7512-update-notification-types.sql new file mode 100644 index 00000000000..c063aba38f8 --- /dev/null +++ b/src/main/resources/db/migration/V5.3.0.2__7512-update-notification-types.sql @@ -0,0 +1,6 @@ +--In support of removing world map we are deleting notifications related to world map +--and updating notifications to reflect the current enumeration of types on UserNotification + +delete from usernotification where type = 5 or type = 6; + +update usernotification set type = type - 2 where type > 6; \ No newline at end of file diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 65a30280ceb..53c383381d4 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -974,6 +974,9 @@
    + - -
    -
    -
    - -

    - - - - - - - - - -

    - -

    - - - - - - - - - - -

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - #{bundle['user.username']} - - #{DataverseUserPage.currentUser.identifier.replaceFirst("@", "")} -
    - #{bundle['user.firstName']} - - #{DataverseUserPage.currentUser.firstName} -
    - #{bundle['user.lastName']} - - #{DataverseUserPage.currentUser.lastName} -
    - #{bundle.email} - -

    - #{DataverseUserPage.currentUser.email} -   - - #{bundle['confirmEmail.notVerified']} - - - #{bundle['confirmEmail.verified']} - -

    -   - - #{bundle['confirmEmail.submitRequest']} - -
    - #{DataverseUserPage.userAuthProvider.persistentIdName} - -   - - - -
    - #{bundle.affiliation} - - #{DataverseUserPage.currentUser.affiliation} -
    - #{bundle['user.position']} - - #{DataverseUserPage.currentUser.position} -
    -
    - @@ -473,6 +327,152 @@
    + +
    +
    + + +
    +
    +
    + +

    + + + + + + + + + +

    + +

    + + + + + + + + + + +

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + #{bundle['user.username']} + + #{DataverseUserPage.currentUser.identifier.replaceFirst("@", "")} +
    + #{bundle['user.firstName']} + + #{DataverseUserPage.currentUser.firstName} +
    + #{bundle['user.lastName']} + + #{DataverseUserPage.currentUser.lastName} +
    + #{bundle.email} + +

    + #{DataverseUserPage.currentUser.email} +   + + #{bundle['confirmEmail.notVerified']} + + + #{bundle['confirmEmail.verified']} + +

    +   + + #{bundle['confirmEmail.submitRequest']} + +
    + #{DataverseUserPage.userAuthProvider.persistentIdName} + +   + + + +
    + #{bundle.affiliation} + + #{DataverseUserPage.currentUser.affiliation} +
    + #{bundle['user.position']} + + #{DataverseUserPage.currentUser.position} +
    +
    +

    From 4d430c266f4b652cfec681deb3e0b9b56108bbe1 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 3 Feb 2021 13:47:45 -0500 Subject: [PATCH 251/576] #7534 fixing default values --- .../authorization/providers/builtin/DataverseUserPage.java | 2 +- src/main/webapp/dataverseuser.xhtml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java index 9d34a6c4e63..d3c2cbf711b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java @@ -132,7 +132,7 @@ public enum EditMode { private Long dataverseId; private List notificationsList; private int activeIndex; - private String selectTab = "somedata"; + private String selectTab = "dataRelatedToMe"; UIInput usernameField; diff --git a/src/main/webapp/dataverseuser.xhtml b/src/main/webapp/dataverseuser.xhtml index a339ba66a69..ba25798e081 100644 --- a/src/main/webapp/dataverseuser.xhtml +++ b/src/main/webapp/dataverseuser.xhtml @@ -39,7 +39,7 @@ - + From 6a2e472bdbd19255aa2587f5f4d1c8cdf95326b6 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 3 Feb 2021 15:13:47 -0500 Subject: [PATCH 252/576] #7534 clean up comments --- .../authorization/providers/builtin/DataverseUserPage.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java index d3c2cbf711b..204d93b5b8f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java @@ -176,17 +176,14 @@ public String init() { mydatapage.init(); activeIndex = 0; break; - // case "groupsRoles": - // activeIndex = 2; - // break; case "accountInfo": activeIndex = 2; - // activeIndex = 3; break; case "apiTokenTab": activeIndex = 3; break; default: + //TODO: Do we need to call mydatapage.init(); here too? activeIndex = 0; break; } From 481c3b4bb03209bca88de6a6b70673e86145f791 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 3 Feb 2021 19:23:03 -0500 Subject: [PATCH 253/576] implements changes to the API path as discussed (#7084) --- .../java/edu/harvard/iq/dataverse/api/Datasets.java | 8 ++++---- .../java/edu/harvard/iq/dataverse/util/FileUtil.java | 12 ++++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index e0ed28a9f3b..52790256bea 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -468,12 +468,12 @@ public Response getVersionFiles( @PathParam("id") String datasetId, @PathParam(" } @GET - @Path("{id}/versions/{versionId}/fileaccess") + @Path("{id}/dirindex") @Produces("text/html") - //public Response getFileAccessFolderView(@PathParam("id") String datasetId, @PathParam("versionId") String versionId, @PathParam("name") String name, @QueryParam("folder") String folderName, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) { - public Response getFileAccessFolderView(@PathParam("id") String datasetId, @PathParam("versionId") String versionId, @QueryParam("folder") String folderName, @QueryParam("original") Boolean originals, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) { + public Response getFileAccessFolderView(@PathParam("id") String datasetId, @QueryParam("version") String versionId, @QueryParam("folder") String folderName, @QueryParam("original") Boolean originals, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) { - folderName = folderName == null ? "" : folderName; + folderName = folderName == null ? "" : folderName; + versionId = versionId == null ? ":latest-published" : versionId; DatasetVersion version; try { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 58a1e7e4c14..04d539ffb98 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -1935,12 +1935,16 @@ public static String formatFolderListingHtml(String folderName, DatasetVersion v StringBuilder sb = new StringBuilder(); - sb.append(HtmlFormatUtil.formatTag("Index of folder /" + folderName + " in dataset " + persistentId, HTML_H1)); + String versionTag = version.getFriendlyVersionNumber(); + versionTag = "DRAFT".equals(versionTag) ? "Draft Version" : "v. " + versionTag; + sb.append(HtmlFormatUtil.formatTag("Index of folder /" + folderName + + " in dataset " + persistentId + + " (" + versionTag + ")", HTML_H1)); sb.append("\n"); sb.append(formatFolderListingTableHtml(folderName, fileMetadatas, apiLocation, originals)); String body = sb.toString(); - + return formatDoc(title, body); } @@ -2034,8 +2038,8 @@ private static String getFolderAccessUrl(DatasetVersion version, String currentF } return apiLocation + "/api/datasets/" + datasetId + - "/versions/" + versionTag + - "/fileaccess?folder=" + subFolder + + "/dirindex/?version=" + versionTag + "&" + + "folder=" + subFolder + (originals ? "&original=true" : ""); } From cdaf121610990e60aa2a0c2eab5c0da0d97a4f73 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 3 Feb 2021 19:26:31 -0500 Subject: [PATCH 254/576] Changed the release note to reflect the modifications to the API path (#7084) --- doc/release-notes/7084-crawlable-file-access.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/7084-crawlable-file-access.md b/doc/release-notes/7084-crawlable-file-access.md index 111dbeb0547..a8d5660566f 100644 --- a/doc/release-notes/7084-crawlable-file-access.md +++ b/doc/release-notes/7084-crawlable-file-access.md @@ -5,7 +5,7 @@ A new api offers *crawlable* access view of the folders and files within a datset: ``` - /api/datasets//versions//fileaccess[?folder=] + /api/datasets//dirindex ``` will output a simple html listing, based on the standard Apache From b774f07e697ac3fb063d158a567f9161bf8e8218 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 3 Feb 2021 20:04:05 -0500 Subject: [PATCH 255/576] changed the api guide to reflect the latest mods. #7084 --- doc/sphinx-guides/source/api/native-api.rst | 43 ++++++++++----------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 926470a93e7..5773e1fdbab 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -786,34 +786,26 @@ The fully expanded example above (without environment variables) looks like this curl https://demo.dataverse.org/api/datasets/24/versions/1.0/files -File Access Tree View in a Dataset -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +View Dataset Files and Folders as a Directory Index +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -|CORS| Lists all the file metadata, for the given dataset and version: +|CORS| Provides a *crawlable* view of files and folders within the given dataset and version: .. code-block:: bash - export SERVER_URL=https://demo.dataverse.org - export ID=24 - export VERSION=1.0 - - curl $SERVER_URL/api/datasets/$ID/versions/$VERSION/fileaccess - -The fully expanded example above (without environment variables) looks like this: - -.. code-block:: bash - - curl https://demo.dataverse.org/api/datasets/24/versions/1.0/fileaccess + curl $SERVER_URL/api/datasets/$ID/dirindex Optional parameters: -* ``folder`` Specifies a subfolder within the dataset. +* ``folder`` - A subfolder within the dataset (default: top-level view of the dataset) +* ``version`` - Specifies the version (default: latest published version) +* ``original=true`` - Download original versions of ingested tabular files. This API outputs a simple html listing, based on the standard Apache directory index, with Access API download links for individual files, and recursive calls to the API above for sub-folders. -Using this API, ``wget -recursive`` (or similar crawling client) can +Using this API, ``wget --recursive`` (or similar crawling client) can be used to download all the files in a dataset, preserving the file names and folder structure; without having to use the download-as-zip API. In addition to being faster (zipping is a relatively @@ -834,7 +826,7 @@ or, as viewed as a tree on the dataset page: .. |image2| image:: ./img/dataset_page_tree_view.png -The fileaccess API for the top-level folder view output will be as follows: +The output of the API for the top-level folder (``/api/datasets/{dataset}/dirindex/``) will be as follows: .. |image3| image:: ./img/index_view_top.png @@ -844,15 +836,15 @@ with the underlying html source: Index of folder / -

    Index of folder / in dataset doi:XXX/YY/ZZZZ

    +

    Index of folder / in dataset doi:XXX/YY/ZZZZ (v. MM)

    - +
    NameLast ModifiedSizeDescription
    subfolder/ - -  
    subfolder/ - -  
    testfile.txt13-January-2021 22:3519 B 
    -The ``/fileaccess?folder=subfolder`` link above will produce the following view: +The ``/dirindex?folder=subfolder`` link above will produce the following view: .. |image4| image:: ./img/index_view_subfolder.png @@ -862,16 +854,23 @@ with the html source as follows: Index of folder /subfolder -

    Index of folder /subfolder in dataset doi:XXX/YY/ZZZZ

    +

    Index of folder /subfolder in dataset doi:XXX/YY/ZZZZ (v. MM)

    - +
    NameLast ModifiedSizeDescription
    50by1000.tab11-January-2021 09:31102.5 KB 
    50by1000.tab11-January-2021 09:31102.5 KB 
    +An example of a ``wget`` command line for crawling ("recursive downloading") of the files and folders in a dataset: +.. code-block:: bash + + wget -r -nH --cut-dirs=3 --content-disposition https://demo.dataverse.org/api/datasets/24/dirindex +.. note:: In addition to the files and folders in the dataset, the command line above will also save the directory index of each folder, in a separate folder "dirindex". +.. note:: When running in recursive mode ``wget`` respects the "Robot Exclusion Standard" rules, as specified in the ``robots.txt``. If a Dataverse installation is running with a restrictive robot access rules, they may need to be adjusted to allow recursive downloading via using this API. + List All Metadata Blocks for a Dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From a853e314312010f95991d7e0012b9715aeb3d5e2 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 3 Feb 2021 20:08:13 -0500 Subject: [PATCH 256/576] a (#7084) --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 5773e1fdbab..d57b4ea5f4d 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -805,7 +805,7 @@ This API outputs a simple html listing, based on the standard Apache directory index, with Access API download links for individual files, and recursive calls to the API above for sub-folders. -Using this API, ``wget --recursive`` (or similar crawling client) can +Using this API, ``wget --recursive`` (or a similar crawling client) can be used to download all the files in a dataset, preserving the file names and folder structure; without having to use the download-as-zip API. In addition to being faster (zipping is a relatively From c0d5c86c1a7f65708af4db04860c0fcc7487d477 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 3 Feb 2021 21:26:18 -0500 Subject: [PATCH 257/576] escape from HTML: the sequel --- doc/release-notes/4259-java-11.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/release-notes/4259-java-11.md b/doc/release-notes/4259-java-11.md index 89af24305cd..eada46f3751 100644 --- a/doc/release-notes/4259-java-11.md +++ b/doc/release-notes/4259-java-11.md @@ -27,20 +27,20 @@ There are some things to note and keep in mind regarding the move to Java 11: For installations that were previously running on Dataverse 4.x, the following JVM Options have been added in support of Java 11: -- [9|]--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED -- [9|]--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED -- [9|]--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED -- [9|]--add-opens=java.base/java.lang=ALL-UNNAMED -- [9|]--add-opens=java.base/java.net=ALL-UNNAMED -- [9|]--add-opens=java.base/java.nio=ALL-UNNAMED -- [9|]--add-opens=java.base/java.util=ALL-UNNAMED -- [9|]--add-opens=java.base/sun.nio.ch=ALL-UNNAMED -- [9|]--add-opens=java.management/sun.management=ALL-UNNAMED -- [9|]--add-opens=java.base/sun.net.www.protocol.jrt=ALL-UNNAMED -- [9|]--add-opens=java.base/sun.net.www.protocol.jar=ALL-UNNAMED -- [9|]--add-opens=java.naming/javax.naming.spi=ALL-UNNAMED -- [9|]--add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED -- [9|]--add-opens=java.logging/java.util.logging=ALL-UNNAMED +- `[9|]--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED` +- `[9|]--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED` +- `[9|]--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED` +- `[9|]--add-opens=java.base/java.lang=ALL-UNNAMED` +- `[9|]--add-opens=java.base/java.net=ALL-UNNAMED` +- `[9|]--add-opens=java.base/java.nio=ALL-UNNAMED` +- `[9|]--add-opens=java.base/java.util=ALL-UNNAMED` +- `[9|]--add-opens=java.base/sun.nio.ch=ALL-UNNAMED` +- `[9|]--add-opens=java.management/sun.management=ALL-UNNAMED` +- `[9|]--add-opens=java.base/sun.net.www.protocol.jrt=ALL-UNNAMED` +- `[9|]--add-opens=java.base/sun.net.www.protocol.jar=ALL-UNNAMED` +- `[9|]--add-opens=java.naming/javax.naming.spi=ALL-UNNAMED` +- `[9|]--add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED` +- `[9|]--add-opens=java.logging/java.util.logging=ALL-UNNAMED` ## Upgrade Instructions From 61c5cba10846f3a67b13c6c12b9dce26b68605f5 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 4 Feb 2021 10:37:23 -0500 Subject: [PATCH 258/576] updated the images for the guide to reflect the minor cosmetic change (version number added to the dir. index) #7084 --- .../source/api/img/index_view_subfolder.png | Bin 23816 -> 25559 bytes .../source/api/img/index_view_top.png | Bin 24369 -> 25521 bytes doc/sphinx-guides/source/api/native-api.rst | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/img/index_view_subfolder.png b/doc/sphinx-guides/source/api/img/index_view_subfolder.png index c793fe6135c1824506a62c37d24319a4e93013a9..6c940ad23541bddd19672927fd0c6be5b9eafbb1 100644 GIT binary patch delta 22429 zcmc$`byQVt_ckhmB2pp>n-)-zZs`_~mPR@Rq`PC$xdD}Kq(eGHx;NcO=cc=J184C( z@B90n_dVk~t&FaL{Jr9Y&8YQZFiN16YVP?0An0*E z=iCou_%8(#G0HHvpvb}<-h8#A5zNJrAZCH`?$!K0f8~1D&$-}Fo$5ltcQ$FLpAka! z6rHKw24eoRjP}=R)r1Z@wjjdh;sPGohBzK1GfpsyWghz|sv@_Kv*5npaBy?JaBW%d z%c9PiAzTCuaIYsH>{JW}X2cj+L(@t1beOM~w-tYYDvt3K!w5 zJ#Y1Q%3u-a^|~&DD<}e#;p~bR;P)S=*IpO69+LKl*hLClZA{|3@BKEpIO=6;!`Jor zCSmTHHna z&xh?xaw=XqEKa0v*sy8c@lwqqx?b}ZoB!lv|6_xoaiLQ`yK%d!qUx{DiMrlV+(2fMMY*IYfBnFi$$rB*qhbMn|~F-Rn`Un)3zfm|$q8n3nrW5;W@i|Yv2gmqKFMS$5Br)Ae2dx@*x zbaQ4N56eli(twkQc{(q6o+mO~(5*_oR(&hmt(umN{5>SNG&Kht+U(*bKjcg+#pc>~ z2@W~cqjJ12y-$eGcKhgO}9vvt)JcY*4w#Ign_`3 zwQM&W>!l#*FA7%{cDW{W0Z;9z6eE{fZ6DIE#k4zwiFVRkgW}(7w21ghA{^`zY~o@G zQrFMB;7TYsHq6+d&&KAtR5cny4Y)l-voEUpvqev@9Rajqa*mx!nsgRj)e!D{*cmY) z2LM-<-fYjZ5tDegsxXBh97@6Xa$z60j7EMR1{ozDFBeJYK$q^=hWKw*Yn7mLf|bW# z_dXFPLUY$3oBiNpZ@d9|$IwIdx6x>%(+O%C6XmWiHf?lF{9RmA_{jj>=9Sg4fqFh` ze?1*RB~R!)^ZG>@-Piq$W{lSNu7piMD{^+b`e7;C02d`@=$?2TBmEOaJ7Iiy9n=(= znh927v|r{_NmHao$tgqC#~|XT>vfr}P)T|$@l^IQB0$$w&SO|jwI<)+@q?(wMQSps z=gII$jiz;jBWcxMqlJ$I0ez>P!4EkI@AJm40eIGV48#$rA%0P8r0B;Eclr$+2mNBh zkgwYs+VeQVN4eB>SqiXyz`A%8#LaM5J!4f!^xk5dT-byt)N9IhJ!NzwX6}RQ$Zb(i z-udaRr?tso{6XokrojrT|UN4YaVdAu>_yrBL0yo)lj{ddgnaWV@%MFMh8ITTIOSzR`c zw_v|xnTbyHum!2u4Tq=m;sAkW3AL>$Wos}i*uDa^qEWZkws`O=2%H;vBK>jI z)XT6p?jbnre>c-tQB*fF;VNLXzveih&DmK|@xgKPw?4+ItBvlT{axUTQqa@Sr1SI#-M=AL*Yho*LFI z=UH0`ALp0+)v8=7nEO~jX0bqVsNF*7gq8908)Lz`){6NFg45KW)^%l;>s>T6v_Uv4 z$!O25XPN)}-g^IuWB?3M28~DO$Z89z&M7ybFR7q+P0{5&HhULIkuSF$nsl}{;+2Nm z`+7*{&}mI9bs&DXVo;#$$?Kc3{aO3LQ;IOUk>z@l*@~f_?!&i$VJS}S>ZP|tLILxs zXn7j$cSCr6BRqo>YqaKL!Q8zVqjvrx-AiMrMA zRbPG!u~hXk7E@Q-eJo8h@pUl!A(h9k1H7`stBbVOEdjb~Avd`Y~J#grMVVJ19I5l(Ppjq7(f;pG+{%?9g*`iR6J9QMGoM&r9r1CQ|}= z?gR(m4F`dEAY^K%kuYk8we)H<{5 zT26S{M&$)biu}EkBzu=@jOTAg({5K347rY34UZE{QbPtqBr)Jx%c_<JOBp7S;kg#Y1eTS(#h?T1EY$-{&3*$0sTf--)L#iF$3u~_}=@IZYjI5m#+%Y(qJef`Kf)l`IP z#!>mNDdVJBI)i~aKI-{LK_pJ?5 zAF-3H8(mY#ICssIf}ptA^DR_J5dwYuG%TI2*0IB2#3fiS^Jom@WDTsNmwY2gPc`+s z@!}W-s<1XL*?*-V#V5~q?Bv3y@kF{U@i#$OU&}#>oCLA$`yIbzCZio(V4?C2pII;6 zv4Ej(?YGo(CDC#@D@mgn_xvsNLg}o>svizTucEfUy17ghWLdnF@=%pejW931YBi(b z4>nzkGwVdOvs-A}yhSdTqn}ef@#8m1aa5gqS}ikJit)DkheuMF3enZ-<{4@&qL^1~ zZlrp}%7K>%XC#5o%PlRn5fFdb=|~aMR)BLu8#(w?AkL}sd)~RjUFpk2GOVmdD8J2V zt-Voi-qW(S#lpQY!iWZsFob%5pGs|ifIw1%`ZnS%emL5RnjJf`#C=Z63(}SUDU&vdo9+Qp;#J ze&yC^q=vQV_(=0;tBWuO$1A62bf@yzNYy=l z2M7d#Yxe#ct!vFzz(iX~|Ak+<$ILj@k({%MN9rl-W=Z*a$mPI79UrnlPWTT7r(nLw z0j(vpC5Gb)twsMVe(L#|gU??FjUr6hLU?HNETeR^F*&j- z4T<4Z)%lHC)WZ$F4j9c-Pb>Dwn+4xQef$$Jx(OZ|0d~pNlx^2FC=XQ>U*BoRNJv1? z7}6{G$nA+}a+{%GwE(@~(ExuYp^y4LNXM)QW+crtBd)SJ7?aV-(a~_zn+URv!+PgA z$B7N=GvP4cbtAu2@{1CO=iS%}?1p2gs&_-)tbwqZEj1lgmA4$Qm6?dmAKzaf>%6@+ zpu$2C-7~Ut6i+E-&EeT?Vimt>M#lchf_5$4dmQ-5fF)PpHbtRi1je12Bp1xQ9Z{m| zstO)Z53r&*(60-k*{-A8JX(;Lz_f-YGnMp@9Q75z*K~Vxjjz&8fVcLdGlU5dG8yBc zo!2_M1$}2Y_y}Vl4<6U(JtxQ!6KkNB-6#f^BId?ZDcjxQlNruB^g1%{$(xDz6l61V zNSA-Qud|8Ra%lHUm$56#$m+g`e1}vx8d76 zPoHca+hUWL98u2QF(vtMSC)L0Hx8*I9*QLGvwUPsEWu*$6CXR9yk#22M@^L|rVQwQ zo9khMqPZv64>|Rj5-e8QfV3p*bv$M>+C!~S2|#otY!zoySB#1gU)oMavnDI7H$&rq zZVscdkTk*OO_XnJTQuT}0AYa$E3c1ZJ?z;h2;kmB0uZ%Z(h8t-Akjj@L>H zdD0TP%H(SmmGus-;<;n^Zlu&kV{?(_iCAobrs{5s1g1`es;YzR^-_)8Y$;M)Q&Z%E zCS5IT{R`{NIy(7HY!oV6Gf&p{^pXStCY3e)t#9XQJ$b;*=>*CAry{JOL&P=3+m7{g zKPU#=ZR1iOa0({Mf|b{)tw-OjEVS>y^{)h2a(CKqEZ;qsmQB-s$MIxHEG%;3H4C zVS$k2uKYS3eyU>0o?=g49Gbvzgw3)h+J}P%Cf?q>DEWjT*!sh(6jv1e%Bb@O-O=;! zu){(@&GrBji}xgg{KhoXVjc|`A`9+Y`MJftf2z@!8zjGOoy=GYjcq=Z<2!73+#De? zv>KWD+Ih;9frFHyOdZwiJQU6<4zO<(Ow$TL{Pq=+bTEhXX1E<~^XhPU(rBho=|*Nq z?PqAK8TY%Xl780wiuu&CHG#@6JwW_X&4GXT7pF=LpGJ_oAyq4wq2u2u z-RL7yx~ywAbHdiqcZ4I{!aBQxSYX7Xnd}YkMkNMubKu|B$tRq}oEvEKfyhFNQ+K|D zGbX~B_A7Vq-$OUWBGro}VaCrlmIdw<^>7Z7C{xrf9;V`&NGq~#8oJGl%uRU^tke9c z8?5Lcz!`X!&|WW64)gEyQY4-1Kb59m9wLTxY};JT>)L*c4zqjqe@9{I=Ub|#!9x<$Em%Xg^B zNaly1w2R7?m5y;0J)YPCLEB`h`hZr=o+DxR&yM$vZ%)nE-kE+0vX|zg+)z+^lPWy0 zw({9Gf2c&aX|`_3vu=9jkxgtM8>(FRK+q55)aUEw2zq=Izb?NDyj|e+)gL@x$4|ca zp9@%gTm#sU>+S2LfSYYy-q))h*eKSf1rFIaf5;U=BzW#PuY6g$-J%ubw^xHAW+Tz? zd~sw#kabkh#OdB(71{(xlVQZ+2GOR(%W!{<`Yx&HD=F>g$v}H*wUuM9D3#!V81)OA zwf|zS_qB+5J5-VyRxbxIHxzx*JD1l<`@No8cVJKk^Z7;F?jxV3HPY5wolz_;M?8}q zlGe7iShP#uE`Re!Y|AGJk)1|HY^IuZRYSm2)W%cMEM*U0w3TA6R_Q)5^7~{F93;%` z@r9Q9@c5M3k7gC>K&ZO5XVJ)veaVKxhJ)zD%Og(P`}dfFB{@W$h`{=h1;QK zl^TU%Co{6CcvNXomCydjCLf9UTAFu^&RPlkxTv!;#{>ab)Pekwh-nPvgNEK1nesTVL$IN@hvmPqIL1D1S{nV*q!C(<*Ra!L zZE+KsKL7kQvv1%j_G{`o_e3ttwC8wF3%*{}JeA_dMWuU1}^!#At!&eun-K*VRM#QkSFkj5CIe0M`_7#lrhmWdQT#xWU zS0I6y;@Hx-`LEE>g?9>COV!BNjMfVhh))#W0WWy3xLcmZHXu*=RN3Pxlt$;6swIu^DUm+BGe(zty|yFHrO$yTp1AI# zD?jvGvZIhqXK8_|XvErwOKL(1Uac}y$^n-fSaIm-gI67Ei-2J%|1dee( z-LoNGfQsqGQcy`)XhC?AUjh2t-%Cq92^>7gN_;BSlFZoAij9yDyxuM<<1{0DyxWZ1 z8zAw$#8?t1PAuz!OhJ3B^D(0hyjyOXu}XqB-X`x=!IT z`0rwq8fV`kcNz`ypVO;^QEDgT*NQuCLjkq2{&*^uP zWlvz0x@-n410qjl^g2x`;Z#PC6{K4$%?a4z*XB~{HTpA6x%2BRBBa`Q;d@d##p1!! zfV5TlWN$T_ovr@XxmQ59Om^~K9#HH1XLFwAyqYE0PbuTT27;jfueAaryPwnge zx;m*aAIZ-w&1{?CA>Gx>iv&O%&JmPsa{WXvB|0#+l6IbQVN`!mMOf&nqq6^LsO3C+ z-=5QYI^{P90~tB(mGn|^jp8j^@C!oSCP^!>t~|+mzndLH*m+jo|o=t zMrKSa0kqgNJN4|}3@Ekx(5P~9yRoFlUw(_st!-Wr_e}O?`I?e!@_}Z*jh)`{7{Dm5 z-70t#B?qyWwc6*DRqB|F@#yCfMc^TC>3#6fC$!5KVB&^kc?|TZ=ogHYnI#t+Bnfiv zrUqRy27cmSY6seKhOj5w0-rr=<|>^WRKathQM-JHib0O?s*Z8;&w3Ji-ka6;w?&+R37`rI{?OY%cI@Bi{>^LKyDaeLB6-Nw`oto0rcdP6 z++0p$Bl4mvPOeO=T#ctUjh8Mt!1*gvR>#Y2HO0@YRH={!HlQWd#|CvDI9%~h_o)cn z@pTA#=DX6K@w<~kYM!8md_MVmqyRGos?(Lx|_XPvP>?G1ZugF@pOfX}}*b*RpiDGl_{XakKaaOgX zdL_QKtn2lKIpv9_QNDmxz(-)v|5!LI+Riat9PSrqb48JJ_pxP*KKmG*)>OMYjwiFa zH4MiXN!+h~1UtVbdT#tlL#l`Wnosp?$85$xV{b4>WR=lcM0e1-VfTzw+w zvwGSEJpuRnfgJ(Y#Nw3>-Uu<~_T?JrP@P2f>s*hZxwa*8EfUrk#yHRDCrkRpY`d4~ zoc-hz4(HJU_MbdnUmMw@AijDUaTq5s5>Ag(m6&@OA$V8OiOKbvr*C_dSc8 zGxvYQ_~_gtQM$2+UEMV%f*QX<^?hRUvp}od?$t*L@nPsLI+C0$|6RQ^R|u$n$2-0b(X+*uRINRI6jQN=eQ9OizFRWjuNv%OmZMf3*w4;Brr(?! zEP?Xo^6_!*#sky2IsKcEk7T@|{Ux{hY%oENw!}!swFlHPZEl3cXXOX!IcfqgqH`)F z>ozzIv9?-+*U18ou}iW{Ey;D;lF8u|r{`r7VlJ_x%(I;bf1usLhI}_RaWL~3rX1^# z-g9b2)|`^JVchu3qnoKEx-8Uzm!dmVGx|l4y@iY~D!_sRG}j2eM_Zk>+r!sa7vgp9 zWXsV_HYL`oY)BYbXTi7}wdc=*7w>LIyG_~Ss1#?4IN>FuvXS!3m>J-7$ip>Sw!k=RUih;!O{?Y17;ovHWAh z)B`7lbyg|MU|f8G{FmH!?YwN%yze!l(R80+$14r!HqWfAc&eNozSb0aX>_m)O|JKv zcPc&@4Da&K7W3*~gDpttNXjwyunVLOy-jZE1y(|L)gOnl37J0fe8@s0r$B%m)N&HK z>J&~B$%K7W>I!0@@oX}`8&=p!t+q@Hchn^;Lfu_>6xN(vLJZUD8HBSk@pY?C_++_Y<>RbDRqXEskydV(hwi!S)Ng2{fk^oLqZu`jo|-JTGA;-!2iaqmit{#~#5l?e?~?mn zee_(4P6CQY-*sDRV4b$*)y&Q4p|JKi%r!D*A$Ks|up8AqK2l~`$)g_D);xuuv#EB* zmmFEEOdf^t(o;eM1^H6s&+*EXA!)ZDki5F2m9OSDh#qJ%yWXkwY{|B)ZXw1)DKP%g z((#}m+4|$V2$aif$m2+l;f>Ynafi>_kTK&^%K&S&(o;TOHteVz3w@*SI~fc)6lxPp zm7GRn`ddm==t$4K=B=Y!)IT3?sAsFKJ%w05p^EKLBlH5}`It;LRp2q~+UJrI2!~sB zK65mTuvXKdx?$*{O_B%;eS-z*_84AqL@9L8k^LUSQv zA?EWa7uch>x+8sQ=4+nR@w?#a=@eK%u9<5z!z^iem|~m27F3OlTftIj4p#ffTu_ue zLp{w3locQ_-#2M=9*3h~?4AL7ilGd^{-K_pY=->WC9(FDVpNF0^AgFJ;?RTPCmv3?2w0`J4wl*OPb8^33?5w~F zq-oK0Cmvn-;Kx&w#JaV))z8*f;++IkQiewr+bIIGPKz0N&oGxdVmkr-fEeGflMvz7 z2`(jZqD>#{40e9o%V)UColv$z7VcbAMdt+N%siWAW{$*Vaw#$@)y@IpXLHp*31h{6 z>rA}fZtc!W`NQCRESw&#PMj6*(%^BBZF^jg5Fp{>pA$3+<1Rr}Yc2dL_%_ zaIna->ij8RnJ;LS>VxppGbMiARTSL)2d19rbf2GPX+tTmVrxs^8a`iNP43KPWQNsy zco5?ZTzts3GxB|+)Av#B$T7ri>LM56fV6K^I(cwOX(>u;5dyp{Z!$hTZh6;V zxWzZSf&RUcVNP=8W@+~klSyk*wn-V%YAfa-byI8onwQt0x(#jH0{oKW<@?`hx}c8{ z6b)ZfD2dbETPb#6;C{xX0s5e6k9kt#Z zuOP>S79;4BqXY66xiRmCGE7ArW``-0z#95CmPMAWdn;h+$MVs7D>MNAWMVXO_T+tn z%F+9UvG|=l>PJ+>cHNF8g)5@T!asl=pAZua2$NB6o~$o~3n2zsf3lNJWYIy)9M_c) zRL&w^SH?X^PUvvP68HFrAK$b`L&*rm4ZRdLjRlY{p ztZ@l^jE?*@uo8yFblOQ;EBS5tg438~OZE6$wYcHP>sp`RF)V%dtxQ#KV;;^HG@8z` zI)iS)W11h^-WxAv7UZ3j^d9F-SbnUnPL#Md(3Anp2pyXV;yO|8eq5RBouKXuXoQ|c}71wkk%eaXYe%I8g!s6TPhzFiT}YeLbTur?e-5q z1H5jo0%nyD$cPhT`+u<|;jlHr4c7HrKTw8`v0~z6T`8hbi9LV#KlCQ=XErEbv5vKY z#|j)MT<|u69k_ zB?bZatgHWdqYxWPdmg8#g#Q1u++PMn(&O&Ck!~(j|L04Q$cHdMQdR$uLi|5F@qV`P z2JGhi`2nr!f6}tpMM1h2o5KsL|7AE5*r6L5*vtQe?ZE&2TmRo1|9`-C;I9$-|7iUG z0o#H9zXtwawgdl~-TOuMzZv*IE4)b}{RNL>*~N8su)Z1u^5cfB+d$gNa{-X3wd_^1 z6o{S~{!YV|Nn()v>v?z_dmnS#J^QT98ujWFQJBG_c-M-b^#bcV{0dIOfI z{2-p;bLk1@o|Y0$J)-vup4o-s4iG8E*- zRT)O|S3il$G{LDkn>I(^X^6Zh?@bF_?~G$b>WK`^ae+ zEDhf;ZNqXrgdXPcgAB54+a%XU@bRh`HJ#sG<{*oWKEoQMTMei2+?84;|t1|=MNue`iQ6cNGww`ESe1RC9 z*l)bHvQVa+y#@@zJn1j&+s9|x5!E2fJvI;0hy;`z)9fd-o#z`6bo8|d+YS*XGBuLt9EZ&LE(*py zdDh^f3TWCDX(z7tik9R-zw{c)fqRYKN) zJSTtTgDpZDS_l38DNBBA}`ph{^!Ql8X zw?7koah~Fseq;!BmWeH(BX$86eU?Bp7VW;W_@lZ55=fQiaU{eu_S*`qALv2V?@1H} z>T0+39wSb<4B9mAdfD7}h+{*AJO*wba!9UnpgZCG#4Pj?^*#*T8Ui~YbH>(-(xRgO zT^KDjX$faBv{2LP_Z#mLGzlJ+N6Hm{G|Y?gj$){^48u8EX>x(&=Pp#>KD%ZAVD-2E z|L|-h(az$#usjbZxE5@{Jp!Z9n=}S_f2AW*bRE|}0tpL>zc@_D29U=)P;E(r`QL0y z)*cI1Z6QrXkQbfXF1T{E7dvX&pG&v1Q(dBQ&f17-J!l@vzG~d)SO7VtOPP@ZPw)AP z@s`!V)ML%gJgiAPPL+7ksMa8mb4Y#9h1Cb&hX#_e3`wdtG{8kZ-lCk`iow?RY#z9~ zDV;Vee57QGM#U1rAyjg`mSl~E&RC<@f>=|@l;6<2dZ}$QlYmzEvs?;lygh2O@=f9~ z#AU{5FbdChhyldvV3f#M$SmMCn&<$st;YOVjGoWcGF$0*9Cql@qALi_4QhZ?7`ov! z8;4DT+>M$@_K^=QuVY$XFO_Ngi{8_6Pm=U}(ONUY7^j8r$QpL;3Bs|$0)D(KR<9tF zwlES-3vl`rs6BHMjGuyVs1`A{QO)|GXXt1!;NT@DCqrM_;`V@og|Z1YbmFH zG?^lhd*(r?9iD;xYnQp>@O(s?se?P1C2FbN89Ui)vyr? z%wS@IOigEKPiu?`kYW;gxE*#oC~*A~0zWyuyFG8x+hVpQ)%pBQ?4DwaKe_sbTK1H=)R#+73aX>u_SZq(D15GtBvKvEc_m=h|%^qfB^3EC4 z+!VeGY-h*W8eeP5htA3(%@ePCIHIt`cWNYHM68bmu0UpsIEP_|q1K6h_k>QPSQ-Rb zH*;`Ujft{)P*zHf&G^R$Q9)h;d!p+xNHRsT?XtfHbUT^MzK?~&@UdaNazJ2_yPNg9S@Fv=M+3*Y0*0hnpT<~2%$>M3*^Co_vT}q-;=p>kgW$;vf}Nr6O=hSg~K~x z)K{a(e)sI}3~hnO^|Le9O$|>ncl!dWKLAj-o!J>ILt-K*j#|m`C3&9{;nP!sfzLvG z)}3Eq>}p!JVk{BiArbaDaCGtZ<7<#hR_6ppq?hNneOX)FP=!=Bkj_sB^c9^iJOroqLa{37zb!z?F`h=-CXqS`P-(1;3hL0DA@cQ_9_V!Sf zp5R{G6LAkHu`n{5p6Ow|(tXnW;ufcy1wIW`_@xbf*b4@@5;-P!&_O_lTC}1f&);C+ z_dP`M+cK&eQE(O~F^-F?0doC5d2_7`0UQQn&hxJ8@sB%E4d`7+EB!oDeDo2`>}?2_ zWMzrDnfDMVPy6IbYUQ&_R}GU9i0(zph7dFJY}=Sn_4dcCB|WB4) z?nI0@Bf;m=aFzkrT8IKT5BJl6;zMrfyFJ>j%4vop*GVjv0#nu5JAj$llz-RSQ z)LT)7pjuLaE*|9a8bW}=;##v=uN07Lhc4*u8Wcq4#r2WuGgEzD>!N1==9%@k4-pTM zl(nF)PtNQxGb4D+l+{Y(tT3Eq!hIso^S({$x8z*`%RJp;;wd)ibXFr96pwfoJF^}c zPG*KG;e6$qTz2*}*}~Ab8re4cqvE@}g2Ky1)TXMgACPzEA5jLAbQ~*;cawp`WOfH3 zzcxN=lBr-B1}^g#Fz4w^A@M~L#)2&IjWd=(*_Rwy6n?^8Es%$P;6&6G|B>fERUWez zC&2`IY+>vAFkWnz6#NPUZa_prwdR@ky!i1qm z`eS4kK~}Piy+;Ia;sG%Dl!B+qAcpEmih}-0k#~WY7w>%DE04dk**UX2YT0t$(#E_n;g6C%}Lg4LAGUMXu183olxhUa7V zjcpYxo^4Lq@lY-B`6x5MJ2_n374Y-j+E$B}&kDfkuw{G^ zlM|shr0?O^NeT`VfEt}n^qwuM4rZ&!yhI;sgL{%9UQb(;;Ll5tc{+T4ufH|oWisD| zjfEfL*3tq?N#^>R;B|sYl_aJm2eqGGW1^Utp-;`P=Sr@Jy^4bqhV5f}i zYE`F{iM%S1XtvORbv(!8)(h3rMn+@7_#(%w`2j^=X0putW0=0V;2Q{?&a9Q>8%+CO zg%52aKk-uIsT=9iI*LCSjnMgZQkcE>?q;56(|~r@f_ikv3~eRk#S~hIKrD-vMo5!1 zT>~bmvkg%1D^Z&%PV_0Bk|YR0Xk9MYnKpG0~ky(-^ol#@G_Jo?G}A{-F0!I9;A~i&3`c7jMGFAjl=CNlAH_ zEG0cP;27oA?z1UM7}sfQj2&etx3{i_n0uW-!VDeO8v}{lu($oKe|V`nZ@;uVxl0XH zwtn!%UPEymZgr@Q;5hu|n^S$TiG4-TpbWurV*Hxb<}kH1SMw%K3P&MLSY)3T!<57Z zXjA{OyeRVkogTv0m4G*!Pv}izh5q5ex7&(5?;+X1OEHp8g2!V{dasAV0*2g;h3z(e z1a-RKc1^&dwq@YfL6(MI#e&-=bO+8ycFb+-?Q+P51T*7K%uH^%W+Id6IsJKlTrES` z#r};P#%w9gWPwS?*u@el@PP}%J}z4<0BH?d+j*?Kc@qE2naZfmJ(Ip8;}bgV)yU{s z`?yL<*UWeW1RoT2nX@}~**C}Mg!g&UJCQqSe_?Z*S+oO_SyV8IfqN5Z;AQVT6Qcj1 z{$wU7beTU4&j~VwX40pGw*rqkCeBw*X72nMsnE2%@30!9)@eeZ?P@aSY7C?+dv^SU zy{{{*n?%U29|)XdbzLFb{s$%?*=Mvxok9_eE8GF*K+tP+dppeoL+a{Bw(8&%rUh~=AR zeD3Z@GCVTA9Z*Wfl+NC5ibIBur+(0dMh8iyUi~fh;%bs1=WFL!&Q>R$>4u@<69Z4# zP=e!@v5cg0Hu3zoSiP@IH5(eELsT$j)>yn67`5^>%@lD2>2(f(kPCLwNs)XSzSfoi zyOi0Hpj<}}NN+*rGxnM%jD8fcyy%hb+y>fq}-*1bgd@AjdVXg(_}%Y@B(+mhCU;h=GlAl3*J10?iyKqL{)7wveCf$yBk8k>H zW@>{sr6@@1w#h$$>&Sfj69BqAFd_|T10?QP52UNFO~YL5Pph)NNq$V zu4GNaNf?wM)!()>s?CRAHt!*G^;Z-^%TzE~7W*xVbePp6&eduoI zB5UyJAOT?FSX$q@hXqpNX!#A1RX@NbdyhAQvg>@wJ(n)*KN4;79q)d2dcYwVBr?|= z_xpFK1_dv9AA08`%;YECsS@E|N0CL}=2yLW*o-9UPJii5A9J9nzq;}zG-o@L4HYgz zD=4{#x%9pYrOR*SVL$K6|NfUN7JT~LXZz*@|-wprzEQxynnX;5t>7Q=}0c3s@dCpN^8Z(^Fsg1x$Y+|osxNg3+@k(;I&GRD}4CZH}D4t8HuN$mFhNm@vC63 z(xLoq$-jp}$hY|!N9Kx-+IIUEjayiD zU}S~ihXZHQG35YwT#7vfdMWR*;2&lw$CX8UnpI^|Nwu24K+zGKGEJ_E<_pGujijC) z`H-pj%_?jjJ*m%4y^bHj@h_<#xsUUjfY3)JnLle&MAYrsjaV*`#xk z^utr>{7;_z^YCA%*mpWh^`*Naxv=_To+WZ@v)-z@n)<^|B7G3U#RfIvhNhnP`dAxK zpBWT@r;!J$FOEUH`6&nt%f#Du62PtR)kpW?`_ouU=ROGLqJgJ>&-i=z=pGLTRJ9$) z@8#KJRr{cH`zdgHoYvNg&O{7?T8%a!o%v5d+gA{ixx>82iy&4tORXS;M7+OEapN>v zuyDi<3a8!F9frjWr+OKFX{9sLD4c=#JWfusT~F!Bh;$H0Yvt7fMTMr(is#!UO{CSJ zs{5Xq%M9w-CC+mJPCnceirM4_jd4naHUup zOk3|tn|1n6IO%NtyqGacs4%x~JW%L^&y;xOS@O)FReAoY$vf*EZ2Hr9eFd9dFyU@A zhPq?i-OiAi=iXiancl)B)^J~mRpU{y`m}P4U0+z;@0;6!h?qtsVkuxvZG=v{DUksx z*H>Zt=1sRp=DUlSOp&K>QZp6ZEFxICzVYIl%3S#S^Omwf_}60MVa{CN519&Xwn1jjsMDu~$b zgJmd9PTswaunLr%vjmQEPG~6!3TU2cep|AR0BIV6Yo{ww^5SGYuNS}tQ5#%)cY7@% zXvmkj00Pu6_9bV6p_dV-p!|^E{MW3Vv?Zt=4l~8#J7@-KUk~yf%UHt0IT<_KuJ@aC z@pYU=?vdO%cvCiUN3tzYk&Un=)>vViZ|~SLDrp)rRviw|x7x6Z6*R zFV7F-bqDeUo$oYs z^-f@djo*KDHe-#RGnTLfYRv_%aMo6!MUoNb_f@UWRI*QZS~!j#HF5k=&pfb~i1DIi zM1ORo4vq=G>4~8}lF}0eDqF-nW%=Tw#9YsYFFIr190-Do4S`M$cLs=GhvU2hi!A;C6%7LRCSdIPwt!al=D-7VT9#WV=G z?XBx5#h49!8W&Mh|AJ-jGYxIAbfp5OI8s-DnZ{w#HNT>b`+63q@$v}O9sfp-i4P$M zG20+-tF)I@2V8C!6qUrz?*@Q^1AA1Sg+aj!r;CGrrdc|~!JPck=y?;Pm*5Sv(8O61 zQ)8r;uGf(z4nAzC4NJv&pL6U{qRFM{ES(JN+@$`+w?MRCx)(Ah<|?{Q#S1Qbnaph- zxIs(e#5rB-cd?{HIqt=7jR(>s=h`IX2trKYgKsIyM_Dk?(i{9DdDHCc`HR_Dx zTZEDX0i&tQ9;tEs!gg%XC4T|2_7AY`;9a82kL{3t?|v@WI>#H};vIytS^@rF9aevmKzIAz$93NYt74y z*lN-5=a_)*4GL_#L}W@eeY*gm)?imi6k!QsIE{)1>t1Ng94J^1I$;)!@vqUhC%J+F zpcspp4-9z2L@{5%N&NW{=uP*^+vM`W*`n34K0n_EOMu%0Jrm_+YT2@Rk4bErk@UOB zfwY`@`JRK5rv#ZKhv}GktTEwqkx4q*44tqmu-Jev^POxD^z+Q17s*vXcm6aZPU^a% z7nTVLI;8tI=U_+$#S1`I?fP*yihQED;C2YN>FBM>_Qr!ekk~W;F z0=#XhNErf~uG+7~_=taLA2_Hx7i6}j7V1Dw8|7!0av-;dyt-nZ2mNb&G&>DfVoND% zk2iW6&&E*odtK$wa(C>cwFSbhblg5Squb3&ikF0mc&b}?oNhY*Q1M)F-yQ9ek!S9J zIJ!bPfhli3T^7%qWWDwh0ow;kC(CX7;v_c{7@YC9$P;z+J9TRvElyOeKHEB^|5q*N z84Xw4?(sw!L`^~Tk`P3h(MyD2l<0{BQAT)(?vaQV>>&Dx7HvdC7eSP0L$pz%6B8tY z7$po*XJoW-wsTJ2=XsxX)_Omk{c+Y_*V^}9_uSXMuHXOq?~-yR-@toE|)R{GDU%`5UiLxKx>PXaN=F7H;LW6i5N2szJ(%GP{Jjpk4@6PSjf@Su-4J&;+bi>km+> zIfEag%qKQBK42eju-zvL!=$pt;tTqj*og^`0w9=(pE$7vYWt32@^$Y1*7dQMkDeCa z{Q0g`1W4Ls+M+dwL;}sWqh>-5Jbslnop8LZS6zDMy@hiDUqTYFX>f`^ls|Q?Q~#2T z_ia9DjJJ73ayan=4g6p63=;IYCtQbil};yBJ~dSzxa~LFD-u@&9y-*%K?d)xXOtQt z!~V#I-HVvCoZFO(irIT|5h2G-bcBsB;~9a zzl@Zkp5pdUd`Hk#BQBvKKih40Xz)*`MJ-{KcwnH92%2*o2(h9OLs{6A@a@$Mc{i84OOSCnT(w3cTGPgHEX9BlUX6+doWHE)3 z_JGth5>$$`6DlweE_W8Ur_>A|@7PpkP*BE1aEz@yXCU6ZT7F@kKQ*#)CMiXxd7U|h zc}#8G!U68YM?9Z#AM3bj^=CV^gIPwkP@09{%^6drr&UqbVX=LEmJhO7$8DedTt*O? zW3CPGmWn6xa3jsP^?N^G?laR!G^zPm84kqXPu3IGOL^QkUE-b|8D6~dwuhP}*_w^L zXqG3_qsIXxxUr6Tl9V-B8w?=Yyf&AtFG=^E&ZD|lxZS5Yzv&9O^u4H5x=ko}5=cn5 zIf`9z3EHRD7cc7P4f0zA=hOA_Wql5Yy1-p>e(jqE#^2?XfvT(acjx^r?|RxeKz&U* z=6Ct$!y0NNgSTZtFR!jCw|(Pt6?HIHbY<8-#02&>PVweA<6){U&1f?emk;>A@WG)} znC&`8;-WlNfAv-pB3s{TYYhtn4)JOa83rGJZI!8IcSbi}iJfpF{#4lV^4NdfV(3XZ zQ+Nc+X9M!%RqZX@(gAAB<6>1dUzzi%x`bRr2)R(w@v1dm&~FE#CvM=ge_}jiwS)#o zz~C#SlL@1W23+jLRt`;^^0ZRQR2hH4tdqwj)JrYKdGN5SxZ1PB4W{AM>KmUaU9Jvr zC=b1-iHxVFdS)i)Ayved?Lt7l0t-KpP@TRtycD#<1n^8tmzK=}h3f-Dv@4bcRnWOIE!K`^!3RwIY^tE%!?ypA5|=8>A5q5h>-7HlvqO1z z|GRT`QvbhxOY4(qTHNM+g-4Z_m&fhU6elovKaxv1$OSzsx)&@{or%R=N}~Z|!^sF#$g&9GzxJNSkEUCfq!g{K_ z(lC!zaJ$KX{P|cke%8jgruu9qf-o!9Lc)F2x;yv?T-0;Zq{xoR+~n_6^Qb!Zl?OM) zX)XAzLCPONR+_7bTUVid1+fVfQFHevjOysHUNEQLCfPKiB?DV@H@o8MPm9Xb0qaVK z&UXXDGSiTO#1e+uWc6aA{b|Uay$GXQdXsCG2y?b~h8-K<+9#igO3~qfm{KjJ3Npsu zZuD;IBoEF2qJ@b5+xE?ELM8f0p&VJo(;HY{bWLO`ee++MpUbar6=zCq*Sn8 zSk`R*lSNFnKSwwYFhABT4I!{iM1tjvfwIGyzf*am90kPXzeH0fAzd3jn8c z$3hcXa!aG7E$59@e67Ws$!j`WBgJnAUpEas58I%UElu1IW+Bgh@Jv7-Hf0~AXN((I z%ip|9Qs^j*9x^r=ndcTBS8KJcRf2(3DZPvn89>s!2%#gk`0YXfk>=B)^Gi zhxyWSr%vR^I|ZvB@M~ticI0W-n4a>7kBf%gOD5ntAE-qY@w!U66gv~GWVzD&m6ae? zg+m?^^MIj(a?X0!tjP#iP5#!r2Dg!~{X{qq+rz{=sw@hLkxf znisz%(aB9Mz5*KN2tD>B>wbefq&a}JTrL%RkTnz}h?MT8<;mFEGo;}Kn93!qJR7O@ zYnUM=ymq}pg-U4|($ZhjEqJo;w-B4J%(s6gl52F4_X=A&kMK1c9jd+~tYKe3NF*f&M-}0M&lMu;RtRj!H(Hf)<+<&Ej0h>r{%zfm;`BU0E_!Qpf2^?N>`L4MB52iBOVDSKgUt zRNPm&JZG4@vAsg=wrUStP0v5gRmGR&<3dd?I+}lI@4RGi4$-k-Cx-53Nw&YJhu~j0 z=P5PTns;VCTT=1bwO0GO5#Y0(2s0~-8GmB;qsl}$-ToAJ^y6AuHqPbN{26sct$St? zBagI+p+Sk87!x#SeEM|h>@{Ab;+~!4$Q1p|iH=5OtZ7O7l=7Ikd}T=ret1V{1pFqi zC!w$GwAcrQR_gvo;o=N3n z)92p5%FLd__$=X0GEiSV_y_}aNvQl?W|`n5e}(YYt(zNUv_ zE5&?4Hd?^mX1SDWQ9h5FU}vg-P%pNuK`QNe|8s zv3KY{oNj8!ry4T%A^{#c`&|F9eiv)=Zel$1@FNh7Q8mAgSQVF?(Qaw`I3eS4sxybj zw(P()ANELYEJd;-ekB@VOdvR#-R@^$0m%sPsV1B)MeOm`0)eG zv@2bru|7PbO8i&^2|JdP>1W@OzRC^f2%^Ilq|2pPOc)sEI}O;xE1O*3n{4k!t#s(r zO8}g%-B5M}lh82eW%RxzN98bF*=1U_Z0(HXi#xmeY89H6LC(<$^BHD}b(#AbpiIi$ zC2)HoU$dKv9bS*G7sPonrQjw8N)5%i%0#jfSPMI8!j^$P>Bwen)XQB^SZJYhT#~@U zFM(%BL#7fabzjX~{EhPWJGkiZHkQkE>H0z%sKe1I#52<#PFMrB!BoQsW5e7m&U7wd0N{E^%PNN^8uqDqm zor>(Wh&#Grr z$z331RbNa%5*4m*WIL%JB?u{N3l=}B_5uWRQM+<1R4?Q-)|W( zFNLR%nHm25`QP6Hr1~=1&d~pH*tZc5XIved1VJAT=Vjn$;(x!%WMBMOdH?+W|Lchm zD>?vTH;Q`pv2sjsZ0LNf*X6;yAASDfv^J7@_+o_I7BDoFU%Rn_eQoX_)`1 z+4FMNbTKDZZWl|(Nr8QcV=b<1;4{>#1todjjn?~9>AF8(5E^GqzWqjG9)#qsS^kvg zf0#fb$xUZ5#&UQGbBY&h$>(501&Rh1U_pY9#6#^w1p#_+t) zN8ed!MOayvNW9e}0`Bdn+W?@h2hXy;FfXNu<71c(|IIqa)c`}A@;Xx=S#14E5K(K# zB7Q&Vh?Es@cfjsCZ(YZ^G~Mcb*I3bmSFGIb^KhYdb<#^Tjl?}Fg3+)N_-@GixE+8E zFmOKfdV088Oa~q{aKs3x=aY;gcld6%NyQt&%uf3#U5R|I4aUdtJhr8hV%Ld63T{n$#dhAZ}qaFZR~a09&h*ZCJL1~ITo7mtQ&Xo z@b7ib>Q4J<)WVj?uPD>Km@qzmIMwyOm~4u(X%cC@n9>qn%)v13IKJ7!R8=RAMb@cG z8XK)7Pt<1Wn)Y-{%hB4-;bn3S)Y|uTfND%>7n_fKvLy~> z|Hxi>o9@ig+@ZGQQ2@5?uTO&Mm=Wdv1|>m7H8?xed?mCV2xlf<7{nq<Z3;$#e z^Gtj+gU%bD9`D2pO_FI9x#Nd;uNE-I3|i&RsLv2>7+H(u>{BKWby!iHvCEoK@pa$a zYsz{aHkcw$Ed68&bs|%v)Tl9OI%@T_O$#~6_!CU2Gw_87yT&YyD@=!Jr{)4HAkJm#@tccnG<52~8xx$zD-L@qN1 z7o;^*U21yX@iYk5OEP)UA@SV?@|h+Hb(k|9FLrQ>)NsC4iX?A>V=I(JVsG=CZJEU8 zdaOLMfrdcq!{t0N5H7J&nd&k}eQoJpf7rNJM*My0XSfB9KT;IWsl+9CMV?0{dAT<` ztOVli(%5U7gk|q+>NCX(>viqBF*El~)4g4K)Ztl~OIeg~X#1VFGlM+JRn&A?;|$6g zHv6f1!QM;?Ss!7(5e>(`cpIWmzf(Vy0i!SmcM@DDD^yQVOL8qnWRoMiw|m)Aw3!5+A9hD zlI}8Rt`cKac6Ly=oHo!j+E>?*Yc6NOebnq&CYQrC%yS0K4)fJQL_Cyx7@qR`*G1=P z9Zq&G>+H9HE7usq0L!KS-; z@;$OrnF(VApG+IHlr zePcJXr?cEkaGj;j$}Y7-%_jn}~(e~$Hs6_7LeK;TYvC;udl8rwk#i2^r=-Y*oxt?Hu&Ae^^G~8Cva6l=(E7%xB2NljSTO?Gth6&zB7ei`#Ec3<8Z~VTj2G- zwW*m@-HmG+n=d^ z=rVt-y(dP7-@0IkznbQG$Z1Q?eYN1&gsG-I8nO-%>BiH}30~2#y}j7mvfcXbC}&HY zf4CFR)juUaq@l_ac@3yfq4DR56M0XB5QchgCYyeTC*`#PtfymXaCJIetjo4HrWwe-4Q~c=m~aya}8a6A6DYgba+!3U2PJ~ z#bfxu`R_zMpV>A_63G*z5X~TV#ig%y?Xsp7))B|-CQ z=0%$RZ4r997o}ofoJK`Chgu~VLh5SH`{M$Lr8rbB;2PHv-u^F|7~)il}x8hkxxcyy6Rb50piPG(mvio7?F>g^iZR1I0U7 zI*f$dh_yMBzc>cJqfV>YN92ly6PzRo07({FMeF4;l(ICfk4L~0sf4*u9o<1&3Sp6K zTq!LpX{p7UnKsPvG}CT8*S6$)oXkIwX!5)ks+tyIP_~LuZt2wv9=Z1gKUstdTV9%X zJ31y;uLtLaR(_SEd(5C(1ea>rCiD663Q;vgE-vv7_bq`8Mnm*I*`ajo+lpR7;H(!h zhe5ud#So_arqD-lp>GE=cbrpbaDgW;S0pc3&jjs=#az>mSoIr`Im7+OB3uQpfhyz52%S|ozZ*T$+}w9?dTl6>LVRQZ-39udhsGb4zd zthp>LuFKwPEj*+-=Jv)ind#dkai3Eq)v&L_OZ(49ube81Wk_up9di$X?Uxn}hWlfE ztp&}Q4zJm%v5YIMo$x6vSIr2{@t7kjV?VAia1+1IeVnHz)MJ$`JE=c*-JWwLnJxX5 zUac?pF1IXCg(VUzK2t)XOFbx;!m+(txiYzqAS zM&D|z+m1ZhZ{w^W2prQGz!HiWB;^0)o5^J1`<)VV86zv%98A@rBtd%P$GGV1k2||D z%rD|cfRXe19vRdKw7c%H95+aWGu4D9bz~5IC+F7pwei*0G}1}Ey*pi8G&|8Why|#K z*7!kpVpPACa8+bq=_GP^M?!s+BJcT*S*RRljvsT7%|)dNOF>xD4sd!1Ehb$Ld_`Vu z_Ln-be{=HXC4gxBOE_69QhtP|p+|;vW$pE=$CLH1Oc9J2U3aJqAA-X~b7E>L&IVIg z6%InXcx=+$zD5lTk|QF#J>^+J;RW^^dKnKo3K$1K%;$Sx-`twC<` z4=|Y{`0tN}yT7X(;eT8LjDHz~fs&33pGk(V&#B5Gd6N-(LiykAOBUx;QPuHIq z;NG}xx@0QTtAl+gYOhpRme<`>W{rAv!a|f5=Jv8@Ps~p2V;8WhcQ#?k8^}s0&m|s< zjDw}2ns2|hWiml6$Ix2NbE+dXOS?MZ2KVsOJU^8!vBXOB+Ehbh>zp{(al=Xc_IxKc z44eO#$UFWJH`4>(}aUx6_CmV1Eq0Kh2B{tt7-|(t8RRo+W556J@&&H3Sg-Rlu zMi6PaFev`ko>Ky37)9L@py(DH{p7@W@x~5f6JT5Los_9*8U?pa`=>ndN>()}*j#6g z6Wy=Wml0jb$G#Z^;x_XYA)@LIizZ32U(Og=eodLyybUw&s^>rD%eht9=1L&9T&t9L zAV6-ukXG!Vy+0s!-F}lmtl_o!L8E_w%up(X=@qG(zm2<42wsYm`Oy{?&c{ge3G^T@@$I8XP4XuQZ(Z6FHBFRtj*4o)n zz+4O750zZkDlUYv;A0GC9tL#`xJiGQC_IcG<^;OmGKbgqied9)+wqY}sH!wrtzxs% zVIZpit20cO+_#YAb~;Gi-Y)2ud|_gH`%bq{iWfTtLeg^Czu;vdMOMc7q{^Fv@v-P! zn?}!aO>=wV%F=%}N~zQA14U9vCE2aY(hH>K0YA?3Z6Pl3U$@ZBi)dPY)Mq-^nvB;CgJ$yysGcgnv#g)L8v z0vfBzFnkjSR(~vM!tDt!`FgGSqLq1vFBu`#DiHo{S!DI*R{mC~kX06^=7SCBGWLKXI&Mj`(0dAp~Wi zfoI=^<|sIetcLOQUA+AjQYsU2NdhNf@Ij|XSMR2g6z;gP;YXJNX1S`-yClJhdi$WT zI+IEMc)p@0-KJu1mZu@@_OwJ;P19884>1B+zXX~=kVECVBEs)&`j*gnnj`!tR3Vv68V+#1HnnOL_EYG*vW2_1W`YDO z5OSga(nx|}hn>;r%kB&GJ(3#wUaNOmj2|um+x1pv^THw&t??a3rxbQ^ z;tC0=5bGT`SGk913I}Brg#08oJ;OlVha~Y(I2F!F`_N>IO?9j|>Xn4ed;@(e1DUnd z3X@66vjP|ELOG)v8KbN7*Q?*NE91)SL{FNnCWLfeor{XxG%4fluyL7|b^H4W$?WWs zU0P1@A{rI}G`%&#`6J3nc!(de0kH(|} zKKQM|JYz(G;^H-}$}QfOQl5Y0ryAF9RSE`DYdZY>12+|nXn7K+5;&Yz3+c-#L%)9) zC}{@*7FUZyEP0JO`0oj%yVR*wvYmbkz;iB^39v9sSw7lX%EH2Df9KwY^;2f*+sz(S zC7!Q7$#@fsE-ksS2bYrCR!PoBM)g}9De@P0>lWwaQ72vQp;>x0Vgz3mD|CfyN9Gpe zN{~HLS1pf+wjt)OvnDG(4I5QFM?OYO;L#u}!0INDUrVd;E>X0oVP)0tR+4j3v2=<* z33XM{_fl9kOChuudqRvWmfqOJGMSEtf}VnREEEMvbtk*%){K+Wj)VTyVZp_=f6>Z0 zUJ!YdCKxTXIFXs<-tPz+a`fUmDo^O_b-Ivxq&mC|dV2A>v6Cl9ktUqITS(VWS-Tw_ z5N4kvogx-xN1-;68RVR-RfM0Od`1WCImhnY9#z}Ka!&DrEVIoSdn?}isTW|Pd8C{O zuH#WGx+}Z7!KrAZm}P}b%mUCPlcM5ZYe3*C5ORW4i;1({)g(o`8rng6LW@WGCuoE} z&=96p!#Q+7>q1xcqjShB9dh4=s7^rJNGi(VeX~;>s)%l@wpV1!#~2mcA$e8xr7-5M zbsxeRe#_m#2248^Yl?w2S^bv-t z-Xcxg*a1$Vo$7L&?fuWMRz%TpoBi!*&=IHyCEg@_Uqr|M)D!MXE(c520z$3$D^QrJ z-Bbu~j>PYA7!G$_r3wKLZF#Rp0_VuP&ovcU>UaIPzV`3B#4U5ZQy)va?DgzUAB?T~VPNIqdhgK=hJj+*+f0gC;EN zJ$4+v;~H=-vr!lb2hepClyGgl(O%2kB=&_^pW+sW zCWcA9BxB$*RZ=iZ4F`-~R5K6kvMDP+7`Qm0N)F4e<*tUIP&4P-pN#i$^G3;bZpSWf2kTn26?cGN(Im=u=jg6PwF#Yae0}Xe5e7!=L{W>zf-x5ND;)7L z;nm4>nvKwOjbq?Aa&M*pA%g?)>igbovXjwfcsKkqi;2~UqlKi_9mFjMA*mWaODWzc z!6Lqnur;+^b=>L`K7|}w?65da+zAG2f_Br#qa<_x_Harsf82WWrf692=m42e+~IIc zh%YK4CW0@A_S547^X%p2<2@rk#HY*PJdICe)~wegmmE;6_^LL_f%gV$@#jPiVsipo z$*MhTsH)riY_;;WU>hAtG)16?`E30uy?sxG#LEZ6;PUI7Bi6N6^l&}=bkAG9t%%Fm z@{o4RAL==Jd0$ZZTweC=kYk#EMdtZZKzYrE=^TWp3%|)pKof^%tD0JEb5X{c+>=x+ z3pHl?LJ|ciP~k6wq@Fl>G4y9;dNm6mR#xz%H(r0%P~Cu_dA%k#xm8!elwJ4V7_l!1 zW?U!#lo5OueiWwanfY@RDSc)fW|C&-hv7&_=pQc@UOE1j=}IxlAT5}$EME76E7Oea z1wBb-4fzW5Ui4v zjaL6$#VNqTV$p;$V8FyuD&oKpNhFK)lQ!xb{0XL1X%%>TWn{VLV2#*nzZ`tDQziO} z%fJnGtn+BCUSet&4WsI^g5waX(lK+5{2{nDmzEawE8bggOaw8ckkm`kv237gEHq1$ zF&26P+yknNsZTuh@$1T15JFnHe7c+?u^(FnPSuK=@$%+sm}h#-;|YRr->rFcU7a8! zoCV$bCNlZgtae@t?Fea&OO`_v3%~J=W5xfaZE6m5*rSm(P> zHjZuWDVUB6apbs@@=#@2e4&a^+k1lnk13f6SawT4teX*}BkQ?KKH#}ey_RF%T}Mzp zwWaEi6rBG84Q-qAxfNU0<7FU*SbrbFHb*%_OUk zS4{Q)MFYrSf6&9hE)VO4gjuJr@DXnt(a^7p3oeDD*R_=TKS zWE=mVni%-MYU559wVuh-2T(mgz@MuiPB0k(Vo$f*kJoD`SCeXb0b6KdIPDRPJvb|~ zHkv^Kb8IQ&cAZFDpj?608-I2BsP%3L6hLsSbMOD?Yy?qPZk}HDw)@ktcPxO}$Nlvl zd`B5}keJ?D*yX*y*-AeIWete!!SY-qM{Q4sgdW?(?05TBO(2;->A$iGGP-uUIw|%PB>T$Wfykrx+tDmn> z>P4}2ChGL)7UoMe^#I_Z=7`e1A&}6+Y}vM<_T5P&@6CFwO|&SMeR#v~4AV6C9>!Ks zz;XUtVrUGUtG>j(0#hbVAX96scy#;?WaleEc?5CXY0b3mfF0=|sEJ5Ob?B#VAGXm9 z`&nAIg!f~bEybd!uoC3n>T_`ipud5t3s{vij_nM;%9CDzi2oe7pJGtM;9AFZkri8x zqJ2MJO_v%`Yry{JXQ;bR?_+3qH#yk(y|y85H4VZJ5rgPQ4~NZw0z zfPc|xeB;&?PJ1=5m1z2ICqJDYt$6Eobbw*)xE!~8Gf33KJ`D{s5A!@eg$XmY5<09&n}g{uIpM*mhcH|ZlC$LI=u@>Y%>{3#^{0k7(L8gEjxMNygU_cdz_wC zJ=?f&^)9rVbae~pFmGWnDKw(@*F)b_IZSDoZY=C)Ucfb9EqON7x`QeitrTjR=@@DH z<=AbpXecY=QLOVfq7d|uyUthP5|+6lKH}?SlKU45~kj7 zH2F@+2oK|Yi4by!&@U#OJZQ$<+FxT|a}k2nJQM{)+aLh5xrr%4v_1&V+O5-2dfw(Y z>-0JK+dli+tu#-VEAVEk=B0ORDcg;lH5T)!=XW&L6OSovYY z79nhFEI?~9QI$J$guvfU3`d(veLoO>l!QKSSjG(#lWmx?`RDbKc?HMZ|Mr;fo=DqrE3X?o& z^4-2iTfJY5b6bgIr~l^U_?euZRXx82!i&D$ffSpk>-}g;PI?vwXFpXoNAw?#Xw+Me z6%<}D#4TB`hETdl(^iNCv{HF^Jxjob{hU0wM{VCH6~a>_{j_e_NYD+kPtY*2&rkCx zIzkbx&p+}GS4!ve7MdU9tLCnrP?UDp8Ryvl+G@Hm3r0q7{`W@&t_!0K0zf@>U(*p_ z+j^5n^YD%$?gT|l3tMV||N4*LVQ!-CsdER$^ZtquD zxOVPp^Ss$e^o}Xgg8HTGCL0q{_9K@uzP)J>Fd(#Zp z3a&ju8b91x#M6woy04IKgNp=?AOBFI&;00)KxXK=qmW4lq8vMs*!_KI5mSknG4&uM z-}&^sPP%6sw$k3EU8IK8jqGGMW6vEmWOZJHl(_$C452+PJOAx=wv0=tP)Zz>2PkD? zcJh+k3#BPLHmJv@&cBK}A<>?D3b?sg*&ha%cW^=0+GY7LE>VL^l}zS@YLtOmm~Q$$ z&ivEMpF}{<%a}ot4F&*tqvT_UfkmEQQB z%9C}6ThiqzW&J$9mdWLv6X9@B7y?Kbdn^{I$A0Gi+uwLFut|6$iV9v8bu{U6iduiMX)c;G?~wIB6AP&57393 zW&rW#7{xy6hzNv9Vz))~tEl0TU2-v1_{8pZ{UTfS19nC!d48zCbcjbztRe2f^q}8m6If& z235;(xULV(Ck+tbS?&C6G9`9|5S?HG4D331JXrnzpe_DD%D=Ub!(D|VrYi>2skx)= zxEXIxe4cKn+nOz2a+%{%OqtUP}sY^fx~#LhZm0s4t*`hy_C_ z+8Jq`FS;%x9fWX+x{gVZM#KRFx>&fG4+@2uzz-E9Mm{FOtQaVGj4k?>bH!%?MiZD} z_1IbuWcA;fKP7ZmEWdmj1|HWIRWO4dX%iL+ZI9J$TP~V`+x!)KY_YLoO-SCDDryli zsG^YPsQra&No=715N~Z8d|>KCL~Lzj03k4P=F#mH*T4CBX$pq}yp$EsI7k+=Ky-8k z)2YDz>^Qplkrmuj5g1uc&p26F{lx)KhHB;m0QH&h}RcD<~bH9uk&-L+g&ec z)8R%HutlK=f``sSC7_|89x^g|`MFx>@oxFaMI;6k)-EpxDTpz-vACUq`UdW$nN*bleH3RtZYR9^ zVQf-c)!s7m9Z4;^GWZ*eEGd2hTJJi#R78li zYKgvES@sM)60SkRc#;W96o38DMKcl#hs3e{LhLOk_wOgX;KwM$&`-#PiQh}No>e{! z#L)*@K>2-S7b67@PZET}Vbu8)gD{*-acKx$Cf;hmJsW%Mv%P;vshEUKoMA}y_Mp4z zeKZrzPFL~p??$E*<`pHM(Ve1{5~d}USLaM9WEv6}(Z@!mz@&;=Lj>F0FXEE`!$7`L zE_5!=c}9+Or!mPi>M!lV^QrcvtR8Qo8&?ATfWdeOMF_v$i_YFJB;>c8&L1VY!%*Z> z)%AT}3w;b%5P4$bVZVcEjyLq0ES+OTE)LhfA97~L&?{Zow1K-HN8Lt?5wm_vTYxK| z^o4P>RJc}ktTc+ni*(q_S%`Yc!N3yEk}sR*rGrVD=|Y8-@s}w&6*5zZFg4=|DYZe2 z2oQr%;r#njOGf5YhQuNo|8A$nX8L4Mfm{l5C#n}bDjBloQEhcHZFsJG-tM57sQzX` z21c$9SrTI>1wBTNRz@Y%uSv?j8X2#aUALWd$vh*H7CjnI$m0T~kLW?Z7Y2)dUrIkD zibbJLFvMXi)ip3lqH+8Yn2#v0A4IuMy2WtdQZZ~kN| z9_XqNqLZRd5(M3p+|_}yD#2p>$l`sPyNpHGc^aPFLeIsR zPpG&wlDEHi0vE|Uy8Qaaink$j*t~fG5XgH8>QzX;n_aknOydrwE3b)AIH?x^N$LSr ze80<2Ii+KaNFU9+64Yh-1==S*qlkAwwmV-G$`P(aBX^rxj}cZgd&tOyN;pMBI$i{? zL@%?2<|!3<~*7%}!kGaAsO zWT>4=o*2pgG7{^@$p_}6_sjs0Q~V(5yl;msOq`y!t5rx(e<+oLB3?{I?3>Hr$d-nK z>9*xYlg@ltz^qj(1xaj)9~oYwA|pQzx)Ih>y*N}{*a6OQ$eUBNdYef{%!d{ik$$Z$`ZU6Dx#VdJcp)7@d8cqOT~-`_0ltbUnV#SO+yJ+z zXX&{jlnL40>hFm;1N0*`;`%Zd9&ust`e?ZP&)rYIR0CO0@1zkupUV-Cmu@+2 zq)T`!n5JC*NK@tXq6|o0G=cK|S8*f?Z&oUxyV2RmOC(tV*;Azf!#Q=js1VX3q=r^e z{5f?BNX5qmnd<<1UwcSuSm_V8adc?I>Zfkb+7!y z3XNiKC({0AjiKh&Y$sgs@rGS zL+LA8bEy_VgMhw##JME7SpNlMYD3EWDZa|ldzJ

    NFC4BS-~Dm+{l8o{W|N#aav zyY12rrpqgxtv+PXar}NU;7a9f%0WTLI~0lm6OIT42xmCMA&QQjR$g%k^?wN{Mx<@s zmuOTcuiEhULDY(uAek8pBUjZaq$)N$6)X>BP{f%Of6*d9)}xMBtinC44rjC&dPrjJ zKY>P#iY}D=S*XKF{vIOL{v&g;%Z0!j0X{^=1XIgOg8SoKJ}rhUOLW~(uiN{@&^;;6 z8pLf6V7I-;-eYb$`se$_92wF1Bm3`ZUQwSyU6FTU4uwY8wm-A^eJwF~>&Hif)pTGJ zAa_R!AGM6R#O%~mp)a;83#_+27X2u!A3gs6wk)Xr1%|~5JR<=#9 z^kpz|#GNE2>k3Ec_;l-9!n{$%@f4 z-fr4H@6AJ=)7M%`Wz`S=ItvQ*V~g-1s9?Fw!04-r27$kGU1*uH8PEAC3zKI!O<)}D z=L}*t-2_S8j7ub4>*V4;;prbF)U_smvga?%3-C>NuRlLd=R1l1A2{YeNa&G;d~w9z zz#^R>hB`M$-BU06+nwOf{{bv@4$Ny9|3-AlpRrqXbvlLr)g1r4c{~H5mt~jLbAMZ=z|1q;+zP%V zq2Rwu#d+EC46u5td1_iekH3FAu~5z6!6Is*Ri!8XY9HUFfS*5%i^uD1mn#M7zW=)z zNEaFoD7s&K)}?fv)gU7I0&iaCCX4Czobkh%yn-R<;^5 zSt!E$ME|x$P4rL2nVfemX8ErFyi%h`_i$$X*XVdGgku?GX}#Fbw+Wk-M``Y-iR!yF?Nzw6VBx+|%~#)v%bzLwjrAYwT+8$Qn?E!X`` zc1gZEH8SB=(?b^^ao)z)aiR7seGOUT3QzpTut-H0>1yHOsL}h$IeFeb`pip{J@qli zgLPw0MSBT&^8ROA0Oky(KX{%IUv3ajTp$UfRovEF2R9!KjiBRU={L{W;Nw4_am&?G z6O~!Hz31IQ-6DO{{whV9n<4u%4$V5H-suev;@*baXI<{IZNm{n1`D*n)NcWlmByY0 zaW)=zAS^0M(``Ik1E=%liJrbAoKFk4xJa*k2kl)pA>;|i?&bBXp^=qcB1@n{u=$(E zZsDTC0NvJRs;jOm=yK^1d~>wydBj@>;)PA%>==&VKCFLy1ifQra0sM;_j9reVqi<& zV8(>`D2jY>tJ}hbgvauWz^%x zrJCE#Lj9J@O~7@*KFC)3;@OwC0OlXNhs|el%UHW*YPa2wcSlE{2Z9n|6rDyPJ?sJZ zIrfYQqkaW^ONqIhS}d6i;Hr2NT7ErUYCo>-`%S~%viRISCRz<}$_*nQda7LZ9swr& zPsf(EH&*$^9U>l2?fefZsdYy!vvU|kvAo{H^GF+7Z?8{D3Hx2tueH;cLbR3@(bl(R zZGW+SzwFu%67;L$f4XMowus&HzHtlOjZcIewcY~_@=v$1glnVI@u$o7k5!KiQ_J(i zfaU=Awe#ylqexB)43n?3)bj>G1VSkujv#hr`s@R`O1Ej;nP`|$$x)eowjr*)aa8^w zCy&Hp_vfe8GgQL{BH~u*-j{tnM1Ysxk^5#cB{%?A0dIEEs30kKQRk}?9&`*sGATsQ&$x-aojs8hy!GbJ2gJBB_7XRy_<1p`R;5(b`HGELf)TJRlI`M1c^#4`v>HSbAReN39^9)!Tls z^iK#PjCxVLGIl=ex%}y!Jm7G7XA8XTX;}Rg#Orr)8%qSde$pE9=x^fnTz+xnaz#FT zw`Br}Aft*AO4sdYX}z5ZT6!olx(%vqewq_Fa#?}8@~mUp2%y$=tz4^Y=YPZ-^R*AO zzu#%n`=aIHxK=3tiPF~9a##DJfu#n;urJbI`sIYrK4X@P$4ZJ6(qtC1f7<Nu-J>@Mm%7<_zxN*2y%DUhV+6IU8s2F}BMP>PSF3)&O1%Z*YZVP-6fOTGAv zlf)vFCa(G?cKRsO&+FS~K2VeT^x&EnD0Ktz$LG#Tw_+05Y9PYn8SK(eo$q$94tX2R zltRLaymbD0?hBC{-oNIH7OI(<=j|K@p(9DEqbIFrMbr8p*oCO&2a>hbXtAXLRhI0Np;(MRFTw!)#uUF8o zW!_cq;cbTl^Zs$?dfJ6=rNCJ!ODb1Gp9#)%1V_JJ*>XICXPw7}vFpx*8f;@S*7lkh zj(h#rp(na&h5SeM7OaKUZ{oH-i#jgX!iPb&uy>6Qr{lKm?*`NAALu9*B`C*c{tzy| zT|vLgzv@J~a-CJ=_cm#&VP+WwAOzRDD&qpChoj;|7yJfU6SH~&g7&t}CxLBGAi1^x z(kK{C&|tEq)kXj^_c8^#z2TlduV4^=<#xV0nI-W8_mm&*?MRP6O0K^mbtv9i$)~ZU zKcT2`ax8>&0rcGBp5mD2M1P0)KPJ;Dy01LA{D9aX^OM*5W`cMtFxdk7ObQ$oToH;mw_|c+$Gy&7t&KOOdgJ-2KMRQWb=xC*(7xP14TKQ?+M0%7NYq&M!MP= z!^3AnP@eZ9&edcdq&L!$wDRMkN}b3dhSC}XrH38_KG!Q3wdXYjNAZOi^H(5PSpUvr zyDf;?I+lb7s1fKmcjqa1wF@KqAl2EUw%9n;>a$yz&lPRg`1o+wglqbp5gG3=Y?lpO zXSfU|ym4;JpL0$yje70dCccKw>T#8e%$tX@d_A`4Fg0#7z&T_mzj)yi%=xE#c)>09 zk=J0A(I=g;`H6cN!0NDfe8OaCK+5CnBlyu1uM5Z@hwB`jKI-}0w@ec)SR67h3R>!I z@cA?e*IE1wOK-B?;+{`!+2U<*D&lRvaq89QZU1Dn`Q;-a&hk;))X9+bzL4`@)|Gt}X@xPRoI#J%WU(@2$DhfVCB0e?-fU*j zT~-ecrSZL$%peM_)_>>*{@L4ez<}}a>w!snE7J$!*li@F#fBFzP*R>BMqmO4x2tgV z_4OW{?ChfsNX8W`GkHudNANepQ0n_LFv4z`BpJY9GrPF`x@^5lw=8S8uVDtB>6s`t zki*@|lIPrrd&CXIHNls{CA0I>@6QAS=5*bnb=~4-dc49aRABTakb_()d-R}np_V7N z{XQ!lH+Gd;YqG-eP`5Vpfq9V8JCLRNX#J(CyFOF}h=;rL>SKwWPl^SLB8T6Cq?Yoi z9-d@x$3NSF5gRy`E*3)1Ws1X?Iv1y>rlN8`d!aXO_X+Gx<$MoI(ohwT1m+XXWQN-Y zTyKZl2(FFF4uPZA^8oJSIp>9T_L40zLTl*g$Ch=%h7)J@J{F^<#UZW9AOiPuw1v0K zt|c@1Dc(Zt=Mdqh19@G>M7f$E_VgWnylC1UmQ^Z(4ZrAOHQjKZ_x)(GQzrvaZDFYo z4^-$bb&lNqQE_J(d(+O$L+pi~r;&B|FswAze05sR-vD==1snht8y%>4v;KrWCcn?&n7avyFCac~HRe+KXThR@Dyf~f<;DpbJtyDH^OBE|MRejDzED4d(SCX~HvJQYRC;sY zDvSN=s0Kk#()L&XFv7gp11BPAAcWMr8j;w(#|}}rAkMrLD_yh~cswJr_3~3d3a_(z zfW0s7W_JD89t-^?5@UpNxb>U{cG_U-@t-^!-}wBin0nZ9_wEoMz26?4$KB`Y!U;~} zL#)x?43;DpOoHe(S`M*$&{8yUx^_0xW{6Y0j^Y`b=d;;c4mYE%R{*zfGoH#f@9w7) zAn2`0fq0F6ArB7>VTUset@+LT$h!rx+iShL9nCH|3f7i9*w^K-yA2(Ajt#Te=^aLw z5mxQOM6D;2dc<7xHFkZk_w$0lr#%dWq@~Tftwngl{~!x+;>H^=O2VbB5rrLLb}cTc z3`Oy)M(%jSMsD0XK>!3|z&L^A=t(%GRu{y?%6zRBOnL7zAaKtv(nN_$I=EQ)!cQUA z8(?bu3)C}sEVb+t#Yo|x)P5so^f*=<3zMv43ahn_UT8lKRS5iMRoS;{hP0?Qpq1=AoPxC>_z!296?0wP&~pU1LVQ_ifvLNx!|Y>XghU;7)9m+Oq>h zOi@FlIyFqAkn>*fN^LJm>)8!hpI%+&x|W=<4~b-ZEGxsFn$D&Knk2a@0;MWTC%5-1 z`NL<Lnv}YDw4y$a4I9=c*DbK-ldEIadoJ;j#C=vzw+pEjq z(*zXQC&5P@^hqNQ$( zn(faLOJ`T>yo~xF5^e;{Y|=?%$jxR3B{2nTGsO&38HNAZjaPHVZhI^-a+qp{^`14j zW}FRQD^yxx^*oG%Fd(g zC5kk<6PgjS@9X11g&|@HnZ|^WvW%@s$TpTIWgCOx+)h{Lbgpxq7w5Xp+xxm-e1HEJ z_x-=VpYQKGxA~Qj^k@9PoyT{_PBx6@>i2LfJMXD)nS1Q|QU(M^u$1 z%4FXm+*Z2k6w9RLY4r@PrLy#Wjk@5mfSq?P4&f?3PMN8r+G0_&^44scuRl!ogc!KN z(u3%cX)l^xr+3X!{vP)vO+KoS;l z!{C(v5fdzF5>pOzlv{t`blUrP_2_GGx~kTV`->l6CAUK%Dr;>6p2z+~l1ykXB_Z&E zzy#}=+Ff+Rx3Zw%n=I6qAF%aX=a*fUTRwJ$HZShITzLbm=W6HZgbCoe%qglu@4gH_ zar~q=oV+49Ms*XUup`3)gaB7lDXg407Acll{~4Hxrnh}r)XT0}n(eJK87XC^`ZvcO zb8VYZzM3glZti)3`$+r(yW;B57HByW4`V!h~IQTu#yQhE)pj zD-%&bDmFx9LT}E|*zC*PI4@p0-ZNnsuiwqO}61Fnet+?Pp~uB3SxdFoClMnPshNrdXvxF%ei1{ zN9547?>G7yfHflvu6J{h?A4EIyt9wf)MaFT9XFn{|d$ux7ex}@hM4T)QmTA=6;>bDQoc-604~2JYZHqXtyxrB|s_`>VSNKq?2*Y&?gobH8_z{=@LuHC$*> zNdDXF5A`k3SYJx{fvA>V9VYet8vd5Du4XYUBOJf_42=@l)EJQyL8~_JU+}tRwoZQ;m<{G(FUKHJ-Drl&RX{`KZf+4-Ej9XQq zSk(y#oF}*(#j`%;FjrVNb)UnbPvk45c-Y4TpNeH-uR!FE*snjny*w5GZZa)6(udEQ z4pNZnv^wiP4QV@zLa3Et6le^6OgvVc?UncMO3o7*gi7fH^OCg^ zJ;KQKl;FNS^eJFu*KI<35TZP_$*erxjXSLGQxw9@dwx`L5*LnQtO_WbZaM84O}wSz z&)aCWQjxU~+xhC`8QIu(5~C!GOMEn0od=A~CmfCE@LT7v#@#4v?<+ibc8z_H*rdPLjLLk#_yXp%(Yrd*z2Tnr z#L;btn^EM@CztMs?|2;kMGv4NNqZ!bVCsB|+Nr4RJ#6F)U?Blsil^Y^i3G}Y6M2aL zijv-@8}1-OHbQuJYUlFkX~oN??I?nyWtq`#v4GoLdzN^jiA>~8g1gLx!;zCo?qQjK zd5F46`5vsubg+SuG46iC`#ISbp94!-R7lL#+yNrLNd%uHcX9SFz&-gyUl~*EPQ&Xw z^*I8+lW}Ewyp&QpmSiQ6l&G4bzl5^E5WrvFvWxw%fvReIb zewQ0D6z~3~vc|BAAO@p-9 zIc8u|t&#Il=!JT2p+Ce%%oR#zznW0Zpyy6P%mvcSb@>oPiJoZt_w}jgl_3U(_M-lxN+EQ>=ji&978aYAJ`?Owv| zwj&FB7y30jp6bk-HZsIZ(jk6+)yd8px&RamK^&QZHt`qyD(cpxx{+PM(S0MLYYTX| zTD5Xoaoa}Y6u16i=3fP<-dg4tTLn^t&L?^dfwX#u!~>y4u3HH7v%89X9Y6g3okek* zJU?N(0kQs%zAfp7w+?l$oU4Nush=k&iQBM8OFq)-0O%$-epfIbCk(iYoY{NY?(dh^ z`hzWau!f1*M7cNTbAE;@CSY}+PvP85+w;8GyfxL0VQ+7N$ulbYVd%J~b7PnC_A5k# zgwW9d9;^)t3_-GffzD;s9zSE0{TMD^9rIcJW`*kIpPz%TcfkQ%(6AHXXZYYNdc;hM zanI8Jjilg?s0+^8n_2Cjj{R|?lBgqY6ZN6P!t47fey2HP2@w5|HIjJW{y3B)7_fZ= zkGB1wXXF1hDVRSAN&tiHL1Z{m`DfVifdynVZzJ@R+l$FFYkIz7VueC4EZ(w>u1lNAxnr0%RhVe!VCO8 z4gMAQ(~`Q@xtxp?5!ovp<{2#b#`C9-qGS=VVSnB#7*u>gBz*polbw~HDaslND=HO9 z{RIb4)D4m}fJNy1{n<0YXA;666rG;$r@Txi6uIc4BZkMv@Pb2TMDW)rYtusEzFIvv z__notaNytI^XIqL3(sesq@IGoK2EHUm&NNG6IB9i0u2>O3j?lcLspJ)Q>6uk+VAsC zmC)=&{l1gw!sEX9`(=#iMR9Hev4%hLbD>r=W&jrHuUA<97oG64!p#5kC-|-|uShHI z@9nRDis%086y#s*!GHYsfu7rMo*TgCj(OHt5&X}WJRKjLKJc$I|NR9y3b4WG$+B>Z z^_Ks!#t-<;;>8K~_aT41#LJ?8(I{AIxjP9W;Ih)przjEfN3lwE-k?dyVT>4;x@Q^S zUJWkMsIjuOsc?HZ9eTeYM&iB(x%vacwd8hMS2d+QH^9PG;Z6Ux7$E&T5nE@ z>p`7X!?*qUKfk@%#(ohyZjWZ08I@N9kN1~WJLCBY=cc6h$(GeK^I`8FwpvMp>KzXc z)N4u@T!$S`Ngl8KHEcVuDt=Mv*$?vM?vCe20mLl&J_I~=h8VeA75%J1rPOq?V>T(y z8wmo+^A!)}D4npAMFT7m7bid;)1`s!&310X`-iKx5PS|=S>=m_YCm(2@;bGWbhNl~ zgRYxv6~<45B$p0XN6Sjv3YwqtO**45O{TOQ8DffPZ96)Xp$UD?iwpX5)5!!yeDZ@FFMT#idW8530t!mwL9(m`&gs8C$-b|(hF1pB@Ra=jh z0P0_nRC)uZZiWW5-;v$j)*{3rA%+X{wp~Ot^si6|<70|GwE57yJKGu|pFkl>dsXbj zOXC(MaM9ha4C&no0R(G}e|fN6o(z9~4#PT^wK^Dc0(rPv^>Vwy`n9MwTIQixK|iZq z#Ar!y42D<5VUF7uqYLlLPY^fr59rNFg(dZ7#*1aCCaKO;G^;nerJokfW!law$7Ja= zo!4V1`+f?59~sNXx&@nD*oSaTijVqmsc1RGFai15u_wO)IHIqgdv2f?zL%986xT;Z z3OqyaAKm1UTm@Cd7H?}yCw3G@%UHQeTOR7uRyWfCkA&!nfadc_ji8mvWj}(Ag0;Ph zAx8C1kK28fFTuRinejup$##;SO__{qy+)BtDIX`-4CUUIWNcrphJAXzfPwoVf;;<$ zBJw-5yo)#q@OcV8O1+mb?scdO z;zvER!Iey!Tyqu`6b!1VV8H2p&0Ktce>Rq1J)CNWkwi&?KHyt0v#8hRHvz3~4kncj zX^wO8{oO59nUBkM1{yI%yAeoNBEB=H=TGz+&;jlpB6AXr0@ku@+R?=*988yLiQUge zBvM>pDhf=mT8+HZ_?>@==9U~cs9CHwuE%P~U!Q%Ti1CF%;clJkGQQeKvNltEzK1YY zz2#7|@J(v8o$GSZ)jIOQkHG5PXAkV(>~h4x&nhRBH7KuL&UchZ52@4K!UVQoarqQK4o2LwZk^{`O>OM_55$z-)4*?@Z--a26+pQF6OLp zgw-{!z|`_u$L&}IgCusM#(GInqhKOeF++ryv3~3A5|xd)GzNmS(_X)h=zm*^%5WP`Ghovp@*` z6Cp;N0L9c*RyWP22o|ZqJ$}23YtwW#Dup;CaH&ZfERrezTF0e#pbwtY|9GUsJ~!nX zEh3`47uu4OExm>y7A7ek${f_a>c>FB$#h;442Q65n*73ghu}i^HbN$#iuU$b6MF3)k^6GvKq|aM)ax)yh1o zhT(BB5gt#cZiT(7Aw7!Nh8O7pB(t?t#J{-7Ip5+iw=5D(gMPzm0Jbvx-CQ3|(;nYQ zjixUjwmf<;@VBlH3!yToS!3IOg5Vy<<7c8kVDm}x-O5o*Yqf5Y%}qw)gs73HikK7` zDif!!M63{kbsN>bPq|+%Y_@2tsjlMiwV!t2=t#?`E4~qb=6ZRs-xkIP9Mm#9AGlJy zUcTBef=#-)@+ruLBXKLECwA*e`8q10%?eYkkRwf#l$(K37T{+9b>|S@OTsYD)}9QAQR;qj^zag1!1VM6IXRsJJHh-n_Ia_RZ;{>)~~s=j+1go4Q1!-_MLBravP4 zO|Ltj6Zg|^OUacxvPlVj03P4p?p3IFn5s{<=#TRC$E{qxGlo{`j*f~!YJ)~NrZiXk zyz3O+q7JDUJcrocGn*b87Y7q#4ASctt=j#Xbd5J24M27-){hsm#p5>#QR28!EPO8a zERudFzdiUZ?a#$9ed}G83}hD0{!M+KwNHEB&DVQwNKJ!zzia}Jw}8%4>hNBM>TTPR zCZE)#zJ9PSyupJjcA?WR(*q~H;iDt`z_a<*%K&!EMsLUfzs9>zkfN9bAK|dOk^&+GuT= z{{5CR0`{m#Z{u!B1Jv+=Sd_XX$prU0{!JK|8r0i9+>? zMXB(UZQU}PYlh@HD*sU{bAjApjD$kN5L%iIfK1RKM!p@t3Fx^TNU?Y!tHJG$eH%fM z9zKzmDAl8($8o=2kCWB@si@AM@omxB#2+rl4Uex}&IXab>^o*aDkhc?Z87A}#PwEY z*i9uOLuELoYq2yFn+%C|e(t9<$RZ3**c1SUucOuz5b^Yvu34h)#F;+BUSy%SKI@0H(C4WSDTcD!aVpfoG+m~-O14%Jt`H)odgXcI9lUND)J@O za#^ndSqr!M+%%7!Ch6ZY<5G(iDUQx54wEBUkW|MM9)?I_xY)PHN2dw`j&!Eh@;03z zZlraO_lJON7TIEwoi{S4&75_MyJ+w12j?Zr>u#I=w4&>v&Gg7=*E(<&PP!Tdu64CU z;p1pCH6DGWCi%07sM4mECS1REw6T_YsT~hI4`FmEHdZUzUa?sxcKYL~=nk&OFieip zhW+ZfajH zek#>~gXI>l>ZJ&DM>XsV`K?&>7K;iH$JckI3mIDZRnhDuPQq8v zUZA(f^id4acAaVOS07_D(GM4vaNjZ^LD*u;;X-`1C9v+uFU_V;N~TXo%P;IsJt;p# z?C6-5FX;8EHf>T<*-H;ZkNjT5kMBn-iJ(+Ii;5m51Al>%7+czLWnP;M2$yAa!6`}a z<$Z8Xa6PUidl(cF7r!>kA>8BXMHgKQjL<&c4YduP`%Tzm#^y|Ci_uyOkz&Z;NfD%8{9wvkvLmNuS{KcPq~(RGbKMwa_W9Ec?lPU8VCiCZ zaG1qafZ;R#2@g7J*Q)sqUB^lS&O>Wy9GT+=c=aU%Y2Nt z40Mfe5ej|4jq#hwaYMRg+a6_@@!O#Wywsxf^zO3Una`{9 zhczvF5i_PC50y}-v`WS%d6O;i&m~AJ21$0k@`NXA)FF~hl|(KKK-H#Uj8-v3?%TV3 zlqJj++|r`UfOFX?yT#`z{!m?KYTOfL$v|Xdm3aGioGu-&-cwAzS@9V;`JE_RT5sax z3~qzPOzghhd{d)*L9uVN`dGXuPz5^PpuIwHrMpg)@RV{AcVagsE|GrL*uott2&cYk zAj-^6uRIrM6S3^81Teo-=o>NbR($t8V^eK<@QT=v&QLMtw@3#;!P2&vKW0>0&}-7pC(@wyHC(1$q-{~ ztFtbu(MA`s>Wk94QqzVAK}t$ z$^#R|z^;LEW(7IIxi_bPgpQ&}s_uqN{5B@SU~oq)g4N5%7qQVVb9T%F+)GWZN9%%< za1lGhF9}wiia(W-k*jVu$V{w_{5r$8vUz)-hZPoQLGyY^jUmcBzv`!|^8~$AR$bN2 z#|B;Q)6a{J3cy5}-cnEDViR+aGmRAUbC^8Q3$=@HwlnpRWZObqS?-2yuA6FpUwby+x$eQ_=9z~j5HR$%XJhj-jZTUO4Psitn_U zwVlPengRN`pZeDkv1S6AZE?J@pGiYLqgo=(w7FICp@RLHbH1D2(#o~WHJR_ZZ;+Ck zpZXSsH#=j0+{7R=1-7J;u-5*rf*&e>+sS#8?sJqjhs?gQMYO1+)N^WjwmrlP z`SP!fhOn!b^g|6Y3$_Vl!R4S4GA@t)o;;ZMpElg2J3R;T)QY~ z)^(oNsxGjsqNTp<(nRiCgJc~UP~X*uB@~C7cDJc2sULu4!{@@3#-vJ;kCItPa&E8) zNa^xFd<(QrDpgLqrfiU(@h$d*2l51spD*O>m7$L3-%q;msQXrhbBNvM*>(UWYKcg_ zoqd(^1;%P_{aqwim2~FhNONl24f#2cXrhPX2sxTW36Yr!J7ib=_R=Fu1^I*7vl1gIgS+CqV)kww zmg*rXC~j7y{$5ao0D!wlrx#t&KGdu`Oc+T&Da-lN`YWMdT>WNl^vB4M_ijb&@!y}j z8-%;%=M+Tv@;Zm6P};nu5epH3*t#lfJFioG0pW+!K{yG)00Kg9W&qwlkcT>^TiaG~ zt~%ycE5LdN`B0o`I~mAF|l+6GR3wL zzGDMm$tBxd(ggDZzqRY|ISTE~px7!d5X+p6L;vQi9NyhTi608sy!Thv`U^Y#g-!H2sag$%I zNF3ifp;*U1SJz-kLklcawNMKgEam4^kq^+aIz*P;xl4hUaLkFoU?YGqc}y z*0OX`TYS>;0QeH_b|T)5GCaD5=_SalYd2$(W%Ppmpt5> zsDW&j#|uEi+H4+~W;Wi3hr@XKl4+)_*5$IGMi5z z<;QmWUp^T30hSFC3OiG*tyf=pX5MCj{-%$*y&?))UEQ2k#%NHRRNc^br%|3WbDn!nKGt7!7WU26_FYj1w3NU3vU)#$H=Vqf}>d@)_6}~yJ6Q14JF&n2asLkmhcjBsjKW|M9J3*LZ z0@rL=auz!WdFX1)M(^RQdM(Cw;oF=gW)F2hLd!yfBU3`DrlD3ZRuVqYOsQ&60{q5l zppVZl;C2ix9lH|vq12HteeR~JA3!Nj`GZYxr*2wEsAVphgEmRvRa!^lA6&RWZ}0GK z(w-Hh0o82(thbX+?Y_zt^t~JX{8`$0k_m0ectHq)(v=*YjdhJlwmRJY7#P;4+_cW3 z`t43W6i{D7!Zw?O!{%Enk~bwVdjPqbJR{G-&#i`mnH1>v`p>IEu1a3{=aE!WAI14M z+1@4uPu-fGN$jGQCbQsU%ofSNF+Wl@iND+j*JKPbR>|}VZ&vMjMo|Sp#4F3N+j%iQdhgc)yrq>c*|CA~~K148#pcNa_ck8Xn*W{;# zfyY9Fv*c3J*9*m?vc;B2)G#}!==MhxSPd1B$$m4a*8glro{?6ypGDPaCV0C09UXt3 z^8Nw$5j%y26_gXrKf%u`QV&o+XsaBT3*>I($Wm1z1R70e{oHDJ!torq{mo0ups({E zzT9ihkd{TYqtMPL>%qr!k(F-;jME@VE~r${cZ!4_9T^0<=VE-be_J;$1{lfsM8M~( z9~#$+`K)i&4fPTfjIFS$a$R?OU0wCQ-6NK(>ulGK64{d?R7M&QxCG`UB8!C5(47c9 zu9hVdk6zt6M!@s#<;Gkw_enCn4~eT2EgSs7mN`Vsnmw@-RSmWx-v!r8VNaClKjYW0ixDH=5Z~@B6M9pYQs^B*L zLrt36RyAI-4l(g?D&S4cDvenDgReGMIdujxOq;V|o(MatO8b;#@~U?pXYgh+wW8IJ zU?YXoMdu$*auR|kQUs8uz*f2Kj-BMju8{O(Kmk*;tzIZbp8nMoOL~7NVVdg8??8PyhJLbxtYN@bUksM( zJ3lehjw|1;h;}o_PEyp4L+n#g2>?Pz$Tn^hPHaKYl_43TluGFPv;7YK)P@Nu=2G`1 zSH#9dVD6y&dO@N;n}868C@D?QCGJNo!l|%XTn@ZN7k&7H)pNHYyM2c91DkFKRh%6P z872Z(+R(2Vb%2x>*zz?TEo4e_cxD{t1+-n*q;a%d_b{z|DJ{`hQ|ZMg-#z)v7Ky~27k#uKpL#LGq2hq zm}Q+v;jr#5)KRp}jv>uK-)J{%i|tvMgZPR0j~Ft$CQ#Z4s9B1o5T;_J=SnLUep1_? zWJa&2*zaF`KE#`GpE02JChbk@qDJT&u|fV&2|y*$g~ydWr|3SJLm;O!PiVKjUB-6w zTy`~3Yiv_aIA6D2SX0y7HWP!26_Meav^4(U`?22h0>ozr3-~!IxVV;vZm$TK1X@ne z*uEsjaRUN^tkoT3>@L|U(>%kXq42{;HY349NaP`RZZbZi7Y0{|#COwN=bx(?HNg&n zu6pSwVQ0sx`fYO8d@cF~e<$7C*HIhudc3!{qe zj{fAy084BF)^8&C*gC7_KEDkXw{CNS?UKVXWp?DPx^0d#pofF3OVCxOuME$^t_-NG z;4TWtU=$0Q){n6AKfI=fyjevhC{DA5U%Cam`q_vMO}l3$7b5&e3YHm*;pjf5NL;oM z7XV?*HWOvz=WL;A!+Eb*bgH$=z%SiM;n^57SCI94{nIW-B2~4etbDxMM`eTj{>`=$ zM%v}V+UoZT`L>g@R8)b*zQ*{@>l5kI3HG1gPgpunmyM}!7hiz`*@>$0Ft z?e5`DF<^vq1G1^t4Axd2k1H_Y&UgSb5Hvo;6|!1Y`Dr=EU2Oc?gRp%(z4ZZ6h_Do^ z7fl6>2Pae2D~J!tJU)pIdgYfi=p~gj4x85>#bDiFwNos|E99~bB8As@0-f>!jg1D$ zdU=+kbOyby-T2es!cVG&R+#eXR-VdBh%(>F4?|b!1iRp-{Y-T!+BT0MUk`w1wN64l zNt;Gf{1&0iukI2>ySvL-UNq;7(V6lrq4nxtsMf0Hak#q;-0Tdzh)vTO!;vhF?of$m zeZ96y|GPHyNNp}r2`f*j7fovPK%#8dBh z%B7NlS;r!Ed0IjFol^IWlm4>*j(mwE?COBT9Ja^TFGS3mywn^-Q%5i(T;dNgW~Dha z-;7u$dC({A>v0u#Z#wxrSMgQ}(|8gt7dG4x`%n*swt3AXJ3ZrAowHAXw*HUDa0#)P zI#MjBF|pYw$hJiMY^Bk^I#~xZ4?%>WB7RI*laYjY{f9H!W}nN(JpuxmSI#OqTTmbZ zJ-bdmT`*ZRB?Ds&Q)rZdz8en<`I}t2%OVnniLEH>&k2A&WaU`oN2LE1T8`z-V!N}h zPg`C0r#@_QifT2d@b?tJq~@f~q$p)2oSJhR^M|wSW?9SFs=HIMZ7N1j5D=%YA#s8}ki;SRYTw&S-S&JS^RSI&e9`Pye9rYnCDZ)Q_< zz;9SqX0!cAtFRIFeR}`aUWkX32=z;5J=u4_KJ1IkJzNmst+E{1*t~h|&q;OtCt<|D z{^tt|d=ZGIFE-$yIPZD@=xq?h=JuSuDpf#WpM~3_-x`VZS@y)LML6bj1>{ZG-blQEC@zLa(U?WcvP1ID zyZ*;~&?L`fcwF*$%y&}8|4>~BN(q6|j#$p0lK{zoE|wYA?yYXHu53De!ao%w49K9q zLu2-bI`B`KMt1a9w-j>xGF_%;)f-LIoH$vm(ZHxxpH*UcG( zY%qndq1uw5QBA^~VVkO*cWlWH8j@kQo7K8ceas!2P7Pxps7MT>@i6_BL z{vSM(TNSaM7iC>0hslAG`4XMu_utBVgL5tRhNa+^F^Yr z#Yq3sKe3{}V<(kKWX~@fm88SUT3I_RX*iIaaO$PWrLZ+GDi>V_RTr`yb3zc zB3`E0Uo)*A1T{%>k1Y-7&BEBoKRWIr!5O#)32LlZb+dfMhE%tcp8hk?C?{I2Fk*q% zYo&->g_ex`3gBN}yqKWgZ(=w$)37=Fh8JYs15=3E?suRcz;p{CkYD4Cd&c@uQaqhLOUQara z(J=6azn4}&cQtA?ol;WKTJOV=aD>JWfyx|TkNX>?rbS7_HweZj8FChoINB&jFI23E zXH-4EHjFV$|GqY~7sw^vB1F|75y?06)d;XG%^2XE>jxDlVpa!rn-|af7#oT-!2X+c z!Z)$2#3;8OtQ2h&RsrAYjhZSM;wa>$p3ReA4)gAe85w0l=X0 z4;T8Xs2O2BYQ_O6ybhATOz$NGF>9Sci`?lxjv>7W<=uJbxZp5F$pDKPu~HB6BRmnj z)XT`WLRw5McfXj_kA;evBBiJ!jot&*3*R&@CscHtCme(-$v!0n=Zd>8jD32)50coM z{A!Wx@$`G3IH`w!>PJyszU?=F<0(<6XP|wt5&cZi3<_uXOP0F zuR4MvHh99oxoApw%V?!R+~-H7nD(F4bOW*Vk%!&nVP8S9UnMHIC-0_+_e8RD&D(hM zt%VEmWB&BCz3s4QZ93_tYliHogL<)fB)0zao_j1RudrZnqh8c=#Kt%ctvy(1xYe~- zDN{0Je1{C+v9_NdE-lR!vmWZA=?RWEqw@8G!ad%-ov$Un`xF9}TVOtvqpvJ6byp&g zyaA4W!|_FUxe#bGuWNP0Vm3&05^D%RScPTwu(iI*U3L;!bML)3@N$O1$s<&@U6dbk z6F&MtBQH55sP%P334y#C1lA~!*C2=?V;nDScXWb8#oQi>zrujx1u})HG!`9VwaO{~aXV%}pai}W+Qx7$W@aciw1x*7XkWUi*9ol10Zs!yyC+3yJ0CxS|;U?KXPit0r@RAoP-Y5GCK4 zEXno*r)uc~@~Ir0gDGx-+>eJ5m0soZCn?kJ>Q$HK1phSHsA)dwx`6r|I~b~2?WUg? zIGt3cZ_Q4dE@tT@90z$euefiz;*U!#D+V|v_zy7^QRvd%t?F?5{lSYY*fpi2HH5zv zsS}@ka*ZoW8Kf>hN(E<#lgf z4dr`1ExA*{9M;Salnpa)Agm6>Fqcfj>VGQSr%Z0_&nT*s$S4Ylr~W(%ZWX(mOqbK? zAI(-J)Utv$o0uyhDffSswCT7nkgdqYJ9^>m!a^b#V>L@_S6;VFmvhsWhhxHJzKYN} z-FiVX3)LGA8*%6+TnQ&7&Xa(|ONw)uV;lbvN&BTor~$lCg2KnP;Zb9jd9pz9UP6gq zi=vInlq83wvwE$e$C$HHspzG=eeu;J0Jh8D_whYk`g!c*{Pb=sZGfT>vkuPh<>?TaJ<~# zTt)fmUI|0Z9WdNd-YoteTYOlkJToaPMF|_f~d_} zcZ7rst`VUI)VUnY>{o8}uOhfA(StXsPIY#(JbnOmGLRkY{7O{BDS?f;64P0$wC;z$ zs_M_Mp0g%S-8&=U6a=-HObuDD^@ba1$q~W{6a~Wzjub^c;+l!SX&KXjKPE|din$$*^5nt<-z*eN{-T%^wC#HV_+AAMYWo`2_+G`9esd%fT~gye<*$@ z4=G)tVn(dcu&V3OQ?{M6tf4%_J|r`n0qj&Mv6P%Jlb*w8ZhoC-VOe$;l|kz9_f)yv zArx(5qQ-Wb8}dfs?G9*XE!XJmf5r>y{*uM37gAjhw$rMXbje9*7kzZYrzNl}5XR38 zu{VY@sNl#a$;Go~AJq4B;?vmka(`otI!Ifp;*m!L-*WdGkm}(tfC|v9N4M{HKv0;$ zs7(X@Qy-HK&(2HUz2WVS8x)ICtw&bArQgyqW4>G3(p*D?xEi!zk|KkXi?!{C>c#SA zBEJDtU^gvucFvWLxFy$Q^9Myk9kU4s%}eTxe?9;G&LHV4C?ll0uygng(?!ZD0iVPO zK`IMV|B~Swi0#kjAIYk-JmUp0Kb!XYXfa#QVwP&AsrM|}CQJ7~_qL;P*;{1X13zW@ z7u9o_vkh-l$Ps6xKZ!@JBP=?NdO}|Rx{WLH=jfZ`N z+8FCe$I}HP$U^!fz;b8E#FNaYhnytq)Yo+Xto% zT+#LVsW9yLnk(r6`Iko?@3xbb;qam5Mgu=>h~7LGRK`I3Yu%H6NVl|``oV>}xLQ74 zVKStiYR?XX5S?e*_?u1JnLD<Py)Wl0mIMBUZ{XrgjgaW_TrU&bNBkxlwCP8WLBF?j%k@e$&19n?f zdIt1&=_BIqZ*)-_izGje4!U5He#}r+@gdu0G_1n7o#W6yXo`XDvZF1TS?Eb}LuhSI z(QO5a3E3;cB5A^_&vHm!!b01R<#5J85A42Y|LS)24Diug*R`O`@?Pg(;;_^zS$2cp zw5$6Ct?ZpA3sHW(+2l7f@zl+FN}_*Z4@mcLFo%&#FwB<>@1?PauJpTeNK6PaBRNsh zE#n3GWM)TAH(w(s9l!HuOO4H2JlYZ&eaopt{FgfWWt5fdGLvZ*Mz>IRx?Jfr$2En% z`3Ar_t?SKQ2Pm7q3NDO{#hmngt(wi4rH~&~EpRqEszx`e13HZJkI&Y@Or=pBu;1Ee9eP&EdS0Z5Fye=DOB|3Avqe3fhl!2+klCL zOrlYqM^BjWdtP(A!PS89GvA^m$&oNr`vJ`pbiW87`V~ieAz-FU{#T>E6C?;Gy}W}fnw+D)ct7%hRx4@>Y(1^Ba8WW(D6zzL z8|wSBpbbziUhtu(v-EoUhXbZEfuIrLXK$MF<&4+7hmEI%7cvZm`nP3Y=tGEx@Q@~D2jNU!=Ztz!la?s zpI3-CVE829MbYTP1@%z7BiI2#gr7cV(V13?N|M}NQHGWmi6-h=rgVqjG43~~DaDhi z?l|rQA^JjrNMPefM zoWR~jnGlv3#ZbJyPVX@n;euB?ru;-XH{{wxajjoygXLRczq9z}OIl#dY5xVami5^V z+xLhKLbt`3Ikm=ZwKupxXQJedS7BKo9JW#gB8$QRmR&5)>DmlvT+L@W3!T@7Jv%urrW9ww6eERx!KZpW{w6_wkrPh;VJfOl*I=Oa)4vRONFv$fUC*^1ni- zoluQKuYQElzJQ^Oh|W3L{*)sJx%TlK&yrc8jZ|A=WPYx}Z&L>c!-F_d6XA$^_l9Ie z=uivl47^dfL5OMVrGEXeY+I<^sE(tu0ZUx#G#~hN*7f$)%f-f7J zOmTqV_p9R0ZwG)gkv|gs$KrbKCvsc6f7-p6lOcZQDq9QEyt0g~QmUuJ#vMLoM>uM0 z zTp~m^^W>?k$_;wZpawrlzkWgqPl@=<>;4H7o_}mrY&-WFV*Te7*svQqh)G_Drc}-U z6O(iS$HR1+{#5jX^I!it2fNM_df04W!oc+p^e|l;1Ta;ZuIMZNb0qGI0UQvNOgfxa z>jN$NH(Z;C10s!+h>$*ue}bK-PjhS=eBUPR{&OTV?2a!8V4eg2TZ8{U+v{2Fob*)s zZ^)Q!{|%%j!oL8XMz$Ju3JMd&&5Q8BB4~IVt$P6eSNvCyV9&2wnl<>xm3$}rEC#EN z<+Bq%_3NL=uP!{h0c^i$86IjX$HG7Td_xNQ*?qX8@t+_qcr57VPHz(#(9QqApCM0v zCOM-o{VzY`KKc3hkJ{LO`5E=e&u{{L=KruxkWCcyvoXKZV#JJwfJGO#OOT-wKSN@mt5Mc3r(eKt9A`b2zU8nC|m!%fSRhqS` zSGiJ;8u^U+8Wy(a<~L98ZtlAyorxQ6h#ddCFL8t`weu!88vK-C3gC11r_{^|Sxt#B z+J|WZ$I8Zu^{~X{7YrqjN$GJ>g1@Y|4%w3h{cBV+x=Q;=3|~@BKm?-g|KLqx(g(P{?5V*@lIHjiZNyf!L?7 z6%Gj_VqK=x!AWb)yN^jprq!X^FYr1Qs5!r%ng4+knqQ$CNMiOV6}a=AJCCjcR%!QV z{<&m~Jq1OA|1jh>{?YxTJlE*oa1g&aZ)fi4=$caDS#!I4*^Ohh05GhgG#VOQPTUI3 zTjn-xiH6})(=I-qMhb|mCWQgV7jA8S!51G8=|@BqTlt)^=$zk?oHICl=vw?$Mew*O z>^S53$%V=$fddDyv3`FalVlW%w{TozwYm{^*8%_P{LJ#eLBPj3Nej0w?(ngQ%CvP` zz`hUeAYga#RC@oD4yq#}16wZN{sWPKz3Y&3Y27EC+eOQ=UAu!AI0z2bxnWITfO%ap zs#*tIL(AhCI3MaQ5~yUo{9lo53TwB^4(QFNHk}LAjMIc}?m?wf>n8-PMzrl4GOedafqjRI z8>>DTm5YJTkfSynxVlHJqxy!mY=MRB(!KR>kse4l7I_UyxTDTEDQ&;AJ~DfXE!Ae% z*lRXleuNY)Wx~w=f<}LfFHZrgf&aQZE?7G@uYJC8c+^tvHJYJD2c3e#xi3Sh%`KLX zfrxx**(g-RRCYiIGLG#j(_#ma`%Rq6{y}XsR z9HNRO5ex$Uj{4qfIWqDY@9V(r+acieL!OtAx>fOkwq+13Q&6+pp$W;nK}TKG*2Rj? zi>ezbiXTm6gT!-0&1R8)1Twk7oAjfX;oSac94E9!{4s6(->l);*A+*%&zJR5o8KF2=?_loOMdVnkW-KvxO@*-6`LAV?KJv6djs6rE zQXe|M>f%30Bu{fFgKdX>A#?Pj@xB7?^1-gGQ;Ljz5usD zdnx(|&YrDl+u5O!1e4S{<%S$eS>>94ZBwmF@N34f02R_zTw` z$hz1WE)Rcvtry^!Ja(05okkKVWG(*UWMV zKwZe=FCDk#gHLV(cW3k7aZT%EZf7T?JdDP9b35tyz-B}gXNv~E|B}Pd!o^E3&Fp8- zu!Wu;Mv%d08q}~5NjJ1x-_Iu{KY93m!ihU7!A|$P%EF!XoRCi5zfj1(i?8$qvf&M+ z41UWEF0ZuuZFhfzuCNn(OOmtuNmsN&3QN&rVyeB6u9i0ryWBN7-153hT$?W{c!4aJ z3df|L`?~>~%Rx36fr@T9MsA~Ix66wMCp-`8t6wmwHt&%M50cnEEB;RL9oeP2ju6Cn zZ0)eF)qx8;YvE_r#1#!q*=9F~yF8O2sfA;5Z<9%7pK!&o?kwAInpA*UAb<)& zJoXz>_3n*^#cRsMAF)+`7Mg5a6$85CgWQ3wnk#Qk4x=V7-0PNWA*nPX`ag&ah8%G0 zN(Ih*y%UtESW&;l{vH2+U9V@(55YpOVHtSPhr;Tz;rpLTjG6k{slZBluGQ00+K zb!a8CLl%uO4sH(eVLndM`}jVdjRc-xTIWyhDVvEsxgXpIBDlza5vxJMz3NsAqdE4q zIdrIv?6LNc9v-+x|HJnS+6z4eP0L10ZQXhC$gJz*r!IDNE#-gJ_J1e6iI1c;!e#1M z>OB9lp z;{SbecDAQN(ip~g;4c#Ce*!`Msg7izC?@>p_a)+i6(eWG^NNjsmVuuc0N6x!@~bYA zmt6Y77pQ4yTuw*hX3N8W)}wEq$Bkq@J_xM7vX*8m)m>>1Fmh#fFuSgc$W8u*Wkm5# zfI}4iBSBE; zKGOoRd&-7S1C|5eY~#ZXIQjXiE@&y$WhdVX92RwyVrr^<)Bx_!7eJg)m0t+)hco#0 zLd)a5HK^8s3!4I;NwC=&Mv}HOGr+a9Hwg|5)?5Iy=6(D#<4rd^h51)+vEOxO8wYYFx3tuz|0fI#Oj_{wJOWGNlS2C@c^8o z3=AqbpD)#UUjxF94PdgIVW{8wdC+(|SQukGUu%1kT9n)mxAD?h-RQp4AiQB5xbr!l z2cI*sfFVV{3}Rt0VY^)<0O6YK5(Fe(At8ejO{~~Yn2YOncAO+@^7iOn&)+VULklmx z58N92M6D9tthgeVnS=cbH8oO_Rkzg;{W%Jg+ohI_)8j)Fq7(y1!O`6*raSL-(<)So zN#?{B+jCO&~H|X+5tXOFS-5~pT2Xf&zCJ~`noHK@)Vb_3~Ns5xHRo!}|9Vo!{&q@YS z_@Xk92EDz>TLoE8iIrbgC(*aykSP}UoPlPjw(GtU_jXp)ovOlxBwev$JiDp9aydjg z{&U(!{cZ_ex0qK8^%4kKr}o-|$m$!Hv#sS@F_LR?cN$u*_aIv-q_fl=LGI2{nRI>q zM6ly9T%Zb{k3t^6aq%WE=KAL1C;DGcjM&+5q!C;1Uuj_Y3jWA9H3@R_#V$joRIY9; zPO7$#(cpC$zkJ&t)7dtZTLsRoQHQ&uwl_11emFaA@K7Jd#-ncVX%0{|Wx*KzXWqYa zaM`>p9zYD&IuN0D>t~SLtctcXC7=={Ji9(tv0d|ZRzHa0HHrI~+~~m5LsKq^I$>XY zvRMQP{z!J8-)_Hs1VQXjRu5bR(kuhfQqACeEoVv0DTj_!O6sO(B& zPVCJy({vI58AL>|&alT2)~du2RSpPy7T*ezyJnvc%*(H7MjO=6e*nT`{`;VKjj|6U zD1F^|BRHwHuWBlZ&N$w9Gco5u~|2bc&Q5273o*19)DcoV%+Jv zJS^8*Gy;)^qJj^8GAgPdK(2)@Z{m$@D^p6oE;19spr`oQ7@l_YikCG~$e;JRgKkm+ zcULe#jK6Wjw`j*iMj(f=iRyL@s|+SE;2zK>n>VA{POv(HCDyEEh?6~nN+#sebtCOV zUm6a`M;%K0B0a^b7QtKau9l5GXeKzxCGoCc`~_)ICyoO%JCL!n`gQq9r&zFX$jb39WO^rL0N{gi<7jevOOHk$rXA3Ft_M>nx_&L&oNVD1 zJeqz%qs5RWZXvkzv-9NoQ}F>7g5_J;P7OZ*AqRH?2=~bB*WRgtR3Y9e{iFLB2%H|g zM~Tg6T6%2jck#?}+|7sUMu?I!uzeahk_Yh7@D=htqT_LEB`=ohrW^ZNOh z{X2gZj5FFFb_gWnv&$lHDv){*YyG!HDDqT)5htv?@S}#kg ztb02*UDOa}QriVUq@czw!#mr{ws2>{+o+nTr2{gGks_f)xL5AUmU0`j+a(_WA zGbnH>)7_H92;YP;AI#W}58tz=Jtcu|!btl~pzYMIg7Qs>!0?gHs+{bjEkS!ZOSdgXJXyf9J)2B?8m_!1szJ+dVc%Rp3G(VzSFj;WbJH{-q7fc>%5R< zV|ksBlV3>WR;NfkwPejl2{8Icj~CW_a7eGbr#in?@BwYLtf9aLsnvhyIAm(Rjkc`; z+EeS4Nj)-KUv3QcNpR@@)=p_cg~L>Z&UNxCX&1m-TTw==+rtzUoVW_^qdg+mp|a_! z^3JFx<0g%i!EyKNjdpGV`&ttaR-Pn=Fj7u8qfM?xuB#+HvK7;aCT%DX|}8^BF_ zt?E)yyVKpg@MU(RC5jO!iL77;@i|s`5w{b)u?(bq*9kH&(k#eQF;SG}< zE_=!GW`jc-a&@z;^*6WNs>UAaojS<9sN&$!2gu2}94-CXfrn^aUU=bX%NJ$1Kh91F zBVWBI+lr#!^~WiuZn##`Kw&KJ4V(i6QOMy>;5xrbn*W7b<(yLixhiF995TCYQ@|7tf-j4$9yQaE@EYaMsDc+3q6N7l@)FE( zq6K&7?y@aexv`wa=fJ;QIs0?Tb#ReG9Rh@aPQGa4_2c8fW}lUN$&NX}$3I2qR;xXvl2W0?UdR3Jg)J?p^hb~9 z_+pF0qs2!O-Zbnfi<@V{3h1+fm~8hZ4wkC{@5aCpGw(LX!39p%SMsVg;GQXtc|J{R zX|C=VFrV%eP<|hd!#?0QJ-JdGj#vJ2`+U96^1Kiddv8BnVd>+($buc~A{armCKOY@ z{-NIJl_(K}lKNGMW8-g~i~ytWiH5wu(PqMjnN1XWtd|Mr<`x;cyb@z+!WI#g{+lHl^&C+b*%l2X-*gEqe5r%HuS);-SRb8k^b9&;w zRl-J$wNsYQ#A&2Ox4PEp()$8{i=~PU%={y6;$l@BYckO)*;lB8xRD+F03xU#+L&XX zVCLSqJzh(D!)?L6{z-1&k5_sviak?1V?{fWR^KI4Ty=!zc9({<2NQX;Ocg$`4Or;i z^dHD2)As4g@nbm5(|WTXhA$a?0LFCb!A7eP6Er%XSn^J(N+1udJRY4|>~!hP%?SOm zr#?&Y)}whP?}V)c`56k%0aOEyY5O|M@L8GYak)nyHPvSy`WkEHwSe%|p`+mnkwilK zVdRDTW{#%u;Wzx}$=YdC-sv_l4K1Fd7M~i3Lb{8VUa#&t<0=S^86AX~1rN4ZWLuM- zAA(6ZH6RxTOrp^^lpIhY22R%@>0tFC1?SI~flPx;^1{vMCWeeMvAaa2 zBwD93ElhEGsE*}xJ#yV@?%}e22B(ReZUu=&g#89FPUZleWA*IbmETAIP%^E*?X=(A z$>5^oDOI=c*bkYkuV#zZugjtkqX?W&PmeeYTv~rZQL(Ra7fclZWD*vwd;&#B5};3F zQn~dn=n1KFAwsYfe0W1dEA7`h_sRaS0fz7yA#>hvHKksmfadZ8MRr-&Oe$5zpV!TXDFO6+`d8J&7vL4+Xf{45v?h0HY@nF_EWJ-;33JLzLC zA{tOV6-I4lCBoDTW|y_wldrW=P`YNzP{q!PG`NnGS#x1s5?j=DeZ+*)RMG-gIBX$* zbf1@G=NJ#oKEUk8p9@;}0%Opb2}oy2#O~y-hu(Kn^E;l@5`*J`Y`ZIC?L0^;otui) zjbEMfzN}Stw-<4!DF!d!o*SYg+C<-CQut6@m&{JNR(QRql^PXU8#!-YKfYP* zFm&mj+ew4N>dEo?Nz&uQe*QbN8^k)}Y%k30bVE2!?q|DJrGDeXbQE5AAo@WbH9@I| zAe8~pmVW_{l?T$eU3jzmymoA}VoY703}=>^gueK5!}vx4$5Z$&4|jn~mV?Lny=HQg z3F|-~2K1NBZ}rHI#IVt;?_bz>rHOyby9i@MCJ|x6@+;(0O+nf)+xKkkxU5u){-v@9 zuK8|8DoT0{VV1E9Z!Y7Cs*;d6Zkc(H;=E%#ka=85;g6y%vDKC#BSGdv+ zmU0)vdGeHG!VSF>7R*7@xl~fVCZvdU!aRQ{g&Kdu@e0~z$rh=nBoDhLw%C_p8~Y5% zV&4g4uzsui_fITS0uFY!ld)eDp5jL<;1gaPalbWzF0ACw zysY0=zaa--mnIbllW+8l;B;Btl4!a{;4_%kIxM{3=HP!kB)*qqVzqGb# zy!8UX%ZuE1qy)zVzF5o60&U<~SPC5F#s++UY=Ow%sQwuWP0jV1Yk?2=&Yw*(K7Po1 zbGQ&VIgTzBSn^7f^mGvCAQz0E!4f}_$p>5%=1Wn~3^@pMU^cyxbc5rm0j=@()I2+q zOb?+UBDA1kv{2fBpu~3#yUMSKhS3rw;<$1)Bd>OM{BwPbzI-_Y<_^>?;Wk@Br==Ad zHm1VmzBPeg>PsM<>zO`J-#d8`<} zt-7-Ntz^@)Fq4Sp11x_GIOY1;Cos?xRFZbqtG;sB(7Tk?7S4<4Rzj|lU2fX>D5cY3 ze5is*yXyIul%PSJpU|%U%tpJQb-rs`3~9RMhtU28Z_20Ik5ZObDM`e}zlxKu?_Z?q zpJojLvm@CeI!QiEg-y=6tF0x;2b)a8PyN_qd9rq|v!ytwWdatnBlp@5HC}e27D`6Z z`!Ssa#}sy`XJsV5MEtZMBmlQpjmy1$crWyp9_6sLr@x({o)+?(S(p;?;3BXFD%x^s z1(*o%&=8lrY+mzNuqA2s;2vdG6s}g9$H0qyfqL2C3RgC=w*D7XIzus^II65i&zq@d zQbsv=f=9iQlz1?;Z8(22N*C1&pd|5uLcckh=|J$omk;aFQ#J_wPAVCDTiKgu=*G4X9E2J_oB z{=ei^C~>a#$%Xjo^s4hiOpgf(w3gEOIY$jIr9&lb;idHdsBxb-Pm6o6<-c!qx%^kE zg>Mig-4ZPIOd(HYsvitZnPSt&A<7phm=YM=$wrEwK23ZS1+2L#d71b=Dpr}^0 zvibhof8Ai?(>&H#b$iA?RZlJJcW$Rz-5Ale@!xJJ(8|9GfEC!DwO{!k4RGvK1FR&B z1UUUyg@l0>#Hr%7{uL%cx4QDw$8|)PZ}yFEm7dR90fqZv76Us kIqN?LGv_%i#y#LA-#zWg(bhOHH1~|2mZ4^u`opmQ11q8^mjD0& delta 21230 zcmc$`byQSg`}ZrMl(ck*gVHDsk|N>I-QCjC1Dg(!P(Zqrk`(Fg?nY|p&Y}BkpXc|) zS!b=Y&U)ATuJg~to;~;6`_5~w&-c5_x{=D7km$md6r`~+$uS>2dW0=2^H$~2BV-8p z`WN~W@KMMLBy+#$G>#0>Mv?4<0=ddnEf-T+IV%KMgHay%NzaNu-STg`O9csK0!roR3x5TQmLo z)ZtJ+qfvCX{9^H@{Nj^IYl#+~l(I4_EtR<7`~$3f=+FFKtzB*ZqPe9=#)6B2!Yf(d zURlWN*m(#UouwH%DRPJ(ozdSXvB$I{707)?Qkge*K=l!6o;m(d6NjXZ^8+?KnSc?Gz+vJH?zgf_t*B5>YA>D$viBAjqfN$U^OX%t_333 zETn<1>^4^WE^wcVTJgXqwQSY%V|I1hxwQrB?Qr11XSx{xs{7gNYYrBG`prmP5{^H$ z53A9lQ$`EfvV_&AWACTYtNq6CF88b9^A*9Xn6M(E3xD4_%u?-1qQeJe^ z05uqdkd9GI5!ETjf#FF$EG|r^Bw^3rhd}UH$Kh&S6qQgFt1Iy7aO1xK0TFwK3jmi2ZwrPpQnO~Qn{p!&NY%!h`vDP7C#z)iOF zj|~GBDR$qor@gO6UPTKdWELx|IeJCyH!U-Kc-d?SrebELpY%PQQR-&OfW7)DIXl>R zv6vFv8AB&I_I(ntPK2r?QNQsri(zHiEI zNxi+zEN2FZA&hI;sh_Qy5nkD~>~;Rf(z2xfpsu0e#jq!^nU-+$wbR(<4d0AWAftkZ z%iYOkzOVlwNqj z(QqW8q5VQpAiR|#t13%LpkK&hAd!4#=%-N|V<9}pAnEAS^S(G+XWxw^OPgP{=d-r4 zw{GAp7;uOCBy_xF;uwXPMF}0$Fme=AOcN>!bM*qM4tVL@I9Mg_i=IrcT{fUTX|9pY zgJAQp$QBCMEP8IHX>Xs3a|3il<^L_a@4RCfOWe|(rztws$~i0(Fm2!N7H+-!9>bW?jj3kp_O3v;+FBQ{g&Lbn=f z`|W7RZt~lYhEQ5Ev+jk%<)D9ODkfy9hY`OjDUm`hAVKO~a!A0WEtDzMFFiIMx&B}6P-^!Iyu4{c;7y?_kfriga;#93gT@!p%M>~q(PY_4ROL_>Ae&RIn9uCH zSJsWWLgzL)u$`B2pjl!hWII(9V7sI8!=q3D1svp#!ax}qXY>{q)5ZU?8gc)W2k?XO)w zVZc^y@ySitzG6~a`Hh8DMO|e_UZ6X55Ylvzf7p`b!c8BaEueHQQ1&B0K}302r_G_p z65!vn2+}CdbH%C7D+4^j4IwsymzOGaD`h@guetC6SsWn|t% zPc!65pgX@C!1r;50F`qOoow^uaX>ik!>jAZ7ts5%m$YmT&IqrgRy5nV2!t`U-vqEZ z$OnhA$Z2RbIh<6@*%y8p23MWr`1H{7A;JD`N!kG^E1qY7WMhDS(cq|*MsgMXT5&|n z!|kT12?a_)Wibf^IM^0FhN2jsfQ(Zb~*N zO++L$`Q#s==%~tA_a$?M8>`w{1lOQobG+FJ+n;9g2u$lUOl1mUSQONUm3%q za}v!>UKlu=P!+MSwqHn{xlwJb;qmo&o}p>r*?H(_EZr-5e~im?e$v5`zQ7Vk7x`As zZDJYhOnHNHl$9QaXlVM$LQlbLf-a@R0yxlj8jFtE;ox47`Z0*{W&P*36kmmR<(qx) zZuiM;i%@AFtqrmr z$CKXEpGxYUj3Y5^O5509>=*Juk z2dHKn!L2MRjp)?yT9NL%erMcfnehc`z0izo8KRHnMxSmuNby-vCga)ilx6v( zhHkBSg8mddp-r>@MIk#=l`Eg)l2~&D%KTfZJ@=?%HizH65c0|9P2D`jez_U{oFa6e zhlw<tPvC!3WMjO^G7*{n!Y+VmvG-(^d~I`dUpq2 z51+!V8Da~!R z*U;K!RPRzB=5%WsB>=%e@aOD{m}t(dCm0?i-bk2KE0WVVDm>c?iPi-JXI<2hozTu z6)%WSAy|$db>{$k7LbNyc64J3Pu?4T@QinuB?=Y46KSurX-021QB`;e=Us(r8Lm^5 z_-;o#32nY6+rrfG4A$g+oTDV*xm=A0VCnD zSL7f@SX@V_V))AlhN=^zC*WVNO7M=~SQ~gfR~+cfQQC_|2RJ&jv_yV4HoVhbigg&0 zUDYoU%sE!TN-@IiOf)Is9;u-36`i}|Q+A#Fmm4Qy%T8X_CjOWD-vg<1LCahMH zO%ADd+@g&k0kS{gyxOtvJhyqCLEqZyuD+>PLMUGjGAou`b$!yi?dyGCq0h4;QH$sI zDa)i;o+l;9m8tk3UEJg4w=v>AbjCf403f;f-GFX)C>T2P`Np3WHE|f%*5Y=@7NH7I zYb|vMdjQ1pDpAk2TN7JNM6M`d<27-G3f2uS!RFB(%0&|7gx^D{J2Cy_OrLr!FZFv9 z5zA1NTJbYTVt8jDR>>7AxD_th^@u_PZLMCbMXz&GxrtR;BU*{xe`v?N3OYA9dp3RX z^cS8)Y{+)y%xxIS=FhemL`zj6toHW;^re0G(GOsVDRE%%JRFT4LhnXa#-_X*Osn*C zTP)QgWhkE7?(0T752efs5oICI0#dN4L1L6Mi6j_LJ89JYvUhq20c4DQp)GR9WH?#l z9J8M|^>qZ=FFKDRgF_uBUv50l$iSpsB123cBKtpoI_+Pbh_Go~Wm#41d0m8cey-(> zj|o7I-8BZPOLkweQ;&??ED^1)K7ab3K zv9`giRtAT-h-*06FFi{BwOhcM^p58jYh$w6MCA1xVzfSu#Jm# zh($;ieRmCrs!pYkc@eA?ChCXFp4*Tt(iCF)Xy#ZxB(DFevI@@I&@LF0H>G=G&cbF& zXBsu73}3Iywf0&bWRFt$ncASG*9BlH^buj6Sg*63&8dwGH?)feEvm?pWYU52`LP~* zdh)lbe(4p{cy)!rh$9u5lkro%ZNqDQqV|Xyr;H%n&7wF7a20>)N#GupYMAr=u`Jar z_xH7J2dTj#@%v9FMrDkI>KbaP38yt)sXK9w$m%PJu9Ovy-_^dhw)fJmrvdrgMI-S) z5+uKdzkn4jTZKz5&C`BaEC2n)pTRRKP1NZV+gq^hRuDyv=d-dS&i!l&^CWexqhx4& zFA~aK>8NuW92Of+6=l>Ow1*qM5pf)3ZWHXgB4O z6{>R8*Rq+WCxxz{6%idy2pG?Y8*NkJp5+m&96k!w%{fYG(@s0|R6y@Oig9e>2x)W1 zYfgoq;|T37U9tqfO_Yg@imgQVk#fNElLePx*r#y^=!*AxTD9psx4}<11gB($C8zgJ zy`skVwUmYtKCZPwC8MhJ6pYnTA=FIBe#i=DJ0S`g<@E?X(?UJq{^KF!#c<`z+am`SRB8Qw{Vf z^Io6U$5E+vM(qnOpv&clPaE=95OD~RODXlk?`!wvk}#98p<;gPW_N;%Ntn4fI!T^| zG+YADXenFXMIQaiBlys!T@#w%OQO^h`umq^AQjiG@6$Vu^iwJCn5PxXd@LIgG3vz! ze~<`O%22$X`^=$JLzSQ(_Chd7O!r<9hzd7ju@+gD+&Cjlfcd340_?npI|#A2Tt$7N z%{&UOaHhESv`Vx5sL~}I3Clg!oQrkY|1oMkXQ!Et1oP$$Po9wC2?iG{rd2}BCG{}> z>SxXuwU(P-5}RRnU&U&(8e6#-M@O#H3XDlFzWXEU-DXA!&z(@E%aA)p?+&snoYbL{ z_tnR5z0>1!0X^3~6P-Ot5u*13P=T^pHM^c_xUcuOS(bC&AByhop)DR{4n!>rONQA)4H*eP*VuDdN8GYhDbB5CDR}I9UJE}e zBpsKryHv673g@omGH=bJJ!ha3-WLwV3pDAY=)yfTeT`!AIc9}R)(2{HW7jTJ0(|Jh zIN|s>tbn=G!q-}Ze-rIbb0{^#(!s2%U~sv)#38>D<{5y=@E!dG)UCcJEWWe#+QyzDdEPC~#GLfQVaX)6JT%WbGkeN*Dll+cmqLbNB zAl{7?W$HC$N-Do7J>$?ix-9t{SW&EZw$a2#I-pF{5Mx|iCV$EFMBy_1=GrEshB8OE zzvGqWAb(CA+h>Y&uJ6nApSSE5S@!!%X?AyFdrsjdJifJw(;*zqE6h>3i)LM)(p>~i zim^8hG8-eB17|8?3-u~742^?{et!*jF`aTL39Y-Y64;f2&`^dr6SY3fvTZ7h6=`X2 zn*zR3>PJNdSZUR7Q@%xztS=AXR3KYzZMCOrKd(>oh?k1v>1xP=rDzj_Q z^CVJDcroFJLNZ`iat2A<@>kHY!Z{}RsRPis+z0XEuOPimox4^1$P~W*B>o5ggQsgk zh-upw7lGTQOtif%A+=%3@2X*()7V1)~ zf+;1i>wnd>4$UlkBzit$8q2?oeINUs%PH%D3M;tc(mBp|hqRSHP?Yo;rYtxVwHCh~ zp8V8YS>R|BZeNzB>(qS`PFT>rqA##~NZpZsUALcbf~REK_WnVM@1+xeRKXr__U2|l z`O6RcIPod(=5l#FBKquRi=WIaYDZ<)L}wmcK}q5#>wN+Ya+FOuhxzqS^Y4kwryLu1 zgyd5W$sHtd+>2wPP*Tj6E|~0*(XP-Df@&D(e1)Yn4?3pOqlyP zGOHAakI<`=yDx+16ir`~*(p5*I^o23?_{IraONIY3S^_b={)50>40EXH8GEhwsFt* zqaM14`K4G1*)%W>!(fxA_zyw#P$&jRlPUVRe5$Ag^~CJnhr0JeXG@Lg9BTOuS4~tn zK3FFvbH!bo^9e^(7t0Fmv#`0(?#tC^iagq??gN5O>DG6;08b)fr$!3!%BkR`H(c;4 zKCbx|t6B4d$xiyIuGQ-G@c{qR!QN`JRZQN>oJ)$l&-5?W%<_D`deWzkb0b;sZsgP) z)Vg^h8jT6YF8k1+Jgg0f8!Kzn5^|1tx*oa=@)FIa=8E5@_ZaTW+w4QmlP`B|>)J@>3fUEB?$DT{-TJ znjYrMt?t^a$Hkq?9bLs zYWH}lp{OWOMw?~Wg~gA<70j<7zWdA|$-U!tV@@0zNVZ{!TH#TogjB5OD%V~Z_e-sn zF)q7JF683TGn%buP13(GGTLuOwBvD^sWV5pN3OWB19=FE^3p;A9;5JJ7agC<=?!W6 zj|Jh1WM4`j1Ke!W73$*T`IOTo*hQLS3^_X;Qi4OTFs`~UbfzkVNl{lWmtUjB1P5_G zGx@$QF*6=`$nD$1)UX++ol|F@!M)7Gigg%4_XhRFgvGPiD1Kw#TU+@(z_&W87)qX17gjOW@}^jvc=PM*G1vb!wk zVx(rR6v@IX@}zxal`-L=*Zw$yBj&bW(;zZ-_mf|?5BYn^5*mprb>?RS?_@*k_Pz8dR1G}HaHNtm%f z*v)cUB=J#TXAO%x^dULM08_t|F_V9Yx@Bl6ni7Lw0b={WV3=yz8lAFEPo{LIp$${} z^{4oEAG7w5SnXImi{2AbLwhn|y_@s>#s}F@94er%vuO(UYd=D18Q{i-I!3LbVcJT9 z7~-<*4J>Gj<2agpDtE}`HTGPnbzx3|!K{OKUfQLxTvCgQU3(n%rUd27ZI?@jEjN1+ z%hXnL^PT3Lndy*^4V(4LXs0F*aVnQ4ftyfDUk|)Iha|JR==EipZrrQ&*^ZDO67z)EvaIdGr|HB4ok)^%vq0V0BT zbi)rRR;pjlS{=5FH^<2~D_3ehKh64t2W3V=X9@f#;`~cGCq$9Ys#x;w3+P}++ z4>R@uznAsDQWWsNv8?}gURcE z`0QpUT(?H(SPDh&FWcqQ1bHxQ;Z2RUiw#4cV%J`CECy+Xp6Oa;1<%7y21%X_R^G_Q zLTYGxw2Jx-RQ#L#d;rn;A^xs1x~zGuBm86N5V z9L+F9dP>r3%evC0>%Kb>JeQbuN7*Nte{+gp{Vk)BYC^L$eror8M@*mj%ezSO^`^@e zEItqz8(KPEEr-%j-dXfMPdJsL=KYX(17>IEy>^Q?lfg{OxlMNE8z8B|YP9Vy!8RW* zrF?1)#}&ODv8xz;XMWSyDZPo3T^ncu$c)5H5p6JaBBJ+;~=O9&QDl#`Y zU;i+kE5FglQaW(DGuhu=4>I4c{5kUt*E@yUJ`V+jCCtyS^dh`L;CbbBzCF(5vKq-} zQDX!5JROzx5dakkUxF6E+PSJ!foR<9FBoM6XxLCz(d^o096@Yu_I9K=_SUxjBPc64 zTVL=#Y_yFPKDoPqTfVK49oyJRkwvZNLM10dE-Bf$*~|d0#ih{5OK5yfl{MTz@*PG4 zdt3gM$R0O`!Eweo^m*V7$Xc}o0|`JXx9FK9admsWTPmAm^#JM-UhyO>HhNoj0)+cg zHVq~y47~=}~7a)Cpx^och+e0&=-M5^*{RVT|5F!U0 zkE`{Bg{9U&pA=h(Bu1(Y60)L&o-YhOM_>vkuhiBxKHlxC!XPg29p4e`R2RFsD%ai8 z<|aTLlNQ7_c^$!jZOpuH~mp#-XATiRGvebc@K%37R6?@nVkCx4s2ukMAU@Q=26 zc$enGs0d&Avi<(;4D|qczP<^|Mfe*0=CJwjilb@E729LR9vOw+h{L^5qB$YB^Uy}wL)N`%7=i<^!h&ZS6g*-3)!_=xLaFz z661C~dhRJznD_NWi$97_GcBd~Rv@m3qlp2+>4>lG@QFC3u-k&Grl_0JL;4%k1f(9J&(zk)-+6pM=3kFfef3v=qG3?b*R7Y`MEXkE zqQhyhP|f`N&qy_w@K34)PcaB4|0V+vAXwMRzbAc-&Jzj>+CrJ!}Dm$xoV-W4b3b z4bwUb<0v>s5j#YL-=Gz;_*^bjjx9*C4d5&pY5w6`K?a{45;{z6*z^v-G+t-Oxy72p z6P8ptAvf<_Ls@?K?)w20xqNY;>GQ)Nt6TZR0{g?>c&cKkG4{!E4?gPt_ScbhItpBG8y?^wtOur)CnQ{_b)hL`?E+<`4|}Nb)h^L7DPycmLU}ZRr^e z4NV~d3f3VWCGD6kIhCV?AtvQBCwi=4B1%^CQ0cEYbZBF}f&Kup30?qSz(9jo4mu>* zvCEqHLt?&DN${^chD0&JEn)S_p+bZI6(Cy6^S&y@BV~QcOrz^rxal zpN3dPkv}I{I7hftmXo!-dkoBppTg;TGpa%p<0^n&%<0hJyJxL$E3kSqO9>&dnA8P?*v8>^r{;=0r{;70c^jL4O2wbDD(KOL&E?NGszr z;uj1RhfO!vi1Hb~7c}cFKdH0iZU{}0g5dZ87_NDSgbAEhpa$oBc-9ws+OUb6CG!|_ zmgxJ_2Gv%x+sk8F7%Cs7YrAjRw1M~4N5Lmwcu%>)Zve=h;Rz!U)+8^8ATCbB#p-@@ zT^nVsV^7n2Plk+*=b!A&m!moTxHo8}BjSxQ8*-rL8wiqfU>~tce0>2oM*D|H!MdN; z121fJCN5&0zB7pI)li%m#+##~`!I=TqX?CFm3V16@h3l>f{>_9+vm*@E&)tr_sJ&!W1Ih@)9MMiy3)ad0w-?Ev(Fzcl?-*`JSQUg;)r^1R?8 z3AuxKCOMB6X_pr%L4!YJ1)Hgy_(%^v&Ob7zS6pE!VJU5JCLm7s{@UWzlm(4`bDoF- zByD{iGkSu@7RD!wpBVB7l(TJf1}^$sEc8})x&vWkId~ef@18GqSP0<@r&b9;jGHV6 z1Y^((7NI|UNL^C*&`+Brr74|)2qP6%E<&{N1VPC}n$`V0D9kv_s9ZE+2q@>f!X^?; z(Gz)2p3y;H#6O^8)Zu{|J~eFUa)&ieh~`3|B_SB~6Ze@NW3uD}jteiN#AjXbqVFPkZuGIFVQvtBarU?G3>3}C2>#(t>0ZnISnry=5pwb*EH;!;l}%pQd0vE zK^K>TQXTlCQ6EKF@&D*l@N=5=T-VlcYGMv{Ty$*5jzz+6#hyDKm8MC%)SY;OOPNMi zK?g&|Q!6WV-<@J(5=vE#S0hCq<~5c8+7i%B2uA&N#^YA|g0h25U-+QGk*=|kEAqF` zTvxE-QPs8v{g6-^q6fj^U91VOG3^4zU-_)XNa7sJ;wj+?%Ai>T%hb*IR$xr#W|Dh+&nU7~3Zw$-0V6G! zzd^}mXz8aqO*(R8dXaG^8k8p7=lkP3cu@W(CF^HVfxl%GB20|IHCZh4YM%9ZRL~x} zmT4r9jYqC3j>gt^jWZlQkH*y(6j&UUY#f0^fzJpNXFk~W=eFjWI(8W2E(Z~5i=;Yv zbX+C&Zh~5z6W6XQ$51M_o-x2H9<$4-n`)DaSvC}f6(`^-Z1UGv8|u>$k+$}D?z2Ly zv2dhCG#c2i2dtCUGR>E)DIw}BMB+icl-*KbzIgb+UEA zzLA_sz&gfUE%@bMrSNj^g4{w=(#Sv}bBG60$E-Y} zt!%Q1!P4TH*5Tf!NLwLIm-?sIuOOA`bHFI>><%O6udFWah_No(cX2`_f#7)4I zhHm5CYfD!;k~#@gTbu9CIB32()C-WK9a|!fIn~2Z^4co`t_tupw8@CCT5w1O9u0Np zie_RMKUE$}dvQIWK%r15H6nr61!WPDzOTG2xOBGBtGcr>WGDjWlT{T4f;*xIZS`;4 z4Ig?E?#ZL08#KnbFRbsV%Ok(gadI#y^o-^hluMO}_rA^EP9*)PqC`WG)}y4$!u9A!T_dPHq&h$L^p`MxFVAb#Du+Lg zT`2XW!o<;nB6|-ALIeuVJ^w5A@D8i`V?hDfO_!uKfA!J2~yEs?(@t< zH;>=~8gltZ3>{+Bib(|!RMD0{R>RmQ&`p`F5a*v$9tR*a@~VcKeSLhHZ}0wC;D!_C zH-{kCuY4y*rw=>HzsTdi*(>*Sm0ZG*D_24c{O7Fe$}=`tzqWW$oQ-2-`K?9ceSuwQ%+wa;&njn}2K+pVwLV z{dMTOL|-7fGy)TH`AaNI;7O|^C`3950Ot9)oJ@QJNJZ)PyDta)FyOqp({y$e`_n-= z@sW-Ul9}SPw1E9qBI3{)gQlo=mxbW%ChMg z3H}h<8){>Zss7A(qG&1>rJBj;nzhl9YOHc0WZ=UyrpDjpVgnl5%XJjFctd*xMw%$w z=6I*|=`*=6Q8K%aLCw?qOXpO6Av_ZTbuW$kx_UphpGWbnGfc)LGBx#nh?DgF7(88} zKT_hJW|eyOAn)#H$v_btTWneI8Qum$lJ^+(})xyI?a+{ zY(U!a)JIMw0(lh;!8ayU_t}L0?|x`>sB>#J=@}tc?Kv(}KV4%bHhkA)e54BVpgD-5 zOFfi!%1H-9Sy=WGjgZdAgwGqD@gl7_ByJU9=R=3N7+=lbwhRT`C2~(lLe`$v|9n9L zREX}Qi;kXq5bsSG3&9dISXE%ud+1Nwe*&ibZo&1TVckPwZp&8WeMO0Q#OaAF5e3<$ zs?=`VVUBS`8N83YJ!}k2Bu;(e)2!%OuLSXOJ)PGp^nPKsZ(V?aDj5G@1b-XSFBMr>Dw)EJx?_5UegW_6VTBE5)w)xg4%b&Y~;(?b< z5G$b>s^AmDFj2IxqrR_NhKSI$-i+gpEIE4FLQe@bYEB6!9*SOQvn9l8A;se65e3T2 z%$9P7FENN|ySi(WmJ%FnKkS`n-W|b>ItP*TDPL%?_l2t0BK&aok3or^*Xyt7sOdbp z&nYdqu8h3C!%YxaK1N|-79kZ@i@-+q!BU>WpS@f0n>K%M*lQ=loDW7g_b}67^g#|0 zQJAIISLZD5^F=(XE%b>(qp^|Jas#M3oa5OkV=kkIoyA} z44KBXnZC4f-xh*)evEO-IeFkAoUDGUVT7{lh-t|1JWwMnqO^7q9%= zWwB0Y6d=1^86MyBudJd~1$v^<*nbxVp#SeH>wl%FfBHZEpY{4%)*u$qPsI$8;O{fu z|1^d8V@hVK<|!P2X^hfeD%ws@{oBx=J%et|Qwh1T4)mJ*J5>rc0cR*WcQ5)N9q#*o zx7uIli8dZ5P3-R$E9n1tNdEqQCMDP^Dmc#~L;sVzje86xTlKRP91Firas>vLDfznYj6wa*{yiFL72Fmd&-6aW5fYc?w_FjC&_R zbiSiZ74a^E$EAuw$V6q4Kd0f=rA>{`4)Iy#tCrq>Ro_o64HLrPOL@qC)yy>CTxYnR z-f$3zEah}irE2JEd2r+>WVspYlMDyQGx!F_`CwZ(4>xQFJ+AhyfugrOJO`zcT{RKF zh3EW!t@EWlcKxc zT0flM#l`-Z(Db`h=;9ct>-=VYXaP-$=lgRWTD(5BXi?kXRQJ*0Fie^(nU_P+H58kA z9}=0dqcLZ;-QYR*{vX%!&++#H+kYcRii6GVH9qRMd)@Bu1U^#^`6&+f#3k0{kgvJ1 z^($w0n7IrGU*8C>MM|d*>Cav94<^S;b9!$6EM3@XtP|MzZ->*mWTsWCSzCVZLH%$@ zeU8T73*@&hd=CL^RJr8&gKa1EZfV4hi3YifX~1?n@qX1W+tJ@i&A1O)c8PCnt9F@T z$=r4;7SN3KuJFTd_Y53&(8b`!5GsE(D#TtJV>_13yiQ{ZsW)t*T1A({BK+Eg(O%On zZN8PzT;EHWSMNrpb2IHUZ>?%kLxUFSj8$d3&cnUzjpzss^1>$LoM&sES_!8`oD&+> z1C7J;{6k~u|4Nc0NgCnczmVgsJsFs$MDFyOJ9FfFxG}HZ>C#_7BC%h-dv`5*RXKN> z!BXE|I^XAW)9dJd@u##td^I;rcH^O8L$d$kZj+%wv?p66<}1T3gHN?CfDFbm->ZZOkrRTSptc zR>N^mg1RzF?`o$IT8*W{{S20NvoGJ2t8Q(q(9IVSgBh=*mFk6-YQR}{ud%@UWVEy+ zn|plUb@rWo-jAu{w&^Xy2S;1`GHA&j+x&lKsZF?;xGrh{(DB~gXXl*XQ?h#9*S6+0 z&FsRBFYAb#NcHAR3Jdf4QFfL(Y&RV`7AQ^A-5Q_!NT;p`p?owC$w@%Zo6U2V2=;}c zG_OPU`&k=*;`?;V0Y*(~#OG-Zy_+HbgPy|b^Lr*!-;}MP*=;&}oiXH72G)dx#BK#h@0b^EgIjY+;9VtZF|;ep+!TmO#(;69mxR`VvCTC$mP`+xw+`q-Sr z30)pfqBp#Hf9cLlV|Q{e^bD~1r7IPbeWg=&qm0UJOgRz$2t)AAdM!Q{pS!0A0Zyfw|^CQ1Zj7W`NB#N6n?*kHu^SW^U@ zi>0_aJJB*XeB>yBn9<~D<453p&}i&J-5h&HgZMb%SRhIB`r`~;MbA#H9JfHUgu4+HpFr`2&WRn-Zh3kYC7Gp1CaZ!$IVJ z5bZ*%{5_&BhksH+{$3?%sWjkZr=+_irDOTs>QCAUjU8%&%}k?n?YtI+OwPy!3S`&F z7LW{VfO@)LMIVOvfz_wgLxQ_z!GrsuO+GhuU^11rx5|1v_h8Zc(A93v;wJn#{$AHV znpQkB(+FAaCZjoN_xPf7KC-)cZP9Cw<>CkW#f81Tot@CW~2$l5SIdzK$pc53R+OlfTuKzqc6?kkf-A7@XS$SKXZQ=-8w*r-Z$mAfdJ9 z#_^f!U)7zT%X<%i{;Nd;u&VxI-BDBvAe+^p(cT;+jFk7WwL=h(7?6Y8trWe|NVnO> zXYGSpU6J!SuT6*XF}vEs9RuCboSQb^ZY%(j6m595m?huRyqZ%ldO~!!l|!eERv39( zYeT`w0r#=^1(0AO@&7TIGQQ#L_`|Sk*W3WdKN%Vi^dWt4)YH#?pVBdG8ek=oDZ5%2 zl?~tFcu1J0zHHlUS3Zl@ziWy<`9mRT?tLUvy^60s^efNd!f$Wjs;3clk#}&qPXjoz zlr+}t{_>a$0vC!xH@q*TBIstj_L}?kD@n7W|Fz#yAP)`^^>s(WYdr7I6)B?%ntP59 zP_FNn+_&;`vnuz7==K&YoA6J1o>gb@hpfZCL9Fy3WX z$h8#yl>%27IGoq>nE}cVrs+(5xI)6Va~sjA)1JQ}Ctnox>a0JwFkZdu{befUXPk13 zj5WAvP~+q0y;tDAFhEv*cR)47-?~7}dhJa8I3y}@N*y8Yi0;0bW zhuh!KY5o1DaMP+=Cr9#F0rKEP{`}7uR*EBDzv4BjH$t57Uxmf?mok2)Sd8TmVL%s( zV4Z@%;xE!~XF2LJqXTQh*@ozp?T3W78`SCp%*pTD`SJZMuJonoaO?U06Hy`P0C0S) zWxO{~ZkpfVQi#62KJ+Ylr$P0vcF*+1f057i6TYH-@Sl}qBoXh$*6nO!%da|cjQz_~ zd`AyPXLlOSXSch#hSo}8BT%X zT27qo-tGkV%9jpW#Q$unOf^(yInM1l_mDZ8LfL~L!rYqk{qFk8>GZ7-+){F*n~RKX zFmvz~guTZH=W}`SI>WWp51=8+8O+Sr+*~w|D~5vX&U*n&AaBi9+3l7pzByAY!q{+GW1>!K96>j#g*s7wE*nEx{7LP7U=r_i~N zMCO#qsWL1@dEK^$C#4%{RJY9a}dHw4K3?oHxDfvf4 zBhG2dLA#ju0O(S`9RP`Q^3fG2$tM&6t=#4tEhxCBS6{lr@EN|kTk~}uQSsQQS%g3T z)e#+@1R2K`j2|`syM0|9U8FX}y2;uZ7d?_PBc! zYRmDDr34+{=(%c)$jt|fT+3mM$E!pX#-)uA^K{s6xup^AZpxE9DCyM+aCx<<{i-K& za3I_$K9uEXSlJsGF7w3j%!yWC$RI29%ke&9S4o$8CM`G`)cD0^7f_!XdIqLzzK-J6Fcah8vDuOUmvyOzV^93 z?Il=QdS91fU-wcr58z{dL_`4nwCoMK_RR1Y;~|Raspuli{M+?MUbEJD<{+=HW(;e1 z!4VKoAn{V77bKzF`CGOn_b+E7$*;n?x+z%HQx(V%|1h~d2&L+#J2xZHe`f)3_|f4ABK# zQML6qCLJG`PJ+Oh_NUkIQrOu{J?X?avtiX13?)`*A3ifbeABf*X7-7sMX4g(4+Z5+ zXWYPx!%FM%0*i55QnCyX?s`fc-L?&;3cQA`gHn^&L_D*zT{DUIi>6gAYb`VzPm{!A zH<#N(nT~7x4?*g``w&JF3#}1BH%NTCR173y&_cP$A!*~Goew?I+=`1Ia}MwCFPept z9yL1P9HQY;?-_0J zmJSkDT!FQipkYvTa>(s4ASTBk9W`JR{TB(kZjVVB-kdKP=cT#HwHbIUKmNwx={ zj77Cm5Z^N%VlJ%hWY`dQ=;fm>CdR&f{T%Mu7kaPx_Sy}K#a@$nI2RNFWD+%3S^i_Y zv7uE`;%v=8@kd=ohDTdw6q6^4T$z2-bzcZSh*Qn2QP;ea+#4mF&lFdW^0~aYTcwGe z*LRn9G4vI9|0MUg!~vcZ#XT%)7=OR$o#*Xq+BM)NO)c=JnL^m(o6t&F@2eO^x7|`v zWk@E*K^MizP3#(rv>q_Wl#xnzPpTe!=D8{Fq^Zv%4`D}StPtye3>a1={@JA0&vp?? zlDYH+D=-l#8_v3;K@#VN9Dv%KkF}5plgn4Ne~r5K!*4E536H^*^}q+)G2j+tL!IuH zEYn$%XiO{~p}=F`N55y?zE553;nVZZ7@<_pJNFAH_un)F6b=oeqM^R8g1)3<9E(0& zMeCZ!tEXBK3*+wiMRqXup2PJoMEm*<3c2@f@k1I*C`Gug2{L8d`H$qh z)t=IK*MA@l*}Z{*a{{JJOvm5@muTSvQ*dD1LW^9h_R`oRnws!8RKT=*h}iX%zf3^$ zDh+n;84rXdo22`C)v@|_TUgw$e{-I*Q+*u`2)`OEU+b&iOz-DikEzlu75)0*n1^oh zXP(AQ>@DJb)yq630RoHGL1pQzI>c~t!U5y;xWKz zr6mEhQhRB}qJQbop6(_Saw}3~h>E}0R=*5tm%MxIdwVSX@L3TpNEWG?WF^)6pq30K z+OfkT|6NQm1J#77mzGh79^B|U;-&zqN9VscClEJ9H6ValapLbvCFu>DoeCZd# zdm)hjmT4TG6s7UKG|rf&_TKvvZx7zf;&gc_BZBk)s^!eXp=|d!uEr8s<|RwAL=3Mb zgpkQvBn%n5ye5XBys|YYc`#!cjBRknG`6C$G_vPKMI(`|7umNFz4k#QlJivOT%AA8 zb*^)+>-7Kcx_-~|+|PaAzx#WCeqY5jeXpv$tx=V=#AaWZZcLzTmH)TNR0##^wzR1G z?Bm!`jq8E+n{RzqwrAvSbsYkIHIgK90vde@CRcXU)c3FSR2`^D+X z^o!`jr`J;IJGB~>Y?D*n!X?);yCiJl*uszeVlXy$r_R6W=76nv8+`8F(afjX!GD0I z&Y3qdk4Fn@@r0SijqZ&G8`S_5EUN!94hG@h7b;htAFHn#^hNDV^Ia|Cd7np3v=jxG zkjH~hmH==O(~G5qf^Emg_B*@NG~YVDm_%w-Yk@u*3HNAAFlouo15m~5)sQ9mUt@ad zsrz6P{Q{|AJgU%PxAY5=2+<95H`0|sw5@=h$|VI|@iSTLm$}^*taibsa5ks1*5>|I z_JkTm)HX;35I8GS2I2Wd&G5qhGAH%PTn3;>LMgV8pPu}bz<#)kct{=*?m^wI-L6cS zTI|wUQc^g#bY=uivK%|JST+-O#in7oc66N*#kE%b5v)UdL!l1`mUchAuR=ffnFE;{ z70z?aKX=ay`7ln8mc=ZNH^emTuFS8jwP9YNPe0TBE1x1jr?y;vC|1N=9j?kI^8n5S zACId%QYA<9T3@#M;)-$l!v!;S=?rz2cA9tyxVOQ|m_*bqHJ8MoOCYuT4fR0_$K}0Z zIz_F2VgoHSkDf#g}Tc@ElnWKfrE8X#Us$jLpdTTJgeLl@pB26zK z)v8B-BGCjECWNt<(?zFa06(aVsuaQGxY`Y+ntmI-_zNa=B*{54MIQ69hMs7TEsROoJLN8e;rGsTY zsIw;sSx_`f|3pO|5sl~;o*$_xF4L~6@K7@yN>n*1f3V$l6&Kxeh?3EVC>26iA_rEA zt#v4xV2#{QhMEgiFgl%Pf#^K*)rl_G_ua!>MNr%r)f~X#j(6X=Nn;n##auXYw>O(a zNV8i2$+eZ-usPu9M^Mdf0B7|BXkz-LWZ1j32*IvlL2o1GE&14oC!z%eBuI-O!nDk$ zYMHnn#f8Kd@)yM=O0G+iMT8-qeVLfsifm*1_$oB z$G6)D11VPN3h9twHKLq~gHutq_O)&%jEdudK&pMcIXO653is$pjJVW)u>YfF;cT577*?)52w^P@XkpxHfdU7BZ6G)C6~fL zXO(2}zvq_D7*IsytS_g^6py`EGk-l*`b4-2kjP&+yAmFDUaQkq%+JW|D z@zQFjN~5YVc3eYn^R|g?YnSY@ua=JuNW7*L4wBr=biFJx{R>yQGI1Kd`{UmC2&c?6 z|3)adk-l%P(v@mfN)R>~3JC7f>_d+RxeU@=1?mQWQ{GFa#z|b25^ysgirtWj~E*mR~B^;Gu~6ak>C00A=C{I??C7qAnB4V+@oX; z9?^{L4%CFvAr{k-Xsm(2iR8A4&Nh1&Ji-D6oi9F2k+eX1!=NH+Jnnvq=jwMbvVk1S#sbI5jRZSikOthD4DSq3m=B|7V5q0vs0 zP{!rbOFZ9o7+%TX{4K_Nq}s)KQf7fUasewF-b+34Bf+yKZT*==yk=-+SZJ|Gt`?R~bZi!X>dkHAQN`acOr$*!L zqwYyM*Oa`yisGGyR&Ooe+1B@LIcmWeo!1m&)Xi%O>?A9G5t9b!uNG=A-r|2++I5>2 zD6gWSOrCYAqbW2bjaM$voK6z+7AdiYv?rfh)QD3-k>xC3WQw4e^n4v3aDoxh;A@s6 zD!ql6mhmAs`!9&FO_fUtBw8iLp3gyRiYhWx0WmSB3Sr|V3B$uSW!EpN!8LhDM2}nuk%Ot121Lo@9VB!ci+>5m_3n8?8zlYIEZgPQIM{vRqFsEH9lfo;HXDCb9Lzxr1(EePVc^)J%d zKf-|jGVW)E0f+w6)BUWhKav9f=9~QdtUr NameLast ModifiedSizeDescription
    -     subfolder/ - -   + subfolder/ - -   testfile.txt13-January-2021 22:3519 B  From 75e6e35e3c3be9c77a25cfd90534408717340e25 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 4 Feb 2021 10:48:29 -0500 Subject: [PATCH 259/576] Changed the RestAssured tests to reflect the API changes. #7084 --- src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java | 9 ++++----- src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index 953989369e7..8d4369fa85b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -1470,8 +1470,7 @@ public void test_ProcessShapeFilePackage() { } /* - First test for the new "crawlable file access" API - (#7084; work in progress) + First test for the new "crawlable file access" API (#7084) */ @Test public void test_CrawlableAccessToDatasetFiles() { @@ -1514,16 +1513,16 @@ public void test_CrawlableAccessToDatasetFiles() { // maybe upload a few more files, in more folders, // and try an actual recursive crawl of a full tree - ? - // Make some calls to the "/fileaccess API: + // Make some calls to the "/dirindex API: // (note that this API outputs HTML!) // Expected values in the output: String expectedTitleTopFolder = "Index of folder /"; String expectedLinkTopFolder = folderName + "/"; - String expectedLinkAhrefTopFolder = "/api/datasets/"+datasetId+"/versions/:draft/fileaccess?folder=subfolder"; + String expectedLinkAhrefTopFolder = "/api/datasets/"+datasetId+"/dirindex/?version=:draft&folder=subfolder"; String expectedTitleSubFolder = "Index of folder /" + folderName; - String expectedLinkAhrefSubFolder = "/api/access/datafile/" + dataFileId; + String expectedLinkAhrefSubFolder = "/api/access/datafile/" + folderName + "/" + dataFileId; // ... with no folder specified: // (with just the one file above, this should show one folder only - "subfolder", and no files) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 9919a1c107e..2b6bbaf0caa 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -646,9 +646,9 @@ static Response uploadFileViaNative(String datasetId, String pathToFile, String static Response getCrawlableFileAccess(String datasetId, String folderName, String apiToken) { RequestSpecification requestSpecification = given() .header(API_TOKEN_HTTP_HEADER, apiToken); - String apiPath = "/api/datasets/" + datasetId + "/versions/:draft/fileaccess"; + String apiPath = "/api/datasets/" + datasetId + "/dirindex?version=:draft"; if (StringUtil.nonEmpty(folderName)) { - apiPath = apiPath.concat("?folder="+folderName); + apiPath = apiPath.concat("&folder="+folderName); } return requestSpecification.get(apiPath); } From 9c07284cf7b0a870b9eedf2d839cbebc75e5dbf7 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Thu, 4 Feb 2021 11:49:39 -0500 Subject: [PATCH 260/576] Added js+css workarounds to get the checksum to display in Bootstrap tooltips [ref #7081, #6685, #5210] --- src/main/webapp/file-info-fragment.xhtml | 24 ++++++++++++------- .../resources/js/dv_rebind_bootstrap_ui.js | 3 +++ src/main/webapp/search-include-fragment.xhtml | 20 +++++++++------- 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index 9a0b286d9e8..9a2add76147 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -71,24 +71,32 @@

    - #{fileMetadata.dataFile.checksumType}: #{fileMetadata.dataFile.checksumValue} - + title="#{bundle['file.metaData.checksum.copy']}<br/>#{fileMetadata.dataFile.checksumValue}">#{fileMetadata.dataFile.checksumType}: #{fileMetadata.dataFile.checksumValue} +
    - #{fileMetadata.dataFile.unf} - + title="#{bundle['file.metaData.checksum.copy']}<br/>#{fileMetadata.dataFile.unf}">#{fileMetadata.dataFile.unf} +
    diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index 5b8986111f6..b027d75cef5 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -186,6 +186,9 @@ function checksumTruncate(){ } } }); + $('span.checksum-tooltip').on('inserted.bs.tooltip', function () { + $("body div.tooltip-inner").css("word-break", "break-all"); + }); } function clickCopyClipboard(){ diff --git a/src/main/webapp/search-include-fragment.xhtml b/src/main/webapp/search-include-fragment.xhtml index 1b875fea375..0167a53ca48 100644 --- a/src/main/webapp/search-include-fragment.xhtml +++ b/src/main/webapp/search-include-fragment.xhtml @@ -614,25 +614,29 @@ - #{result.fileChecksumType}: #{result.fileChecksumValue} - #{result.fileChecksumType}: #{result.fileChecksumValue} + + data-clipboard-text="#{result.fileChecksumValue}" title="#{bundle['file.metaData.checksum.copy']}<br/>#{result.fileChecksumValue}"> - #{SearchIncludeFragment.tabularDataUnfDisplay(result.entity)} - #{SearchIncludeFragment.tabularDataUnfDisplay(result.entity)} + + data-clipboard-text="#{SearchIncludeFragment.tabularDataUnfDisplay(result.entity)}" title="#{bundle['file.metaData.checksum.copy']}<br/>#{SearchIncludeFragment.tabularDataUnfDisplay(result.entity)}">
    From 638c30dccbffeed0c85c40da3dcaec7d7bb4d1a7 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 4 Feb 2021 11:49:51 -0500 Subject: [PATCH 261/576] update naming --- doc/release-notes/4259-java-11.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/release-notes/4259-java-11.md b/doc/release-notes/4259-java-11.md index eada46f3751..1e5ee19230a 100644 --- a/doc/release-notes/4259-java-11.md +++ b/doc/release-notes/4259-java-11.md @@ -19,13 +19,13 @@ There are some things to note and keep in mind regarding the move to Java 11: The `remove` command may provide an error message if -headless isn't installed. - We targeted and tested Java 11, but 11+ will likely work. Java 11 was targeted because of its long term support. -- If you're moving from a Dataverse installation that was previously running Glassfish 4.x (typically this would be Dataverse 4.x), you will need to adjust some JVM options in domain.xml as part of the upgrade process. We've provided these optional steps below. These steps are not required if your first installed Dataverse version was running Payara 5.x (typically Dataverse 5.x). +- If you're moving from a Dataverse installation that was previously running Glassfish 4.x (typically this would be Dataverse Software 4.x), you will need to adjust some JVM options in domain.xml as part of the upgrade process. We've provided these optional steps below. These steps are not required if your first installed Dataverse version was running Payara 5.x (typically Dataverse Software 5.x). ### New JVM and DB Options #### New JVM Options -For installations that were previously running on Dataverse 4.x, the following JVM Options have been added in support of Java 11: +For installations that were previously running on Dataverse Software 4.x, the following JVM Options have been added in support of Java 11: - `[9|]--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED` - `[9|]--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED` @@ -57,7 +57,7 @@ For installations that were previously running on Dataverse 4.x, the following J - remove the generated directory: `rm -rf /payara/domains/domain1/generated` -3\. (only required for installations previously running on Dataverse 4.x) +3\. (only required for installations previously running on Dataverse Software 4.x) In domain.xml: From e1b691b550bfcb011ae8ce38c38fa61ae703d9f8 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 4 Feb 2021 16:10:01 -0500 Subject: [PATCH 262/576] test passing FacesContext --- .../impl/FinalizeDatasetPublicationCommand.java | 2 +- .../command/impl/PublishDatasetCommand.java | 5 ++++- .../dataverse/workflow/WorkflowServiceBean.java | 16 +++++++++++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java index 7a32f515d36..df389e74f1a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java @@ -183,7 +183,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { ctxt.workflows().getDefaultWorkflow(TriggerType.PostPublishDataset).ifPresent(wf -> { try { - ctxt.workflows().start(wf, buildContext(ds, TriggerType.PostPublishDataset, datasetExternallyReleased)); + ctxt.workflows().start(wf, buildContext(ds, TriggerType.PostPublishDataset, datasetExternallyReleased), null); } catch (CommandException ex) { logger.log(Level.SEVERE, "Error invoking post-publish workflow: " + ex.getMessage(), ex); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java index ae555faee58..5141325cdcc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java @@ -20,6 +20,9 @@ import java.util.List; import java.util.Optional; import java.util.logging.Logger; + +import javax.faces.context.FacesContext; + import static java.util.stream.Collectors.joining; import static edu.harvard.iq.dataverse.engine.command.impl.PublishDatasetResult.Status; @@ -91,7 +94,7 @@ public PublishDatasetResult execute(CommandContext ctxt) throws CommandException // We start a workflow theDataset = ctxt.em().merge(theDataset); ctxt.em().flush(); - ctxt.workflows().start(prePubWf.get(), buildContext(theDataset, TriggerType.PrePublishDataset, datasetExternallyReleased)); + ctxt.workflows().start(prePubWf.get(), buildContext(theDataset, TriggerType.PrePublishDataset, datasetExternallyReleased), FacesContext.getCurrentInstance()); return new PublishDatasetResult(theDataset, Status.Workflow); } else{ diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java index bf9eb68612b..a8ce879a3e7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java @@ -31,6 +31,8 @@ import javax.ejb.Stateless; import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; +import javax.faces.application.FacesMessage; +import javax.faces.context.FacesContext; import javax.inject.Inject; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; @@ -97,9 +99,21 @@ public WorkflowServiceBean() { * @throws CommandException If the dataset could not be locked. */ @Asynchronous - public void start(Workflow wf, WorkflowContext ctxt) throws CommandException { + public void start(Workflow wf, WorkflowContext ctxt, FacesContext facesCtxt) throws CommandException { ctxt = refresh(ctxt, retrieveRequestedSettings( wf.getRequiredSettings()), getCurrentApiToken(ctxt.getRequest().getAuthenticatedUser())); lockDataset(ctxt); + if (facesCtxt != null) { + try { + facesCtxt.addMessage(null, new FacesMessage("Starting Workflow")); + } catch (Throwable t) { + logger.warning(t.getLocalizedMessage()); + logger.warning(t.getClass().toString()); + + t.printStackTrace(); + } + } else { + logger.warning("FacesCtxt was null"); + } forward(wf, ctxt); } From 1f4842644330c9aeedf426883257929027b28ec7 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 4 Feb 2021 23:53:48 -0500 Subject: [PATCH 263/576] docs for disable/delete/anon, need to move to user admin section --- doc/sphinx-guides/source/api/native-api.rst | 67 ++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 44d099a1685..36b3193cdb4 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2961,6 +2961,8 @@ Example: ``curl -H "X-Dataverse-key: $API_TOKEN" -X POST http://demo.dataverse.o This action moves account data from jsmith2 into the account jsmith and deletes the account of jsmith2. +Note: User accounts can only be merged if they are either both enabled or both disabled. See :ref:`disable a user`. + .. _change-identifier-label: Change User Identifier @@ -2992,9 +2994,60 @@ Deletes an ``AuthenticatedUser`` whose ``id`` is passed. :: DELETE http://$SERVER/api/admin/authenticatedUsers/id/$id -Note: If the user has performed certain actions such as creating or contributing to a Dataset or downloading a file they cannot be deleted. +Note: If the user has performed certain actions such as creating or contributing to a Dataset or downloading a file they cannot be deleted. If a user cannot be deleted due to performing certain actions, you can choose to :ref:`disable a user`. +.. _disable-a-user: + +Disable a User +~~~~~~~~~~~~~~ + +Disables an ``AuthenticatedUser`` whose ``identifier`` (without the ``@`` sign) is passed. :: + + PUT http://$SERVER/api/admin/authenticatedUsers/disable/$identifier + +Disables an ``AuthenticatedUser`` whose ``id`` is passed. :: + + PUT http://$SERVER/api/admin/authenticatedUsers/disable/id/$id +Note: Since the primary purpose of Dataverse is to serve as an archive and to track data accesses and the modifications of data and metadata, a simple mechanism to delete users that have performed certain actions in the system is not provided by design. + +Disabling a user with this endpoint will: + +- Disable the user's ability to log in to the Dataverse installation +- Remove the user's access from all datasets and files +- Cancel any pending file access requests generated by the user +- Remove the user from all groups +- No longer have notifications generated or sent by the Dataverse installation + +Disabling a user with this endpoint will keep: + +- The user's contributions to datasets, including dataset creation, file uploads, publishing, submitting for review, and returning to author. +- Any Dataverse Collections where the user is the sole administrator +- The user's access to datafiles in the Dataverse installation, including guestbook records and acceptance of terms of use. +- The user's account information (specifically name, email, affiliation, and position) + +If it is determined that disabling a user is not sufficient, it is possible anonymize a user through use of the :ref:`merge-accounts-label` feature. See :ref:`Anonymize a User ` for more information about this process. + +.. _anonymize-a-user: + +Anonymize a User +~~~~~~~~~~~~~~~~ + +In some cases, due to privacy laws or other organizational policies, it may be required to anonymize a user. This process should only be used in cases where a Dataverse installation has the need to completely remove the traces of a user from the Dataverse installation. This process is only provided because it is not feasible for the Dataverse Software to provide deletion and deactivation options that will satisfy all use cases and all privacy laws, so the most complete deletion option is provided here. + +Note: Before going through these steps, please note that you will be permanently losing: + +- The user's contributions to datasets, including dataset creation, file uploads, publishing, submitting for review, and returning to author. This will all be marked as "Anonymous User" in your Dataverse installation, and it will not be possible to track who made changes as part of the archival record. +- Records of the user's access to datafiles in the Dataverse installation, including guestbook records and acceptance of terms of use. This could potentially introduce issues where it is no longer possible for data curators and owners to follow up with those who have accessed data on the application, which may be a legal requirement or data sharing agreement. It will be up to the Dataverse installation administrators to determine which datasets the user has accessed to determine if anonymization is feasible in these situations. +- The user's account information (specifically name, email, affiliation, and position). This will make it impossible to follow up with the user being deleted. You may find that you need to contact a user about a security breach or something else in the future, and you will not able to do so. + +To anonymize a user in your Dataverse installation: + +- Create a user called "Anonymous User" +- Use the :ref:`merge-accounts-label` functionality to merge the user you wish to anonymize INTO the Anonymous User +- Remove All Roles from the Anonymous User, either through the Dashboard or the :ref:`Remove Role Assignments ` API. + +Once you've created the Anonymous User once, you will not need to create it again. List Role Assignments of a Role Assignee ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -3005,6 +3058,18 @@ List all role assignments of a role assignee (i.e. a user or a group):: Note that ``identifier`` can contain slashes (e.g. ``&ip/localhost-users``). +.. _remove-role-assignments: + +Remove Role Assignments of Role Assignee +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Delete all role assignments of a role assignee (i.e. a user or a group):: + + DELETE http://$SERVER/api/admin/assignments/assignees/$identifier + +Note that ``identifier`` can contain slashes (e.g. ``&ip/-users``). + + List Permissions a User Has on a Dataverse Collection or Dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From c3d8bbfbd4552298bf380c6c12af69f31bafb8b7 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Feb 2021 10:50:28 -0500 Subject: [PATCH 264/576] Revert "test passing FacesContext" This reverts commit e1b691b550bfcb011ae8ce38c38fa61ae703d9f8. --- .../impl/FinalizeDatasetPublicationCommand.java | 2 +- .../command/impl/PublishDatasetCommand.java | 5 +---- .../dataverse/workflow/WorkflowServiceBean.java | 16 +--------------- 3 files changed, 3 insertions(+), 20 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java index df389e74f1a..7a32f515d36 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java @@ -183,7 +183,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { ctxt.workflows().getDefaultWorkflow(TriggerType.PostPublishDataset).ifPresent(wf -> { try { - ctxt.workflows().start(wf, buildContext(ds, TriggerType.PostPublishDataset, datasetExternallyReleased), null); + ctxt.workflows().start(wf, buildContext(ds, TriggerType.PostPublishDataset, datasetExternallyReleased)); } catch (CommandException ex) { logger.log(Level.SEVERE, "Error invoking post-publish workflow: " + ex.getMessage(), ex); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java index 5141325cdcc..ae555faee58 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java @@ -20,9 +20,6 @@ import java.util.List; import java.util.Optional; import java.util.logging.Logger; - -import javax.faces.context.FacesContext; - import static java.util.stream.Collectors.joining; import static edu.harvard.iq.dataverse.engine.command.impl.PublishDatasetResult.Status; @@ -94,7 +91,7 @@ public PublishDatasetResult execute(CommandContext ctxt) throws CommandException // We start a workflow theDataset = ctxt.em().merge(theDataset); ctxt.em().flush(); - ctxt.workflows().start(prePubWf.get(), buildContext(theDataset, TriggerType.PrePublishDataset, datasetExternallyReleased), FacesContext.getCurrentInstance()); + ctxt.workflows().start(prePubWf.get(), buildContext(theDataset, TriggerType.PrePublishDataset, datasetExternallyReleased)); return new PublishDatasetResult(theDataset, Status.Workflow); } else{ diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java index a8ce879a3e7..bf9eb68612b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java @@ -31,8 +31,6 @@ import javax.ejb.Stateless; import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; -import javax.faces.application.FacesMessage; -import javax.faces.context.FacesContext; import javax.inject.Inject; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; @@ -99,21 +97,9 @@ public WorkflowServiceBean() { * @throws CommandException If the dataset could not be locked. */ @Asynchronous - public void start(Workflow wf, WorkflowContext ctxt, FacesContext facesCtxt) throws CommandException { + public void start(Workflow wf, WorkflowContext ctxt) throws CommandException { ctxt = refresh(ctxt, retrieveRequestedSettings( wf.getRequiredSettings()), getCurrentApiToken(ctxt.getRequest().getAuthenticatedUser())); lockDataset(ctxt); - if (facesCtxt != null) { - try { - facesCtxt.addMessage(null, new FacesMessage("Starting Workflow")); - } catch (Throwable t) { - logger.warning(t.getLocalizedMessage()); - logger.warning(t.getClass().toString()); - - t.printStackTrace(); - } - } else { - logger.warning("FacesCtxt was null"); - } forward(wf, ctxt); } From edee04e3d98804adf86dc93b1d7966d3442e00d5 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 5 Feb 2021 10:52:38 -0500 Subject: [PATCH 265/576] #7205 fix dataset page load --- .../java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java index 69e35b8e8c8..12a2cf58feb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java @@ -469,8 +469,8 @@ public static String getDownloadSize(DatasetVersion dsv, boolean original) { long bytes = 0l; for (FileMetadata fileMetadata : dsv.getFileMetadatas()) { DataFile dataFile = fileMetadata.getDataFile(); - if (original && dataFile.isTabularData()) { - bytes += dataFile.getOriginalFileSize(); + if (original && dataFile.isTabularData()) { + bytes += dataFile.getOriginalFileSize() == null ? 0 : dataFile.getOriginalFileSize(); } else { bytes += dataFile.getFilesize(); } From dba965ccc5caf4046a2fcf9e3aa0f30dd10971f1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Feb 2021 11:39:23 -0500 Subject: [PATCH 266/576] update to use a foreign-key reference to datasetlock, add flyway --- .../iq/dataverse/PermissionServiceBean.java | 2 +- .../workflow/PendingWorkflowInvocation.java | 16 +++++++++------- .../iq/dataverse/workflow/WorkflowContext.java | 17 +++++++++-------- .../dataverse/workflow/WorkflowServiceBean.java | 6 +++--- .../db/migration/V5.4.0.1__7563-workflow.sql | 4 ++++ 5 files changed, 26 insertions(+), 19 deletions(-) create mode 100644 src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index aaf38af1b36..f38efa7b66b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -816,7 +816,7 @@ public boolean isMatchingWorkflowLock(Dataset d, String userId, String invocatio if (invocationId != null) { PendingWorkflowInvocation pwfi = workflowService.getPendingWorkflow(invocationId); for (DatasetLock dl : d.getLocks()) { - if (dl.getId().equals(pwfi.getLockId()) && pwfi.getUserId().equals(userId)) { + if (dl.equals(pwfi.getLock()) && pwfi.getUserId().equals(userId)) { return true; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java b/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java index 6efccf4de03..227176dc4b5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.workflow; import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetLock; import edu.harvard.iq.dataverse.RoleAssigneeServiceBean; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; import edu.harvard.iq.dataverse.authorization.users.User; @@ -52,7 +53,8 @@ public class PendingWorkflowInvocation implements Serializable { int typeOrdinal; boolean datasetExternallyReleased; - long lockId; + @OneToOne + DatasetLock lock; /** Empty constructor for JPA */ public PendingWorkflowInvocation(){ @@ -70,13 +72,13 @@ public PendingWorkflowInvocation(Workflow wf, WorkflowContext ctxt, Pending resu localData = new HashMap<>(result.getData()); typeOrdinal = ctxt.getType().ordinal(); datasetExternallyReleased=ctxt.getDatasetExternallyReleased(); - lockId=ctxt.getLockId(); + lock=ctxt.getLock(); } public WorkflowContext reCreateContext(RoleAssigneeServiceBean roleAssignees) { DataverseRequest aRequest = new DataverseRequest((User)roleAssignees.getRoleAssignee(userId), IpAddress.valueOf(ipAddress)); final WorkflowContext workflowContext = new WorkflowContext(aRequest, dataset, nextVersionNumber, - nextMinorVersionNumber, WorkflowContext.TriggerType.values()[typeOrdinal], null, null, datasetExternallyReleased, invocationId, lockId); + nextMinorVersionNumber, WorkflowContext.TriggerType.values()[typeOrdinal], null, null, datasetExternallyReleased, invocationId, lock); return workflowContext; } @@ -160,10 +162,10 @@ public void setTypeOrdinal(int typeOrdinal) { this.typeOrdinal = typeOrdinal; } - public long getLockId() { - return lockId; + public DatasetLock getLock() { + return lock; } - public void setLockId(long lockId) { - this.lockId = lockId; + public void setLockId(DatasetLock lock) { + this.lock = lock; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowContext.java b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowContext.java index 76ca212976f..6208acb0999 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowContext.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowContext.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.workflow; import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetLock; import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.workflow.step.WorkflowStep; @@ -31,7 +32,7 @@ public enum TriggerType { private final ApiToken apiToken; private final boolean datasetExternallyReleased; private Map settings; - private long lockId = -1; + private DatasetLock lock = null; private String invocationId = UUID.randomUUID().toString(); @@ -47,7 +48,7 @@ public WorkflowContext(DataverseRequest request, Dataset dataset, long nextVersi } public WorkflowContext(DataverseRequest request, Dataset dataset, long nextVersionNumber, - long nextMinorVersionNumber, TriggerType type, Map settings, ApiToken apiToken, boolean datasetExternallyReleased, String invocationId, Long lockId) { + long nextMinorVersionNumber, TriggerType type, Map settings, ApiToken apiToken, boolean datasetExternallyReleased, String invocationId, DatasetLock lock) { this.request = request; this.dataset = dataset; this.nextVersionNumber = nextVersionNumber; @@ -60,8 +61,8 @@ public WorkflowContext(DataverseRequest request, Dataset dataset, long nextVersi if(invocationId!=null) { setInvocationId(invocationId); } - if(lockId != null) { - this.setLockId(lockId); + if(lock != null) { + this.setLock(lock); } } @@ -108,11 +109,11 @@ public ApiToken getApiToken() { public boolean getDatasetExternallyReleased() { return datasetExternallyReleased; } - public long getLockId() { - return lockId; + public DatasetLock getLock() { + return lock; } - public void setLockId(long lockId) { - this.lockId = lockId; + public void setLock(DatasetLock lock) { + this.lock = lock; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java index bf9eb68612b..2c20bd54151 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java @@ -282,7 +282,7 @@ void lockDataset( WorkflowContext ctxt ) throws CommandException { em.persist(datasetLock); //flush creates the id em.flush(); - ctxt.setLockId(datasetLock.getId()); + ctxt.setLock(datasetLock); } @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) @@ -295,7 +295,7 @@ void unlockDataset( WorkflowContext ctxt ) throws CommandException { lockCounter.setParameter("datasetId", ctxt.getDataset().getId()); List locks = lockCounter.getResultList(); for(DatasetLock lock: locks) { - if(lock.getReason() == DatasetLock.Reason.Workflow && lock.getId()==ctxt.getLockId()) { + if(lock.getReason() == DatasetLock.Reason.Workflow && lock==ctxt.getLock()) { logger.fine("Removing lock"); em.remove(lock); } @@ -458,7 +458,7 @@ private WorkflowContext refresh( WorkflowContext ctxt, Map setti */ WorkflowContext newCtxt =new WorkflowContext( ctxt.getRequest(), em.merge(ctxt.getDataset()), ctxt.getNextVersionNumber(), - ctxt.getNextMinorVersionNumber(), ctxt.getType(), settings, apiToken, ctxt.getDatasetExternallyReleased(), ctxt.getInvocationId(), ctxt.getLockId()); + ctxt.getNextMinorVersionNumber(), ctxt.getType(), settings, apiToken, ctxt.getDatasetExternallyReleased(), ctxt.getInvocationId(), ctxt.getLock()); return newCtxt; } diff --git a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql new file mode 100644 index 00000000000..a7ee9b8478b --- /dev/null +++ b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql @@ -0,0 +1,4 @@ +ALTER TABLE pendingworkflowinvocation +ADD COLUMN IFNOTEXISTS datasetlock_id bigint, +ADD CONSTRAINT fk_pendingworkflowinvocation_datasetlock_id FOREIGN KEY (datasetlock_id) + REFERENCES datasetlock (id); \ No newline at end of file From 80f886e48c6655a7bd87ceed530e25ec668e0b5f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Feb 2021 11:58:18 -0500 Subject: [PATCH 267/576] fix name --- src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql index a7ee9b8478b..6a50e89c1ea 100644 --- a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql +++ b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql @@ -1,4 +1,4 @@ ALTER TABLE pendingworkflowinvocation -ADD COLUMN IFNOTEXISTS datasetlock_id bigint, -ADD CONSTRAINT fk_pendingworkflowinvocation_datasetlock_id FOREIGN KEY (datasetlock_id) +ADD COLUMN IFNOTEXISTS lock_id bigint, +ADD CONSTRAINT fk_pendingworkflowinvocation_lock_id FOREIGN KEY (datasetlock_id) REFERENCES datasetlock (id); \ No newline at end of file From 77677625ed83b0ccb1bd0eb5fc75369cc7c1b394 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Feb 2021 11:59:25 -0500 Subject: [PATCH 268/576] one more --- src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql index 6a50e89c1ea..8b852f840bb 100644 --- a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql +++ b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql @@ -1,4 +1,4 @@ ALTER TABLE pendingworkflowinvocation ADD COLUMN IFNOTEXISTS lock_id bigint, -ADD CONSTRAINT fk_pendingworkflowinvocation_lock_id FOREIGN KEY (datasetlock_id) +ADD CONSTRAINT fk_pendingworkflowinvocation_lock_id FOREIGN KEY (lock_id) REFERENCES datasetlock (id); \ No newline at end of file From 7527eb461de91f5eb0708ef243ac64f57546754a Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 5 Feb 2021 12:14:36 -0500 Subject: [PATCH 269/576] #7205 suppress orig button if size 0 B --- src/main/webapp/dataset.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 34b49a8658c..4f23d4c7b44 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -158,7 +158,7 @@ -
  • +
  • #{bundle.downloadOriginal} From dc50df56060f56b5f166462dcf065e5f06ee8627 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 5 Feb 2021 12:27:31 -0500 Subject: [PATCH 270/576] adding release note --- doc/release-notes/7205-orig-file-size.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/release-notes/7205-orig-file-size.md diff --git a/doc/release-notes/7205-orig-file-size.md b/doc/release-notes/7205-orig-file-size.md new file mode 100644 index 00000000000..2f7af23b6f3 --- /dev/null +++ b/doc/release-notes/7205-orig-file-size.md @@ -0,0 +1,11 @@ +## Notes to Admins + +Beginning in Dataverse Software 4.10, the size of the saved original file (for an ingested tabular datafile) was stored in the database. For files added before this change, we provide an API that retrieves and permanently stores the sizes for any already existing saved originals. See [Datafile Integrity API](https://guides.dataverse.org/en/5.4/api/native-api.html#datafile-integrity) for more information. + +This was documented as a step in previous release notes, but we are noting it in these release notes to give it more visibility. + +## Upgrade Instructions + +X./ Retroactively store original file size + +Use the [Datafile Integrity API](https://guides.dataverse.org/en/5.4/api/native-api.html#datafile-integrity) to ensure that the sizes of all original files are stored in the database. \ No newline at end of file From 87e89a2fc222fda152a9e0e440ff80c344bd8a68 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Feb 2021 12:30:41 -0500 Subject: [PATCH 271/576] spaces --- src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql index 8b852f840bb..fb3714bb573 100644 --- a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql +++ b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql @@ -1,4 +1,4 @@ ALTER TABLE pendingworkflowinvocation -ADD COLUMN IFNOTEXISTS lock_id bigint, +ADD COLUMN IF NOT EXISTS lock_id bigint, ADD CONSTRAINT fk_pendingworkflowinvocation_lock_id FOREIGN KEY (lock_id) REFERENCES datasetlock (id); \ No newline at end of file From 911de9a4817aa66670414409202d78611126e278 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Feb 2021 12:51:03 -0500 Subject: [PATCH 272/576] make idempotent/fix syntax --- .../resources/db/migration/V5.4.0.1__7563-workflow.sql | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql index fb3714bb573..246acca3964 100644 --- a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql +++ b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql @@ -1,4 +1,10 @@ ALTER TABLE pendingworkflowinvocation ADD COLUMN IF NOT EXISTS lock_id bigint, -ADD CONSTRAINT fk_pendingworkflowinvocation_lock_id FOREIGN KEY (lock_id) - REFERENCES datasetlock (id); \ No newline at end of file + +DO $$ +BEGIN +IF NOT EXISTS (SELECT 1 FROM information_schema.constraint_column_usage WHERE constraint_name='fk_pendingworkflowinvocation_lock_id') +THEN +ALTER TABLE pendingworkflowinvocation ADD CONSTRAINT fk_pendingworkflowinvocation_lock_id FOREIGN KEY (lock_id) REFERENCES datasetlock (id); +END IF; +END$$; \ No newline at end of file From 127977e6bfd7311e748c3f98338c7dc0018a4f95 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Feb 2021 13:11:34 -0500 Subject: [PATCH 273/576] one more typo --- src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql index 246acca3964..57ecec383a7 100644 --- a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql +++ b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql @@ -1,5 +1,5 @@ ALTER TABLE pendingworkflowinvocation -ADD COLUMN IF NOT EXISTS lock_id bigint, +ADD COLUMN IF NOT EXISTS lock_id bigint; DO $$ BEGIN From 5fe37496231a16d3178850dedcea8f3e44fd4058 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Fri, 5 Feb 2021 13:50:33 -0500 Subject: [PATCH 274/576] Had to remove a negative z-index and add a position:relative to the content block to fix footer overlap bug [ref #7558] --- src/main/webapp/resources/css/structure.css | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index 605342db8d7..e70bb2f7b93 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -14,8 +14,8 @@ body .ui-widget {font-size: inherit;} .ui-widget-content a:hover, .ui-widget-content a:focus {color: #23527C;} /* -------- TEMPLATE -------- */ -/* TODO FIX DROPDOWN MENU LAYERED UNDER FOOTER */ -#content {padding-bottom:0px;} +/* STICKY FOOTER VS DROPDOWN MENU POSITION + Z-INDEX */ +#content {position:relative; z-index:50;} .panelgridHeaderTable {width:100%;} .panelgridHeaderTable .ui-widget-content, .panelgridHeaderTable .ui-widget-content > td {border:0; background:none;} .panelgridHeaderTable .ui-widget-content > td:last-child table {margin-left:auto;} @@ -63,7 +63,7 @@ body .ui-widget {font-size: inherit;} #status-alert {margin-top:0; margin-bottom:0;} #status-alert div.alert {border:0; box-shadow:none;} -#footer {position:absolute; z-index:-1; bottom:0; width:100%; height:60px; padding-bottom:100px; color:#808080;} +#footer {position:absolute; bottom:0; width:100%; height:60px; padding-bottom:100px; color:#808080;} #footer.widget-view {position:fixed; left:0; bottom:0; margin:0; padding:4px 0 0 0; min-height:44px; height:auto; background:#fff;} #footer .poweredbylogo {text-align:right;} #footer .poweredbylogo span {font-size:.85em;margin-right:.3em;} From 2af33fd9662328d86518c6f8c985c187398d1a0b Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 5 Feb 2021 15:04:20 -0500 Subject: [PATCH 275/576] draft code review feedback --- doc/sphinx-guides/source/api/native-api.rst | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 36b3193cdb4..1d00ea80206 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2994,7 +2994,7 @@ Deletes an ``AuthenticatedUser`` whose ``id`` is passed. :: DELETE http://$SERVER/api/admin/authenticatedUsers/id/$id -Note: If the user has performed certain actions such as creating or contributing to a Dataset or downloading a file they cannot be deleted. If a user cannot be deleted due to performing certain actions, you can choose to :ref:`disable a user`. +Note: If the user has performed certain actions such as creating or contributing to a Dataset or downloading a file they cannot be deleted. If a user cannot be deleted for this reason, you can choose to :ref:`disable a user`. .. _disable-a-user: @@ -3022,7 +3022,6 @@ Disabling a user with this endpoint will: Disabling a user with this endpoint will keep: - The user's contributions to datasets, including dataset creation, file uploads, publishing, submitting for review, and returning to author. -- Any Dataverse Collections where the user is the sole administrator - The user's access to datafiles in the Dataverse installation, including guestbook records and acceptance of terms of use. - The user's account information (specifically name, email, affiliation, and position) @@ -3037,15 +3036,18 @@ In some cases, due to privacy laws or other organizational policies, it may be r Note: Before going through these steps, please note that you will be permanently losing: -- The user's contributions to datasets, including dataset creation, file uploads, publishing, submitting for review, and returning to author. This will all be marked as "Anonymous User" in your Dataverse installation, and it will not be possible to track who made changes as part of the archival record. -- Records of the user's access to datafiles in the Dataverse installation, including guestbook records and acceptance of terms of use. This could potentially introduce issues where it is no longer possible for data curators and owners to follow up with those who have accessed data on the application, which may be a legal requirement or data sharing agreement. It will be up to the Dataverse installation administrators to determine which datasets the user has accessed to determine if anonymization is feasible in these situations. +- The user's contributions to datasets, including dataset creation, file uploads, publishing, submitting for review, and returning to author. This will all be marked as "Anonymous User" in your Dataverse installation, and it will not be possible to track who made changes as part of the archival record. - The user's account information (specifically name, email, affiliation, and position). This will make it impossible to follow up with the user being deleted. You may find that you need to contact a user about a security breach or something else in the future, and you will not able to do so. +You will not lose: + +- Email, First Name, and Last Name in Guestbook records. + To anonymize a user in your Dataverse installation: -- Create a user called "Anonymous User" +- Create a user called "Anonymous User". This is a one-time step. - Use the :ref:`merge-accounts-label` functionality to merge the user you wish to anonymize INTO the Anonymous User -- Remove All Roles from the Anonymous User, either through the Dashboard or the :ref:`Remove Role Assignments ` API. +- Disable the user using the :ref:`disable a user` API. Once you've created the Anonymous User once, you will not need to create it again. From 1a2d175f1968f4734b33fc4a669ba85fab5c716c Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Feb 2021 15:29:31 -0500 Subject: [PATCH 276/576] fix/cleanup lock deletion removes the pendingworkflowinvocation from the db, but not the context, then, in workflowCompleted, the lock is deleted and we refresh the context to make sure the dataset is up-to-date before passing on to Finalize --- .../workflow/PendingWorkflowInvocation.java | 4 +++- .../workflow/WorkflowServiceBean.java | 20 ++++++------------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java b/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java index 227176dc4b5..3ec83fa8036 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java @@ -10,6 +10,8 @@ import java.io.Serializable; import java.util.HashMap; import java.util.Map; + +import javax.persistence.CascadeType; import javax.persistence.ElementCollection; import javax.persistence.Entity; import javax.persistence.FetchType; @@ -165,7 +167,7 @@ public void setTypeOrdinal(int typeOrdinal) { public DatasetLock getLock() { return lock; } - public void setLockId(DatasetLock lock) { + public void setLock(DatasetLock lock) { this.lock = lock; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java index 2c20bd54151..46b9ac5cfe4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java @@ -154,6 +154,7 @@ private Map retrieveRequestedSettings(Map requir @Asynchronous public void resume(PendingWorkflowInvocation pending, String body) { em.remove(em.merge(pending)); + em.flush(); doResume(pending, body); } @@ -287,19 +288,9 @@ void lockDataset( WorkflowContext ctxt ) throws CommandException { @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) void unlockDataset( WorkflowContext ctxt ) throws CommandException { - /* Since the lockDataset command above directly persists a lock to the database, - * the ctxt.getDataset() is not updated and its list of locks can't be used. Using the named query below will find the workflow - * lock and remove it (actually all workflow locks for this Dataset but only one workflow should be active). - */ - TypedQuery lockCounter = em.createNamedQuery("DatasetLock.getLocksByDatasetId", DatasetLock.class); - lockCounter.setParameter("datasetId", ctxt.getDataset().getId()); - List locks = lockCounter.getResultList(); - for(DatasetLock lock: locks) { - if(lock.getReason() == DatasetLock.Reason.Workflow && lock==ctxt.getLock()) { - logger.fine("Removing lock"); - em.remove(lock); - } - } + + em.remove(em.merge(ctxt.getLock())); + em.flush(); } @@ -319,7 +310,8 @@ private void workflowCompleted(Workflow wf, WorkflowContext ctxt) { try { if ( ctxt.getType() == TriggerType.PrePublishDataset ) { unlockDataset(ctxt); - + ctxt.setLock(null); + ctxt = refresh(ctxt); //Now lock for FinalizePublication - this block mirrors that in PublishDatasetCommand AuthenticatedUser user = ctxt.getRequest().getAuthenticatedUser(); DatasetLock lock = new DatasetLock(DatasetLock.Reason.finalizePublication, user); From ecaaa9bda382951eb6e4a62a81d0aac575fc6a8c Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 5 Feb 2021 16:06:15 -0500 Subject: [PATCH 277/576] removing anon and documentation for remove roles API --- doc/sphinx-guides/source/api/native-api.rst | 42 ++------------------- 1 file changed, 3 insertions(+), 39 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 1d00ea80206..43b9e668cb6 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3014,43 +3014,19 @@ Note: Since the primary purpose of Dataverse is to serve as an archive and to tr Disabling a user with this endpoint will: - Disable the user's ability to log in to the Dataverse installation -- Remove the user's access from all datasets and files +- Remove the user's access from all Dataverse collections, datasets and files - Cancel any pending file access requests generated by the user - Remove the user from all groups - No longer have notifications generated or sent by the Dataverse installation Disabling a user with this endpoint will keep: -- The user's contributions to datasets, including dataset creation, file uploads, publishing, submitting for review, and returning to author. -- The user's access to datafiles in the Dataverse installation, including guestbook records and acceptance of terms of use. +- The user's contributions to datasets, including dataset creation, file uploads, and publishing. +- The user's access history to datafiles in the Dataverse installation, including guestbook records and acceptance of terms of use. - The user's account information (specifically name, email, affiliation, and position) If it is determined that disabling a user is not sufficient, it is possible anonymize a user through use of the :ref:`merge-accounts-label` feature. See :ref:`Anonymize a User ` for more information about this process. -.. _anonymize-a-user: - -Anonymize a User -~~~~~~~~~~~~~~~~ - -In some cases, due to privacy laws or other organizational policies, it may be required to anonymize a user. This process should only be used in cases where a Dataverse installation has the need to completely remove the traces of a user from the Dataverse installation. This process is only provided because it is not feasible for the Dataverse Software to provide deletion and deactivation options that will satisfy all use cases and all privacy laws, so the most complete deletion option is provided here. - -Note: Before going through these steps, please note that you will be permanently losing: - -- The user's contributions to datasets, including dataset creation, file uploads, publishing, submitting for review, and returning to author. This will all be marked as "Anonymous User" in your Dataverse installation, and it will not be possible to track who made changes as part of the archival record. -- The user's account information (specifically name, email, affiliation, and position). This will make it impossible to follow up with the user being deleted. You may find that you need to contact a user about a security breach or something else in the future, and you will not able to do so. - -You will not lose: - -- Email, First Name, and Last Name in Guestbook records. - -To anonymize a user in your Dataverse installation: - -- Create a user called "Anonymous User". This is a one-time step. -- Use the :ref:`merge-accounts-label` functionality to merge the user you wish to anonymize INTO the Anonymous User -- Disable the user using the :ref:`disable a user` API. - -Once you've created the Anonymous User once, you will not need to create it again. - List Role Assignments of a Role Assignee ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -3060,18 +3036,6 @@ List all role assignments of a role assignee (i.e. a user or a group):: Note that ``identifier`` can contain slashes (e.g. ``&ip/localhost-users``). -.. _remove-role-assignments: - -Remove Role Assignments of Role Assignee -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Delete all role assignments of a role assignee (i.e. a user or a group):: - - DELETE http://$SERVER/api/admin/assignments/assignees/$identifier - -Note that ``identifier`` can contain slashes (e.g. ``&ip/-users``). - - List Permissions a User Has on a Dataverse Collection or Dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 9ee84f482d042c839efd1d6c7eec9d12651aee5b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Feb 2021 16:55:31 -0500 Subject: [PATCH 278/576] fix editMetadata wrappedResponse handling --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 655cdafe04c..9826ecfe795 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -771,8 +771,12 @@ private String getCompoundDisplayValue (DatasetFieldCompoundValue dscv){ public Response editVersionMetadata(String jsonBody, @PathParam("id") String id, @QueryParam("replace") Boolean replace) throws WrappedResponse{ Boolean replaceData = replace != null; - + try { DataverseRequest req = createDataverseRequest(findUserOrDie()); + } catch (WrappedResponse ex) { + logger.log(Level.SEVERE, "Edit metdata error: " + ex.getMessage(), ex); + return ex.getResponse(); + } return processDatasetUpdate(jsonBody, id, req, replaceData); } From 308bae13d425b1faafaac37864bbe37941495194 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Feb 2021 16:56:57 -0500 Subject: [PATCH 279/576] log the stack trace for Throwable/500 errors --- .../harvard/iq/dataverse/api/errorhandlers/ThrowableHandler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/ThrowableHandler.java b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/ThrowableHandler.java index 7b7f1f08fad..d0e1637243a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/ThrowableHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/ThrowableHandler.java @@ -27,6 +27,7 @@ public class ThrowableHandler implements ExceptionMapper{ @Override public Response toResponse(Throwable ex){ + ex.printStackTrace(); return JsonResponseBuilder.error(Response.Status.INTERNAL_SERVER_ERROR) .randomIncidentId() .internalError(ex) From 1c5d63277cf58d2c6d012b2bc57881da44889e4a Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 5 Feb 2021 16:57:06 -0500 Subject: [PATCH 280/576] user merge and delete cleanup, get traces #4475 #7575 Primarily this has been an investigation into existing code for deleting and merging users. Additionally, a "get user traces" command was added to get a sense of why a user can't be deleted or what would be merged. The top of DeleteUsersIT has a lengthy comment about which database tables are in play and a variety of scenarios involving users to be deleted, merged or (in the future) disabled (#2419). The following bugs where fixed: - Unable to merge (or delete) a user when they have initiated a password reset. #7575 - Delete OAuth2 tokens on delete. --- .../edu/harvard/iq/dataverse/DataFile.java | 4 + .../iq/dataverse/DataFileServiceBean.java | 10 +- .../edu/harvard/iq/dataverse/Dataset.java | 4 + .../iq/dataverse/DatasetServiceBean.java | 8 + .../edu/harvard/iq/dataverse/Dataverse.java | 2 + .../iq/dataverse/DataverseServiceBean.java | 8 + .../edu/harvard/iq/dataverse/api/Users.java | 30 + .../AuthenticationServiceBean.java | 15 +- .../builtin/BuiltinUserServiceBean.java | 2 + .../command/impl/GetUserTracesCommand.java | 228 ++++++ .../command/impl/MergeInAccountCommand.java | 5 +- .../passwordreset/PasswordResetData.java | 4 +- .../PasswordResetServiceBean.java | 6 + src/main/java/propertyFiles/Bundle.properties | 3 +- .../iq/dataverse/api/DeleteUsersIT.java | 693 ++++++++++++++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 14 + 16 files changed, 1027 insertions(+), 9 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java create mode 100644 src/test/java/edu/harvard/iq/dataverse/api/DeleteUsersIT.java diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java index cb2b6e221e8..707cbc9daf0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java @@ -57,6 +57,10 @@ @NamedQueries({ @NamedQuery( name="DataFile.removeFromDatasetVersion", query="DELETE FROM FileMetadata f WHERE f.datasetVersion.id=:versionId and f.dataFile.id=:fileId"), + @NamedQuery(name = "DataFile.findByCreatorId", + query = "SELECT o FROM DataFile o WHERE o.creator.id=:creatorId"), + @NamedQuery(name = "DataFile.findByReleaseUserId", + query = "SELECT o FROM DataFile o WHERE o.releaseUser.id=:releaseUserId"), @NamedQuery(name="DataFile.findDataFileByIdProtocolAuth", query="SELECT s FROM DataFile s WHERE s.identifier=:identifier AND s.protocol=:protocol AND s.authority=:authority"), @NamedQuery(name="DataFile.findDataFileThatReplacedId", diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java index 094e20edf28..bc1dab51d2e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java @@ -155,7 +155,15 @@ public DataFile find(Object pk) { public DataFile findByGlobalId(String globalId) { return (DataFile) dvObjectService.findByGlobalId(globalId, DataFile.DATAFILE_DTYPE_STRING); } - + + public List findByCreatorId(Long creatorId) { + return em.createNamedQuery("DataFile.findByCreatorId").setParameter("creatorId", creatorId).getResultList(); + } + + public List findByReleaseUserId(Long releaseUserId) { + return em.createNamedQuery("DataFile.findByReleaseUserId").setParameter("releaseUserId", releaseUserId).getResultList(); + } + public DataFile findReplacementFile(Long previousFileId){ Query query = em.createQuery("select object(o) from DataFile as o where o.previousDataFileId = :previousFileId"); query.setParameter("previousFileId", previousFileId); diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java index 4cf95dda250..cd40e76a304 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java @@ -53,6 +53,10 @@ query = "SELECT o.id FROM Dataset o WHERE o.owner.id=:ownerId"), @NamedQuery(name = "Dataset.findByOwnerId", query = "SELECT o FROM Dataset o WHERE o.owner.id=:ownerId"), + @NamedQuery(name = "Dataset.findByCreatorId", + query = "SELECT o FROM Dataset o WHERE o.creator.id=:creatorId"), + @NamedQuery(name = "Dataset.findByReleaseUserId", + query = "SELECT o FROM Dataset o WHERE o.releaseUser.id=:releaseUserId"), }) /* diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index c1efe119fd2..4b36f466c61 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -150,6 +150,14 @@ private List findIdsByOwnerId(Long ownerId, boolean onlyPublished) { } } + public List findByCreatorId(Long creatorId) { + return em.createNamedQuery("Dataset.findByCreatorId").setParameter("creatorId", creatorId).getResultList(); + } + + public List findByReleaseUserId(Long releaseUserId) { + return em.createNamedQuery("Dataset.findByReleaseUserId").setParameter("releaseUserId", releaseUserId).getResultList(); + } + public List filterByPidQuery(String filterQuery) { // finds only exact matches Dataset ds = findByGlobalId(filterQuery); diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java index 5aab9ef9a9e..b46333a4287 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java @@ -50,6 +50,8 @@ @NamedQuery(name = "Dataverse.findRoot", query = "SELECT d FROM Dataverse d where d.owner.id=null"), @NamedQuery(name = "Dataverse.findByAlias", query="SELECT dv FROM Dataverse dv WHERE LOWER(dv.alias)=:alias"), @NamedQuery(name = "Dataverse.findByOwnerId", query="select object(o) from Dataverse as o where o.owner.id =:ownerId order by o.name"), + @NamedQuery(name = "Dataverse.findByCreatorId", query="select object(o) from Dataverse as o where o.creator.id =:creatorId order by o.name"), + @NamedQuery(name = "Dataverse.findByReleaseUserId", query="select object(o) from Dataverse as o where o.releaseUser.id =:releaseUserId order by o.name"), @NamedQuery(name = "Dataverse.filterByAlias", query="SELECT dv FROM Dataverse dv WHERE LOWER(dv.alias) LIKE :alias order by dv.alias"), @NamedQuery(name = "Dataverse.filterByAliasNameAffiliation", query="SELECT dv FROM Dataverse dv WHERE (LOWER(dv.alias) LIKE :alias) OR (LOWER(dv.name) LIKE :name) OR (LOWER(dv.affiliation) LIKE :affiliation) order by dv.alias"), @NamedQuery(name = "Dataverse.filterByName", query="SELECT dv FROM Dataverse dv WHERE LOWER(dv.name) LIKE :name order by dv.alias") diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index 96963bb9cb4..9df4ca52938 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -172,6 +172,14 @@ public List findDataverseIdsForIndexing(boolean skipIndexed) { } + public List findByCreatorId(Long creatorId) { + return em.createNamedQuery("Dataverse.findByCreatorId").setParameter("creatorId", creatorId).getResultList(); + } + + public List findByReleaseUserId(Long releaseUserId) { + return em.createNamedQuery("Dataverse.findByReleaseUserId").setParameter("releaseUserId", releaseUserId).getResultList(); + } + public List findByOwnerId(Long ownerId) { return em.createNamedQuery("Dataverse.findByOwnerId").setParameter("ownerId", ownerId).getResultList(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Users.java b/src/main/java/edu/harvard/iq/dataverse/api/Users.java index 37eedbe7714..e8558f88d6e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Users.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Users.java @@ -10,10 +10,13 @@ import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.engine.command.impl.ChangeUserIdentifierCommand; +import edu.harvard.iq.dataverse.engine.command.impl.GetUserTracesCommand; import edu.harvard.iq.dataverse.engine.command.impl.MergeInAccountCommand; +import edu.harvard.iq.dataverse.engine.command.impl.RevokeAllRolesCommand; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; import java.util.logging.Logger; import javax.ejb.Stateless; +import javax.json.JsonObjectBuilder; import javax.ws.rs.DELETE; import javax.ws.rs.GET; import javax.ws.rs.POST; @@ -191,4 +194,31 @@ public Response getAuthenticatedUserByToken() { } + @POST + @Path("{identifier}/removeRoles") + public Response removeUserRoles(@PathParam("identifier") String identifier) { + try { + AuthenticatedUser userToModify = authSvc.getAuthenticatedUser(identifier); + if (userToModify == null) { + return error(Response.Status.BAD_REQUEST, "Cannot find user based on " + identifier + "."); + } + execCommand(new RevokeAllRolesCommand(userToModify, createDataverseRequest(findUserOrDie()))); + return ok("Roles removed for user " + identifier + "."); + } catch (Exception ex) { + return error(Response.Status.BAD_REQUEST, "Unable to revoke all roles: " + ex.getLocalizedMessage()); + } + } + + @GET + @Path("{identifier}/traces") + public Response getTraces(@PathParam("identifier") String identifier) { + try { + AuthenticatedUser userToQuery = authSvc.getAuthenticatedUser(identifier); // TODO: do lookup here? What if null? + JsonObjectBuilder jsonObj = execCommand(new GetUserTracesCommand(createDataverseRequest(findUserOrDie()), userToQuery)); + return ok(jsonObj); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java index af644f8a4de..c9a54dba078 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java @@ -30,8 +30,10 @@ import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.confirmemail.ConfirmEmailData; import edu.harvard.iq.dataverse.confirmemail.ConfirmEmailServiceBean; +import edu.harvard.iq.dataverse.engine.command.impl.RevokeAllRolesCommand; import edu.harvard.iq.dataverse.passwordreset.PasswordResetData; import edu.harvard.iq.dataverse.passwordreset.PasswordResetServiceBean; +import edu.harvard.iq.dataverse.search.savedsearch.SavedSearchServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.validation.PasswordValidatorServiceBean; import edu.harvard.iq.dataverse.workflows.WorkflowComment; @@ -120,7 +122,10 @@ public class AuthenticationServiceBean { @EJB ExplicitGroupServiceBean explicitGroupService; - + + @EJB + SavedSearchServiceBean savedSearchService; + @PersistenceContext(unitName = "VDCNet-ejbPU") private EntityManager em; @@ -193,7 +198,7 @@ public boolean isOrcidEnabled() { * * Before calling this method, make sure you've deleted all the stuff tied * to the user, including stuff they've created, role assignments, group - * assignments, etc. + * assignments, etc. See the "removeAuthentictedUserItems" (sic) method. * * Longer term, the intention is to have a "disableAuthenticatedUser" * method/command. See https://github.com/IQSS/dataverse/issues/2419 @@ -215,6 +220,7 @@ public void deleteAuthenticatedUser(Object pk) { em.remove(confirmEmailData); } userNotificationService.findByUser(user.getId()).forEach(userNotificationService::delete); + em.createNativeQuery("delete from OAuth2TokenData where user_id =" + user.getId()).executeUpdate(); AuthenticationProvider prv = lookupProvider(user); if ( prv != null && prv.isUserDeletionAllowed() ) { @@ -478,6 +484,10 @@ public String getDeleteUserErrorMessages(AuthenticatedUser au) { if (!datasetVersionService.getDatasetVersionUsersByAuthenticatedUser(au).isEmpty()) { reasons.add(BundleUtil.getStringFromBundle("admin.api.deleteUser.failure.versionUser")); } + + if (!savedSearchService.findByAuthenticatedUser(au).isEmpty()) { + reasons.add(BundleUtil.getStringFromBundle("admin.api.deleteUser.failure.savedSearches")); + } if (!reasons.isEmpty()) { retVal = BundleUtil.getStringFromBundle("admin.api.deleteUser.failure.prefix", Arrays.asList(au.getIdentifier())); @@ -518,7 +528,6 @@ private void deletePendingAccessRequests(AuthenticatedUser au){ } - public AuthenticatedUser save( AuthenticatedUser user ) { em.persist(user); em.flush(); diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/BuiltinUserServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/BuiltinUserServiceBean.java index c39c7cb2985..7804716561b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/BuiltinUserServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/BuiltinUserServiceBean.java @@ -79,6 +79,8 @@ public BuiltinUser find(Long pk) { public void removeUser( String userName ) { final BuiltinUser user = findByUserName(userName); if ( user != null ) { + // TODO: Consider adding a cascade delete instead. + passwordResetService.deleteResetDataByDataverseUser(user); em.remove(user); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java new file mode 100644 index 00000000000..41a1708e4c5 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java @@ -0,0 +1,228 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetVersionUser; +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.GuestbookResponse; +import edu.harvard.iq.dataverse.RoleAssignment; +import edu.harvard.iq.dataverse.authorization.groups.impl.explicit.ExplicitGroup; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; +import edu.harvard.iq.dataverse.search.savedsearch.SavedSearch; +import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; +import java.math.BigDecimal; +import java.util.List; +import java.util.Set; +import javax.json.Json; +import javax.json.JsonArrayBuilder; +import javax.json.JsonObjectBuilder; + +// Superuser-only enforced below. +@RequiredPermissions({}) +public class GetUserTracesCommand extends AbstractCommand { + + private DataverseRequest request; + private AuthenticatedUser user; + + public GetUserTracesCommand(DataverseRequest request, AuthenticatedUser user) { + super(request, (DvObject) null); + this.request = request; + this.user = user; + } + + @Override + public JsonObjectBuilder execute(CommandContext ctxt) throws CommandException { + if (!(getUser() instanceof AuthenticatedUser) || !getUser().isSuperuser()) { + throw new PermissionException("Get user traces command can only be called by superusers.", this, null, null); + } + if (user == null) { + throw new CommandException("Cannot get traces. User not found.", this); + } + Long userId = user.getId(); + JsonObjectBuilder traces = Json.createObjectBuilder(); +// List roleAssignments = ctxt.permissions().getDvObjectsUserHasRoleOn(user); + List roleAssignments = ctxt.roleAssignees().getAssignmentsFor(user.getIdentifier()); + if (roleAssignments != null && !roleAssignments.isEmpty()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (RoleAssignment roleAssignment : roleAssignments) { + jab.add(NullSafeJsonBuilder.jsonObjectBuilder() + .add("id", roleAssignment.getId()) + .add("definitionPointName", roleAssignment.getDefinitionPoint().getCurrentName()) + .add("definitionPointIdentifier", roleAssignment.getDefinitionPoint().getIdentifier()) + .add("definitionPointId", roleAssignment.getDefinitionPoint().getId()) + .add("roleAlias", roleAssignment.getRole().getAlias()) + .add("roleName", roleAssignment.getRole().getName()) + ); + } + job.add("count", roleAssignments.size()); + job.add("items", jab); + traces.add("roleAssignments", job); + } + List dataversesCreated = ctxt.dataverses().findByCreatorId(userId); + if (dataversesCreated != null && !dataversesCreated.isEmpty()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (Dataverse dataverse : dataversesCreated) { + jab.add(Json.createObjectBuilder() + .add("id", dataverse.getId()) + .add("alias", dataverse.getAlias()) + ); + } + job.add("count", dataversesCreated.size()); + job.add("items", jab); + traces.add("dataverseCreator", job); + } + List dataversesPublished = ctxt.dataverses().findByReleaseUserId(userId); + if (dataversesPublished != null && !dataversesPublished.isEmpty()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (Dataverse dataverse : dataversesPublished) { + jab.add(Json.createObjectBuilder() + .add("id", dataverse.getId()) + .add("alias", dataverse.getAlias()) + ); + } + job.add("count", dataversesPublished.size()); + job.add("items", jab); + traces.add("dataversePublisher", job); + } + List datasetsCreated = ctxt.datasets().findByCreatorId(userId); + if (datasetsCreated != null && !datasetsCreated.isEmpty()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (Dataset dataset : datasetsCreated) { + jab.add(Json.createObjectBuilder() + .add("id", dataset.getId()) + .add("pid", dataset.getGlobalId().asString()) + ); + } + job.add("count", datasetsCreated.size()); + job.add("items", jab); + traces.add("datasetCreator", job); + } + List datasetsPublished = ctxt.datasets().findByReleaseUserId(userId); + if (datasetsPublished != null && !datasetsPublished.isEmpty()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (Dataset dataset : datasetsPublished) { + jab.add(Json.createObjectBuilder() + .add("id", dataset.getId()) + .add("pid", dataset.getGlobalId().asString()) + ); + } + job.add("count", datasetsPublished.size()); + job.add("items", jab); + traces.add("datasetPublisher", job); + } + List dataFilesCreated = ctxt.files().findByCreatorId(userId); + if (dataFilesCreated != null && !dataFilesCreated.isEmpty()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (DataFile dataFile : dataFilesCreated) { + jab.add(Json.createObjectBuilder() + .add("id", dataFile.getId()) + .add("filename", dataFile.getCurrentName()) + .add("datasetPid", dataFile.getOwner().getGlobalId().asString()) + ); + } + job.add("count", dataFilesCreated.size()); + job.add("items", jab); + traces.add("dataFileCreator", job); + } + // TODO: Consider removing this because we don't seem to populate releaseuser_id for files. + List dataFilesPublished = ctxt.files().findByReleaseUserId(userId); + if (dataFilesPublished != null && !dataFilesPublished.isEmpty()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (DataFile dataFile : dataFilesPublished) { + jab.add(Json.createObjectBuilder() + .add("id", dataFile.getId()) + .add("filename", dataFile.getCurrentName()) + .add("datasetPid", dataFile.getOwner().getGlobalId().asString()) + ); + } + job.add("count", dataFilesPublished.size()); + job.add("items", jab); + traces.add("dataFileCreator", job); + } + // These are the users who have published a version (or created a draft). + List datasetVersionUsers = ctxt.datasetVersion().getDatasetVersionUsersByAuthenticatedUser(user); + if (datasetVersionUsers != null && !datasetVersionUsers.isEmpty()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (DatasetVersionUser datasetVersionUser : datasetVersionUsers) { + jab.add(Json.createObjectBuilder() + .add("id", datasetVersionUser.getId()) + .add("dataset", datasetVersionUser.getDatasetVersion().getDataset().getGlobalId().asString()) + .add("version", datasetVersionUser.getDatasetVersion().getSemanticVersion()) + ); + } + job.add("count", datasetVersionUsers.size()); + job.add("items", jab); + traces.add("datasetVersionUsers", job); + } + Set explicitGroups = ctxt.explicitGroups().findDirectlyContainingGroups(user); + if (explicitGroups != null && !explicitGroups.isEmpty()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (ExplicitGroup explicitGroup : explicitGroups) { + jab.add(Json.createObjectBuilder() + .add("id", explicitGroup.getId()) + .add("name", explicitGroup.getDisplayName()) + ); + } + job.add("count", explicitGroups.size()); + job.add("items", jab); + traces.add("explicitGroups", job); + } + List guestbookResponses = ctxt.responses().findByAuthenticatedUserId(user); + if (guestbookResponses != null && !guestbookResponses.isEmpty()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + // The feeling is that this is too much detail for now so we only show a count. +// JsonArrayBuilder jab = Json.createArrayBuilder(); +// for (GuestbookResponse guestbookResponse : guestbookResponses) { +// jab.add(Json.createObjectBuilder() +// .add("id", guestbookResponse.getId()) +// .add("downloadType", guestbookResponse.getDownloadtype()) +// .add("filename", guestbookResponse.getDataFile().getCurrentName()) +// .add("date", guestbookResponse.getResponseDate()) +// .add("guestbookName", guestbookResponse.getGuestbook().getName()) +// .add("dataset", guestbookResponse.getDatasetVersion().getDataset().getGlobalId().asString()) +// .add("version", guestbookResponse.getDatasetVersion().getSemanticVersion()) +// ); +// } + job.add("count", guestbookResponses.size()); +// job.add("items", jab); + traces.add("guestbookEntries", job); + } + List savedSearchs = ctxt.savedSearches().findByAuthenticatedUser(user); + if (savedSearchs != null && !savedSearchs.isEmpty()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (SavedSearch savedSearch : savedSearchs) { + jab.add(Json.createObjectBuilder() + .add("id", savedSearch.getId()) + ); + } + job.add("count", savedSearchs.size()); + job.add("items", jab); + traces.add("savedSearches", job); + } + JsonObjectBuilder result = Json.createObjectBuilder(); + result.add("user", Json.createObjectBuilder() + .add("identifier", user.getIdentifier()) + .add("name", user.getName()) + ); + result.add("traces", traces); + return result; + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java index 28db9b890e9..a13cf27fcd6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java @@ -25,6 +25,7 @@ import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; +import edu.harvard.iq.dataverse.passwordreset.PasswordResetData; import edu.harvard.iq.dataverse.search.IndexResponse; import edu.harvard.iq.dataverse.search.savedsearch.SavedSearch; import edu.harvard.iq.dataverse.workflows.WorkflowComment; @@ -185,8 +186,8 @@ protected void executeImpl(CommandContext ctxt) throws CommandException { ctxt.em().remove(consumedAUL); ctxt.em().remove(consumedAU); BuiltinUser consumedBuiltinUser = ctxt.builtinUsers().findByUserName(consumedAU.getUserIdentifier()); - if (consumedBuiltinUser != null){ - ctxt.em().remove(consumedBuiltinUser); + if (consumedBuiltinUser != null) { + ctxt.builtinUsers().removeUser(consumedBuiltinUser.getUserName()); } diff --git a/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetData.java b/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetData.java index 78af2a31dc2..a3150161c52 100644 --- a/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetData.java +++ b/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetData.java @@ -28,7 +28,9 @@ @NamedQuery(name="PasswordResetData.findByUser", query="SELECT prd FROM PasswordResetData prd WHERE prd.builtinUser = :user"), @NamedQuery(name="PasswordResetData.findByToken", - query="SELECT prd FROM PasswordResetData prd WHERE prd.token = :token") + query="SELECT prd FROM PasswordResetData prd WHERE prd.token = :token"), + @NamedQuery(name="PasswordResetData.deleteByUser", + query="DELETE FROM PasswordResetData prd WHERE prd.builtinUser = :user"), }) @Entity public class PasswordResetData implements Serializable { diff --git a/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetServiceBean.java index 507c31f5595..c3f0f7f2abc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetServiceBean.java @@ -186,6 +186,12 @@ private long deleteAllExpiredTokens() { return numDeleted; } + public void deleteResetDataByDataverseUser(BuiltinUser user) { + TypedQuery typedQuery = em.createNamedQuery("PasswordResetData.deleteByUser", PasswordResetData.class); + typedQuery.setParameter("user", user); + int numRowsAffected = typedQuery.executeUpdate(); + } + public PasswordChangeAttemptResponse attemptPasswordReset(BuiltinUser user, String newPassword, String token) { final String messageSummarySuccess = "Password Reset Successfully"; diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 1fd7c300ab4..6cdf8b93550 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2280,8 +2280,7 @@ admin.api.deleteUser.failure.dvobjects= the user has created Dataverse object(s) admin.api.deleteUser.failure.gbResps= the user is associated with file download (Guestbook Response) record(s) admin.api.deleteUser.failure.roleAssignments=the user is associated with role assignment record(s) admin.api.deleteUser.failure.versionUser=the user has contributed to dataset version(s) -admin.api.deleteUser.failure.groupMember=the user is a member of Explicit Group(s) -admin.api.deleteUser.failure.pendingRequests=the user has pending File Access Request(s) +admin.api.deleteUser.failure.savedSearches=the user has created saved searches admin.api.deleteUser.success=Authenticated User {0} deleted. #Files.java diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DeleteUsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DeleteUsersIT.java new file mode 100644 index 00000000000..cbfe761234d --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/api/DeleteUsersIT.java @@ -0,0 +1,693 @@ +package edu.harvard.iq.dataverse.api; + +import com.jayway.restassured.RestAssured; +import com.jayway.restassured.path.json.JsonPath; +import com.jayway.restassured.response.Response; +import edu.harvard.iq.dataverse.authorization.DataverseRole; +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import javax.json.Json; +import javax.json.JsonObjectBuilder; +import static javax.ws.rs.core.Response.Status.BAD_REQUEST; +import static javax.ws.rs.core.Response.Status.CREATED; +import static javax.ws.rs.core.Response.Status.OK; +import static junit.framework.Assert.assertEquals; +import static org.hamcrest.CoreMatchers.equalTo; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * The following query has been helpful in discovering places where user ids + * appear throughout the database. Here's a summary of how user deletion affects + * these tables. + * + * - apitoken: Not a concern. Tokens are deleted. + * + * - authenticateduserlookup: Not a concern. Rows are deleted. + * + * - confirmemaildata: Not a concern. Rows are deleted. + * + * - datasetlock: Not a concern, locks are deleted. + * + * - datasetversionuser: Definitely a concern. This table is what feeds the + * "Contributors" list under the "Version" tab on the dataset page. You can't + * delete the user. You can merge the user but the name under "Contributors" + * will change to the user you merged into. There is talk of implementing the + * concept of disabling users to handle this. + * + * - dvobject (creator_id): Definitely a concern. You can't delete a user. You + * have to merge instead. + * + * - dvobject (releaseuser_id): Definitely a concern. You can't delete a user. + * You have to merge instead. It seems that for files, releaseuser_id is not + * populated. + * + * - explicitgroup: Not a concern. Group membership is deleted. + * + * - fileaccessrequests: Not a concern. File requests are deleted. + * + * - guestbookresponse: Definitely a concern but it's possible to null out the + * user id. You can't delete a user but you can merge instead. There is talk of + * disable which would probably null out the id. In all cases the name and email + * address in the rows are left alone. + * + * - oauth2tokendata: Not a concern. Rows are deleted. + * + * - savedsearch: Definitely a concern. You can't delete a user. You have to + * merge. + * + * - userbannermessage: Not a concern. Rows are deleted. + * + * - usernotification (user_id): Not a concern. Deleted by a cascade. + * + * - usernotification (requestor_id): Not a big concern because of other + * constraints. This is only populated by "submit for review" (so that the + * curator has the name and email address of the author). All these + * notifications would be deleted by a cascade but deleting the user itself is + * prevented because the user recorded in the datasetversionuser table. (Both + * "submit for review" and "return to author" add you to that table.) So the + * bottom line is that the user can't be deleted. It has to be merged. + * + * - workflowcomment: Not a big concern because of other constraints. A workflow + * comment is optionally added as part of "return to author" but this also + * creates a row in the datasetversionuser table which means the user can't be + * deleted. It has to be merged instead. + * + * + * The tables that aren't captured above are actionlogrecord and roleassignment + * because the relationship is to the identifier (username) rather than the id. + * So we'll list them separately: + * + * - actionlogrecord: Not a concern. Delete can go through. On merge, they are + * changed from one user identifier to another. + * + * - roleassignment: Not a concern. Delete can go through. On merge, they are + * changed from one user identifier to another. + */ +/* + table_name | constraint_name +---------------------------------+---------------------------------------------------------------- + apitoken | fk_apitoken_authenticateduser_id + authenticateduserlookup | fk_authenticateduserlookup_authenticateduser_id + confirmemaildata | fk_confirmemaildata_authenticateduser_id + datasetlock | fk_datasetlock_user_id + datasetversionuser | fk_datasetversionuser_authenticateduser_id + dvobject | fk_dvobject_creator_id + dvobject | fk_dvobject_releaseuser_id + explicitgroup_authenticateduser | explicitgroup_authenticateduser_containedauthenticatedusers_id + fileaccessrequests | fk_fileaccessrequests_authenticated_user_id + guestbookresponse | fk_guestbookresponse_authenticateduser_id + oauth2tokendata | fk_oauth2tokendata_user_id + savedsearch | fk_savedsearch_creator_id + userbannermessage | fk_userbannermessage_user_id + usernotification | fk_usernotification_user_id + usernotification | fk_usernotification_requestor_id + workflowcomment | fk_workflowcomment_authenticateduser_id +(16 rows) + +-- https://stackoverflow.com/questions/5347050/postgresql-sql-script-to-get-a-list-of-all-tables-that-has-a-particular-column +select R.TABLE_NAME, R.CONSTRAINT_NAME +from INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE u +inner join INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS FK + on U.CONSTRAINT_CATALOG = FK.UNIQUE_CONSTRAINT_CATALOG + and U.CONSTRAINT_SCHEMA = FK.UNIQUE_CONSTRAINT_SCHEMA + and U.CONSTRAINT_NAME = FK.UNIQUE_CONSTRAINT_NAME +inner join INFORMATION_SCHEMA.KEY_COLUMN_USAGE R + ON R.CONSTRAINT_CATALOG = FK.CONSTRAINT_CATALOG + AND R.CONSTRAINT_SCHEMA = FK.CONSTRAINT_SCHEMA + AND R.CONSTRAINT_NAME = FK.CONSTRAINT_NAME +WHERE U.COLUMN_NAME = 'id' +-- AND U.TABLE_CATALOG = 'b' +-- AND U.TABLE_SCHEMA = 'c' + AND U.TABLE_NAME = 'authenticateduser' +ORDER BY R.TABLE_NAME; + */ +public class DeleteUsersIT { + + @BeforeClass + public static void setUp() { + RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); + } + + @Test + public void testDeleteRolesAndUnpublishedDataverse() { + + Response createSuperuser = UtilIT.createRandomUser(); + String superuserUsername = UtilIT.getUsernameFromResponse(createSuperuser); + String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser); + Response toggleSuperuser = UtilIT.makeSuperUser(superuserUsername); + toggleSuperuser.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String usernameForCreateDV = UtilIT.getUsernameFromResponse(createUser); + String normalApiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response getTraces1 = UtilIT.getUserTraces(usernameForCreateDV, superuserApiToken); + getTraces1.prettyPrint(); + getTraces1.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.user.identifier", equalTo("@" + usernameForCreateDV)) + // traces is {} when user hasn't left a trace + .body("data.traces", equalTo(Collections.emptyMap())); + + Response createDataverse = UtilIT.createRandomDataverse(normalApiToken); + createDataverse.prettyPrint(); + createDataverse.then().assertThat() + .statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverse); + + Response getTraces2 = UtilIT.getUserTraces(usernameForCreateDV, superuserApiToken); + getTraces2.prettyPrint(); + getTraces2.then().assertThat().statusCode(OK.getStatusCode()); + + if (true) { + return; + } + + createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String usernameForAssignedRole = UtilIT.getUsernameFromResponse(createUser); + String roleApiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response assignRole = UtilIT.grantRoleOnDataverse(dataverseAlias, DataverseRole.EDITOR.toString(), + "@" + usernameForAssignedRole, superuserApiToken); + + // Shouldn't be able to delete user with a role + Response deleteUserRole = UtilIT.deleteUser(usernameForAssignedRole); + + deleteUserRole.prettyPrint(); + deleteUserRole.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo("Could not delete Authenticated User @" + usernameForAssignedRole + " because the user is associated with role assignment record(s).")); + + // Now remove that role + Response removeRoles1 = UtilIT.deleteUserRoles(usernameForAssignedRole, superuserApiToken); + removeRoles1.prettyPrint(); + removeRoles1.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", equalTo("Roles removed for user " + usernameForAssignedRole + ".")); + + // Now the delete should work + Response deleteUserRole2 = UtilIT.deleteUser(usernameForAssignedRole); + deleteUserRole2.prettyPrint(); + deleteUserRole2.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", equalTo("AuthenticatedUser @" + usernameForAssignedRole + " deleted. ")); + + // The owner of the dataverse that was just created is dataverseAdmin because it created the parent dataverse (root). + Response getTraces3 = UtilIT.getUserTraces(usernameForCreateDV, superuserApiToken); + getTraces3.prettyPrint(); + getTraces3.then().assertThat().statusCode(OK.getStatusCode()); + + // Removing roles here but could equally just delete the dataverse. + Response removeRoles2 = UtilIT.deleteUserRoles(usernameForCreateDV, superuserApiToken); + removeRoles2.prettyPrint(); + removeRoles2.then().assertThat() + .statusCode(OK.getStatusCode()); + + // Shouldn't be able to delete a user who has created a DV + Response deleteUserCreateDV = UtilIT.deleteUser(usernameForCreateDV); + deleteUserCreateDV.prettyPrint(); + deleteUserCreateDV.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo("Could not delete Authenticated User @" + usernameForCreateDV + " because the user has created Dataverse object(s).")); + + Response deleteDataverse = UtilIT.deleteDataverse(dataverseAlias, superuserApiToken); + deleteDataverse.prettyPrint(); + deleteDataverse.then().assertThat() + .statusCode(OK.getStatusCode()); + + // Should be able to delete user after dv is deleted + Response deleteUserAfterDeleteDV = UtilIT.deleteUser(usernameForCreateDV); + deleteUserAfterDeleteDV.prettyPrint(); + deleteUserAfterDeleteDV.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response deleteSuperuser = UtilIT.deleteUser(superuserUsername); + deleteSuperuser.prettyPrint(); + assertEquals(200, deleteSuperuser.getStatusCode()); + + } + + @Test + public void testDeleteUserWithUnPublishedDataverse() { + + Response createSuperuser = UtilIT.createRandomUser(); + String superuserUsername = UtilIT.getUsernameFromResponse(createSuperuser); + String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser); + Response toggleSuperuser = UtilIT.makeSuperUser(superuserUsername); + toggleSuperuser.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverse = UtilIT.createRandomDataverse(apiToken); + createDataverse.prettyPrint(); + createDataverse.then().assertThat() + .statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverse); + + Response removeRoles1 = UtilIT.deleteUserRoles(username, superuserApiToken); + removeRoles1.prettyPrint(); + removeRoles1.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", equalTo("Roles removed for user " + username + ".")); + + Response deleteUser1 = UtilIT.deleteUser(username); + deleteUser1.prettyPrint(); + deleteUser1.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo("Could not delete Authenticated User @" + username + " because the user has created Dataverse object(s).")); + + Response traces = UtilIT.getUserTraces(username, superuserApiToken); + traces.prettyPrint(); + traces.then().assertThat().statusCode(OK.getStatusCode()); + + // You can't delete. You have to merge. + Response mergeAccounts = UtilIT.mergeAccounts(superuserUsername, username, superuserApiToken); + mergeAccounts.prettyPrint(); + mergeAccounts.then().assertThat().statusCode(OK.getStatusCode()); + } + + /** + * You can't delete an account with guestbook entries so you have to merge + * it instead. + */ + @Test + public void testDeleteUserWithGuestbookEntries() throws IOException { + + Response createSuperuser = UtilIT.createRandomUser(); + String superuserUsername = UtilIT.getUsernameFromResponse(createSuperuser); + String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser); + Response toggleSuperuser = UtilIT.makeSuperUser(superuserUsername); + toggleSuperuser.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String authorUsername = UtilIT.getUsernameFromResponse(createUser); + String authorApiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response downloader = UtilIT.createRandomUser(); + downloader.prettyPrint(); + String downloaderUsername = UtilIT.getUsernameFromResponse(downloader); + String downloaderApiToken = UtilIT.getApiTokenFromResponse(downloader); + + Response createDataverse = UtilIT.createRandomDataverse(authorApiToken); + createDataverse.prettyPrint(); + createDataverse.then().assertThat() + .statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverse); + + Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, authorApiToken); + createDataset.prettyPrint(); + createDataset.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset); + String datasetPid = JsonPath.from(createDataset.asString()).getString("data.persistentId"); + + Path pathtoReadme = Paths.get(java.nio.file.Files.createTempDirectory(null) + File.separator + "README.md"); + java.nio.file.Files.write(pathtoReadme, "In the beginning...".getBytes()); + + Response uploadReadme = UtilIT.uploadFileViaNative(datasetId.toString(), pathtoReadme.toString(), authorApiToken); + uploadReadme.prettyPrint(); + uploadReadme.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.files[0].label", equalTo("README.md")); + + int fileId = JsonPath.from(uploadReadme.body().asString()).getInt("data.files[0].dataFile.id"); + + Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, authorApiToken); + publishDataverse.then().assertThat().statusCode(OK.getStatusCode()); + Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetPid, "major", authorApiToken); + publishDataset.then().assertThat().statusCode(OK.getStatusCode()); + // This download creates a guestbook entry. + Response downloadFile = UtilIT.downloadFile(fileId, downloaderApiToken); + downloadFile.then().assertThat().statusCode(OK.getStatusCode()); + + // We can't delete the downloader because a guestbook record (a download) has been created. + Response deleteDownloaderFail = UtilIT.deleteUser(downloaderUsername); + deleteDownloaderFail.prettyPrint(); + deleteDownloaderFail.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()); + + // Let's see why we can't download. + Response getTraces = UtilIT.getUserTraces(downloaderUsername, superuserApiToken); + getTraces.prettyPrint(); + getTraces.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.traces.guestbookEntries.count", equalTo(1)); + + // We can't delete so we do a merge instead. + Response mergeAccounts = UtilIT.mergeAccounts(superuserUsername, downloaderUsername, superuserApiToken); + mergeAccounts.prettyPrint(); + mergeAccounts.then().assertThat().statusCode(OK.getStatusCode()); + + } + + @Test + public void testDatasetLocks() throws IOException { + + Response createSuperuser = UtilIT.createRandomUser(); + String superuserUsername = UtilIT.getUsernameFromResponse(createSuperuser); + String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser); + Response toggleSuperuser = UtilIT.makeSuperUser(superuserUsername); + toggleSuperuser.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String authorUsername = UtilIT.getUsernameFromResponse(createUser); + String authorApiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response downloader = UtilIT.createRandomUser(); + downloader.prettyPrint(); + String downloaderUsername = UtilIT.getUsernameFromResponse(downloader); + String downloaderApiToken = UtilIT.getApiTokenFromResponse(downloader); + + Response createDataverse = UtilIT.createRandomDataverse(authorApiToken); + createDataverse.prettyPrint(); + createDataverse.then().assertThat() + .statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverse); + + Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, authorApiToken); + createDataset.prettyPrint(); + createDataset.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset); + String datasetPid = JsonPath.from(createDataset.asString()).getString("data.persistentId"); + + Response lockDatasetResponse = UtilIT.lockDataset(datasetId.longValue(), "Ingest", superuserApiToken); + lockDatasetResponse.prettyPrint(); + lockDatasetResponse.then().assertThat() + .body("data.message", equalTo("dataset locked with lock type Ingest")) + .statusCode(200); + + Response checkDatasetLocks = UtilIT.checkDatasetLocks(datasetId.longValue(), "Ingest", superuserApiToken); + checkDatasetLocks.prettyPrint(); + checkDatasetLocks.then().assertThat() + .body("data[0].lockType", equalTo("Ingest")) + .statusCode(200); + Response deleteUserWhoCreatedLock = UtilIT.deleteUser(superuserUsername); + deleteUserWhoCreatedLock.prettyPrint(); + deleteUserWhoCreatedLock.then().assertThat() + .statusCode(OK.getStatusCode()); + } + + @Test + public void testDeleteUserWhoIsMemberOfGroup() throws IOException { + + Response createSuperuser = UtilIT.createRandomUser(); + String superuserUsername = UtilIT.getUsernameFromResponse(createSuperuser); + String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser); + Response toggleSuperuser = UtilIT.makeSuperUser(superuserUsername); + toggleSuperuser.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String authorUsername = UtilIT.getUsernameFromResponse(createUser); + String authorApiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response downloader = UtilIT.createRandomUser(); + downloader.prettyPrint(); + String downloaderUsername = UtilIT.getUsernameFromResponse(downloader); + String downloaderApiToken = UtilIT.getApiTokenFromResponse(downloader); + + Response createDataverse = UtilIT.createRandomDataverse(authorApiToken); + createDataverse.prettyPrint(); + createDataverse.then().assertThat() + .statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverse); + + Response createGroupMember = UtilIT.createRandomUser(); + createGroupMember.prettyPrint(); + String groupMemberUsername = UtilIT.getUsernameFromResponse(createGroupMember); + String groupMemberApiToken = UtilIT.getApiTokenFromResponse(createGroupMember); + + String aliasInOwner = "groupFor" + dataverseAlias; + String displayName = "Group for " + dataverseAlias; + String user2identifier = "@" + groupMemberUsername; + Response createGroup = UtilIT.createGroup(dataverseAlias, aliasInOwner, displayName, superuserApiToken); + createGroup.prettyPrint(); + createGroup.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + String groupIdentifier = JsonPath.from(createGroup.asString()).getString("data.identifier"); + + List roleAssigneesToAdd = new ArrayList<>(); + roleAssigneesToAdd.add(user2identifier); + Response addToGroup = UtilIT.addToGroup(dataverseAlias, aliasInOwner, roleAssigneesToAdd, superuserApiToken); + addToGroup.prettyPrint(); + addToGroup.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response getTraces = UtilIT.getUserTraces(groupMemberUsername, superuserApiToken); + getTraces.prettyPrint(); + getTraces.then().assertThat().statusCode(OK.getStatusCode()); + + Response deleteUserInGroup = UtilIT.deleteUser(groupMemberUsername); + deleteUserInGroup.prettyPrint(); + deleteUserInGroup.then().assertThat() + .statusCode(OK.getStatusCode()); + + } + + @Test + public void testDeleteUserWithFileAccessRequests() throws IOException { + + Response createSuperuser = UtilIT.createRandomUser(); + String superuserUsername = UtilIT.getUsernameFromResponse(createSuperuser); + String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser); + Response toggleSuperuser = UtilIT.makeSuperUser(superuserUsername); + toggleSuperuser.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String authorUsername = UtilIT.getUsernameFromResponse(createUser); + String authorApiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response fileRequester = UtilIT.createRandomUser(); + fileRequester.prettyPrint(); + String fileRequesterUsername = UtilIT.getUsernameFromResponse(fileRequester); + String fileRequesterApiToken = UtilIT.getApiTokenFromResponse(fileRequester); + + Response createDataverse = UtilIT.createRandomDataverse(authorApiToken); + createDataverse.prettyPrint(); + createDataverse.then().assertThat() + .statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverse); + + Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, authorApiToken); + createDataset.prettyPrint(); + createDataset.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset); + String datasetPid = JsonPath.from(createDataset.asString()).getString("data.persistentId"); + + Path pathtoReadme = Paths.get(java.nio.file.Files.createTempDirectory(null) + File.separator + "README.md"); + java.nio.file.Files.write(pathtoReadme, "In the beginning...".getBytes()); + + Response uploadReadme = UtilIT.uploadFileViaNative(datasetId.toString(), pathtoReadme.toString(), authorApiToken); + uploadReadme.prettyPrint(); + uploadReadme.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.files[0].label", equalTo("README.md")); + + Integer fileId = JsonPath.from(uploadReadme.body().asString()).getInt("data.files[0].dataFile.id"); + + Response restrictResponse = UtilIT.restrictFile(fileId.toString(), true, authorApiToken); + restrictResponse.prettyPrint(); + restrictResponse.then().assertThat().statusCode(OK.getStatusCode()); + + //Update Dataset to allow requests + Response allowAccessRequestsResponse = UtilIT.allowAccessRequests(datasetPid, true, authorApiToken); + allowAccessRequestsResponse.prettyPrint(); + allowAccessRequestsResponse.then().assertThat().statusCode(OK.getStatusCode()); + + Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, authorApiToken); + publishDataverse.then().assertThat().statusCode(OK.getStatusCode()); + Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetPid, "major", authorApiToken); + publishDataset.then().assertThat().statusCode(OK.getStatusCode()); + + Response requestFileAccessResponse = UtilIT.requestFileAccess(fileId.toString(), fileRequesterApiToken); + requestFileAccessResponse.prettyPrint(); + requestFileAccessResponse.then().assertThat().statusCode(OK.getStatusCode()); + + // Let's see why we can't download. + Response getTraces = UtilIT.getUserTraces(fileRequesterUsername, superuserApiToken); + getTraces.prettyPrint(); + getTraces.then().assertThat() + .statusCode(OK.getStatusCode()); + + // Even if users have outstanding file requests, they can be deleted. + Response deleteDownloaderSuccess = UtilIT.deleteUser(fileRequesterUsername); + deleteDownloaderSuccess.prettyPrint(); + deleteDownloaderSuccess.then().assertThat() + .statusCode(OK.getStatusCode()); + } + + @Test + public void testCuratorSendsCommentsToAuthor() throws InterruptedException { + Response createSuperuser = UtilIT.createRandomUser(); + String superuserUsername = UtilIT.getUsernameFromResponse(createSuperuser); + String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser); + Response toggleSuperuser = UtilIT.makeSuperUser(superuserUsername); + toggleSuperuser.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response createCurator1 = UtilIT.createRandomUser(); + createCurator1.prettyPrint(); + createCurator1.then().assertThat() + .statusCode(OK.getStatusCode()); + String curator1Username = UtilIT.getUsernameFromResponse(createCurator1); + String curator1ApiToken = UtilIT.getApiTokenFromResponse(createCurator1); + + Response createCurator2 = UtilIT.createRandomUser(); + createCurator2.prettyPrint(); + createCurator2.then().assertThat() + .statusCode(OK.getStatusCode()); + String curator2Username = UtilIT.getUsernameFromResponse(createCurator2); + String curator2ApiToken = UtilIT.getApiTokenFromResponse(createCurator2); + + Response createDataverseResponse = UtilIT.createRandomDataverse(curator1ApiToken); + createDataverseResponse.prettyPrint(); + createDataverseResponse.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response makeCurator2Admin = UtilIT.grantRoleOnDataverse(dataverseAlias, DataverseRole.ADMIN.toString(), "@" + curator2Username, curator1ApiToken); + makeCurator2Admin.prettyPrint(); + makeCurator2Admin.then().assertThat() + .body("data.assignee", equalTo("@" + curator2Username)) + .body("data._roleAlias", equalTo("admin")) + .statusCode(OK.getStatusCode()); + + Response createAuthor1 = UtilIT.createRandomUser(); + createAuthor1.prettyPrint(); + createAuthor1.then().assertThat() + .statusCode(OK.getStatusCode()); + String author1Username = UtilIT.getUsernameFromResponse(createAuthor1); + String author1ApiToken = UtilIT.getApiTokenFromResponse(createAuthor1); + + Response createAuthor2 = UtilIT.createRandomUser(); + createAuthor2.prettyPrint(); + createAuthor2.then().assertThat() + .statusCode(OK.getStatusCode()); + String author2Username = UtilIT.getUsernameFromResponse(createAuthor2); + String author2ApiToken = UtilIT.getApiTokenFromResponse(createAuthor2); + + Response grantAuthor1AddDataset = UtilIT.grantRoleOnDataverse(dataverseAlias, DataverseRole.DS_CONTRIBUTOR.toString(), "@" + author1Username, curator1ApiToken); + grantAuthor1AddDataset.prettyPrint(); + grantAuthor1AddDataset.then().assertThat() + .body("data.assignee", equalTo("@" + author1Username)) + .body("data._roleAlias", equalTo("dsContributor")) + .statusCode(OK.getStatusCode()); + + Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, author1ApiToken); + createDataset.prettyPrint(); + createDataset.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset); + + // FIXME: have the initial create return the DOI or Handle to obviate the need for this call. + Response getDatasetJsonBeforePublishing = UtilIT.nativeGet(datasetId, author1ApiToken); + getDatasetJsonBeforePublishing.prettyPrint(); + String protocol = JsonPath.from(getDatasetJsonBeforePublishing.getBody().asString()).getString("data.protocol"); + String authority = JsonPath.from(getDatasetJsonBeforePublishing.getBody().asString()).getString("data.authority"); + String identifier = JsonPath.from(getDatasetJsonBeforePublishing.getBody().asString()).getString("data.identifier"); + + String datasetPersistentId = protocol + ":" + authority + "/" + identifier; + System.out.println("datasetPersistentId: " + datasetPersistentId); + +// Response grantAuthor2ContributorOnDataset = UtilIT.grantRoleOnDataset(datasetPersistentId, DataverseRole.DS_CONTRIBUTOR.toString(), "@" + author2Username, curatorApiToken); + // TODO: Tighten this down to something more realistic than ADMIN. + Response grantAuthor2ContributorOnDataset = UtilIT.grantRoleOnDataset(datasetPersistentId, DataverseRole.ADMIN.toString(), "@" + author2Username, curator1ApiToken); + grantAuthor2ContributorOnDataset.prettyPrint(); + grantAuthor2ContributorOnDataset.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.assignee", equalTo("@" + author2Username)) + .body("data._roleAlias", equalTo("admin")); + +// // Whoops, the author tries to publish but isn't allowed. The curator will take a look. +// Response noPermToPublish = UtilIT.publishDatasetViaNativeApi(datasetPersistentId, "major", author1ApiToken); +// noPermToPublish.prettyPrint(); +// noPermToPublish.then().assertThat() +// .body("message", equalTo("User @" + author1Username + " is not permitted to perform requested action.")) +// .statusCode(UNAUTHORIZED.getStatusCode()); + Response submitForReview = UtilIT.submitDatasetForReview(datasetPersistentId, author2ApiToken); + submitForReview.prettyPrint(); + submitForReview.then().assertThat() + .statusCode(OK.getStatusCode()); + + // curator2 returns dataset to author. This makes curator2 a contributor. + String comments = "You forgot to upload any files."; + JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder(); + jsonObjectBuilder.add("reasonForReturn", comments); + Response returnToAuthor = UtilIT.returnDatasetToAuthor(datasetPersistentId, jsonObjectBuilder.build(), curator2ApiToken); + returnToAuthor.prettyPrint(); + returnToAuthor.then().assertThat() + .body("data.inReview", equalTo(false)) + .statusCode(OK.getStatusCode()); + + Response getTracesForCurator2 = UtilIT.getUserTraces(curator2Username, superuserApiToken); + getTracesForCurator2.prettyPrint(); + getTracesForCurator2.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response removeRolesFromCurator2 = UtilIT.deleteUserRoles(curator2Username, superuserApiToken); + removeRolesFromCurator2.prettyPrint(); + removeRolesFromCurator2.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", equalTo("Roles removed for user " + curator2Username + ".")); + + // Because curator2 returned the dataset to the authors, curator2 is now a contributor + // and cannot be deleted. + Response deleteCurator2Fail = UtilIT.deleteUser(curator2Username); + deleteCurator2Fail.prettyPrint(); + deleteCurator2Fail.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo("Could not delete Authenticated User @" + curator2Username + + " because the user has contributed to dataset version(s).")); + + // What should we do with curator2 instead of deleting? The only option is to merge + // curator2 into some other account. Once implemented, we'll disable curator2's account + // so that curator2 continues to be displayed as a contributor. + // + // TODO: disable curator2 here + // + Response removeRolesFromAuthor2 = UtilIT.deleteUserRoles(author2Username, superuserApiToken); + removeRolesFromAuthor2.prettyPrint(); + removeRolesFromAuthor2.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", equalTo("Roles removed for user " + author2Username + ".")); + + // Similarly, we can't delete author2 because author2 submitted + // the dataset for review, which makes one a contributor. + Response deleteAuthor2Fail = UtilIT.deleteUser(author2Username); + deleteAuthor2Fail.prettyPrint(); + deleteAuthor2Fail.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo("Could not delete Authenticated User @" + author2Username + + " because the user has contributed to dataset version(s).")); + + } + +} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 27b34aba5e1..dee35556729 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -956,6 +956,20 @@ public static Response deleteUser(String username) { return deleteUserResponse; } + public static Response deleteUserRoles(String username, String apiToken) { + Response deleteUserResponse = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .post("/api/users/" + username + "/removeRoles"); + return deleteUserResponse; + } + + public static Response getUserTraces(String username, String apiToken) { + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/users/" + username + "/traces"); + return response; + } + public static Response reingestFile(Long fileId, String apiToken) { Response response = given() .header(API_TOKEN_HTTP_HEADER, apiToken) From 07dff1a2a9e24ce3bca970593bdbaa00f095549d Mon Sep 17 00:00:00 2001 From: Stefan Kasberger Date: Fri, 5 Feb 2021 23:05:14 +0100 Subject: [PATCH 281/576] Update pyDataverse section --- doc/sphinx-guides/source/api/client-libraries.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/client-libraries.rst b/doc/sphinx-guides/source/api/client-libraries.rst index 64726c77f58..e611f6b2565 100755 --- a/doc/sphinx-guides/source/api/client-libraries.rst +++ b/doc/sphinx-guides/source/api/client-libraries.rst @@ -13,7 +13,8 @@ Python There are two Python modules for interacting with Dataverse APIs. -`pyDataverse `_ had its initial release in 2019 and can be installed with ``pip install pyDataverse``. The module is developed by `Stefan Kasberger `_ from `AUSSDA - The Austrian Social Science Data Archive `_. +`pyDataverse `_ helps with the Dataverse API's and data types (Dataverse, Dataset, Datafile). It's intention is to help with data migrations, DevOps activities like testing and configuration and to build services upon. +The module is developed by `Stefan Kasberger `_ from `AUSSDA - The Austrian Social Science Data Archive `_. `dataverse-client-python `_ had its initial release in 2015. `Robert Liebowitz `_ created this library while at the `Center for Open Science (COS) `_ and the COS uses it to integrate the `Open Science Framework (OSF) `_ with Dataverse via an add-on which itself is open source and listed on the :doc:`/api/apps` page. From 3190fd971d841bdf8a26b2b48ad18d8987b16580 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Feb 2021 17:44:46 -0500 Subject: [PATCH 282/576] var outside try{} --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 9826ecfe795..4021969bf6e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -768,11 +768,12 @@ private String getCompoundDisplayValue (DatasetFieldCompoundValue dscv){ @PUT @Path("{id}/editMetadata") - public Response editVersionMetadata(String jsonBody, @PathParam("id") String id, @QueryParam("replace") Boolean replace) throws WrappedResponse{ + public Response editVersionMetadata(String jsonBody, @PathParam("id") String id, @QueryParam("replace") Boolean replace) { Boolean replaceData = replace != null; + DataverseRequest req = null; try { - DataverseRequest req = createDataverseRequest(findUserOrDie()); + req = createDataverseRequest(findUserOrDie()); } catch (WrappedResponse ex) { logger.log(Level.SEVERE, "Edit metdata error: " + ex.getMessage(), ex); return ex.getResponse(); From 9e1d6f543f2d02d6daa7ff5b66eebb1c38e0236a Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Mon, 8 Feb 2021 09:15:11 -0500 Subject: [PATCH 283/576] #7582 tidy 5.0 release note upgrade instruction formatting --- doc/release-notes/5.0-release-notes.md | 41 +++++++++++++++++--------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/doc/release-notes/5.0-release-notes.md b/doc/release-notes/5.0-release-notes.md index e3d13eaca7c..c65900846b1 100644 --- a/doc/release-notes/5.0-release-notes.md +++ b/doc/release-notes/5.0-release-notes.md @@ -250,33 +250,46 @@ The instructions below describe the upgrade procedure based on moving an existin `sudo rm -rf /usr/local/payara5/glassfish/domains/domain1/generated/` `sudo rm -rf /usr/local/payara5/glassfish/domains/domain1/osgi-cache/` -8. In domain.xml: +8. Make the following changes in **domain.xml**: - Replace the -XX:PermSize and -XX:MaxPermSize JVM options with -XX:MetaspaceSize and -XX:MaxMetaspaceSize. + Replace the `-XX:PermSize` and `-XX:MaxPermSize` JVM options with `-XX:MetaspaceSize` and `-XX:MaxMetaspaceSize` + ``` -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m + ``` -Add the below JVM options beneath the -Ddataverse settings: + Add the below JVM options beneath the -Ddataverse settings: + ``` -Dfish.payara.classloading.delegate=false -XX:+UseG1GC -XX:+UseStringDeduplication -XX:+DisableExplicitGC + ``` -Also in domain.xml, replace the follow element: + Replace the following element: - ` ` + ``` + + + + +``` -with + with - ` ` + ``` + + + + ``` -9. Change any full pathnames /usr/local/glassfish4/... to /usr/local/payara5/... or whatever it is in your case. (Specifically check the -Ddataverse.files.directory and -Ddataverse.files.file.directory JVM options) +9. Change any full pathnames matching `/usr/local/glassfish4/...` to `/usr/local/payara5/...` or whatever it is in your case. Specifically check the `-Ddataverse.files.directory` and `-Ddataverse.files.file.directory` JVM options. 10. In domain1/config/jhove.conf, change the hard-coded /usr/local/glassfish4 path, as above. -(Optional): If you renamed your service account from glassfish to payara or appserver, update the ownership permissions. The Installation Guide recommends a service account of `dataverse`: + (Optional): If you renamed your service account from glassfish to payara or appserver, update the ownership permissions. The Installation Guide recommends a service account of `dataverse`: `sudo chown -R dataverse /usr/local/payara5/glassfish/domains/domain1` `sudo chown -R dataverse /usr/local/payara5/glassfish/lib` @@ -309,20 +322,20 @@ with 3. (Required for installations using DataCite) Add the JVM option doi.dataciterestapiurlstring -For production environments: + For production environments: `/usr/local/payara5/bin/asadmin create-jvm-options "\-Ddoi.dataciterestapiurlstring=https\://api.datacite.org"` -For test environments: + For test environments: `/usr/local/payara5/bin/asadmin create-jvm-options "\-Ddoi.dataciterestapiurlstring=https\://api.test.datacite.org"` -The JVM option `doi.mdcbaseurlstring` should be deleted if it was previously set, for example: + The JVM option `doi.mdcbaseurlstring` should be deleted if it was previously set, for example: `/usr/local/payara5/bin/asadmin delete-jvm-options "\-Ddoi.mdcbaseurlstring=https\://api.test.datacite.org"` 4. (Recommended for installations using DataCite) Pre-register DOIs -Execute the script described in the section "Dataverse Installations Using DataCite: Upgrade Action Recommended" earlier in the Release Note. + Execute the script described in the section "Dataverse Installations Using DataCite: Upgrade Action Recommended" earlier in the Release Note. -Please consult the earlier sections of the Release Note for any additional configuration options that may apply to your installation. + Please consult the earlier sections of the Release Note for any additional configuration options that may apply to your installation. From 71a46e92bc29966f0339f847e5a91de9982dad7b Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Mon, 8 Feb 2021 12:53:00 -0500 Subject: [PATCH 284/576] #7582 reformat the reformat to correct indentation --- doc/release-notes/5.0-release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/5.0-release-notes.md b/doc/release-notes/5.0-release-notes.md index c65900846b1..3bbd33db3ae 100644 --- a/doc/release-notes/5.0-release-notes.md +++ b/doc/release-notes/5.0-release-notes.md @@ -275,7 +275,7 @@ The instructions below describe the upgrade procedure based on moving an existin -``` + ``` with From fbe12d2922ea5308d21472de7f756a319ee1578e Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Mon, 8 Feb 2021 14:25:54 -0500 Subject: [PATCH 285/576] Added position CSS for tabView component to fix dropdown z-index layer, cleaned up other CSS on structure stylesheet [ref #7558] --- src/main/webapp/resources/css/structure.css | 113 +++++--------------- 1 file changed, 28 insertions(+), 85 deletions(-) diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index e70bb2f7b93..3269cf7e12f 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -15,27 +15,15 @@ body .ui-widget {font-size: inherit;} /* -------- TEMPLATE -------- */ /* STICKY FOOTER VS DROPDOWN MENU POSITION + Z-INDEX */ -#content {position:relative; z-index:50;} +/*#content {position:relative;}*/ .panelgridHeaderTable {width:100%;} .panelgridHeaderTable .ui-widget-content, .panelgridHeaderTable .ui-widget-content > td {border:0; background:none;} .panelgridHeaderTable .ui-widget-content > td:last-child table {margin-left:auto;} -.bg-dataverse, bg-brand {background:#C55B28;} -.bg-dataset {background:#337AB7;} -.bg-file {background:#777777;} -.bg-muted {background:#f5f5f5;} -.text-dataverse, .text-brand {color:#C55B28;} -.text-dataset {color:#31708F;} -.text-file {color:#777777;} - -#navbarFixed {margin-bottom:0;} -.navbar-fixed-top, .navbar-fixed-bottom { - z-index: 930; -} +#navbarFixed.navbar-fixed-top {margin-bottom:0; z-index:9999;} #navbarFixed .navbar-brand {color:#C55B28; padding-left:32px;} #navbarFixed .navbar-brand.custom-logo {height:auto; padding:0;} #navbarFixed .icon-dataverse {color:#C55B28; font-size:28px; margin: -4px 0 0 -27px; position: absolute;} - #navbarFixed .dropdown.accountName a:focus {outline: -webkit-focus-ring-color 5px auto;} #navbarFixed form.navbar-form.navbar-form-link {margin:0; padding:0;} #navbarFixed form.navbar-form-link a.ui-commandlink, #navbarFixed ul.navbar-nav li.dropdown span.dropdown-toggle {display:block; padding:15px; color: #777;} @@ -70,40 +58,8 @@ body .ui-widget {font-size: inherit;} #footer .version {vertical-align:bottom;white-space:nowrap;} #footer.widget-view .widgetBrandMsg {line-height:42px;} -.panelLayoutBlock, .panelLayoutTitleBlock, .panelLayoutButtonBlock {border:0;} -.panelLayoutBlock .ui-panelgrid.panelgridLayoutTable .ui-widget-content, .panelLayoutBlock .ui-panelgrid.panelgridLayoutTable .ui-widget-content > td {border:0;} .form-group table.noBorders tr.ui-widget-content, .form-group table.noBorders td {border:0; padding-left:0;} -.panelgridLayoutTable, .panelgridFormTable {width:100%;} -.panelgridFormTable > tbody > tr.ui-widget-content > td:first-child {width:20%;} -.panelgridFormTable.ui-panelgrid td table.ui-picklist td {border: 0; padding:0;} -.panelgridFormTable.ui-panelgrid td table.ui-picklist td ul.ui-widget-content.ui-corner-all {border:1px solid #DDDDDD; border-radius: 4px;} - -.panelLayoutButtonBlock button {/*margin-right:2em;*/} - -.panelLayoutBlock > .ui-widget-content {border-bottom:1px solid #DDDDDD;} -.panelLayoutBlock.ui-panel.padding-none > .ui-widget-content.ui-panel-content {padding:0;} - -.panelLayoutBlock.breadcrumbNavBlock > .ui-widget-content {border-bottom:0;} -.panelLayoutTitleBlock > .ui-widget-content {border:0;} - -.panelLayoutBlock .ui-panelgrid.panelgridLayoutTable .ui-widget-content .ui-panelgrid {margin-bottom: 1em;} -.panelLayoutBlock .ui-panelgrid.panelgridLayoutTable .ui-widget-content .ui-panelgrid .ui-widget-content .ui-panelgrid {margin-bottom: 0} -.panelLayoutBlock .ui-panelgrid.panelgridLayoutTable .ui-widget-content .ui-panelgrid .ui-widget-content .ui-panelgrid td:first-child {padding-left:0;} - -.btn span.glyphicon {margin-right:.3em;} -.btn.btn-link.bootstrap-button-tooltip span.glyphicon {margin:0;} /* OR JUST USE NO-TEXT... */ -.btn.btn-link.bootstrap-button-tooltip span.caret {margin-left:6px;} -.btn span.glyphicon.no-text {margin:0;} -.btn-group .btn span.glyphicon.no-text {margin-left:10px;margin-right:10px;} -.col-manage-action .btn-group .btn span.glyphicon.no-text {margin:0;} -.icon-inline-action {margin-left:.3em;} -.icon-inline-action:first-of-type {margin-left:1em;} - -.btn-group .text-button {display:inline-block;padding:6px;} - -.text-icon-inline {font-size:1.3em;line-height:1.1em;margin-right:4px;} - #messagePanel div.messagePanel {margin-top:1em;} .messagePanel .glyphicon {font-size: 1.5em; vertical-align: bottom;} @@ -128,6 +84,14 @@ td.col-select-width, th.col-select-width {width:36px;} #ajaxStatusPanel_start {width:31px;height:31px;position:fixed;right:50%;bottom:50%;z-index:111111;} #ajaxStatusPanel_complete {position:absolute;top:-9999px;left:-9999px;} +.bg-dataverse, bg-brand {background:#C55B28;} +.bg-dataset {background:#337AB7;} +.bg-file {background:#777777;} +.bg-muted {background:#f5f5f5;} +.text-dataverse, .text-brand {color:#C55B28;} +.text-dataset {color:#31708F;} +.text-file {color:#777777;} + .italic {font-style: italic;} .margin-top {margin-top:1em;} .margin-top-half {margin-top:.5em;} @@ -148,6 +112,16 @@ table.selectManyCheckboxBlock label {font-weight:normal;} hr {border-color: #ddd;} +.btn span.glyphicon {margin-right:.3em;} +.btn.btn-link.bootstrap-button-tooltip span.glyphicon {margin:0;} /* OR JUST USE NO-TEXT... */ +.btn.btn-link.bootstrap-button-tooltip span.caret {margin-left:6px;} +.btn span.glyphicon.no-text {margin:0;} +.btn-group .btn span.glyphicon.no-text {margin-left:10px;margin-right:10px;} +.col-manage-action .btn-group .btn span.glyphicon.no-text {margin:0;} +.icon-inline-action {margin-left:.3em;} +.icon-inline-action:first-of-type {margin-left:1em;} +.btn-group .text-button {display:inline-block;padding:6px;} +.text-icon-inline {font-size:1.3em;line-height:1.1em;margin-right:4px;} span.glyphicon[data-toggle='tooltip'] {cursor: default;} span.glyphicon-question-sign.tooltip-icon {color: #99bcdb !important;} span.glyphicon-question-sign.tooltip-icon:hover {color: #337AB7 !important;} @@ -261,17 +235,11 @@ body.widget-view #dataverse-header-block {padding-top:0px;} #breadcrumbNavBlock .breadcrumbActive {font-weight:bold;} #breadcrumbNavBlock a[id^='breadcrumbLnkTree'] {cursor: pointer; cursor: hand;} #breadcrumbNavBlock a[id^='breadcrumbLnkTree'] span.glyphicon-chevron-down {vertical-align:middle;padding-bottom:2px;} - #breadcrumbNavBlock > .ui-panel-content {overflow:hidden;} - #breadcrumbNavBlock.panelLayoutBlock > .ui-widget-content {border-bottom: 0;} - #breadcrumbNavBlock .breadcrumbCarrot {float:left; margin:0 .4em; color:#B3B3B3;} - #breadcrumbNavBlock div.dropdown-menu {padding-right:.5em; min-width:280px;} - #breadcrumbNavBlock .ui-tree {width: auto;} - #breadcrumbNavBlock div.ui-tree.ui-widget-content {border:0;} #breadcrumbNavBlock div.ui-tree.ui-widget-content span.ui-treenode-content span.ui-icon.ui-tree-toggler {background-size: auto 284px; height:20px; width:20px;} #breadcrumbNavBlock div.ui-tree.ui-widget-content span.ui-treenode-content span.ui-icon.ui-tree-toggler.ui-icon-triangle-1-s {background-position: -72px -16px;} @@ -511,9 +479,7 @@ form[id$='facetType'] > div.clearfix:last-child {margin-bottom:0;} form[id$='facetType'] > div.clearfix .facetTypeChBox {display:block; float:left;} form[id$='facetType'] > div.clearfix .facetTypeChBox > div.ui-chkbox {padding-top:2px;padding-right:4px;} form[id$='facetType'] > div.clearfix a.facetTypeLink {display:block; float:left; text-decoration:none;} - form[id$='facetType'] > div.clearfix a.facetTypeLink span {padding:4px 0px;} - form[id$='facetCategoryForm'] {margin-top:1em;} div[id$='facetCategoryList'] ul, div[id$='facetCategoryList'] li, div[id$='facetCategoryList'] div {margin:0; padding:0; list-style: none; border:0;} div[id$='facetCategoryList'] > div > ul > li {border-bottom:1px solid #DDD; margin-bottom:.5em;} @@ -522,7 +488,6 @@ div[id$='facetCategoryList'] div li div {padding:4px 0px;} div[id$='facetCategoryList'] div li div.ui-datalist-empty-message {padding:4px 0px; font-style:italic; color:#AAA;} div[id$='facetCategoryList'] div.ui-datalist, div[id$='facetCategoryList'] div.ui-datalist div.ui-datalist-content {padding-top:0;} .facetCategoryName {font-weight:bold;} - .facetCategoryName, .facetLink { -ms-word-break: break-all; @@ -572,7 +537,7 @@ div.ui-dialog.smallPopUp {max-width:600px;} div.ui-dialog.largePopUp {min-width:60%; max-width:80%; max-height:80%; overflow-y:scroll;} div.ui-dialog .ui-datatable-scrollable table {table-layout: auto !important;} div.ui-dialog .ui-datatable-scrollable table colgroup col {max-width:1000px;} - +.ui-dialog.ui-widget.ui-widget-content {min-width:600px;} div.ui-dialog div.highlight { background-color: #f7f7f9; border: 1px solid #e1e1e8; @@ -583,8 +548,6 @@ div.ui-dialog div.highlight { .ui-dialog.dialog-submenu, .ui-dialog.dialog-submenu .ui-dialog-content, .ui-dialog.dialog-submenu .form-inline, .ui-dialog.dialog-submenu .form-group {overflow:visible !important;} .ui-dialog.dialog-submenu .ui-menu-list.ui-menu-child {z-index: 9999 !important; width: auto; min-width: 12.5em;} -.ui-dialog.ui-widget.ui-widget-content {min-width:600px;} - /* -------- SUBTABS -------- */ div.popupSubTabParent > div.ui-tabs-panels > div.ui-tabs-panel.ui-widget-content {padding-left:0; padding-right:0;} div.ui-tabs-panels .ui-tabs .ui-tabs-nav.ui-widget-header li {border:0;} @@ -742,15 +705,10 @@ div[id$="filesHeaderCount"] {padding:6px 12px;} div[id$="filesHeaderCount"] span.highlightBold {display:block; padding:0; line-height:1.65;} div[id$="filesTable"] .ui-selection-column {width:20px !important;} div[id$="filesTable"] .col-file-metadata {padding-left:0; padding-right:0; border-left:0; border-right:0; text-align:left; font-weight:normal;} - -/*div[id$="filesTable"] .col-file-thumb {padding:4px 10px; width:84px;}*/ div[id$="filesTable"] .col-file-thumb div.thumbnail-block {position:relative;width:64px;height:64px;margin:0 auto;} div[id$="filesTable"] .col-file-thumb div.thumbnail-block span.file-thumbnail-icon {font-size:62px;line-height:1.05;} div[id$="filesTable"] .col-file-thumb div.thumbnail-block span.file-thumbnail-preview-img {display:block;} div[id$="filesTable"] .col-file-thumb div.thumbnail-block div.file-icon-restricted-block {display:block;position:absolute;bottom:-4px;right:-1px;} -/*div[id$="filesTable"] .file-metadata-block {-webkit-flex: 1 100%; - -moz-flex: 1 100%; - -ms-flex: 1 100%; flex: 1 100%;}*/ div[id$="filesTable"] div.file-tags-block {padding-bottom:2px;} div[id$="filesTable"] div.file-tags-block a.ui-commandlink {font-size:12px;} div[id$="filesTable"] div.file-tags-block span.label {display:inline-block;} @@ -837,8 +795,6 @@ div[id$="filesTable"] .ui-paginator .ui-paginator-pages .ui-state-active { /* FILES TREE */ div[id$="filesTree"] span.ui-treenode-label span[class^='icon'] {margin-left: -2px;} -/* span.ingest-warning {font-size:2em !important; line-height:1; vertical-align: middle;} */ - /* DOWNLOAD POPUP */ div.panel-body.read-terms{max-height:220px; overflow-y:scroll; width:100%; background-color:#eeeeee;} @@ -848,11 +804,9 @@ div.panel-body.read-terms{max-height:220px; overflow-y:scroll; width:100%; backg /* VERSIONS */ div[id$="versionsTable"] th.col-select-width * {display:none;} - #version-details-block > div {width:35%; padding:4px 10px; border: 1px solid #DDDDDD;} #version-details-block > div:first-of-type {width:30%; border:0;} #version-details-block > div:last-of-type {border-left-width:0;} - div.dvnDifferanceTable > div.ui-datatable-tablewrapper > table > thead {display:none;} div.dvnDifferanceTable > div.ui-datatable-tablewrapper > table, div.dvnDifferanceTable > div.ui-datatable-tablewrapper > table > tbody {border:0;} div.dvnDifferanceTable > div.ui-datatable-tablewrapper > table > tbody > tr:first-child, div.dvnDifferanceTable > div.ui-datatable-tablewrapper > table > tbody > tr:first-child > td {border-top-width:0;} @@ -861,22 +815,14 @@ div.dvnDifferanceTable > div.ui-datatable-tablewrapper > table > tbody > tr, div div.dvnDifferanceTable > div.ui-datatable-tablewrapper > table > tbody > tr, div.dvnDifferanceTable > div.ui-datatable-tablewrapper > table > tbody > tr > td:last-child {border-right-width:0;} div.dvnDifferanceTable div.ui-datatable-tablewrapper table td.versionValue {width:30%;} div.dvnDifferanceTable div.ui-datatable-tablewrapper table td.versionDetails {width:35%;} - div.dvnDifferanceTable .diffDetailBlock {display:block;} -/* MANAGE TABLE + DROPDOWN BUTTON */ -div.manageTable.ui-datatable div.ui-datatable-tablewrapper, div.manageTable.ui-datatable td.col-manage-action {overflow:visible;} -div[id$="filesTable"].ui-datatable thead th.col-file-action, div[id$="filesTable"].ui-datatable tbody td.col-file-action {overflow:visible !important;} +/* DATATABLE + DROPDOWN BUTTON + OVERFLOW VISIBLE */ thead.ui-datatable-scrollable-theadclone {display:none} - -/* DATATABE + OVERFLOW VISIBLE */ -.ui-datatable thead th, .ui-datatable tbody td, .ui-datatable tfoot td, .ui-datatable tfoot th {overflow:visible;} +.ui-datatable thead th, .ui-datatable tbody td, .ui-datatable tfoot td, .ui-datatable tfoot th {overflow:visible !important;} /* FILE */ -#file-preview-icon-block > #border-block {/*border:1px solid #DDD;border-radius:4px; padding: 10px;*/} -#file-preview-icon-block > #border-block > #preview-square {/*position: relative; overflow:hidden; padding-bottom:100%;*/} -#file-preview-icon-block > #border-block > #preview-square > span.text-muted {font-size:6.5em; line-height:1 !important; /*text-align: center; position: absolute; width: 100%; height: 100%; margin:0;*/} -#file-preview-icon-block > #border-block > #preview-square > img.img-responsive {/*position: absolute; max-width: 100%; max-height: 100%; top: 50%; left: 50%; transform: translateX(-50%) translateY(-50%);*/} +#file-preview-icon-block > #border-block > #preview-square > span.text-muted {font-size:6.5em; line-height:1 !important;} @media(min-width:768px){ #file-preview-icon-block > #border-block > #preview-square > span.text-muted { font-size: 9.5em; @@ -905,10 +851,11 @@ thead.ui-datatable-scrollable-theadclone {display:none} div.ui-dialog.ui-widget-content {z-index:100001 !important;} /* POPUP DROPDOWN MENUS + TOOLTIPS + BLOCK UI */ -div.ui-selectcheckboxmenu-panel, div.ui-selectonemenu-panel, div.ui-autocomplete-panel, -.ui-selectonemenu-list, .ui-selectcheckboxmenu-list, div.tooltip, div.popover {z-index:100002 !important;} +.ui-selectcheckboxmenu-panel, .ui-selectonemenu-panel, .ui-autocomplete-panel, +.ui-selectonemenu-list, .ui-selectcheckboxmenu-list, .tooltip, .popover {z-index:100002 !important;} /* Overwrite Primefaces */ +.ui-tabs.ui-widget.ui-widget-content {position:relative; z-index:999;} .ui-tabs.ui-widget-content .ui-tabs-nav li.ui-tabs-selected a {color: #555555;} button.btn-default.ui-button-text-only .ui-button-text {padding:0; font-size:14px;} button.btn-default span.ui-button-text {line-height:inherit !important;} @@ -922,12 +869,8 @@ div.ui-widget-overlay {background:#ccc; opacity:.6;} .ui-button.btn-link:hover, .ui-button.btn-link:focus {color: #23527c; text-decoration: underline !important; background-color: transparent;} .ui-button.btn-link[disabled]:hover, fieldset[disabled] .ui-button.btn-link:hover, .ui-button.btn-link[disabled]:focus, fieldset[disabled] .ui-button.btn-link:focus {color: #777;text-decoration: none;} .ui-widget-header .form-group {font-weight:normal;} -/* .ui-messages-info-detail, .ui-messages-warn-detail, .ui-messages-error-detail, .ui-messages-fatal-detail, .ui-message-info-detail, .ui-message-warn-detail, .ui-message-error-detail, .ui-message-fatal-detail { - margin-left: 0px; -} */ + .ui-state-disabled, .ui-widget-content .ui-state-disabled, .ui-widget-header .ui-state-disabled {background: #e0e0e0; border-color: #ccc; opacity: 0.65;} -/* .ui-autocomplete-input-token input, .ui-autocomplete-input-token {width:100%;} */ -.ui-autocomplete-panel {z-index:100002 !important;} div.ui-autocomplete {width:100%;} div.ui-autocomplete > ul, div.ui-autocomplete > ul > li > input {width:100%;} /* FIX WIDTH FOR AUTOCOMPLETE WITHOUT P:FRAGMENT */ span.ui-autocomplete {width:100%;} /* FIX WIDTH FOR AUTOCOMPLETE IN P:FRAGMENT */ From 754b3f9d71aa0b9ab51a3028180e6e95bf6115c3 Mon Sep 17 00:00:00 2001 From: Michael Heppler <687227+mheppler@users.noreply.github.com> Date: Mon, 8 Feb 2021 14:31:37 -0500 Subject: [PATCH 286/576] Removing commented out styles not in use [ref #7758] --- src/main/webapp/resources/css/structure.css | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index 3269cf7e12f..2603fd6ba0b 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -14,8 +14,6 @@ body .ui-widget {font-size: inherit;} .ui-widget-content a:hover, .ui-widget-content a:focus {color: #23527C;} /* -------- TEMPLATE -------- */ -/* STICKY FOOTER VS DROPDOWN MENU POSITION + Z-INDEX */ -/*#content {position:relative;}*/ .panelgridHeaderTable {width:100%;} .panelgridHeaderTable .ui-widget-content, .panelgridHeaderTable .ui-widget-content > td {border:0; background:none;} .panelgridHeaderTable .ui-widget-content > td:last-child table {margin-left:auto;} From 51707f5ad01b5a59b28225ebc20fe289c8d6b5d1 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Mon, 8 Feb 2021 16:28:23 -0500 Subject: [PATCH 287/576] Removed extra link from Feature Dataverse carousel [ref #7476] --- src/main/webapp/dataverse.xhtml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index fbbe7563baf..14dc7ee22cb 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -580,17 +580,13 @@
    • +
    +

    VIEW ALL DATA

    +
    From ba623ad830b66c66c836c651e96d63d8c0216ee7 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 9 Feb 2021 15:17:18 -0500 Subject: [PATCH 289/576] #6972 allow pdf download if "original" is selected. --- src/main/java/edu/harvard/iq/dataverse/api/Access.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index df9e39547b9..e98bc8aa1fa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -331,6 +331,14 @@ public DownloadInstance datafile(@PathParam("fileId") String fileId, @QueryParam // So we need to identify when a service is being called and then let checkIfServiceSupportedAndSetConverter see if the required one exists if (key.equals("imageThumb") || key.equals("format") || key.equals("variables") || key.equals("noVarHeader")) { serviceRequested = true; + //In the dataset file table context a user is allowed to select original as the format + //for download + // if the dataset has tabular files - it should not be applied to instances + // where the file selected is not tabular see #6972 + if("format".equals(key) && "original".equals(value) && !df.isTabularData()) { + serviceRequested = false; + break; + } //Only need to check if this key is associated with a service if (downloadInstance.checkIfServiceSupportedAndSetConverter(key, value)) { // this automatically sets the conversion parameters in From b70c3aac407eccc4eb72a97b9c37156efd4d9c79 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 9 Feb 2021 15:18:25 -0500 Subject: [PATCH 290/576] initial doc for s3 upload api --- doc/sphinx-guides/source/developers/index.rst | 1 + .../developers/s3-direct-upload-api.rst | 103 ++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 doc/sphinx-guides/source/developers/s3-direct-upload-api.rst diff --git a/doc/sphinx-guides/source/developers/index.rst b/doc/sphinx-guides/source/developers/index.rst index 184d8aff85a..eebfd50ba35 100755 --- a/doc/sphinx-guides/source/developers/index.rst +++ b/doc/sphinx-guides/source/developers/index.rst @@ -34,4 +34,5 @@ Developer Guide selinux big-data-support aux-file-support + s3-direct-upload-api workflows diff --git a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst new file mode 100644 index 00000000000..8c6f2521000 --- /dev/null +++ b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst @@ -0,0 +1,103 @@ +Direct DataFile Upload API +========================== + +The direct Datafile Upload API is used internally to support direct upload of files to S3 storage and by tools such as the DVUploader. + +Direct upload involves a series of three activities: +- Requesting initiation of a transfer from the Dataverse server +- Use of the pre-signed URL(s) returned in that call to perform an upload/multipart-upload of the file to S3 +- A call to the Dataverse server to register the file as part of the dataset and/or to cancel the transfer + +This API is only enabled when a Dataset is configured with a data store supporting direct S3 upload. +Administrators should be aware that partial transfers, where a client starts uploading the file/parts of the file and does not contact the Dataverse server to complete/cancel the transfer, will result in data stored in S3 that is not referenced in Dataverse (e.g. should be considered temporary and deleted.) + + +Requesting Direct Upload of a DataFile +-------------------------------------- +To initiate a transfer of a file to S3, make a call to the Dataverse Server indicating the size of the file to upload. The response will include a pre-signed URL(s) that allow the client to transfer the file. Pre-signed URLs include a short-lived token authorizing the action represented by the URL. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV + export SIZE=1000000000 + + curl -H X-Dataverse-key:$API_TOKEN "$SERVER_URL/api/datasets/:persistentId/uploadurls?PersistentId=$PERSISTENT_IDENTIFIER&size=$SIZE" + +The response to this call, assuming direct uploads are enabled, will be one of two forms: + +Single URL: when the file is smaller than the size at which uploads must be broken into multiple parts + +.. code-block:: bash + + { + "status":"OK", + "data":{ + "url":"...", + "partSize":1073741824, + "storageIdentifier":"s3://demo-dataverse-bucket:177883619b8-892ca9f7112e" + } + } + +Multiple URLs: when the file must be uploaded in multiple parts. The part size is set by the Dataverse server and, for AWS-based storage, range from 5 MB to 5 GB + +.. code-block:: bash + + { + "status":"OK", + "data":{ + "urls":{ + "1":"...", + "2":"...", + "3":"...", + "4":"...", + "5":"..." + } + "abort":"/api/datasets/mpupload?...", + "complete":"/api/datasets/mpupload?..." + "partSize":1073741824, + "storageIdentifier":"s3://demo-dataverse-bucket:177883b000e-49cedef268ac" + } + +In the example responses above, the URLs, which are very long, have been omitted. These URLs reference the S3 server and the specific object identifier that will be used, starting with, for example, "https://demo-dataverse-bucket.s3.amazonaws.com/10.5072/FK2FOQPJS/177883b000e-49cedef268ac?..." + +The client must then use the URL(s) to POST the file, or if the file is larger than the specified partSize, parts of the file. + +In the multipart case, the client must send each part and collect the 'eTag' responses from the server. To successfully conclude the multipart upload, the client must call the 'complete' URI, sending a json object including the part eTags: + +.. code-block:: bash + + curl -X PUT "$SERVER_URL/api/datasets/mpload?..." -d '{"1":"\","2":"\","3":"\","4":"\","5":"\"}' + +If the client is unable to complete the multipart upload, it should call the abort URL: + +.. code-block:: bash + + curl -X DELETE "$SERVER_URL/api/datasets/mpload?..." + + +Adding the Uploaded file to the Dataset +--------------------------------------- + +Once the file exists in the s3 bucket, a final API call is needed to add it to the Dataset. This call is the same call used to upload a file to Dataverse but, rather than sending the file bytes, additional metadata is added the "jsonData" parameter. +jsonData normally includes information such as a file description, tags, provenance, whether the file is restricted, etc. For direct uploads, the jsonData object must also include values for: +- "storageIdentifier" - as specified in prior calls +- "fileName" +- "mimeType" +- "md5Hash" +- "fileSize" + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV + + curl -X POST -H 'Content-Type: multipart/formm-data' "$SERVER_URL/api/datasets/:persistentId/add?persistentId=#PERSISTENT_IDENTIFIER" --upload-file jsonData.json + +Note that this API call can be used independently of the others, e.g. supporting use cases in which the file already exists in S3/has been uploaded via some out-of-band method. +With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifer must be prefaced with the Dataverse store identifier used, as with the internally generated examples above. + + + \ No newline at end of file From c274ee7d3988049e86497d477ef6e6982925ad98 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 9 Feb 2021 15:33:48 -0500 Subject: [PATCH 291/576] formatting fixes --- .../source/developers/s3-direct-upload-api.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst index 8c6f2521000..67a04786571 100644 --- a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst +++ b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst @@ -4,9 +4,9 @@ Direct DataFile Upload API The direct Datafile Upload API is used internally to support direct upload of files to S3 storage and by tools such as the DVUploader. Direct upload involves a series of three activities: -- Requesting initiation of a transfer from the Dataverse server -- Use of the pre-signed URL(s) returned in that call to perform an upload/multipart-upload of the file to S3 -- A call to the Dataverse server to register the file as part of the dataset and/or to cancel the transfer +* Requesting initiation of a transfer from the Dataverse server +* Use of the pre-signed URL(s) returned in that call to perform an upload/multipart-upload of the file to S3 +* A call to the Dataverse server to register the file as part of the dataset and/or to cancel the transfer This API is only enabled when a Dataset is configured with a data store supporting direct S3 upload. Administrators should be aware that partial transfers, where a client starts uploading the file/parts of the file and does not contact the Dataverse server to complete/cancel the transfer, will result in data stored in S3 that is not referenced in Dataverse (e.g. should be considered temporary and deleted.) @@ -60,7 +60,7 @@ Multiple URLs: when the file must be uploaded in multiple parts. The part size i "storageIdentifier":"s3://demo-dataverse-bucket:177883b000e-49cedef268ac" } -In the example responses above, the URLs, which are very long, have been omitted. These URLs reference the S3 server and the specific object identifier that will be used, starting with, for example, "https://demo-dataverse-bucket.s3.amazonaws.com/10.5072/FK2FOQPJS/177883b000e-49cedef268ac?..." +In the example responses above, the URLs, which are very long, have been omitted. These URLs reference the S3 server and the specific object identifier that will be used, starting with, for example, https://demo-dataverse-bucket.s3.amazonaws.com/10.5072/FK2FOQPJS/177883b000e-49cedef268ac?... The client must then use the URL(s) to POST the file, or if the file is larger than the specified partSize, parts of the file. @@ -82,11 +82,11 @@ Adding the Uploaded file to the Dataset Once the file exists in the s3 bucket, a final API call is needed to add it to the Dataset. This call is the same call used to upload a file to Dataverse but, rather than sending the file bytes, additional metadata is added the "jsonData" parameter. jsonData normally includes information such as a file description, tags, provenance, whether the file is restricted, etc. For direct uploads, the jsonData object must also include values for: -- "storageIdentifier" - as specified in prior calls -- "fileName" -- "mimeType" -- "md5Hash" -- "fileSize" +* "storageIdentifier" - as specified in prior calls +* "fileName" +* "mimeType" +* "md5Hash" +* "fileSize" .. code-block:: bash From 9a7df8cc083b130910935cace774e1cc4914d487 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 9 Feb 2021 15:35:04 -0500 Subject: [PATCH 292/576] more fixes --- doc/sphinx-guides/source/developers/s3-direct-upload-api.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst index 67a04786571..4b2ffb08d65 100644 --- a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst +++ b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst @@ -4,6 +4,7 @@ Direct DataFile Upload API The direct Datafile Upload API is used internally to support direct upload of files to S3 storage and by tools such as the DVUploader. Direct upload involves a series of three activities: + * Requesting initiation of a transfer from the Dataverse server * Use of the pre-signed URL(s) returned in that call to perform an upload/multipart-upload of the file to S3 * A call to the Dataverse server to register the file as part of the dataset and/or to cancel the transfer @@ -82,6 +83,7 @@ Adding the Uploaded file to the Dataset Once the file exists in the s3 bucket, a final API call is needed to add it to the Dataset. This call is the same call used to upload a file to Dataverse but, rather than sending the file bytes, additional metadata is added the "jsonData" parameter. jsonData normally includes information such as a file description, tags, provenance, whether the file is restricted, etc. For direct uploads, the jsonData object must also include values for: + * "storageIdentifier" - as specified in prior calls * "fileName" * "mimeType" From d44c6d8d8794ce20beb6a74606df7f6052b7f08e Mon Sep 17 00:00:00 2001 From: gdurand Date: Tue, 9 Feb 2021 17:10:51 -0500 Subject: [PATCH 293/576] n ew logic to handle the conditally required validation --- .../harvard/iq/dataverse/DatasetField.java | 74 +++++++++----- .../iq/dataverse/DatasetFieldType.java | 2 + .../iq/dataverse/DatasetFieldValidator.java | 47 +++++++-- src/main/java/propertyFiles/Bundle.properties | 3 +- src/main/webapp/dataverse.xhtml | 4 +- src/main/webapp/metadataFragment.xhtml | 8 +- .../dataverse/DatasetFieldValidatorTest.java | 99 +++++++++++++------ 7 files changed, 169 insertions(+), 68 deletions(-) mode change 100755 => 100644 src/main/java/propertyFiles/Bundle.properties diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java index 6d4b56cf6cf..0bca1b50319 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java @@ -407,20 +407,23 @@ public void setValidationMessage(String validationMessage) { this.validationMessage = validationMessage; } + + // these two booleans are used to wrap around the field type isRequired and isHasChildren methods to also check + // if the dataverse has overriden the default values @Transient private Boolean required; + @Transient + private Boolean hasRequiredChildren; public boolean isRequired() { if (required == null) { required = false; - if (this.datasetFieldType.isPrimitive() && this.datasetFieldType.isRequired()) { - required = true; - } - - if (this.datasetFieldType.isHasRequiredChildren()) { + + if (this.datasetFieldType.isRequired()) { required = true; } - + + // otherwise check if overridden at the dataverse level Dataverse dv = getDataverse(); while (!dv.isMetadataBlockRoot()) { if (dv.getOwner() == null) { @@ -430,29 +433,54 @@ public boolean isRequired() { } List dftilListFirst = dv.getDataverseFieldTypeInputLevels(); - if (!getDatasetFieldType().isHasChildren()) { - for (DataverseFieldTypeInputLevel dsftil : dftilListFirst) { - if (dsftil.getDatasetFieldType().equals(this.datasetFieldType)) { - required = dsftil.isRequired(); - } + for (DataverseFieldTypeInputLevel dsftil : dftilListFirst) { + if (dsftil.getDatasetFieldType().equals(this.datasetFieldType)) { + required = dsftil.isRequired(); } } - - if (getDatasetFieldType().isHasChildren() && (!dftilListFirst.isEmpty())) { - for (DatasetFieldType child : getDatasetFieldType().getChildDatasetFieldTypes()) { - for (DataverseFieldTypeInputLevel dftilTest : dftilListFirst) { - if (child.equals(dftilTest.getDatasetFieldType())) { - if (dftilTest.isRequired()) { - required = true; - } + + // since we don't currently support the use case where the parent is required + // but no specific children are (the "true/false" case), we override this as false + if (required && this.datasetFieldType.isCompound() && !isHasRequiredChildren()) + { + required = false; + } + } + + return required; + } + + public boolean isHasRequiredChildren() { + if (hasRequiredChildren == null) { + hasRequiredChildren = false; + } + + if (this.datasetFieldType.isHasRequiredChildren()) { + hasRequiredChildren = true; + } + + Dataverse dv = getDataverse(); + while (!dv.isMetadataBlockRoot()) { + if (dv.getOwner() == null) { + break; // we are at the root; which by defintion is metadata blcok root, regarldess of the value + } + dv = dv.getOwner(); + } + + List dftilListFirst = dv.getDataverseFieldTypeInputLevels(); + + if (getDatasetFieldType().isHasChildren() && (!dftilListFirst.isEmpty())) { + for (DatasetFieldType child : getDatasetFieldType().getChildDatasetFieldTypes()) { + for (DataverseFieldTypeInputLevel dftilTest : dftilListFirst) { + if (child.equals(dftilTest.getDatasetFieldType())) { + if (dftilTest.isRequired()) { + hasRequiredChildren = true; } } } } - - } - // logger.fine("at return " + this.datasetFieldType.getDisplayName() + " " + required); - return required; + } + return hasRequiredChildren; } public Dataverse getDataverse() { diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java index 04901264455..842b9490784 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java @@ -141,7 +141,9 @@ public void setOptionSelectItems(List optionSelectItems) { public DatasetFieldType() {} public DatasetFieldType(String name, FieldType fieldType, boolean allowMultiples) { + // use the name for both default name and title this.name = name; + this.title = name; this.fieldType = fieldType; this.allowMultiples = allowMultiples; childDatasetFieldTypes = new LinkedList<>(); diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValidator.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValidator.java index 425ed0b22ef..00b7be97b83 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValidator.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValidator.java @@ -9,6 +9,8 @@ import javax.validation.ConstraintValidatorContext; import edu.harvard.iq.dataverse.util.BundleUtil; +import java.util.Collections; +import java.util.List; import org.apache.commons.lang.StringUtils; @@ -25,22 +27,38 @@ public void initialize(ValidateDatasetFieldType constraintAnnotation) { @Override public boolean isValid(DatasetField value, ConstraintValidatorContext context) { context.disableDefaultConstraintViolation(); // we do this so we can have different messages depending on the different issue - + DatasetFieldType dsfType = value.getDatasetFieldType(); //SEK Additional logic turns off validation for templates if (isTemplateDatasetField(value)){ return true; } - if (((dsfType.isPrimitive() && dsfType.isRequired()) || (dsfType.isPrimitive() && value.isRequired())) - && StringUtils.isBlank(value.getValue())) { - try{ - context.buildConstraintViolationWithTemplate(dsfType.getDisplayName() + " " + BundleUtil.getStringFromBundle("isrequired")).addConstraintViolation(); - } catch (NullPointerException npe){ - //if there's no context for the error we can't put it anywhere.... + + // if value is not primitive or not empty + if (!dsfType.isPrimitive() || !StringUtils.isBlank(value.getValue())) { + return true; + } + + if (value.isRequired()) { + String errorMessage = null; + DatasetFieldCompoundValue parent = value.getParentDatasetFieldCompoundValue(); + if (parent == null || parent.getParentDatasetField().isRequired()) { + errorMessage = BundleUtil.getStringFromBundle("isrequired", List.of(dsfType.getDisplayName())); + } else if (areSiblingsPopulated(value)) { + errorMessage = BundleUtil.getStringFromBundle("isrequired.conditional", List.of(dsfType.getDisplayName(),parent.getParentDatasetField().getDatasetFieldType().getDisplayName())); } - return false; + if (errorMessage != null) { + try { + context.buildConstraintViolationWithTemplate(errorMessage).addConstraintViolation(); + } catch (NullPointerException npe){ + //if there's no context for the error we can't put it anywhere.... + } + + return false; + } } + return true; } @@ -51,4 +69,17 @@ private boolean isTemplateDatasetField(DatasetField dsf) { return dsf.getTemplate() != null; } } + + private boolean areSiblingsPopulated(DatasetField dsf) { + if (dsf.getParentDatasetFieldCompoundValue() != null) { + DatasetFieldCompoundValue compound = dsf.getParentDatasetFieldCompoundValue(); + for (DatasetField sibling : compound.getChildDatasetFields()) { + if (!StringUtils.isBlank(sibling.getValue())) { + return true; + } + } + } + + return false; + } } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties old mode 100755 new mode 100644 index 2a73ea97307..fd46e16b0fe --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2446,7 +2446,8 @@ editfilesfragment.label4.item2=Unicode (UTF-16LE) editfilesfragment.label4.item3=Unicode (UTF-16BE) editfilesfragment.label5=US-ASCII -isrequired=is required. +isrequired={0} is required. +isrequired.conditional={0} is required if you choose to enter text in any of the {1} fields. draftversion=DRAFT VERSION deaccessionedversion=DEACCESSIONED VERSION diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index fbbe7563baf..1921eb324e8 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -331,10 +331,10 @@ + rendered="#{!dsft.required}"> - + #{bundle['dataverse.field.requiredByDataverse']} diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index 908a216fb12..231e59d133e 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -105,14 +105,14 @@
    #{dsf.datasetFieldType.localeTitle} - + @@ -185,7 +185,9 @@
    #{dsf.datasetFieldType.localeTitle} - + + +
    diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValidatorTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValidatorTest.java index 9369c1b6930..e82fc02e37a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValidatorTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValidatorTest.java @@ -47,43 +47,80 @@ public void tearDown() { */ @Test public void testIsValid() { - DatasetField dataSetField = new DatasetField(); + + Dataverse dataverse = new Dataverse(); + Dataset dataset = new Dataset(); + dataset.setOwner(dataverse); + DatasetVersion datasetVersion = new DatasetVersion(); + datasetVersion.setDataset(dataset); + + testPrimitiveDatasetField("test isValid() on template field", true, "", null, true); + testPrimitiveDatasetField("test isValid() for required primitive with empty value", true, "", datasetVersion, false); + testPrimitiveDatasetField("test isValid() for required primitive with nonempty value", true, "test", datasetVersion, true); + testPrimitiveDatasetField("test isValid() for not required primitive", false, "", datasetVersion, true); + + // the compound tests are defined by parent/child1 required and child1/child2 values + testCompoundDatasetField("test isValid() for false/false compound with empty/empty children", false, false, "", "", datasetVersion, true); + testCompoundDatasetField("test isValid() for true/true compound with empty/empty children", true, true, "", "", datasetVersion, false); + testCompoundDatasetField("test isValid() for true/true compound with empty/nonempty children", true, true, "", "test", datasetVersion, false); + testCompoundDatasetField("test isValid() for true/true compound with nonempty/empty children", true, true, "test", "", datasetVersion, true); + testCompoundDatasetField("test isValid() for false/true compound with empty/empty children", false, true, "", "", datasetVersion, true); + testCompoundDatasetField("test isValid() for false/true compound with empty/nonempty children", false, true, "", "test", datasetVersion, false); + testCompoundDatasetField("test isValid() for false/true compound with nonempty/empty children", false, true, "test", "", datasetVersion, true); + } + + private void testPrimitiveDatasetField(String test, boolean required, String value, + DatasetVersion dsv, boolean expectedOutcome) { + DatasetFieldType primitiveDSFType = new DatasetFieldType("primitive", DatasetFieldType.FieldType.TEXT, false); + primitiveDSFType.setRequired(required); + + DatasetField testDatasetField = new DatasetField(); + if (dsv != null) { + testDatasetField.setDatasetVersion(dsv); + } else { + testDatasetField.setTemplate(new Template()); + } + testDatasetField.setDatasetFieldType(primitiveDSFType); + testDatasetField.setSingleValue(value); + DatasetFieldValidator datasetFieldValidator = new DatasetFieldValidator(); + assertEquals( test, expectedOutcome, datasetFieldValidator.isValid(testDatasetField, constraintValidatorContext)); + + } + + private void testCompoundDatasetField(String test, boolean requiredParent, boolean requiredChild1, + String valueChild1, String valueChild2, DatasetVersion dsv, boolean expectedOutcome) { - // if it is a template field it is always valid - dataSetField.setTemplate(new Template()); - boolean expResult = true; - boolean result = datasetFieldValidator.isValid(dataSetField, constraintValidatorContext); - assertEquals("test isValid() on template field", expResult, result); + DatasetFieldType parentDSFType = new DatasetFieldType("parent", DatasetFieldType.FieldType.TEXT, false); + DatasetFieldType child1DSFType = new DatasetFieldType("child1", DatasetFieldType.FieldType.TEXT, false); + DatasetFieldType child2DSFType = new DatasetFieldType("child2", DatasetFieldType.FieldType.TEXT, false); - // if not a template field and required - dataSetField.setTemplate(null); - DatasetVersion datasetVersion = new DatasetVersion(); - Dataset dataset = new Dataset(); - Dataverse dataverse = new Dataverse(); - dataset.setOwner(dataverse); - datasetVersion.setDataset(dataset); - dataSetField.setDatasetVersion(datasetVersion); + parentDSFType.setRequired(requiredParent); + child1DSFType.setRequired(requiredChild1); + child2DSFType.setRequired(false); + + DatasetField parentDatasetField = new DatasetField(); + parentDatasetField.setDatasetVersion(dsv); + parentDatasetField.setDatasetFieldType(parentDSFType); + + DatasetFieldCompoundValue compound = new DatasetFieldCompoundValue(); + compound.setParentDatasetField(parentDatasetField); - DatasetFieldValue datasetFieldValue = new DatasetFieldValue(); - DatasetFieldType datasetFieldType = new DatasetFieldType("test", DatasetFieldType.FieldType.TEXT, false); - datasetFieldType.setRequired(true); - dataSetField.setDatasetFieldType(datasetFieldType); - dataSetField.setSingleValue(""); - datasetFieldValue.setValue(""); - result = datasetFieldValidator.isValid(dataSetField, constraintValidatorContext); - assertEquals("test isValid() if not template field and required with empty DataSetField", false, result); + DatasetField child1DatasetField = new DatasetField(); + child1DatasetField.setParentDatasetFieldCompoundValue(compound); + compound.getChildDatasetFields().add(child1DatasetField); + child1DatasetField.setDatasetFieldType(child1DSFType); + child1DatasetField.setSingleValue(valueChild1); - // fill in a value - the required constraint is satisfied now - dataSetField.setSingleValue("value"); - result = datasetFieldValidator.isValid(dataSetField, constraintValidatorContext); - assertEquals("test isValid() if not template field and required with non-empty DataSetField", true, result); + DatasetField child2DatasetField = new DatasetField(); + child2DatasetField.setParentDatasetFieldCompoundValue(compound); + compound.getChildDatasetFields().add(child2DatasetField); + child2DatasetField.setDatasetFieldType(child2DSFType); + child2DatasetField.setSingleValue(valueChild2); + - // if not required the field can be blank - datasetFieldType.setRequired(false); - dataSetField.setSingleValue(""); - result = datasetFieldValidator.isValid(dataSetField, constraintValidatorContext); - assertEquals("test isValid() if not template field and not required", true, result); + DatasetFieldValidator datasetFieldValidator = new DatasetFieldValidator(); + assertEquals( test, expectedOutcome, datasetFieldValidator.isValid(child1DatasetField, constraintValidatorContext)); } } From eafdc73ecde663d04f6790b19bb5be57b8883932 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 9 Feb 2021 22:16:44 -0500 Subject: [PATCH 294/576] updates from draft code review --- doc/sphinx-guides/source/api/native-api.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 43b9e668cb6..ee606ea94bd 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3009,7 +3009,7 @@ Disables an ``AuthenticatedUser`` whose ``id`` is passed. :: PUT http://$SERVER/api/admin/authenticatedUsers/disable/id/$id -Note: Since the primary purpose of Dataverse is to serve as an archive and to track data accesses and the modifications of data and metadata, a simple mechanism to delete users that have performed certain actions in the system is not provided by design. +Note: A primary purpose of most Dataverse installations is to serve an archive. In the archival space, there are best practices around the tracking of data access and the tracking of modifications to data and metadata. In support of these key workflows, a simple mechanism to delete users that have performed edit or access actions in the system is not provided. Providing a Disable User endpoint instead of a Delete User endpoint is by design. Disabling a user with this endpoint will: @@ -3022,11 +3022,9 @@ Disabling a user with this endpoint will: Disabling a user with this endpoint will keep: - The user's contributions to datasets, including dataset creation, file uploads, and publishing. -- The user's access history to datafiles in the Dataverse installation, including guestbook records and acceptance of terms of use. +- The user's access history to datafiles in the Dataverse installation, including guestbook records. - The user's account information (specifically name, email, affiliation, and position) -If it is determined that disabling a user is not sufficient, it is possible anonymize a user through use of the :ref:`merge-accounts-label` feature. See :ref:`Anonymize a User ` for more information about this process. - List Role Assignments of a Role Assignee ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From a3b51d729f12e111b20a0a6928204a6e85ccaab1 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 9 Feb 2021 22:30:56 -0500 Subject: [PATCH 295/576] adding links from user administration --- doc/sphinx-guides/source/admin/user-administration.rst | 10 ++++++++++ doc/sphinx-guides/source/api/native-api.rst | 4 +++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/admin/user-administration.rst b/doc/sphinx-guides/source/admin/user-administration.rst index bc9be64775f..2999c6e0856 100644 --- a/doc/sphinx-guides/source/admin/user-administration.rst +++ b/doc/sphinx-guides/source/admin/user-administration.rst @@ -44,6 +44,16 @@ Change User Identifier See :ref:`change-identifier-label` +Delete a User +------------- + +See :ref:`delete-a-user` + +Disable a User +-------------- + +See :ref:`disable-a-user` + Confirm Email ------------- diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index ee606ea94bd..312175d4c74 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2982,7 +2982,9 @@ Make User a SuperUser Toggles superuser mode on the ``AuthenticatedUser`` whose ``identifier`` (without the ``@`` sign) is passed. :: POST http://$SERVER/api/admin/superuser/$identifier - + +.. _delete-a-user: + Delete a User ~~~~~~~~~~~~~ From 14138e47e32060bee658f5302183e772f39378da Mon Sep 17 00:00:00 2001 From: gdurand Date: Wed, 10 Feb 2021 11:38:26 -0500 Subject: [PATCH 296/576] tweaks for conditionally required validation --- src/main/java/propertyFiles/Bundle.properties | 4 +++- src/main/webapp/metadataFragment.xhtml | 14 +++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index fd46e16b0fe..cd6c542c62b 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -68,6 +68,8 @@ selectedFiles=Selected Files htmlAllowedTitle=Allowed HTML Tags htmlAllowedMsg=This field supports only certain HTML tags. htmlAllowedTags=<a>, <b>, <blockquote>, <br>, <code>, <del>, <dd>, <dl>, <dt>, <em>, <hr>, <h1>-<h3>, <i>, <img>, <kbd>, <li>, <ol>, <p>, <pre>, <s>, <sup>, <sub>, <strong>, <strike>, <u>, <ul> +conditionalRequiredMsg=One or more of these fields may become required if you add to one or more of these optional fields. +conditionalRequiredMsg.tooltip=This field will become required if you choose to enter values in one or more of these optional fields. toggleNavigation=Toggle navigation defaultBody=Default Body loading=Loading... @@ -2447,7 +2449,7 @@ editfilesfragment.label4.item3=Unicode (UTF-16BE) editfilesfragment.label5=US-ASCII isrequired={0} is required. -isrequired.conditional={0} is required if you choose to enter text in any of the {1} fields. +isrequired.conditional={0} is required if you choose to enter a value in any of the optional {1} fields. draftversion=DRAFT VERSION deaccessionedversion=DEACCESSIONED VERSION diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index 231e59d133e..bb04f173bdb 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -185,13 +185,16 @@
    #{dsf.datasetFieldType.localeTitle} - - - +
    + +

    + +

    +
    @@ -204,9 +207,10 @@ From adc5236d7839d3b5a6e794a2323021cfe5efcc10 Mon Sep 17 00:00:00 2001 From: gdurand Date: Wed, 10 Feb 2021 13:34:58 -0500 Subject: [PATCH 297/576] minor formatting change --- src/main/webapp/metadataFragment.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index bb04f173bdb..74e1f531234 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -210,7 +210,7 @@ + data-original-title="#{subdsf.datasetFieldType.localeDescription}#{subdsf.required and !dsf.required?' – '.concat(bundle['conditionalRequiredMsg.tooltip']) : ''}"> From a9725b0149a3e3bb09d0b5e62026edc4899ed890 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Wed, 10 Feb 2021 13:49:01 -0500 Subject: [PATCH 298/576] Fixed accessible input labels on add dv form [ref #7566] --- src/main/java/propertyFiles/Bundle.properties | 1 + src/main/webapp/contactFormFragment.xhtml | 3 +- src/main/webapp/dataverse.xhtml | 222 +++++++++--------- 3 files changed, 114 insertions(+), 112 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 1fd7c300ab4..1d2f6cee9cc 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -140,6 +140,7 @@ contact.msg=Message contact.msg.required=Message text is required. contact.send=Send Message contact.question=Please fill this out to prove you are not a robot. +contact.sum.title=Human Access Validation Answer contact.sum.required=Value is required. contact.sum.invalid=Incorrect sum, please try again. contact.sum.converterMessage=Please enter a number. diff --git a/src/main/webapp/contactFormFragment.xhtml b/src/main/webapp/contactFormFragment.xhtml index 1a28562df1c..284d84eb885 100644 --- a/src/main/webapp/contactFormFragment.xhtml +++ b/src/main/webapp/contactFormFragment.xhtml @@ -63,7 +63,8 @@ + validatorMessage="#{bundle['contact.sum.invalid']}" validator="#{sendFeedbackDialog.validateUserSum}" + title="#{bundle['contact.sum.title']}"> diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index fbbe7563baf..67ecd4017e7 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -61,14 +61,14 @@
    - + #{bundle.hostDataverse} - +
    -

    #{DataversePage.dataverse.getOwner().name}

    +

    #{DataversePage.dataverse.getOwner().name}

     - + #{bundle['dataverse.email']} - +
    - + +
    @@ -674,121 +674,121 @@ - - -

    #{bundle['dataverse.share.dataverseShare.tip']}

    -
    -
    - -
    -     -     - - - - - - -

    #{bundle['dataverse.link.dataverse.choose']}

    -     - -

    #{bundle['dataverse.savedsearch.dataverse.choose']}

    -     -
    - -
    - - - - - - -
    - #{u.id}
    - #{u.affiliation} -
    -     -     -     -
    -
    - -
    + + +

    #{bundle['dataverse.share.dataverseShare.tip']}

    +
    +
    + +
    +     +     + + + + + + +

    #{bundle['dataverse.link.dataverse.choose']}

    +     + +

    #{bundle['dataverse.savedsearch.dataverse.choose']}

    +     +
    -

    - #{SearchIncludeFragment.query.toString()} -

    + + + + + + +
    + #{u.id}
    + #{u.affiliation} +
    +     +     +
    -
    - -
    - -

    -     + +
    + +
    +

    + #{SearchIncludeFragment.query.toString()} +

    +
    +
    +
    + +
    + +

    +     +
    +     +
    + + +
    -
    - - - -
    - - -

    #{bundle['dataverse.link.no.choice']}

    -
    -
    - -
    -

    #{DataversePage.linkingDataverse.displayName}

    + +

    #{bundle['dataverse.link.no.choice']}

    +
    +
    + +
    +

    #{DataversePage.linkingDataverse.displayName}

    +
    -
    -
    - - -
    - - -

    - #{bundle['dataverse.link.no.linkable']} -

    -
    - -
    -     - - +
    + + +
    + + +

    + #{bundle['dataverse.link.no.linkable']} +

    +
    + +
    +     + + From 3fdbb45277b4a128ad63855479b278b1db5053e7 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 10 Feb 2021 14:54:49 -0500 Subject: [PATCH 299/576] support different checksum algorithms in direct upload --- .../datasetutility/AddReplaceFileHelper.java | 14 +++++--- .../datasetutility/OptionalFileParams.java | 36 ++++++++++++++----- .../harvard/iq/dataverse/util/FileUtil.java | 19 ++++++---- 3 files changed, 50 insertions(+), 19 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index ea1cfc38cfa..a8c52414e56 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -6,6 +6,7 @@ package edu.harvard.iq.dataverse.datasetutility; import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.DataFile.ChecksumType; import edu.harvard.iq.dataverse.DataFileServiceBean; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetServiceBean; @@ -117,11 +118,12 @@ public class AddReplaceFileHelper{ // ----------------------------------- private Dataset dataset; // constructor (for add, not replace) private DataverseRequest dvRequest; // constructor - private InputStream newFileInputStream; // step 20 - private String newFileName; // step 20 - private String newFileContentType; // step 20 - private String newStorageIdentifier; // step 20 - private String newCheckSum; // step 20 + private InputStream newFileInputStream; // step 30 + private String newFileName; // step 30 + private String newFileContentType; // step 30 + private String newStorageIdentifier; // step 30 + private String newCheckSum; // step 30 + private ChecksumType newCheckSumType; //step 30 // -- Optional private DataFile fileToReplace; // step 25 @@ -552,6 +554,7 @@ private boolean runAddReplacePhase1(Dataset owner, if(optionalFileParams != null) { if(optionalFileParams.hasCheckSum()) { newCheckSum = optionalFileParams.getCheckSum(); + newCheckSumType = optionalFileParams.getCheckSumType(); } } @@ -1131,6 +1134,7 @@ private boolean step_030_createNewFilesViaIngest(){ this.newFileContentType, this.newStorageIdentifier, this.newCheckSum, + this.newCheckSumType, this.systemConfig); } catch (IOException ex) { diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java index e48d96e355d..7fb8134630c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java @@ -10,7 +10,9 @@ import com.google.gson.JsonSyntaxException; import com.google.gson.internal.LinkedTreeMap; import com.google.gson.reflect.TypeToken; + import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.DataFile.ChecksumType; import edu.harvard.iq.dataverse.DataFileTag; import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.api.Util; @@ -68,8 +70,12 @@ public class OptionalFileParams { public static final String FILE_NAME_ATTR_NAME = "fileName"; private String mimeType; public static final String MIME_TYPE_ATTR_NAME = "mimeType"; - private String checkSum; - public static final String CHECKSUM_ATTR_NAME = "md5Hash"; + private String checkSumValue; + private ChecksumType checkSumType; + public static final String LEGACY_CHECKSUM_ATTR_NAME = "md5Hash"; + public static final String CHECKSUM_OBJECT_NAME = "checksum"; + public static final String CHECKSUM_OBJECT_TYPE = "@type"; + public static final String CHECKSUM_OBJECT_VALUE = "@value"; public OptionalFileParams(String jsonData) throws DataFileTagException{ @@ -218,16 +224,20 @@ public String getMimeType() { } public void setCheckSum(String checkSum) { - this.checkSum = checkSum; + this.checkSumValue = checkSum; } public boolean hasCheckSum() { - return ((checkSum!=null)&&(!checkSum.isEmpty())); + return ((checkSumValue!=null)&&(!checkSumValue.isEmpty())); } public String getCheckSum() { - return checkSum; + return checkSumValue; } + + public ChecksumType getCheckSumType() { + return checkSumType; + } /** * Set tags @@ -350,11 +360,21 @@ private void loadParamsFromJson(String jsonData) throws DataFileTagException{ } // ------------------------------- - // get checkSum as string + // get md5 checkSum as string + // ------------------------------- + if ((jsonObj.has(LEGACY_CHECKSUM_ATTR_NAME)) && (!jsonObj.get(LEGACY_CHECKSUM_ATTR_NAME).isJsonNull())){ + + this.checkSumValue = jsonObj.get(LEGACY_CHECKSUM_ATTR_NAME).getAsString(); + this.checkSumType= ChecksumType.MD5; + } // ------------------------------- - if ((jsonObj.has(CHECKSUM_ATTR_NAME)) && (!jsonObj.get(CHECKSUM_ATTR_NAME).isJsonNull())){ + // get checkSum type and value + // ------------------------------- + else if ((jsonObj.has(CHECKSUM_OBJECT_NAME)) && (!jsonObj.get(CHECKSUM_OBJECT_NAME).isJsonNull())){ + + this.checkSumValue = ((JsonObject) jsonObj.get(CHECKSUM_OBJECT_NAME)).get(CHECKSUM_OBJECT_VALUE).getAsString(); + this.checkSumType = ChecksumType.fromString(((JsonObject) jsonObj.get(CHECKSUM_OBJECT_NAME)).get(CHECKSUM_OBJECT_TYPE).getAsString()); - this.checkSum = jsonObj.get(CHECKSUM_ATTR_NAME).getAsString(); } // ------------------------------- diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index c30927281f6..ad70b128ea0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -719,7 +719,17 @@ public static String generateOriginalExtension(String fileType) { return ""; } - public static List createDataFiles(DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, SystemConfig systemConfig) throws IOException { + public static List createDataFiles(DatasetVersion version, InputStream inputStream, + String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, + SystemConfig systemConfig) throws IOException { + ChecksumType checkSumType = DataFile.ChecksumType.MD5; + if (newStorageIdentifier == null) { + checkSumType = systemConfig.getFileFixityChecksumAlgorithm(); + } + return createDataFiles(version, inputStream, fileName, suppliedContentType, newStorageIdentifier, newCheckSum, checkSumType, systemConfig); + } + + public static List createDataFiles(DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, ChecksumType newCheckSumType, SystemConfig systemConfig) throws IOException { List datafiles = new ArrayList<>(); String warningMessage = null; @@ -1106,12 +1116,9 @@ public static List createDataFiles(DatasetVersion version, InputStream if (tempFile != null) { newFile = tempFile.toFile(); } - ChecksumType checkSumType = DataFile.ChecksumType.MD5; - if (newStorageIdentifier == null) { - checkSumType = systemConfig.getFileFixityChecksumAlgorithm(); - } + - DataFile datafile = createSingleDataFile(version, newFile, newStorageIdentifier, fileName, finalType, checkSumType, newCheckSum); + DataFile datafile = createSingleDataFile(version, newFile, newStorageIdentifier, fileName, finalType, newCheckSumType, newCheckSum); File f = null; if (tempFile != null) { f = tempFile.toFile(); From d8b4fb3d853dcff1f885e09addc02fe96ab91435 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 10 Feb 2021 14:54:49 -0500 Subject: [PATCH 300/576] support different checksum algorithms in direct upload --- .../datasetutility/AddReplaceFileHelper.java | 14 +++++--- .../datasetutility/OptionalFileParams.java | 36 ++++++++++++++----- .../harvard/iq/dataverse/util/FileUtil.java | 19 ++++++---- 3 files changed, 50 insertions(+), 19 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index ea1cfc38cfa..a8c52414e56 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -6,6 +6,7 @@ package edu.harvard.iq.dataverse.datasetutility; import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.DataFile.ChecksumType; import edu.harvard.iq.dataverse.DataFileServiceBean; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetServiceBean; @@ -117,11 +118,12 @@ public class AddReplaceFileHelper{ // ----------------------------------- private Dataset dataset; // constructor (for add, not replace) private DataverseRequest dvRequest; // constructor - private InputStream newFileInputStream; // step 20 - private String newFileName; // step 20 - private String newFileContentType; // step 20 - private String newStorageIdentifier; // step 20 - private String newCheckSum; // step 20 + private InputStream newFileInputStream; // step 30 + private String newFileName; // step 30 + private String newFileContentType; // step 30 + private String newStorageIdentifier; // step 30 + private String newCheckSum; // step 30 + private ChecksumType newCheckSumType; //step 30 // -- Optional private DataFile fileToReplace; // step 25 @@ -552,6 +554,7 @@ private boolean runAddReplacePhase1(Dataset owner, if(optionalFileParams != null) { if(optionalFileParams.hasCheckSum()) { newCheckSum = optionalFileParams.getCheckSum(); + newCheckSumType = optionalFileParams.getCheckSumType(); } } @@ -1131,6 +1134,7 @@ private boolean step_030_createNewFilesViaIngest(){ this.newFileContentType, this.newStorageIdentifier, this.newCheckSum, + this.newCheckSumType, this.systemConfig); } catch (IOException ex) { diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java index e48d96e355d..7fb8134630c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java @@ -10,7 +10,9 @@ import com.google.gson.JsonSyntaxException; import com.google.gson.internal.LinkedTreeMap; import com.google.gson.reflect.TypeToken; + import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.DataFile.ChecksumType; import edu.harvard.iq.dataverse.DataFileTag; import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.api.Util; @@ -68,8 +70,12 @@ public class OptionalFileParams { public static final String FILE_NAME_ATTR_NAME = "fileName"; private String mimeType; public static final String MIME_TYPE_ATTR_NAME = "mimeType"; - private String checkSum; - public static final String CHECKSUM_ATTR_NAME = "md5Hash"; + private String checkSumValue; + private ChecksumType checkSumType; + public static final String LEGACY_CHECKSUM_ATTR_NAME = "md5Hash"; + public static final String CHECKSUM_OBJECT_NAME = "checksum"; + public static final String CHECKSUM_OBJECT_TYPE = "@type"; + public static final String CHECKSUM_OBJECT_VALUE = "@value"; public OptionalFileParams(String jsonData) throws DataFileTagException{ @@ -218,16 +224,20 @@ public String getMimeType() { } public void setCheckSum(String checkSum) { - this.checkSum = checkSum; + this.checkSumValue = checkSum; } public boolean hasCheckSum() { - return ((checkSum!=null)&&(!checkSum.isEmpty())); + return ((checkSumValue!=null)&&(!checkSumValue.isEmpty())); } public String getCheckSum() { - return checkSum; + return checkSumValue; } + + public ChecksumType getCheckSumType() { + return checkSumType; + } /** * Set tags @@ -350,11 +360,21 @@ private void loadParamsFromJson(String jsonData) throws DataFileTagException{ } // ------------------------------- - // get checkSum as string + // get md5 checkSum as string + // ------------------------------- + if ((jsonObj.has(LEGACY_CHECKSUM_ATTR_NAME)) && (!jsonObj.get(LEGACY_CHECKSUM_ATTR_NAME).isJsonNull())){ + + this.checkSumValue = jsonObj.get(LEGACY_CHECKSUM_ATTR_NAME).getAsString(); + this.checkSumType= ChecksumType.MD5; + } // ------------------------------- - if ((jsonObj.has(CHECKSUM_ATTR_NAME)) && (!jsonObj.get(CHECKSUM_ATTR_NAME).isJsonNull())){ + // get checkSum type and value + // ------------------------------- + else if ((jsonObj.has(CHECKSUM_OBJECT_NAME)) && (!jsonObj.get(CHECKSUM_OBJECT_NAME).isJsonNull())){ + + this.checkSumValue = ((JsonObject) jsonObj.get(CHECKSUM_OBJECT_NAME)).get(CHECKSUM_OBJECT_VALUE).getAsString(); + this.checkSumType = ChecksumType.fromString(((JsonObject) jsonObj.get(CHECKSUM_OBJECT_NAME)).get(CHECKSUM_OBJECT_TYPE).getAsString()); - this.checkSum = jsonObj.get(CHECKSUM_ATTR_NAME).getAsString(); } // ------------------------------- diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index c30927281f6..ad70b128ea0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -719,7 +719,17 @@ public static String generateOriginalExtension(String fileType) { return ""; } - public static List createDataFiles(DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, SystemConfig systemConfig) throws IOException { + public static List createDataFiles(DatasetVersion version, InputStream inputStream, + String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, + SystemConfig systemConfig) throws IOException { + ChecksumType checkSumType = DataFile.ChecksumType.MD5; + if (newStorageIdentifier == null) { + checkSumType = systemConfig.getFileFixityChecksumAlgorithm(); + } + return createDataFiles(version, inputStream, fileName, suppliedContentType, newStorageIdentifier, newCheckSum, checkSumType, systemConfig); + } + + public static List createDataFiles(DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, ChecksumType newCheckSumType, SystemConfig systemConfig) throws IOException { List datafiles = new ArrayList<>(); String warningMessage = null; @@ -1106,12 +1116,9 @@ public static List createDataFiles(DatasetVersion version, InputStream if (tempFile != null) { newFile = tempFile.toFile(); } - ChecksumType checkSumType = DataFile.ChecksumType.MD5; - if (newStorageIdentifier == null) { - checkSumType = systemConfig.getFileFixityChecksumAlgorithm(); - } + - DataFile datafile = createSingleDataFile(version, newFile, newStorageIdentifier, fileName, finalType, checkSumType, newCheckSum); + DataFile datafile = createSingleDataFile(version, newFile, newStorageIdentifier, fileName, finalType, newCheckSumType, newCheckSum); File f = null; if (tempFile != null) { f = tempFile.toFile(); From d10368d55b96f2b8b5f68dab2cda149476269713 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 10 Feb 2021 15:16:43 -0500 Subject: [PATCH 301/576] update to support #7600 --- .../source/developers/s3-direct-upload-api.rst | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst index 4b2ffb08d65..0ab04956261 100644 --- a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst +++ b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst @@ -78,17 +78,21 @@ If the client is unable to complete the multipart upload, it should call the abo curl -X DELETE "$SERVER_URL/api/datasets/mpload?..." -Adding the Uploaded file to the Dataset +Adding the Uploaded file to the Dataset --------------------------------------- Once the file exists in the s3 bucket, a final API call is needed to add it to the Dataset. This call is the same call used to upload a file to Dataverse but, rather than sending the file bytes, additional metadata is added the "jsonData" parameter. jsonData normally includes information such as a file description, tags, provenance, whether the file is restricted, etc. For direct uploads, the jsonData object must also include values for: -* "storageIdentifier" - as specified in prior calls -* "fileName" -* "mimeType" -* "md5Hash" -* "fileSize" +* "storageIdentifier" - String, as specified in prior calls +* "fileName" - String +* "mimeType" - String +* "fileSize" - number of bytes +* fixity/checksum: either: +** "md5Hash" - String with MD5 hash value, or +** "checksum" - Json Object with "@type" field specifying the algorithm used and "@value" field with the value from that algorithm, both Strings + +The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.DataFile.CheckSumType class and currently include MD5, SHA-1, SHA-256, and SHA-512 .. code-block:: bash From 9ddbbead1e1a813d87b16482d9e30b06905df48d Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Wed, 10 Feb 2021 15:18:19 -0500 Subject: [PATCH 302/576] Fixed extra returns copied with api token on account pg [ref #7601] --- src/main/webapp/dataverseuser.xhtml | 2 +- src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/webapp/dataverseuser.xhtml b/src/main/webapp/dataverseuser.xhtml index ba25798e081..5de0154f49c 100644 --- a/src/main/webapp/dataverseuser.xhtml +++ b/src/main/webapp/dataverseuser.xhtml @@ -521,7 +521,7 @@ //]]> diff --git a/src/main/webapp/provenance-popups-fragment.xhtml b/src/main/webapp/provenance-popups-fragment.xhtml index 6e1663fbe05..df993f03950 100644 --- a/src/main/webapp/provenance-popups-fragment.xhtml +++ b/src/main/webapp/provenance-popups-fragment.xhtml @@ -94,9 +94,7 @@ forceSelection="true" converter="provEntityFileDataConverter" styleClass="DropdownPopup" panelStyleClass="DropdownPopupPanel" - onkeypress="if (event.keyCode == 13) { - return false; - }"> + onkeypress="if (event.keyCode == 13) {return false;}"> @@ -107,9 +105,6 @@ -
    diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index 44e86831c69..8b6a7bff5d2 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -41,6 +41,9 @@ function bind_bsui_components(){ // clipboard.js click to copy clickCopyClipboard(); + // Scrolling autoComplete dropdown in popups + handle_dropdown_popup_scroll(); + // Dialog Listener For Calling handleResizeDialog PrimeFaces.widget.Dialog.prototype.postShow = function() { var dialog_id = this.jq.attr('id').split(/[:]+/).pop(); diff --git a/src/main/webapp/roles-assign.xhtml b/src/main/webapp/roles-assign.xhtml index a3f73793d7b..672086a7882 100644 --- a/src/main/webapp/roles-assign.xhtml +++ b/src/main/webapp/roles-assign.xhtml @@ -40,9 +40,6 @@ -
    From c5121a676150bbd32428c832a06e50f374ee9a14 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 18 Feb 2021 10:42:06 -0500 Subject: [PATCH 338/576] #3175 fix input level editing --- src/main/webapp/dataverse.xhtml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index a069d87e9c7..96d523c335d 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -330,8 +330,9 @@
    - + @@ -345,6 +346,7 @@
    From e9845c077ef847e7bb57feb95c67ef7859519389 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 18 Feb 2021 12:31:41 -0500 Subject: [PATCH 339/576] Update native-api.rst --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 2146060ef37..6d7b200f11a 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3017,7 +3017,7 @@ This is an irreversible action. There is no option to undisable a user. Disabling a user with this endpoint will: -- Disable the user's ability to log in to the Dataverse installation. A message will be shown, stating that the account has been disabled. +- Disable the user's ability to log in to the Dataverse installation. A message will be shown, stating that the account has been disabled. The user will not able to create a new account with the same email address, ORCID, Shibboleth, or other login type. - Remove the user's access from all Dataverse collections, datasets and files - Prevent a user from being assigned any roles - Cancel any pending file access requests generated by the user From 51022aa8135c7acd9284bb0ecf5f04cbd9c9f415 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 18 Feb 2021 14:17:32 -0500 Subject: [PATCH 340/576] disable users #2419 #4475 This commit adds an API endpoint for disabling users. Once users are disabled: - They lose all their roles. - They lose pending file access requests. - They lose group memberships. - They lose banner messages. - They lose all notifications. - They get an error on login. (Builtin/Shib/OAuth tested.) - They get an error if they try to use the API. - They cannot be assigned roles. - They cannot be added to groups. - They cannot initiate password reset. - They cannot confirm their email (from a previous link). --- .../source/developers/remote-users.rst | 2 +- .../iq/dataverse/DataverseSession.java | 16 +- .../java/edu/harvard/iq/dataverse/Shib.java | 1 + .../UserNotificationServiceBean.java | 4 + .../iq/dataverse/api/AbstractApiBean.java | 1 + .../edu/harvard/iq/dataverse/api/Admin.java | 34 +++- .../dataverse/api/datadeposit/SwordAuth.java | 1 + .../AuthenticationServiceBean.java | 10 +- .../oauth2/OAuth2LoginBackingBean.java | 1 + .../users/AuthenticatedUser.java | 26 ++- .../authorization/users/GuestUser.java | 7 +- .../authorization/users/PrivateUrlUser.java | 5 + .../dataverse/authorization/users/User.java | 2 + .../confirmemail/ConfirmEmailServiceBean.java | 4 + ...ddRoleAssigneesToExplicitGroupCommand.java | 7 + .../command/impl/AssignRoleCommand.java | 8 + .../command/impl/DisableUserCommand.java | 44 +++++ .../passwordreset/PasswordResetPage.java | 14 +- .../PasswordResetServiceBean.java | 12 +- .../iq/dataverse/util/json/JsonPrinter.java | 2 + src/main/java/propertyFiles/Bundle.properties | 7 +- .../V5.3.0.3__4475-disable-users.sql | 4 + .../iq/dataverse/api/DisableUsersIT.java | 160 ++++++++++++++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 17 ++ 24 files changed, 371 insertions(+), 18 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DisableUserCommand.java create mode 100644 src/main/resources/db/migration/V5.3.0.3__4475-disable-users.sql create mode 100644 src/test/java/edu/harvard/iq/dataverse/api/DisableUsersIT.java diff --git a/doc/sphinx-guides/source/developers/remote-users.rst b/doc/sphinx-guides/source/developers/remote-users.rst index c85571a55c0..3f8dd836661 100755 --- a/doc/sphinx-guides/source/developers/remote-users.rst +++ b/doc/sphinx-guides/source/developers/remote-users.rst @@ -10,7 +10,7 @@ Shibboleth and OAuth If you are working on anything related to users, please keep in mind that your changes will likely affect Shibboleth and OAuth users. For some background on user accounts in the Dataverse Software, see :ref:`auth-modes` section of Configuration in the Installation Guide. -Rather than setting up Shibboleth on your laptop, developers are advised to simply add a value to their database to enable Shibboleth "dev mode" like this: +Rather than setting up Shibboleth on your laptop, developers are advised to add the Shibboleth auth provider (see "Add the Shibboleth Authentication Provider to Your Dataverse Installation" at :doc:`/installation/shibboleth`) and add a value to their database to enable Shibboleth "dev mode" like this: ``curl http://localhost:8080/api/admin/settings/:DebugShibAccountType -X PUT -d RANDOM`` diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java b/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java index 2a2b02c5b18..fcf8da22f7e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java @@ -7,6 +7,8 @@ import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.GuestUser; import edu.harvard.iq.dataverse.authorization.users.User; +import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.JsfHelper; import edu.harvard.iq.dataverse.util.SessionUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import java.io.IOException; @@ -92,7 +94,16 @@ public User getUser() { } public void setUser(User aUser) { - + // We check for disabled status here in "setUser" to ensure a common user + // experience across Builtin, Shib, OAuth, and OIDC users. + // If we want a different user experience for Builtin users, we can + // modify getUpdateAuthenticatedUser in AuthenticationServiceBean + // (and probably other places). + if (aUser instanceof AuthenticatedUser && aUser.isDisabled()) { + logger.info("Login attempt by disabled user " + aUser.getIdentifier() + "."); + JsfHelper.addErrorMessage(BundleUtil.getStringFromBundle("disabled.error")); + return; + } FacesContext context = FacesContext.getCurrentInstance(); // Log the login/logout and Change the session id if we're using the UI and have // a session, versus an API call with no session - (i.e. /admin/submitToArchive() @@ -210,6 +221,9 @@ public void dismissMessage(BannerMessage message){ } public void configureSessionTimeout() { + if (user instanceof GuestUser) { + return; + } HttpSession httpSession = (HttpSession) FacesContext.getCurrentInstance().getExternalContext().getSession(false); if (httpSession != null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/Shib.java b/src/main/java/edu/harvard/iq/dataverse/Shib.java index 889bdaff03a..4bd38d2377d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Shib.java +++ b/src/main/java/edu/harvard/iq/dataverse/Shib.java @@ -358,6 +358,7 @@ public String confirmAndConvertAccount() { private void logInUserAndSetShibAttributes(AuthenticatedUser au) { au.setShibIdentityProvider(shibIdp); + // setUser checks for disabled users. session.setUser(au); session.configureSessionTimeout(); logger.fine("Groups for user " + au.getId() + " (" + au.getIdentifier() + "): " + getGroups(au)); diff --git a/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java index 071805d3d26..f02962498c8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java @@ -103,6 +103,10 @@ public void sendNotification(AuthenticatedUser dataverseUser, Timestamp sendDate } public void sendNotification(AuthenticatedUser dataverseUser, Timestamp sendDate, Type type, Long objectId, String comment, AuthenticatedUser requestor, boolean isHtmlContent) { + if (dataverseUser.isDisabled()) { + logger.info("An attempt was made to send a " + type + " notification to a disabled user: " + dataverseUser); + return; + } UserNotification userNotification = new UserNotification(); userNotification.setUser(dataverseUser); userNotification.setSendDate(sendDate); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 551c0a797fc..fdc0b97aa86 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -372,6 +372,7 @@ protected AuthenticatedUser findAuthenticatedUserOrDie() throws WrappedResponse private AuthenticatedUser findAuthenticatedUserOrDie( String key ) throws WrappedResponse { + // No check for disabled user because it's done in authSvc.lookupUser. AuthenticatedUser authUser = authSvc.lookupUser(key); if ( authUser != null ) { authUser = userSvc.updateLastApiUseTime(authUser); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index b52665a7747..4c39103ce3e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -87,6 +87,7 @@ import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.impl.MergeInAccountCommand; import edu.harvard.iq.dataverse.engine.command.impl.ChangeUserIdentifierCommand; +import edu.harvard.iq.dataverse.engine.command.impl.DisableUserCommand; import edu.harvard.iq.dataverse.engine.command.impl.RegisterDvObjectCommand; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; @@ -381,9 +382,36 @@ private Response deleteAuthenticatedUser(AuthenticatedUser au) { return ok("AuthenticatedUser " + au.getIdentifier() + " deleted. "); } - - + @POST + @Path("authenticatedUsers/{identifier}/disable") + public Response disableAuthenticatedUser(@PathParam("identifier") String identifier) { + AuthenticatedUser user = authSvc.getAuthenticatedUser(identifier); + if (user != null) { + return disableAuthenticatedUser(user); + } + return error(Response.Status.BAD_REQUEST, "User " + identifier + " not found."); + } + + @POST + @Path("authenticatedUsers/id/{id}/disable") + public Response disableAuthenticatedUserById(@PathParam("id") Long id) { + AuthenticatedUser user = authSvc.findByID(id); + if (user != null) { + return disableAuthenticatedUser(user); + } + return error(Response.Status.BAD_REQUEST, "User " + id + " not found."); + } + + private Response disableAuthenticatedUser(AuthenticatedUser userToDisable) { + try { + execCommand(new DisableUserCommand(createDataverseRequest(findUserOrDie()), userToDisable)); + return ok("User " + userToDisable.getIdentifier() + " disabled."); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + } + @POST @Path("publishDataverseAsCreator/{id}") public Response publishDataverseAsCreator(@PathParam("id") long id) { @@ -1686,7 +1714,7 @@ public Response submitDatasetVersionToArchive(@PathParam("id") String dsid, @Pat // DataverseRequest and is sent to the back-end command where it is used to get // the API Token which is then used to retrieve files (e.g. via S3 direct // downloads) to create the Bag - session.setUser(au); + session.setUser(au); // TODO: Stop using session. Use createDataverseRequest instead. Dataset ds = findDatasetOrDie(dsid); DatasetVersion dv = datasetversionService.findByFriendlyVersionNumber(ds.getId(), versionNumber); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SwordAuth.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SwordAuth.java index 4a474f62894..339832989d6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SwordAuth.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SwordAuth.java @@ -36,6 +36,7 @@ public AuthenticatedUser auth(AuthCredentials authCredentials) throws SwordAuthE throw new SwordAuthException(msg); } + // Checking if the user is disabled is done inside findUserByApiToken. AuthenticatedUser authenticatedUserFromToken = findUserByApiToken(username); if (authenticatedUserFromToken == null) { String msg = "User not found based on API token."; diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java index c9a54dba078..c34015a5dda 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java @@ -309,7 +309,7 @@ public AuthenticatedUser getUpdateAuthenticatedUser( String authenticationProvid // yay! see if we already have this user. AuthenticatedUser user = lookupUser(authenticationProviderId, resp.getUserId()); - if (user != null){ + if (user != null && !user.isDisabled()) { user = userService.updateLastLogin(user); } @@ -453,7 +453,13 @@ public AuthenticatedUser lookupUser( String apiToken ) { } } - return tkn.getAuthenticatedUser(); + AuthenticatedUser user = tkn.getAuthenticatedUser(); + if (!user.isDisabled()) { + return user; + } else { + logger.info("attempted access with token from disabled user: " + apiToken); + return null; + } } /* diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java index e42f82d48d8..c034f7ebc8b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java @@ -106,6 +106,7 @@ public void exchangeCodeForToken() throws IOException { } else { // login the user and redirect to HOME of intended page (if any). + // setUser checks for disabled users. session.setUser(dvUser); session.configureSessionTimeout(); final OAuth2TokenData tokenData = oauthUser.getTokenData(); diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java index 12161eb1a59..ae485103be6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java @@ -55,7 +55,8 @@ @NamedQuery( name="AuthenticatedUser.filter", query="select au from AuthenticatedUser au WHERE (" + "LOWER(au.userIdentifier) like LOWER(:query) OR " - + "lower(concat(au.firstName,' ',au.lastName)) like lower(:query))"), + + "lower(concat(au.firstName,' ',au.lastName)) like lower(:query)) " + + "AND au.disabled != true"), @NamedQuery( name="AuthenticatedUser.findAdminUser", query="select au from AuthenticatedUser au WHERE " + "au.superuser = true " @@ -114,6 +115,12 @@ public class AuthenticatedUser implements User, Serializable { private boolean superuser; + @Column(nullable=true) + private boolean disabled; + + @Column(nullable=true) + private Timestamp disabledTime; + /** * @todo Consider storing a hash of *all* potentially interesting Shibboleth * attribute key/value pairs, not just the Identity Provider (IdP). @@ -303,6 +310,23 @@ public void setSuperuser(boolean superuser) { this.superuser = superuser; } + @Override + public boolean isDisabled() { + return disabled; + } + + public void setDisabled(boolean disabled) { + this.disabled = disabled; + } + + public Timestamp getDisabledTime() { + return disabledTime; + } + + public void setDisabledTime(Timestamp disabledTime) { + this.disabledTime = disabledTime; + } + @OneToOne(mappedBy = "authenticatedUser") private AuthenticatedUserLookup authenticatedUserLookup; diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/users/GuestUser.java b/src/main/java/edu/harvard/iq/dataverse/authorization/users/GuestUser.java index f16fa5afe36..6c4d45d6674 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/users/GuestUser.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/users/GuestUser.java @@ -32,7 +32,12 @@ public RoleAssigneeDisplayInfo getDisplayInfo() { public boolean isSuperuser() { return false; } - + + @Override + public boolean isDisabled() { + return false; + } + @Override public boolean equals( Object o ) { return (o instanceof GuestUser); diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/users/PrivateUrlUser.java b/src/main/java/edu/harvard/iq/dataverse/authorization/users/PrivateUrlUser.java index 59c3240fdfa..e318cf5ce8c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/users/PrivateUrlUser.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/users/PrivateUrlUser.java @@ -47,6 +47,11 @@ public boolean isSuperuser() { return false; } + @Override + public boolean isDisabled() { + return false; + } + @Override public String getIdentifier() { return PREFIX + datasetId; diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/users/User.java b/src/main/java/edu/harvard/iq/dataverse/authorization/users/User.java index ea35f87d178..f3291a2a11f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/users/User.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/users/User.java @@ -14,4 +14,6 @@ public interface User extends RoleAssignee, Serializable { public boolean isSuperuser(); + public boolean isDisabled(); + } diff --git a/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailServiceBean.java index bc3b70326f1..574cfa11cde 100644 --- a/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailServiceBean.java @@ -169,6 +169,10 @@ public ConfirmEmailExecResponse processToken(String tokenQueried) { long nowInMilliseconds = new Date().getTime(); Timestamp emailConfirmed = new Timestamp(nowInMilliseconds); AuthenticatedUser authenticatedUser = confirmEmailData.getAuthenticatedUser(); + if (authenticatedUser.isDisabled()) { + logger.fine("User is disabled."); + return null; + } authenticatedUser.setEmailConfirmed(emailConfirmed); em.remove(confirmEmailData); return goodTokenCanProceed; diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AddRoleAssigneesToExplicitGroupCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AddRoleAssigneesToExplicitGroupCommand.java index f6bd1316e44..d4e47a5d449 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AddRoleAssigneesToExplicitGroupCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AddRoleAssigneesToExplicitGroupCommand.java @@ -4,6 +4,7 @@ import edu.harvard.iq.dataverse.authorization.RoleAssignee; import edu.harvard.iq.dataverse.authorization.groups.GroupException; import edu.harvard.iq.dataverse.authorization.groups.impl.explicit.ExplicitGroup; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.engine.command.AbstractCommand; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; @@ -49,6 +50,12 @@ public ExplicitGroup execute(CommandContext ctxt) throws CommandException { if ( ra == null ) { nonexistentRAs.add( rai ); } else { + if (ra instanceof AuthenticatedUser) { + AuthenticatedUser user = (AuthenticatedUser) ra; + if (user.isDisabled()) { + throw new IllegalCommandException("User " + user.getUserIdentifier() + " is disabled and cannot be added to a group.", this); + } + } try { explicitGroup.add(ra); } catch (GroupException ex) { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java index 34263599ff0..274d485c5d2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java @@ -10,10 +10,12 @@ import edu.harvard.iq.dataverse.RoleAssignment; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.RoleAssignee; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.engine.command.AbstractCommand; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; import java.util.Collections; import java.util.Map; import java.util.Set; @@ -49,6 +51,12 @@ public AssignRoleCommand(RoleAssignee anAssignee, DataverseRole aRole, DvObject @Override public RoleAssignment execute(CommandContext ctxt) throws CommandException { + if (grantee instanceof AuthenticatedUser) { + AuthenticatedUser user = (AuthenticatedUser) grantee; + if (user.isDisabled()) { + throw new IllegalCommandException("User " + user.getUserIdentifier() + " is disabled and cannot be given a role.", this); + } + } // TODO make sure the role is defined on the dataverse. RoleAssignment roleAssignment = new RoleAssignment(role, grantee, defPoint, privateUrlToken); return ctxt.roles().save(roleAssignment); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DisableUserCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DisableUserCommand.java new file mode 100644 index 00000000000..58ee7295e22 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DisableUserCommand.java @@ -0,0 +1,44 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; +import java.sql.Timestamp; +import java.util.Date; + +// Superuser-only enforced below. +@RequiredPermissions({}) +public class DisableUserCommand extends AbstractCommand { + + private DataverseRequest request; + private AuthenticatedUser userToDisable; + + public DisableUserCommand(DataverseRequest request, AuthenticatedUser userToDisable) { + super(request, (DvObject) null); + this.request = request; + this.userToDisable = userToDisable; + } + + @Override + public AuthenticatedUser execute(CommandContext ctxt) throws CommandException { + if (!(getUser() instanceof AuthenticatedUser) || !getUser().isSuperuser()) { + throw new PermissionException("Disable user command can only be called by superusers.", this, null, null); + } + if (userToDisable == null) { + throw new CommandException("Cannot disable user. User not found.", this); + } + ctxt.engine().submit(new RevokeAllRolesCommand(userToDisable, request)); + ctxt.authentication().removeAuthentictedUserItems(userToDisable); + ctxt.notifications().findByUser(userToDisable.getId()).forEach(ctxt.notifications()::delete); + userToDisable.setDisabled(true); + userToDisable.setDisabledTime(new Timestamp(new Date().getTime())); + AuthenticatedUser disabledUser = ctxt.authentication().save(userToDisable); + return disabledUser; + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetPage.java b/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetPage.java index 532c0216038..b5cef3731f5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetPage.java @@ -121,13 +121,15 @@ public String sendPasswordResetLink() { actionLogSvc.log( new ActionLogRecord(ActionLogRecord.ActionType.BuiltinUser, "passwordResetSent") .setInfo("Email Address: " + emailAddress) ); } else { - /** - * @todo remove "single" when it's no longer necessary. See - * https://github.com/IQSS/dataverse/issues/844 and - * https://github.com/IQSS/dataverse/issues/1141 - */ - logger.log(Level.INFO, "Couldn''t find single account using {0}", emailAddress); + logger.log(Level.INFO, "Cannot find account (or it's disabled) given {0}", emailAddress); } + /** + * We show this "an email will be sent" message no matter what (if + * the account can be found or not, if the account has been disabled + * or not) to prevent hackers from figuring out if you have an + * account based on your email address. Yes, this is a white lie + * sometimes, in the name of security. + */ FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("passwdVal.passwdReset.resetInitiated"), BundleUtil.getStringFromBundle("passwdReset.successSubmit.tip", Arrays.asList(emailAddress)))); } catch (PasswordResetException ex) { diff --git a/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetServiceBean.java index c3f0f7f2abc..9b28be6527c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetServiceBean.java @@ -52,13 +52,23 @@ public class PasswordResetServiceBean { * Initiate the password reset process. * * @param emailAddress - * @return {@link PasswordResetInitResponse} + * @return {@link PasswordResetInitResponse} with empty PasswordResetData if + * the reset won't continue (no user, disabled user). * @throws edu.harvard.iq.dataverse.passwordreset.PasswordResetException */ // inspired by Troy Hunt: Everything you ever wanted to know about building a secure password reset feature - http://www.troyhunt.com/2012/05/everything-you-ever-wanted-to-know.html public PasswordResetInitResponse requestReset(String emailAddress) throws PasswordResetException { deleteAllExpiredTokens(); AuthenticatedUser authUser = authService.getAuthenticatedUserByEmail(emailAddress); + // This null check is for the NPE reported in https://github.com/IQSS/dataverse/issues/5462 + if (authUser == null) { + logger.info("Cannot find a user based on " + emailAddress + ". Cannot reset password."); + return new PasswordResetInitResponse(false); + } + if (authUser.isDisabled()) { + logger.info("Cannot reset password for " + emailAddress + " because account is disabled."); + return new PasswordResetInitResponse(false); + } BuiltinUser user = dataverseUserService.findByUserName(authUser.getUserIdentifier()); if (user != null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index c37efc3178f..668f3cbf771 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -107,6 +107,8 @@ public static JsonObjectBuilder json(AuthenticatedUser authenticatedUser) { .add("lastName", authenticatedUser.getLastName()) .add("email", authenticatedUser.getEmail()) .add("superuser", authenticatedUser.isSuperuser()) + .add("disabled", authenticatedUser.isDisabled()) + .add("disabledTime", authenticatedUser.getDisabledTime()) .add("affiliation", authenticatedUser.getAffiliation()) .add("position", authenticatedUser.getPosition()) .add("persistentUserId", authenticatedUser.getAuthenticatedUserLookup().getPersistentUserId()) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 6cdf8b93550..227234da279 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -361,7 +361,7 @@ shib.dataverseUsername=Dataverse Username shib.currentDataversePassword=Current Dataverse Password shib.accountInformation=Account Information shib.offerToCreateNewAccount=This information is provided by your institution and will be used to create your Dataverse account. -shib.passwordRejected=Validation Error - Your account can only be converted if you provide the correct password for your existing account. +shib.passwordRejected=Validation Error - Your account can only be converted if you provide the correct password for your existing account. If your existing account has been disabled by an administrator, you cannot convert your account. # oauth2/firstLogin.xhtml oauth2.btn.convertAccount=Convert Existing Account @@ -396,7 +396,7 @@ oauth2.newAccount.emailInvalid=Invalid email address. oauth2.convertAccount.explanation=Please enter your {0} account username or email and password to convert your account to the {1} log in option. Learn more about converting your account. oauth2.convertAccount.username=Existing username oauth2.convertAccount.password=Password -oauth2.convertAccount.authenticationFailed=Authentication failed - bad username or password. +oauth2.convertAccount.authenticationFailed=Your account can only be converted if you provide the correct username and password for your existing account. If your existing account has been disabled by an administrator, you cannot convert your account. oauth2.convertAccount.buttonTitle=Convert Account oauth2.convertAccount.success=Your Dataverse account is now associated with your {0} account. @@ -404,6 +404,9 @@ oauth2.convertAccount.success=Your Dataverse account is now associated with your oauth2.callback.page.title=OAuth Callback oauth2.callback.message=Authentication Error - Dataverse could not authenticate your login with the provider that you selected. Please make sure you authorize your account to connect with Dataverse. For more details about the information being requested, see the User Guide. +# disabled user accounts +disabled.error=Sorry, your account has been disabled. + # tab on dataverseuser.xhtml apitoken.title=API Token apitoken.message=Your API Token is valid for a year. Check out our {0}API Guide{1} for more information on using your API Token with the Dataverse APIs. diff --git a/src/main/resources/db/migration/V5.3.0.3__4475-disable-users.sql b/src/main/resources/db/migration/V5.3.0.3__4475-disable-users.sql new file mode 100644 index 00000000000..48823a72247 --- /dev/null +++ b/src/main/resources/db/migration/V5.3.0.3__4475-disable-users.sql @@ -0,0 +1,4 @@ +-- Users can be disabled. +ALTER TABLE authenticateduser ADD COLUMN IF NOT EXISTS disabled BOOLEAN; +-- A timestamp of when the user was disabled. +ALTER TABLE authenticateduser ADD COLUMN IF NOT EXISTS disabledtime timestamp without time zone; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DisableUsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DisableUsersIT.java new file mode 100644 index 00000000000..fdec2480db6 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/api/DisableUsersIT.java @@ -0,0 +1,160 @@ +package edu.harvard.iq.dataverse.api; + +import com.jayway.restassured.RestAssured; +import com.jayway.restassured.path.json.JsonPath; +import com.jayway.restassured.response.Response; +import edu.harvard.iq.dataverse.authorization.DataverseRole; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import static javax.ws.rs.core.Response.Status.BAD_REQUEST; +import static javax.ws.rs.core.Response.Status.CREATED; +import static javax.ws.rs.core.Response.Status.FORBIDDEN; +import static javax.ws.rs.core.Response.Status.OK; +import static javax.ws.rs.core.Response.Status.UNAUTHORIZED; +import static org.hamcrest.CoreMatchers.equalTo; +import org.junit.BeforeClass; +import org.junit.Test; + +public class DisableUsersIT { + + @BeforeClass + public static void setUp() { + RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); + } + + @Test + public void testDisableUser() { + + Response createSuperuser = UtilIT.createRandomUser(); + createSuperuser.then().assertThat().statusCode(OK.getStatusCode()); + String superuserUsername = UtilIT.getUsernameFromResponse(createSuperuser); + String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser); + Response toggleSuperuser = UtilIT.makeSuperUser(superuserUsername); + toggleSuperuser.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response createDataverse = UtilIT.createRandomDataverse(superuserApiToken); + createDataverse.then().assertThat().statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverse); + Integer dataverseId = UtilIT.getDataverseIdFromResponse(createDataverse); + + Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, superuserApiToken); + createDataset.prettyPrint(); + createDataset.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + String datasetPersistentId = UtilIT.getDatasetPersistentIdFromResponse(createDataset); + + Response createUser = UtilIT.createRandomUser(); + createUser.then().assertThat().statusCode(OK.getStatusCode()); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response grantRoleBeforeDisable = UtilIT.grantRoleOnDataverse(dataverseAlias, DataverseRole.ADMIN.toString(), "@" + username, superuserApiToken); + grantRoleBeforeDisable.prettyPrint(); + grantRoleBeforeDisable.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.assignee", equalTo("@" + username)) + .body("data._roleAlias", equalTo("admin")); + + String aliasInOwner = "groupFor" + dataverseAlias; + String displayName = "Group for " + dataverseAlias; + String user2identifier = "@" + username; + Response createGroup = UtilIT.createGroup(dataverseAlias, aliasInOwner, displayName, superuserApiToken); + createGroup.prettyPrint(); + createGroup.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + String groupIdentifier = JsonPath.from(createGroup.asString()).getString("data.identifier"); + + List roleAssigneesToAdd = new ArrayList<>(); + roleAssigneesToAdd.add(user2identifier); + Response addToGroup = UtilIT.addToGroup(dataverseAlias, aliasInOwner, roleAssigneesToAdd, superuserApiToken); + addToGroup.prettyPrint(); + addToGroup.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response userTracesBeforeDisable = UtilIT.getUserTraces(username, superuserApiToken); + userTracesBeforeDisable.prettyPrint(); + userTracesBeforeDisable.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.traces.roleAssignments.items[0].definitionPointName", equalTo(dataverseAlias)) + .body("data.traces.roleAssignments.items[0].definitionPointId", equalTo(dataverseId)) + .body("data.traces.explicitGroups.items[0].name", equalTo("Group for " + dataverseAlias)); + + Response failToDisableUserWithoutSuperuserApiToken = UtilIT.disableUser(username, apiToken); + failToDisableUserWithoutSuperuserApiToken.prettyPrint(); + failToDisableUserWithoutSuperuserApiToken.then().assertThat().statusCode(UNAUTHORIZED.getStatusCode()); + + Response disableUser = UtilIT.disableUser(username, superuserApiToken); + disableUser.prettyPrint(); + disableUser.then().assertThat().statusCode(OK.getStatusCode()); + + Response getUser = UtilIT.getAuthenticatedUser(username, superuserApiToken); + getUser.prettyPrint(); + getUser.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.disabled", equalTo(true)); + + Response getUserDisabled = UtilIT.getAuthenticatedUserByToken(apiToken); + getUserDisabled.prettyPrint(); + getUserDisabled.then().assertThat().statusCode(BAD_REQUEST.getStatusCode()); + + Response userTracesAfterDisable = UtilIT.getUserTraces(username, superuserApiToken); + userTracesAfterDisable.prettyPrint(); + userTracesAfterDisable.then().assertThat() + .statusCode(OK.getStatusCode()) + /** + * Here we are showing the the following were deleted: + * + * - role assignments + * + * - membership in explict groups. + */ + .body("data.traces", equalTo(Collections.EMPTY_MAP)); + + Response grantRoleAfterDisable = UtilIT.grantRoleOnDataverse(dataverseAlias, DataverseRole.ADMIN.toString(), "@" + username, superuserApiToken); + grantRoleAfterDisable.prettyPrint(); + grantRoleAfterDisable.then().assertThat() + .statusCode(FORBIDDEN.getStatusCode()) + .body("message", equalTo("User " + username + " is disabled and cannot be given a role.")); + + Response addToGroupAfter = UtilIT.addToGroup(dataverseAlias, aliasInOwner, roleAssigneesToAdd, superuserApiToken); + addToGroupAfter.prettyPrint(); + addToGroupAfter.then().assertThat() + .statusCode(FORBIDDEN.getStatusCode()) + .body("message", equalTo("User " + username + " is disabled and cannot be added to a group.")); + + Response grantRoleOnDataset = UtilIT.grantRoleOnDataset(datasetPersistentId, DataverseRole.ADMIN.toString(), "@" + username, superuserApiToken); + grantRoleOnDataset.prettyPrint(); + grantRoleOnDataset.then().assertThat() + .statusCode(FORBIDDEN.getStatusCode()) + .body("message", equalTo("User " + username + " is disabled and cannot be given a role.")); + + } + + @Test + public void testDisableUserById() { + + Response createSuperuser = UtilIT.createRandomUser(); + createSuperuser.then().assertThat().statusCode(OK.getStatusCode()); + String superuserUsername = UtilIT.getUsernameFromResponse(createSuperuser); + String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser); + Response toggleSuperuser = UtilIT.makeSuperUser(superuserUsername); + toggleSuperuser.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + createUser.then().assertThat().statusCode(OK.getStatusCode()); + String username = UtilIT.getUsernameFromResponse(createUser); + Long userId = JsonPath.from(createUser.body().asString()).getLong("data.user.id"); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response disableUser = UtilIT.disableUser(userId, superuserApiToken); + disableUser.prettyPrint(); + disableUser.then().assertThat().statusCode(OK.getStatusCode()); + } + +} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index dee35556729..4bf55054ba6 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -950,6 +950,22 @@ static public Response grantRoleOnDataverse(String definitionPoint, String role, .post("api/dataverses/" + definitionPoint + "/assignments?key=" + apiToken); } + public static Response disableUser(String username, String apiToken) { +// public static Response disableUser(String username, String apiToken) { + Response disableUserResponse = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) +// .post("/api/users/" + username + "/disable"); + .post("/api/admin/authenticatedUsers/" + username + "/disable"); + return disableUserResponse; + } + + public static Response disableUser(Long userId, String apiToken) { + Response disableUserResponse = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .post("/api/admin/authenticatedUsers/id/" + userId + "/disable"); + return disableUserResponse; + } + public static Response deleteUser(String username) { Response deleteUserResponse = given() .delete("/api/admin/authenticatedUsers/" + username + "/"); @@ -1188,6 +1204,7 @@ static Response listAuthenticatedUsers(String apiToken) { return response; } + // TODO: Consider removing apiToken since it isn't used by the API itself. static Response getAuthenticatedUser(String userIdentifier, String apiToken) { Response response = given() .header(API_TOKEN_HTTP_HEADER, apiToken) From cb964bd4379fdf1532bddb9eb9983a4f92309285 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 18 Feb 2021 15:44:09 -0500 Subject: [PATCH 341/576] #3175 fixes 1198 --- src/main/webapp/dataverse.xhtml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index 96d523c335d..62f03b64be1 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -287,6 +287,7 @@ From 0f2e74bcfa0c6e000070d14e4944232e42d4431b Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 18 Feb 2021 16:48:55 -0500 Subject: [PATCH 342/576] tweaks to API guide, docs for traces, removeRoles #4475 --- doc/sphinx-guides/source/api/native-api.rst | 82 ++++++++++++++++++--- 1 file changed, 73 insertions(+), 9 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 646961e1526..fbf9ed266fc 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3079,21 +3079,41 @@ Deletes an ``AuthenticatedUser`` whose ``id`` is passed. :: DELETE http://$SERVER/api/admin/authenticatedUsers/id/$id -Note: If the user has performed certain actions such as creating or contributing to a Dataset or downloading a file they cannot be deleted. If a user cannot be deleted for this reason, you can choose to :ref:`disable a user`. - +Note: If the user has performed certain actions such as creating or contributing to a Dataset or downloading a file they cannot be deleted. To see where in the database these actions are stored you can use the :ref:`show-user-traces-api` API. If a user cannot be deleted for this reason, you can choose to :ref:`disable a user`. + .. _disable-a-user: Disable a User ~~~~~~~~~~~~~~ -Disables an ``AuthenticatedUser`` whose ``identifier`` (without the ``@`` sign) is passed. :: +Disables a user. - PUT http://$SERVER/api/admin/authenticatedUsers/disable/$identifier - -Disables an ``AuthenticatedUser`` whose ``id`` is passed. :: +.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=http://localhost:8080 + export USERNAME=jdoe + + curl -H "X-Dataverse-key:$API_TOKEN" -X POST $SERVER_URL/api/admin/authenticatedUsers/$USERNAME/disable + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST http://localhost:8080/api/admin/authenticatedUsers/jdoe/disable + +The database ID of the user can be passed instead of the username. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=http://localhost:8080 + export USERID=42 + + curl -H "X-Dataverse-key:$API_TOKEN" -X POST $SERVER_URL/api/admin/authenticatedUsers/id/$USERID/disable - PUT http://$SERVER/api/admin/authenticatedUsers/disable/id/$id - Note: A primary purpose of most Dataverse installations is to serve an archive. In the archival space, there are best practices around the tracking of data access and the tracking of modifications to data and metadata. In support of these key workflows, a simple mechanism to delete users that have performed edit or access actions in the system is not provided. Providing a Disable User endpoint for users who have taken certain actions in the system alongside a Delete User endpoint to remove users that haven't taken certain actions in the system is by design. This is an irreversible action. There is no option to undisable a user. @@ -3109,10 +3129,54 @@ Disabling a user with this endpoint will: Disabling a user with this endpoint will keep: -- The user's contributions to datasets, including dataset creation, file uploads, and publishing. +- The user's contributions to datasets, including dataset creation, file uploads, and publishing. - The user's access history to datafiles in the Dataverse installation, including guestbook records. - The user's account information (specifically name, email, affiliation, and position) +.. _show-user-traces-api: + +Show User Traces +~~~~~~~~~~~~~~~~ + +Show the traces that the user has left in the system, such as datasets created, guestbooks filled out, etc. This can be useful for understanding why a user cannot be deleted. A superuser API token is required. + +.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export USERNAME=jdoe + + curl -H "X-Dataverse-key:$API_TOKEN" -X GET $SERVER_URL/api/users/$USERNAME/traces + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X GET https://demo.dataverse.org/api/users/jdoe/traces + +Remove All Roles from a User +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Removes all roles from the user. This is equivalent of clicking the "Remove All Roles" button in the superuser dashboard. Note that you can preview the roles that will be removed with the :ref:`show-user-traces-api` API. A superuser API token is required. + +.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export USERNAME=jdoe + + curl -H "X-Dataverse-key:$API_TOKEN" -X POST $SERVER_URL/api/users/$USERNAME/removeRoles + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST http://localhost:8080/api/users/jdoe/removeRoles + List Role Assignments of a Role Assignee ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 96fc087486c9a883cb69987dfad90b5150faa053 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 19 Feb 2021 11:01:38 -0500 Subject: [PATCH 343/576] add release note #2419 #4475 --- doc/release-notes/2419-4475-7575-disable-users.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/release-notes/2419-4475-7575-disable-users.md diff --git a/doc/release-notes/2419-4475-7575-disable-users.md b/doc/release-notes/2419-4475-7575-disable-users.md new file mode 100644 index 00000000000..612200905dc --- /dev/null +++ b/doc/release-notes/2419-4475-7575-disable-users.md @@ -0,0 +1,5 @@ +## Release Highlights + +### Disable Users API, Get User Traces API, Revoke Roles API + +A new API has been added to disable users to prevent them from logging in or otherwise being active in the system. Disabling a user is an alternative to deleting a user, especially when the latter is not possible due to the amount of interaction the user has had with the system. In order to learn more about a user before deleting, disabling, or merging, a new "get user traces" API is available that will show objects created, roles, group memberships, and more. Finally, the "remove all roles" button available in the superuser dashboard is now also available via API. From a9a368b9f1981c5c67b3dbf1d4c1fa0d5cbbe24d Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 19 Feb 2021 12:13:29 -0500 Subject: [PATCH 344/576] implement rule on merging disabled accounts #2419 #4475 "User accounts can only be merged if they are either both enabled or both disabled." --- .../command/impl/MergeInAccountCommand.java | 6 +- .../iq/dataverse/api/DisableUsersIT.java | 88 +++++++++++++++++++ 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java index a13cf27fcd6..c0e3b3ea89a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java @@ -58,7 +58,11 @@ public MergeInAccountCommand(DataverseRequest createDataverseRequest, Authentica @Override protected void executeImpl(CommandContext ctxt) throws CommandException { - + + if (consumedAU.isDisabled() && !ongoingAU.isDisabled() || !consumedAU.isDisabled() && ongoingAU.isDisabled()) { + throw new IllegalCommandException("User accounts can only be merged if they are either both enabled or both disabled.", this); + } + List baseRAList = ctxt.roleAssignees().getAssignmentsFor(ongoingAU.getIdentifier()); List consumedRAList = ctxt.roleAssignees().getAssignmentsFor(consumedAU.getIdentifier()); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DisableUsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DisableUsersIT.java index fdec2480db6..bfba9388b93 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DisableUsersIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DisableUsersIT.java @@ -157,4 +157,92 @@ public void testDisableUserById() { disableUser.then().assertThat().statusCode(OK.getStatusCode()); } + @Test + public void testMergeDisabledIntoEnabledUser() { + + Response createSuperuser = UtilIT.createRandomUser(); + String superuserUsername = UtilIT.getUsernameFromResponse(createSuperuser); + String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser); + Response toggleSuperuser = UtilIT.makeSuperUser(superuserUsername); + toggleSuperuser.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response createUserMergeTarget = UtilIT.createRandomUser(); + createUserMergeTarget.prettyPrint(); + String usernameMergeTarget = UtilIT.getUsernameFromResponse(createUserMergeTarget); + + Response createUserToMerge = UtilIT.createRandomUser(); + createUserToMerge.prettyPrint(); + String usernameToMerge = UtilIT.getUsernameFromResponse(createUserToMerge); + + Response disableUser = UtilIT.disableUser(usernameToMerge, superuserApiToken); + disableUser.prettyPrint(); + disableUser.then().assertThat().statusCode(OK.getStatusCode()); + + // User accounts can only be merged if they are either both enabled or both disabled. + Response mergeAccounts = UtilIT.mergeAccounts(usernameMergeTarget, usernameToMerge, superuserApiToken); + mergeAccounts.prettyPrint(); + mergeAccounts.then().assertThat().statusCode(BAD_REQUEST.getStatusCode()); + } + + @Test + public void testMergeEnabledIntoDisabledUser() { + + Response createSuperuser = UtilIT.createRandomUser(); + String superuserUsername = UtilIT.getUsernameFromResponse(createSuperuser); + String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser); + Response toggleSuperuser = UtilIT.makeSuperUser(superuserUsername); + toggleSuperuser.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response createUserMergeTarget = UtilIT.createRandomUser(); + createUserMergeTarget.prettyPrint(); + String usernameMergeTarget = UtilIT.getUsernameFromResponse(createUserMergeTarget); + + Response createUserToMerge = UtilIT.createRandomUser(); + createUserToMerge.prettyPrint(); + String usernameToMerge = UtilIT.getUsernameFromResponse(createUserToMerge); + + Response disableUser = UtilIT.disableUser(usernameMergeTarget, superuserApiToken); + disableUser.prettyPrint(); + disableUser.then().assertThat().statusCode(OK.getStatusCode()); + + // User accounts can only be merged if they are either both enabled or both disabled. + Response mergeAccounts = UtilIT.mergeAccounts(usernameMergeTarget, usernameToMerge, superuserApiToken); + mergeAccounts.prettyPrint(); + mergeAccounts.then().assertThat().statusCode(BAD_REQUEST.getStatusCode()); + } + + @Test + public void testMergeDisabledIntoDisabledUser() { + + Response createSuperuser = UtilIT.createRandomUser(); + String superuserUsername = UtilIT.getUsernameFromResponse(createSuperuser); + String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser); + Response toggleSuperuser = UtilIT.makeSuperUser(superuserUsername); + toggleSuperuser.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response createUserMergeTarget = UtilIT.createRandomUser(); + createUserMergeTarget.prettyPrint(); + String usernameMergeTarget = UtilIT.getUsernameFromResponse(createUserMergeTarget); + + Response createUserToMerge = UtilIT.createRandomUser(); + createUserToMerge.prettyPrint(); + String usernameToMerge = UtilIT.getUsernameFromResponse(createUserToMerge); + + Response disableUserMergeTarget = UtilIT.disableUser(usernameMergeTarget, superuserApiToken); + disableUserMergeTarget.prettyPrint(); + disableUserMergeTarget.then().assertThat().statusCode(OK.getStatusCode()); + + Response disableUserToMerge = UtilIT.disableUser(usernameToMerge, superuserApiToken); + disableUserToMerge.prettyPrint(); + disableUserToMerge.then().assertThat().statusCode(OK.getStatusCode()); + + // User accounts can only be merged if they are either both enabled or both disabled. + Response mergeAccounts = UtilIT.mergeAccounts(usernameMergeTarget, usernameToMerge, superuserApiToken); + mergeAccounts.prettyPrint(); + mergeAccounts.then().assertThat().statusCode(OK.getStatusCode()); + } + } From 8a7ed7b393263afc7f8c7a65d7bacd866b034d4b Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 19 Feb 2021 12:16:03 -0500 Subject: [PATCH 345/576] add new test classes: DeleteUsersIT and DisableUsersIT #2419 #4475 --- conf/docker-aio/run-test-suite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/docker-aio/run-test-suite.sh b/conf/docker-aio/run-test-suite.sh index 2b24f6c90b2..da9e56e0176 100755 --- a/conf/docker-aio/run-test-suite.sh +++ b/conf/docker-aio/run-test-suite.sh @@ -8,4 +8,4 @@ fi # Please note the "dataverse.test.baseurl" is set to run for "all-in-one" Docker environment. # TODO: Rather than hard-coding the list of "IT" classes here, add a profile to pom.xml. -source maven/maven.sh && mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT -Ddataverse.test.baseurl=$dvurl +source maven/maven.sh && mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DisableUsersIT -Ddataverse.test.baseurl=$dvurl From ef1061e7fe05be56317deb8d3dacbfc6fe35ff86 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 19 Feb 2021 15:09:35 -0500 Subject: [PATCH 346/576] disabled users cannot use the API #2419 #4475 --- doc/sphinx-guides/source/api/native-api.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index fbf9ed266fc..9c4f5ffb70d 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3121,6 +3121,7 @@ This is an irreversible action. There is no option to undisable a user. Disabling a user with this endpoint will: - Disable the user's ability to log in to the Dataverse installation. A message will be shown, stating that the account has been disabled. The user will not able to create a new account with the same email address, ORCID, Shibboleth, or other login type. +- Disable the user's ability to use the API - Remove the user's access from all Dataverse collections, datasets and files - Prevent a user from being assigned any roles - Cancel any pending file access requests generated by the user From fa33eee2237f135204cb078742d540145fb463c8 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 19 Feb 2021 15:24:09 -0500 Subject: [PATCH 347/576] cleanup --- src/main/java/edu/harvard/iq/dataverse/api/Users.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Users.java b/src/main/java/edu/harvard/iq/dataverse/api/Users.java index e8558f88d6e..6e0476711ec 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Users.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Users.java @@ -213,7 +213,7 @@ public Response removeUserRoles(@PathParam("identifier") String identifier) { @Path("{identifier}/traces") public Response getTraces(@PathParam("identifier") String identifier) { try { - AuthenticatedUser userToQuery = authSvc.getAuthenticatedUser(identifier); // TODO: do lookup here? What if null? + AuthenticatedUser userToQuery = authSvc.getAuthenticatedUser(identifier); JsonObjectBuilder jsonObj = execCommand(new GetUserTracesCommand(createDataverseRequest(findUserOrDie()), userToQuery)); return ok(jsonObj); } catch (WrappedResponse ex) { From 9b0cab63b60aff0ebc8a902e9bc34bd161bbb000 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 19 Feb 2021 17:13:25 -0500 Subject: [PATCH 348/576] adding note to dataverse management about conditionally required --- doc/sphinx-guides/source/user/dataverse-management.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/user/dataverse-management.rst b/doc/sphinx-guides/source/user/dataverse-management.rst index 29685936b4d..a5bfbf7989b 100755 --- a/doc/sphinx-guides/source/user/dataverse-management.rst +++ b/doc/sphinx-guides/source/user/dataverse-management.rst @@ -28,7 +28,7 @@ Creating a Dataverse collection is easy but first you must be a registered user #. **Choose the sets of Metadata Fields for datasets in this Dataverse collection**: * By default the metadata elements will be from the host Dataverse collection that this new Dataverse collection is created in. * The Dataverse Software offers metadata standards for multiple domains. To learn more about the metadata standards in the Dataverse Software please check out the :doc:`/user/appendix`. - * Metadata fields can be hidden, or selected as required or optional. + * Metadata fields can be hidden, or selected as required, optional, or conditionally required. * Selected metadata elements are also used to pick which metadata fields you would like to use for creating :ref:`Dataset Templates `: after you finish creating your Dataverse collection. #. **Choose which metadata fields will be used as browse/search facets on your Dataverse collection**: * These facets will allow users browsing or searching your Dataverse collection to filter its contents according to the fields you have selected. For example, if you select “Subject” as a facet, users will be able to filter your Dataverse collection’s contents by subject area. From c90befec367479701183d8e1f699dcfff7dec54a Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 22 Feb 2021 11:33:21 -0500 Subject: [PATCH 349/576] #3175 remove unused method/onclick --- .../java/edu/harvard/iq/dataverse/DataversePage.java | 12 ------------ src/main/webapp/dataverse.xhtml | 1 - 2 files changed, 13 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java index 2bc7e7eaf27..d4aede8591c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java @@ -511,18 +511,6 @@ public void hideDatasetFieldTypes(Long mdbId) { } setEditInputLevel(false); } - - public void toggleInputLevel( Long mdbId, long dsftId){ - for (MetadataBlock mdb : allMetadataBlocks) { - if (mdb.getId().equals(mdbId)) { - for (DatasetFieldType dsftTest : mdb.getDatasetFieldTypes()) { - if (dsftTest.getId().equals(dsftId)) { - dsftTest.setRequiredDV(!dsftTest.isRequiredDV()); - } - } - } - } - } public void updateInclude(Long mdbId, long dsftId) { List childDSFT = new ArrayList<>(); diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index 62f03b64be1..284b7135f09 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -332,7 +332,6 @@ From d7c8611a5c58b41715297f2c4f9d4d81c78f4d18 Mon Sep 17 00:00:00 2001 From: Julian Gautier Date: Mon, 22 Feb 2021 11:51:20 -0500 Subject: [PATCH 350/576] Update dataverse-management.rst Revising sentence about controlling which fields are shown, hidden, required and conditionally required in Dataverse collections. --- doc/sphinx-guides/source/user/dataverse-management.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/user/dataverse-management.rst b/doc/sphinx-guides/source/user/dataverse-management.rst index a5bfbf7989b..5cc58a970ac 100755 --- a/doc/sphinx-guides/source/user/dataverse-management.rst +++ b/doc/sphinx-guides/source/user/dataverse-management.rst @@ -28,8 +28,8 @@ Creating a Dataverse collection is easy but first you must be a registered user #. **Choose the sets of Metadata Fields for datasets in this Dataverse collection**: * By default the metadata elements will be from the host Dataverse collection that this new Dataverse collection is created in. * The Dataverse Software offers metadata standards for multiple domains. To learn more about the metadata standards in the Dataverse Software please check out the :doc:`/user/appendix`. - * Metadata fields can be hidden, or selected as required, optional, or conditionally required. - * Selected metadata elements are also used to pick which metadata fields you would like to use for creating :ref:`Dataset Templates `: after you finish creating your Dataverse collection. + * Most metadata fields can be hidden or can be selected as required or optional. Some fields may be selected as conditionally required, depending on metadata options chosen for the Dataverse installation. + * Selected metadata fields are also used to pick which metadata fields you would like to use for creating :ref:`Dataset Templates `: after you finish creating your Dataverse collection. #. **Choose which metadata fields will be used as browse/search facets on your Dataverse collection**: * These facets will allow users browsing or searching your Dataverse collection to filter its contents according to the fields you have selected. For example, if you select “Subject” as a facet, users will be able to filter your Dataverse collection’s contents by subject area. * By default, the facets that will appear on your Dataverse collection's landing page will be from the host Dataverse collection that this new Dataverse collection was created in, but you can add or remove facets from this default. From 444a73cd94e8d2b7c46751dd0855121ca5763f57 Mon Sep 17 00:00:00 2001 From: Julian Gautier Date: Mon, 22 Feb 2021 12:01:34 -0500 Subject: [PATCH 351/576] Added conditionally required metadata field info Added conditionally required metadata field info to Introduction section --- doc/sphinx-guides/source/admin/metadatacustomization.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index bc0b18112ab..f35d46e514e 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -9,7 +9,7 @@ The Dataverse Software has a flexible data-driven metadata system powered by "me Introduction ------------ -Before you embark on customizing metadata in your Dataverse installation you should make sure you are aware of the modest amount of customization that is available with your Dataverse installation's web interface. It's possible to hide fields and make field required by clicking "Edit" at the Dataverse collection level, clicking "General Information" and making adjustments under "Metadata Fields" as described in the :ref:`create-dataverse` section of the Dataverse Collection Management page in the User Guide. +Before you embark on customizing metadata in your Dataverse installation you should make sure you are aware of the modest amount of customization that is available with your Dataverse installation's web interface. It's possible to hide fields and make field required or conditionally required by clicking "Edit" at the Dataverse collection level, clicking "General Information" and making adjustments under "Metadata Fields" as described in the :ref:`create-dataverse` section of the Dataverse Collection Management page in the User Guide. Much more customization of metadata is possible, but this is an advanced topic so feedback on what is written below is very welcome. The possibilities for customization include: From f0fa6d21bb79af1561645af5faf4dca800990db8 Mon Sep 17 00:00:00 2001 From: Julian Gautier Date: Mon, 22 Feb 2021 13:34:28 -0500 Subject: [PATCH 352/576] Update metadatacustomization.rst Added info about conditionally required fields to "required" row of the "datasetField (field) properties" table --- .../source/admin/metadatacustomization.rst | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index f35d46e514e..2ac0ec0347b 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -289,16 +289,37 @@ Each of the three main sections own sets of properties: | | the dataset has been | | | | saved. | | +-----------------------+-----------------------+------------------------+ -| required | Specify whether or | TRUE (required) or | -| | not the field is | FALSE (optional) | -| | required. This means | | -| | that at least one | | -| | instance of the field | | -| | must be present. More | | -| | than one field may be | | -| | allowed, depending on | | -| | the value of | | -| | allowmultiples. | | +| required | For primitive | For primitive | +| | fields, specify | fields, TRUE | +| | whether or not the | (required) or FALSE | +| | field is required. | (optional). | +| | For compound | | +| | fields, also | For compound fields: | +| | specify if one or | | +| | more subfields are | \• To make one or more | +| | required or | subfields optional, | +| | conditionally | the parent field and | +| | required. At least | subfield(s) must be | +| | one instance of a | FALSE (optional). | +| | required field must | | +| | be present. More | \• To make one or more | +| | than one instance | subfields required, | +| | of a field may be | the parent field and | +| | allowed, depending | the required | +| | on the value of | subfield(s) must be | +| | allowmultiples. | TRUE (required). | +| | | | +| | | \• To make one or more | +| | | subfields | +| | | conditionally | +| | | required, make the | +| | | parent field FALSE | +| | | (optional) and make | +| | | TRUE (required) any | +| | | subfield or subfields | +| | | that are required if | +| | | any other subfields | +| | | are filled. | +-----------------------+-----------------------+------------------------+ | parent | For subfields, | \• Must not result in | | | specify the name of | a cyclical | From e81a8770e1e39e6582da75a75f5c6123b8f5b8f9 Mon Sep 17 00:00:00 2001 From: Julian Gautier Date: Mon, 22 Feb 2021 15:44:22 -0500 Subject: [PATCH 353/576] Update metadatacustomization.rst Fixing small typo --- doc/sphinx-guides/source/admin/metadatacustomization.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index 2ac0ec0347b..fdbac996108 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -9,7 +9,7 @@ The Dataverse Software has a flexible data-driven metadata system powered by "me Introduction ------------ -Before you embark on customizing metadata in your Dataverse installation you should make sure you are aware of the modest amount of customization that is available with your Dataverse installation's web interface. It's possible to hide fields and make field required or conditionally required by clicking "Edit" at the Dataverse collection level, clicking "General Information" and making adjustments under "Metadata Fields" as described in the :ref:`create-dataverse` section of the Dataverse Collection Management page in the User Guide. +Before you embark on customizing metadata in your Dataverse installation you should make sure you are aware of the modest amount of customization that is available with your Dataverse installation's web interface. It's possible to hide fields and make fields required or conditionally required by clicking "Edit" at the Dataverse collection level, clicking "General Information" and making adjustments under "Metadata Fields" as described in the :ref:`create-dataverse` section of the Dataverse Collection Management page in the User Guide. Much more customization of metadata is possible, but this is an advanced topic so feedback on what is written below is very welcome. The possibilities for customization include: From acd317cc25b238dbc64c2c346cebddf9624c9910 Mon Sep 17 00:00:00 2001 From: Gustavo Durand Date: Mon, 22 Feb 2021 18:24:31 -0500 Subject: [PATCH 354/576] Update V5.3.0.3__7551-expanded-compound-datasetfield-validation.sql --- ...V5.3.0.3__7551-expanded-compound-datasetfield-validation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/db/migration/V5.3.0.3__7551-expanded-compound-datasetfield-validation.sql b/src/main/resources/db/migration/V5.3.0.3__7551-expanded-compound-datasetfield-validation.sql index 56b699fa366..f9cb9d0e526 100644 --- a/src/main/resources/db/migration/V5.3.0.3__7551-expanded-compound-datasetfield-validation.sql +++ b/src/main/resources/db/migration/V5.3.0.3__7551-expanded-compound-datasetfield-validation.sql @@ -10,7 +10,7 @@ DELETE FROM dataversefieldtypeinputlevel WHERE datasetfieldtype_id in ( SELECT parentdatasetfieldtype_id FROM datasetfieldtype dsft); INSERT INTO dataversefieldtypeinputlevel (include,required,datasetfieldtype_id,dataverse_id) -SELECT true, true, parentdatasetfieldtype_id, dataverse_id FROM +SELECT DISTINCT true, true, parentdatasetfieldtype_id, dataverse_id FROM dataversefieldtypeinputlevel dsftil, datasetfieldtype dsft WHERE dsftil.datasetfieldtype_id = dsft.id AND parentdatasetfieldtype_id IS NOT null; From 78e253418d1b9cbdab12985b4b748f43be07bd9a Mon Sep 17 00:00:00 2001 From: gdurand Date: Mon, 22 Feb 2021 18:50:09 -0500 Subject: [PATCH 355/576] added an else --- .../harvard/iq/dataverse/DatasetField.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java index eee2684bfe1..79f8916deb9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java @@ -421,21 +421,21 @@ public boolean isRequired() { if (this.datasetFieldType.isRequired()) { required = true; - } - - // otherwise check if overridden at the dataverse level - Dataverse dv = getDataverse(); - while (!dv.isMetadataBlockRoot()) { - if (dv.getOwner() == null) { - break; // we are at the root; which by defintion is metadata blcok root, regarldess of the value + } else { + // otherwise check if overridden at the dataverse level + Dataverse dv = getDataverse(); + while (!dv.isMetadataBlockRoot()) { + if (dv.getOwner() == null) { + break; // we are at the root; which by defintion is metadata blcok root, regarldess of the value + } + dv = dv.getOwner(); } - dv = dv.getOwner(); - } - List dftilListFirst = dv.getDataverseFieldTypeInputLevels(); - for (DataverseFieldTypeInputLevel dsftil : dftilListFirst) { - if (dsftil.getDatasetFieldType().equals(this.datasetFieldType)) { - required = dsftil.isRequired(); + List dftilListFirst = dv.getDataverseFieldTypeInputLevels(); + for (DataverseFieldTypeInputLevel dsftil : dftilListFirst) { + if (dsftil.getDatasetFieldType().equals(this.datasetFieldType)) { + required = dsftil.isRequired(); + } } } From 6daa2b4fffd8664f581280dde2eb087557d591e6 Mon Sep 17 00:00:00 2001 From: Gustavo Durand Date: Mon, 22 Feb 2021 19:15:15 -0500 Subject: [PATCH 356/576] Update 7551-expanded-compound-datasetfield-validation.md --- ...551-expanded-compound-datasetfield-validation.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md index 662a5d31fa1..5541f6deb93 100644 --- a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md +++ b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md @@ -14,12 +14,19 @@ Going forward: - For required, the parent compound field would be required = true and at least one child would be required = true. - For conditionally required, the parent compound field would be required = false and at least one child would be required = true. -This release updates the citation .tsv file that us distributed with the software (no other distributed .tsv file has any required values). **If you have created any custom metadata .tsv files**, you will need to make the same (type of) changes there. +This release updates the citation .tsv file that us distributed with the software for the required parent compound fields (e.g. author), as well as sets Producer Name to be conditionally required. No other distributed .tsv files were updated, as they did not have any required compound values. + +**If you have created any custom metadata .tsv files**, you will need to make the same (type of) changes there. For any subfield that has a required value of TRUE, find the corresponding parent field and change its required value to TRUE. -Note: The metadata .tsv files do not have to be reloaded via API, as there is an accompanying Flyway script that updates the values directly in the database. +Note: With the exception of citation.tsv (due to the conditionally required Producer Name), the metadata .tsv files do not have to be reloaded via API, as there is an accompanying Flyway script that updates the values directly in the database. ### Additional Upgrade Steps -1. Update any custom metadata blocks (if used) \ No newline at end of file +1. Update any custom metadata blocks (if used) + +2. Reload Citation Metadata Block: + + `wget https://github.com/IQSS/dataverse/releases/download/v5.4/citation.tsv` + `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @citation.tsv -H "Content-type: text/tab-separated-values"` From 56929541616d6fbf27a82ba7ab883c7138652397 Mon Sep 17 00:00:00 2001 From: Gustavo Durand Date: Mon, 22 Feb 2021 19:16:57 -0500 Subject: [PATCH 357/576] Update 7551-expanded-compound-datasetfield-validation.md --- .../7551-expanded-compound-datasetfield-validation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md index 5541f6deb93..7275a827633 100644 --- a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md +++ b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md @@ -14,7 +14,7 @@ Going forward: - For required, the parent compound field would be required = true and at least one child would be required = true. - For conditionally required, the parent compound field would be required = false and at least one child would be required = true. -This release updates the citation .tsv file that us distributed with the software for the required parent compound fields (e.g. author), as well as sets Producer Name to be conditionally required. No other distributed .tsv files were updated, as they did not have any required compound values. +This release updates the citation .tsv file that is distributed with the software for the required parent compound fields (e.g. author), as well as sets Producer Name to be conditionally required. No other distributed .tsv files were updated, as they did not have any required compound values. **If you have created any custom metadata .tsv files**, you will need to make the same (type of) changes there. From fdfc3ba7685d1fc4f8bb02c523c10f30606a3561 Mon Sep 17 00:00:00 2001 From: Gustavo Durand Date: Tue, 23 Feb 2021 09:56:39 -0500 Subject: [PATCH 358/576] Update 7551-expanded-compound-datasetfield-validation.md --- ...xpanded-compound-datasetfield-validation.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md index 7275a827633..b1ad275c2f2 100644 --- a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md +++ b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md @@ -1,6 +1,6 @@ ## Notes for Dataverse Administrators -Prior to this release, when defining metadata for compound fields, fields could be either be optional or required, i.e. if required you must always have (at least one) value for that field. For example, Author Name being required means you must have at least one Author with an nonempty Author name. +Prior to this release, when defining metadata for compound fields (via their datasrt field types), fields could be either be optional or required, i.e. if required you must always have (at least one) value for that field. For example, Author Name being required means you must have at least one Author with an nonempty Author name. In order to support more robust metadata (and specifically to resolve #7551), we need to allow a third case: Conditionally Required, that is, the field is required if and only if any of its "sibling" fields are required. For example, a user does not have to enter a Producer, but if they do, they have to enter a Producer Name. @@ -18,15 +18,17 @@ This release updates the citation .tsv file that is distributed with the softwar **If you have created any custom metadata .tsv files**, you will need to make the same (type of) changes there. -For any subfield that has a required value of TRUE, find the corresponding parent field and change its required value to TRUE. - -Note: With the exception of citation.tsv (due to the conditionally required Producer Name), the metadata .tsv files do not have to be reloaded via API, as there is an accompanying Flyway script that updates the values directly in the database. - ### Additional Upgrade Steps -1. Update any custom metadata blocks (if used) - -2. Reload Citation Metadata Block: +1. Reload Citation Metadata Block: `wget https://github.com/IQSS/dataverse/releases/download/v5.4/citation.tsv` `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @citation.tsv -H "Content-type: text/tab-separated-values"` + +2. Update any custom metadata blocks (if used): + +For any subfield that has a required value of TRUE, find the corresponding parent field and change its required value to TRUE. + +Note: As there is an accompanying Flyway script that updates the values directly in the database, you do not need to reload these metadata .tsv files via API, unless you make additional changes, e.g set some compound fields to be conditionally required. + + From cfb1fbee13daee5a6aca9678bf7f54a901b2fd1f Mon Sep 17 00:00:00 2001 From: Gustavo Durand Date: Tue, 23 Feb 2021 10:04:59 -0500 Subject: [PATCH 359/576] Update 7551-expanded-compound-datasetfield-validation.md --- .../7551-expanded-compound-datasetfield-validation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md index b1ad275c2f2..14b8e4d1aab 100644 --- a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md +++ b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md @@ -27,8 +27,8 @@ This release updates the citation .tsv file that is distributed with the softwar 2. Update any custom metadata blocks (if used): -For any subfield that has a required value of TRUE, find the corresponding parent field and change its required value to TRUE. + For any subfield that has a required value of TRUE, find the corresponding parent field and change its required value to TRUE. -Note: As there is an accompanying Flyway script that updates the values directly in the database, you do not need to reload these metadata .tsv files via API, unless you make additional changes, e.g set some compound fields to be conditionally required. + Note: As there is an accompanying Flyway script that updates the values directly in the database, you do not need to reload these metadata .tsv files via API, unless you make additional changes, e.g set some compound fields to be conditionally required. From 01e9666b67bf02b54bdedbf0c27c808189c1375c Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 23 Feb 2021 11:18:43 -0500 Subject: [PATCH 360/576] typo, review Looks good @scolapasta. I will add a use case at release time, or you can do so now. --- .../7551-expanded-compound-datasetfield-validation.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md index 14b8e4d1aab..cf8b9a7ba33 100644 --- a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md +++ b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md @@ -1,6 +1,6 @@ ## Notes for Dataverse Administrators -Prior to this release, when defining metadata for compound fields (via their datasrt field types), fields could be either be optional or required, i.e. if required you must always have (at least one) value for that field. For example, Author Name being required means you must have at least one Author with an nonempty Author name. +Prior to this release, when defining metadata for compound fields (via their dataset field types), fields could be either be optional or required, i.e. if required you must always have (at least one) value for that field. For example, Author Name being required means you must have at least one Author with an nonempty Author name. In order to support more robust metadata (and specifically to resolve #7551), we need to allow a third case: Conditionally Required, that is, the field is required if and only if any of its "sibling" fields are required. For example, a user does not have to enter a Producer, but if they do, they have to enter a Producer Name. @@ -31,4 +31,3 @@ This release updates the citation .tsv file that is distributed with the softwar Note: As there is an accompanying Flyway script that updates the values directly in the database, you do not need to reload these metadata .tsv files via API, unless you make additional changes, e.g set some compound fields to be conditionally required. - From e73c43b701956b6be71cfa85d2d82c829624d847 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 23 Feb 2021 12:16:48 -0500 Subject: [PATCH 361/576] rename sql update script #2419 #4475 --- ...3__4475-disable-users.sql => V5.3.0.4__4475-disable-users.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V5.3.0.3__4475-disable-users.sql => V5.3.0.4__4475-disable-users.sql} (100%) diff --git a/src/main/resources/db/migration/V5.3.0.3__4475-disable-users.sql b/src/main/resources/db/migration/V5.3.0.4__4475-disable-users.sql similarity index 100% rename from src/main/resources/db/migration/V5.3.0.3__4475-disable-users.sql rename to src/main/resources/db/migration/V5.3.0.4__4475-disable-users.sql From 3bf665e31a6c89f6ab56e1e5a1521603d46f39d6 Mon Sep 17 00:00:00 2001 From: Gustavo Durand Date: Tue, 23 Feb 2021 12:48:52 -0500 Subject: [PATCH 362/576] Added a use case --- .../7551-expanded-compound-datasetfield-validation.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md index cf8b9a7ba33..19d6d573699 100644 --- a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md +++ b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md @@ -2,7 +2,7 @@ Prior to this release, when defining metadata for compound fields (via their dataset field types), fields could be either be optional or required, i.e. if required you must always have (at least one) value for that field. For example, Author Name being required means you must have at least one Author with an nonempty Author name. -In order to support more robust metadata (and specifically to resolve #7551), we need to allow a third case: Conditionally Required, that is, the field is required if and only if any of its "sibling" fields are required. For example, a user does not have to enter a Producer, but if they do, they have to enter a Producer Name. +In order to support more robust metadata (and specifically to resolve #7551), we need to allow a third case: Conditionally Required, that is, the field is required if and only if any of its "sibling" fields are entered. For example, Producer Name is now conditionally required in the citation metadata block. A user does not have to enter a Producer, but if they do, they have to enter a Producer Name. This change required some modifications to how "required" is defined in the metadata .tsv files (for compound fields). @@ -31,3 +31,6 @@ This release updates the citation .tsv file that is distributed with the softwar Note: As there is an accompanying Flyway script that updates the values directly in the database, you do not need to reload these metadata .tsv files via API, unless you make additional changes, e.g set some compound fields to be conditionally required. +### Use Case + +Metadata designers can now set subfields of compound fields as **conditionally required**, that is, the field is required if and only if any of its "sibling" fields are entered. For example, Producer Name is now conditionally required in the citation metadata block. A user does not have to enter a Producer, but if they do, they have to enter a Producer Name. From 67a28b4406de54219034bed00defadc217036a8e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 24 Feb 2021 09:33:25 -0500 Subject: [PATCH 363/576] sync with 7564 changes sans messaging --- .../edu/harvard/iq/dataverse/DatasetPage.java | 10 --------- .../iq/dataverse/PermissionServiceBean.java | 2 +- .../harvard/iq/dataverse/api/Datasets.java | 6 ++++-- .../FinalizeDatasetPublicationCommand.java | 21 +++++++++---------- .../command/impl/PublishDatasetCommand.java | 11 +++++----- .../harvard/iq/dataverse/util/MailUtil.java | 4 ---- .../workflow/PendingWorkflowInvocation.java | 18 +++++++--------- .../internalspi/InternalWorkflowStepSP.java | 2 ++ .../db/migration/V5.4.0.1__7563-workflow.sql | 10 +-------- 9 files changed, 31 insertions(+), 53 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 04a50bd712d..112c57b7f41 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -1858,16 +1858,6 @@ private String init(boolean initFull) { MakeDataCountEntry entry = new MakeDataCountEntry(FacesContext.getCurrentInstance(), dvRequestService, workingVersion); mdcLogService.logEntry(entry); } - - List comments = workingVersion.getWorkflowComments(); - for(WorkflowComment wfc: comments) { - if(wfc.getType()==WorkflowComment.Type.WORKFLOW_SUCCESS) { - JsfHelper.addSuccessMessage(wfc.getMessage()); - - } else if (wfc.getType()==WorkflowComment.Type.WORKFLOW_FAILURE) { - JsfHelper.addWarningMessage(wfc.getMessage()); - } - } if (initFull) { // init the list of FileMetadatas diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index f38efa7b66b..aaf38af1b36 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -816,7 +816,7 @@ public boolean isMatchingWorkflowLock(Dataset d, String userId, String invocatio if (invocationId != null) { PendingWorkflowInvocation pwfi = workflowService.getPendingWorkflow(invocationId); for (DatasetLock dl : d.getLocks()) { - if (dl.equals(pwfi.getLock()) && pwfi.getUserId().equals(userId)) { + if (dl.getId().equals(pwfi.getLockId()) && pwfi.getUserId().equals(userId)) { return true; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 67f52c9cbe0..5cd87adae82 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -977,7 +977,9 @@ public Response publishDataset(@PathParam("id") String id, @QueryParam("type") S logger.fine("MT: " + ds.getModificationTime()); logger.fine("PIT: " + ds.getPermissionIndexTime()); logger.fine("PMT: " + ds.getPermissionModificationTime()); - logger.fine("ITMT: " + (ds.getIndexTime().compareTo(ds.getModificationTime()) <= 0)); + if (ds.getIndexTime() != null && ds.getModificationTime() != null) { + logger.fine("ITMT: " + (ds.getIndexTime().compareTo(ds.getModificationTime()) <= 0)); + } /* * Some calls, such as the /datasets/actions/:import* commands do not set the * modification or permission modification times. The checks here are trying to @@ -989,7 +991,7 @@ public Response publishDataset(@PathParam("id") String id, @QueryParam("type") S * */ if ((ds.getModificationTime()!=null && (ds.getIndexTime() == null || (ds.getIndexTime().compareTo(ds.getModificationTime()) <= 0))) || - (ds.getPermissionModificationTime()!=null && (ds.getIndexTime() == null || (ds.getIndexTime().compareTo(ds.getModificationTime()) <= 0)))) { + (ds.getPermissionModificationTime()!=null && (ds.getPermissionIndexTime() == null || (ds.getPermissionIndexTime().compareTo(ds.getPermissionModificationTime()) <= 0)))) { return error(Response.Status.CONFLICT, "Dataset is awaiting indexing"); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java index 0da5ee7535d..da9c4515752 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java @@ -183,28 +183,27 @@ public Dataset execute(CommandContext ctxt) throws CommandException { final Dataset ds = ctxt.em().merge(theDataset); //Remove any pre-pub workflow lock (not needed as WorkflowServiceBean.workflowComplete() should already have removed it after setting the finalizePublication lock?) ctxt.datasets().removeDatasetLocks(ds, DatasetLock.Reason.Workflow); + + //Should this be in onSuccess()? ctxt.workflows().getDefaultWorkflow(TriggerType.PostPublishDataset).ifPresent(wf -> { try { - WorkflowContext wfc = buildContext(ds, TriggerType.PostPublishDataset, datasetExternallyReleased); - // Set workflow lock before returning and before releasing the - // finalizePublication lock - ctxt.workflows().lockDataset(wfc); - ctxt.workflows().start(wf, wfc); + ctxt.workflows().start(wf, buildContext(ds, TriggerType.PostPublishDataset, datasetExternallyReleased)); } catch (CommandException ex) { + ctxt.datasets().removeDatasetLocks(ds, DatasetLock.Reason.Workflow); logger.log(Level.SEVERE, "Error invoking post-publish workflow: " + ex.getMessage(), ex); } }); - Dataset readyDataset = ctxt.em().merge(theDataset); + Dataset readyDataset = ctxt.em().merge(ds); // Finally, unlock the dataset (leaving any post-publish workflow lock in place) - ctxt.datasets().removeDatasetLocks(theDataset, DatasetLock.Reason.finalizePublication); - if ( theDataset.isLockedFor(DatasetLock.Reason.InReview) ) { - ctxt.datasets().removeDatasetLocks(theDataset, DatasetLock.Reason.InReview); + ctxt.datasets().removeDatasetLocks(readyDataset, DatasetLock.Reason.finalizePublication); + if (readyDataset.isLockedFor(DatasetLock.Reason.InReview) ) { + ctxt.datasets().removeDatasetLocks(readyDataset, DatasetLock.Reason.InReview); } - logger.info("Successfully published the dataset "+theDataset.getGlobalId().asString()); - + logger.info("Successfully published the dataset "+readyDataset.getGlobalId().asString()); + readyDataset = ctxt.em().merge(readyDataset); return readyDataset; } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java index a47d2a39d87..a2d206ead0d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java @@ -70,6 +70,10 @@ public PublishDatasetResult execute(CommandContext ctxt) throws CommandException Dataset theDataset = getDataset(); + + //ToDo - any reason to set the version in publish versus finalize? Failure in a prepub workflow or finalize will leave draft versions with an assigned version number as is. + //Changing the dataset in this transaction also potentially makes a race condition with a prepub workflow, possibly resulting in an OptimisticLockException there. + // Set the version numbers: if (theDataset.getPublicationDate() == null) { @@ -87,16 +91,13 @@ public PublishDatasetResult execute(CommandContext ctxt) throws CommandException theDataset.getLatestVersion().setMinorVersionNumber(new Long(0)); } + //ToDo - should this be in onSuccess()? May relate to todo above Optional prePubWf = ctxt.workflows().getDefaultWorkflow(TriggerType.PrePublishDataset); if ( prePubWf.isPresent() ) { // We start a workflow theDataset = ctxt.em().merge(theDataset); ctxt.em().flush(); - - WorkflowContext wfc = buildContext(theDataset, TriggerType.PrePublishDataset, datasetExternallyReleased); - //add workflow lock before returning - ctxt.workflows().lockDataset(wfc); - ctxt.workflows().start(prePubWf.get(), wfc); + ctxt.workflows().start(prePubWf.get(), buildContext(theDataset, TriggerType.PrePublishDataset, datasetExternallyReleased)); return new PublishDatasetResult(theDataset, Status.Workflow); } else{ diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java index 2ffec60516c..37667d16b55 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java @@ -54,10 +54,6 @@ public static String getSubjectTextBasedOnNotification(UserNotification userNoti return BundleUtil.getStringFromBundle("notification.email.publishFailure.dataset.subject", rootDvNameAsList); case RETURNEDDS: return BundleUtil.getStringFromBundle("notification.email.returned.dataset.subject", rootDvNameAsList); - case WORKFLOW_SUCCESS: - return BundleUtil.getStringFromBundle("notification.email.workflow.success.dataset.subject", rootDvNameAsList); - case WORKFLOW_FAILURE: - return BundleUtil.getStringFromBundle("notification.email.workflow.failure.dataset.subject", rootDvNameAsList); case CREATEACC: return BundleUtil.getStringFromBundle("notification.email.create.account.subject", rootDvNameAsList); case CHECKSUMFAIL: diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java b/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java index b211347568b..577e0f756de 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java @@ -1,7 +1,6 @@ package edu.harvard.iq.dataverse.workflow; import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetLock; import edu.harvard.iq.dataverse.RoleAssigneeServiceBean; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; import edu.harvard.iq.dataverse.authorization.users.User; @@ -10,8 +9,6 @@ import java.io.Serializable; import java.util.HashMap; import java.util.Map; - -import javax.persistence.CascadeType; import javax.persistence.ElementCollection; import javax.persistence.Entity; import javax.persistence.FetchType; @@ -55,8 +52,7 @@ public class PendingWorkflowInvocation implements Serializable { int typeOrdinal; boolean datasetExternallyReleased; - @OneToOne - DatasetLock lock; + long lockid; public static final String AUTHORIZED= "authorized"; @@ -76,13 +72,13 @@ public PendingWorkflowInvocation(Workflow wf, WorkflowContext ctxt, Pending resu localData = new HashMap<>(result.getData()); typeOrdinal = ctxt.getType().ordinal(); datasetExternallyReleased=ctxt.getDatasetExternallyReleased(); - lock=ctxt.getLock(); + lockid=ctxt.getLockId(); } public WorkflowContext reCreateContext(RoleAssigneeServiceBean roleAssignees) { DataverseRequest aRequest = new DataverseRequest((User)roleAssignees.getRoleAssignee(userId), IpAddress.valueOf(ipAddress)); final WorkflowContext workflowContext = new WorkflowContext(aRequest, dataset, nextVersionNumber, - nextMinorVersionNumber, WorkflowContext.TriggerType.values()[typeOrdinal], null, null, datasetExternallyReleased, invocationId, lock); + nextMinorVersionNumber, WorkflowContext.TriggerType.values()[typeOrdinal], null, null, datasetExternallyReleased, invocationId, lockid); return workflowContext; } @@ -166,10 +162,10 @@ public void setTypeOrdinal(int typeOrdinal) { this.typeOrdinal = typeOrdinal; } - public DatasetLock getLock() { - return lock; + public long getLockId() { + return lockid; } - public void setLock(DatasetLock lock) { - this.lock = lock; + public void setLockId(long lockId) { + this.lockid = lockId; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/InternalWorkflowStepSP.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/InternalWorkflowStepSP.java index 76b7bcd7d44..df5f2de9058 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/InternalWorkflowStepSP.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/InternalWorkflowStepSP.java @@ -19,6 +19,8 @@ public WorkflowStep getStep(String stepType, Map stepParameters) return new PauseStep(stepParameters); case "http/sr": return new HttpSendReceiveClientStep(stepParameters); + case "http/authExt": + return new AuthorizedExternalStep(stepParameters); case "archiver": return new ArchivalSubmissionWorkflowStep(stepParameters); default: diff --git a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql index 57ecec383a7..085a2e4bba3 100644 --- a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql +++ b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql @@ -1,10 +1,2 @@ ALTER TABLE pendingworkflowinvocation -ADD COLUMN IF NOT EXISTS lock_id bigint; - -DO $$ -BEGIN -IF NOT EXISTS (SELECT 1 FROM information_schema.constraint_column_usage WHERE constraint_name='fk_pendingworkflowinvocation_lock_id') -THEN -ALTER TABLE pendingworkflowinvocation ADD CONSTRAINT fk_pendingworkflowinvocation_lock_id FOREIGN KEY (lock_id) REFERENCES datasetlock (id); -END IF; -END$$; \ No newline at end of file +ADD COLUMN IF NOT EXISTS lockid bigint; From dcb0537b3ced4e0dc8bbfdf433dde0e536f45b36 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 24 Feb 2021 10:00:03 -0500 Subject: [PATCH 364/576] more to split from 7564 --- pom.xml | 12 -- .../edu/harvard/iq/dataverse/DatasetPage.java | 4 +- .../iq/dataverse/DatasetServiceBean.java | 6 - .../harvard/iq/dataverse/MailServiceBean.java | 15 --- .../iq/dataverse/UserNotification.java | 3 +- .../harvard/iq/dataverse/api/Datasets.java | 114 ++---------------- .../FinalizeDatasetPublicationCommand.java | 1 - .../command/impl/PublishDatasetCommand.java | 1 - .../dataverse/workflow/WorkflowContext.java | 19 ++- .../workflow/WorkflowServiceBean.java | 81 +++++++------ .../iq/dataverse/workflow/step/Failure.java | 2 +- .../iq/dataverse/workflow/step/Success.java | 47 -------- .../dataverse/workflows/WorkflowComment.java | 20 +-- src/main/java/propertyFiles/Bundle.properties | 4 - 14 files changed, 69 insertions(+), 260 deletions(-) delete mode 100644 src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java diff --git a/pom.xml b/pom.xml index 29ae1c49747..de8ff91863a 100644 --- a/pom.xml +++ b/pom.xml @@ -205,12 +205,6 @@ org.apache.abdera abdera-core 1.1.3 - - - org.apache.geronimo.specs - geronimo-stax-api_1.0_spec - - @@ -504,12 +498,6 @@ com.lyncode xoai-common 4.1.0-header-patch - - - javax.xml.stream - stax-api - - com.lyncode diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 112c57b7f41..5f596d1b556 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -54,8 +54,6 @@ import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.SystemConfig; -import edu.harvard.iq.dataverse.workflows.WorkflowComment; - import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -1858,7 +1856,7 @@ private String init(boolean initFull) { MakeDataCountEntry entry = new MakeDataCountEntry(FacesContext.getCurrentInstance(), dvRequestService, workingVersion); mdcLogService.logEntry(entry); } - + if (initFull) { // init the list of FileMetadatas if (workingVersion.isDraft() && canUpdateDataset()) { diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index 71eef6d6c5f..c1efe119fd2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -787,12 +787,6 @@ public WorkflowComment addWorkflowComment(WorkflowComment workflowComment) { return workflowComment; } - public void markWorkflowCommentAsRead(WorkflowComment workflowComment) { - workflowComment.setToBeShown(false); - em.merge(workflowComment); - } - - /** * This method used to throw CommandException, which was pretty pointless * seeing how it's called asynchronously. As of v5.0 any CommanExceptiom diff --git a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java index 42feaf6d2c2..13a92c9cd27 100644 --- a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java @@ -496,19 +496,6 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio version.getDataset().getOwner().getDisplayName(), getDataverseLink(version.getDataset().getOwner()), optionalReturnReason}; messageText += MessageFormat.format(pattern, paramArrayReturnedDataset); return messageText; - - case WORKFLOW_SUCCESS: - version = (DatasetVersion) targetObject; - pattern = BundleUtil.getStringFromBundle("notification.email.workflow.success"); - String[] paramArrayWorkflowSuccess = {version.getDataset().getDisplayName(), getDatasetLink(version.getDataset()), comment}; - messageText += MessageFormat.format(pattern, paramArrayWorkflowSuccess); - return messageText; - case WORKFLOW_FAILURE: - version = (DatasetVersion) targetObject; - pattern = BundleUtil.getStringFromBundle("notification.email.workflow.failure"); - String[] paramArrayWorkflowFailure = {version.getDataset().getDisplayName(), getDatasetLink(version.getDataset())}; - messageText += MessageFormat.format(pattern, paramArrayWorkflowFailure); - return messageText; case CREATEACC: String rootDataverseName = dataverseService.findRootDataverse().getName(); InternetAddress systemAddress = getSystemAddress(); @@ -607,8 +594,6 @@ private Object getObjectOfNotification (UserNotification userNotification){ case PUBLISHEDDS: case PUBLISHFAILED_PIDREG: case RETURNEDDS: - case WORKFLOW_SUCCESS: - case WORKFLOW_FAILURE: return versionService.find(userNotification.getObjectId()); case CREATEACC: return userNotification.getUser(); diff --git a/src/main/java/edu/harvard/iq/dataverse/UserNotification.java b/src/main/java/edu/harvard/iq/dataverse/UserNotification.java index 445e12f9ef8..e44c5f6333e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/UserNotification.java +++ b/src/main/java/edu/harvard/iq/dataverse/UserNotification.java @@ -30,7 +30,7 @@ public enum Type { ASSIGNROLE, REVOKEROLE, CREATEDV, CREATEDS, CREATEACC, SUBMITTEDDS, RETURNEDDS, PUBLISHEDDS, REQUESTFILEACCESS, GRANTFILEACCESS, REJECTFILEACCESS, FILESYSTEMIMPORT, CHECKSUMIMPORT, CHECKSUMFAIL, CONFIRMEMAIL, APIGENERATED, INGESTCOMPLETED, INGESTCOMPLETEDWITHERRORS, - PUBLISHFAILED_PIDREG, WORKFLOW_SUCCESS, WORKFLOW_FAILURE + PUBLISHFAILED_PIDREG }; private static final long serialVersionUID = 1L; @@ -156,5 +156,4 @@ public void setRoleString(String roleString) { public String getLocaleSendDate() { return DateUtil.formatDate(sendDate); } - } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 5cd87adae82..7341e3f66ed 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -112,8 +112,6 @@ import java.io.StringReader; import java.sql.Timestamp; import java.text.MessageFormat; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; @@ -158,6 +156,7 @@ import org.glassfish.jersey.media.multipart.FormDataParam; import com.amazonaws.services.s3.model.PartETag; +import edu.harvard.iq.dataverse.FileMetadata; import java.util.Map.Entry; @Path("datasets") @@ -770,16 +769,11 @@ private String getCompoundDisplayValue (DatasetFieldCompoundValue dscv){ @PUT @Path("{id}/editMetadata") - public Response editVersionMetadata(String jsonBody, @PathParam("id") String id, @QueryParam("replace") Boolean replace) { + public Response editVersionMetadata(String jsonBody, @PathParam("id") String id, @QueryParam("replace") Boolean replace) throws WrappedResponse{ Boolean replaceData = replace != null; - DataverseRequest req = null; - try { - req = createDataverseRequest(findUserOrDie()); - } catch (WrappedResponse ex) { - logger.log(Level.SEVERE, "Edit metdata error: " + ex.getMessage(), ex); - return ex.getResponse(); - } + + DataverseRequest req = createDataverseRequest(findUserOrDie()); return processDatasetUpdate(jsonBody, id, req, replaceData); } @@ -939,12 +933,12 @@ private String validateDatasetFieldValues(List fields) { @Deprecated public Response publishDataseUsingGetDeprecated( @PathParam("id") String id, @QueryParam("type") String type ) { logger.info("publishDataseUsingGetDeprecated called on id " + id + ". Encourage use of POST rather than GET, which is deprecated."); - return publishDataset(id, type, false); + return publishDataset(id, type); } @POST @Path("{id}/actions/:publish") - public Response publishDataset(@PathParam("id") String id, @QueryParam("type") String type, @QueryParam("assureIsIndexed") boolean mustBeIndexed) { + public Response publishDataset(@PathParam("id") String id, @QueryParam("type") String type) { try { if (type == null) { return error(Response.Status.BAD_REQUEST, "Missing 'type' parameter (either 'major','minor', or 'updatecurrent')."); @@ -972,29 +966,6 @@ public Response publishDataset(@PathParam("id") String id, @QueryParam("type") S } Dataset ds = findDatasetOrDie(id); - if (mustBeIndexed) { - logger.fine("IT: " + ds.getIndexTime()); - logger.fine("MT: " + ds.getModificationTime()); - logger.fine("PIT: " + ds.getPermissionIndexTime()); - logger.fine("PMT: " + ds.getPermissionModificationTime()); - if (ds.getIndexTime() != null && ds.getModificationTime() != null) { - logger.fine("ITMT: " + (ds.getIndexTime().compareTo(ds.getModificationTime()) <= 0)); - } - /* - * Some calls, such as the /datasets/actions/:import* commands do not set the - * modification or permission modification times. The checks here are trying to - * see if indexing or permissionindexing could be pending, so they check to see - * if the relevant modification time is set and if so, whether the index is also - * set and if so, if it after the modification time. If the modification time is - * set and the index time is null or is before the mod time, the 409/conflict - * error is returned. - * - */ - if ((ds.getModificationTime()!=null && (ds.getIndexTime() == null || (ds.getIndexTime().compareTo(ds.getModificationTime()) <= 0))) || - (ds.getPermissionModificationTime()!=null && (ds.getPermissionIndexTime() == null || (ds.getPermissionIndexTime().compareTo(ds.getPermissionModificationTime()) <= 0)))) { - return error(Response.Status.CONFLICT, "Dataset is awaiting indexing"); - } - } if (updateCurrent) { /* * Note: The code here mirrors that in the @@ -2317,76 +2288,5 @@ public Response resetFileStore(@PathParam("identifier") String dvIdtf, datasetService.merge(dataset); return ok("Storage reset to default: " + DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER); } - - @GET - @Path("{identifier}/timestamps") - @Produces(MediaType.APPLICATION_JSON) - public Response getTimestamps(@PathParam("identifier") String id) { - - Dataset dataset = null; - DateTimeFormatter formatter = DateTimeFormatter.ISO_LOCAL_DATE_TIME; - try { - dataset = findDatasetOrDie(id); - User u = findUserOrDie(); - Set perms = new HashSet(); - perms.add(Permission.ViewUnpublishedDataset); - boolean canSeeDraft = permissionSvc.hasPermissionsFor(u, dataset, perms); - JsonObjectBuilder timestamps = Json.createObjectBuilder(); - logger.fine("CSD: " + canSeeDraft); - logger.fine("IT: " + dataset.getIndexTime()); - logger.fine("MT: " + dataset.getModificationTime()); - logger.fine("PIT: " + dataset.getPermissionIndexTime()); - logger.fine("PMT: " + dataset.getPermissionModificationTime()); - // Basic info if it's released - if (dataset.isReleased() || canSeeDraft) { - timestamps.add("createTime", formatter.format(dataset.getCreateDate().toLocalDateTime())); - if (dataset.getPublicationDate() != null) { - timestamps.add("publicationTime", formatter.format(dataset.getPublicationDate().toLocalDateTime())); - } - - if (dataset.getLastExportTime() != null) { - timestamps.add("lastMetadataExportTime", - formatter.format(dataset.getLastExportTime().toInstant().atZone(ZoneId.systemDefault()))); - } - - if (dataset.getMostRecentMajorVersionReleaseDate() != null) { - timestamps.add("lastMajorVersionReleaseTime", formatter.format( - dataset.getMostRecentMajorVersionReleaseDate().toInstant().atZone(ZoneId.systemDefault()))); - } - // If the modification/permissionmodification time is - // set and the index time is null or is before the mod time, the relevant index is stale - timestamps.add("hasStaleIndex", - (dataset.getModificationTime() != null && (dataset.getIndexTime() == null - || (dataset.getIndexTime().compareTo(dataset.getModificationTime()) <= 0))) ? true - : false); - timestamps.add("hasStalePermissionIndex", - (dataset.getPermissionModificationTime() != null && (dataset.getIndexTime() == null - || (dataset.getIndexTime().compareTo(dataset.getModificationTime()) <= 0))) ? true - : false); - } - // More detail if you can see a draft - if (canSeeDraft) { - timestamps.add("lastUpdateTime", formatter.format(dataset.getModificationTime().toLocalDateTime())); - if (dataset.getIndexTime() != null) { - timestamps.add("lastIndexTime", formatter.format(dataset.getIndexTime().toLocalDateTime())); - } - if (dataset.getPermissionModificationTime() != null) { - timestamps.add("lastPermissionUpdateTime", - formatter.format(dataset.getPermissionModificationTime().toLocalDateTime())); - } - if (dataset.getPermissionIndexTime() != null) { - timestamps.add("lastPermissionIndexTime", - formatter.format(dataset.getPermissionIndexTime().toLocalDateTime())); - } - if (dataset.getGlobalIdCreateTime() != null) { - timestamps.add("globalIdCreateTime", formatter - .format(dataset.getGlobalIdCreateTime().toInstant().atZone(ZoneId.systemDefault()))); - } - - } - return ok(timestamps); - } catch (WrappedResponse wr) { - return wr.getResponse(); - } - } } + diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java index da9c4515752..c2f186f1e8c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java @@ -21,7 +21,6 @@ import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; -import edu.harvard.iq.dataverse.workflow.WorkflowContext; import edu.harvard.iq.dataverse.workflow.WorkflowContext.TriggerType; import java.io.IOException; import java.sql.Timestamp; diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java index a2d206ead0d..41645078a25 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java @@ -15,7 +15,6 @@ import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.workflow.Workflow; -import edu.harvard.iq.dataverse.workflow.WorkflowContext; import edu.harvard.iq.dataverse.workflow.WorkflowContext.TriggerType; import java.util.Date; import java.util.List; diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowContext.java b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowContext.java index 6208acb0999..a0aee4cb4c9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowContext.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowContext.java @@ -1,7 +1,6 @@ package edu.harvard.iq.dataverse.workflow; import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetLock; import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.workflow.step.WorkflowStep; @@ -32,7 +31,7 @@ public enum TriggerType { private final ApiToken apiToken; private final boolean datasetExternallyReleased; private Map settings; - private DatasetLock lock = null; + private Long lockId = null; private String invocationId = UUID.randomUUID().toString(); @@ -48,7 +47,7 @@ public WorkflowContext(DataverseRequest request, Dataset dataset, long nextVersi } public WorkflowContext(DataverseRequest request, Dataset dataset, long nextVersionNumber, - long nextMinorVersionNumber, TriggerType type, Map settings, ApiToken apiToken, boolean datasetExternallyReleased, String invocationId, DatasetLock lock) { + long nextMinorVersionNumber, TriggerType type, Map settings, ApiToken apiToken, boolean datasetExternallyReleased, String invocationId, Long lockId) { this.request = request; this.dataset = dataset; this.nextVersionNumber = nextVersionNumber; @@ -57,12 +56,12 @@ public WorkflowContext(DataverseRequest request, Dataset dataset, long nextVersi this.settings = settings; this.apiToken = apiToken; this.datasetExternallyReleased = datasetExternallyReleased; - //If null, we'll keel the randomly generated one + //If null, we'll keep the randomly generated one if(invocationId!=null) { setInvocationId(invocationId); } - if(lock != null) { - this.setLock(lock); + if(lockId != null) { + this.setLockId(lockId); } } @@ -109,11 +108,11 @@ public ApiToken getApiToken() { public boolean getDatasetExternallyReleased() { return datasetExternallyReleased; } - public DatasetLock getLock() { - return lock; + public Long getLockId() { + return lockId; } - public void setLock(DatasetLock lock) { - this.lock = lock; + public void setLockId(Long lockId) { + this.lockId = lockId; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java index e7358cdbcba..b6aa35ede7b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java @@ -5,9 +5,6 @@ import edu.harvard.iq.dataverse.DataverseRequestServiceBean; import edu.harvard.iq.dataverse.EjbDataverseEngine; import edu.harvard.iq.dataverse.RoleAssigneeServiceBean; -import edu.harvard.iq.dataverse.UserNotification; -import edu.harvard.iq.dataverse.UserNotification.Type; -import edu.harvard.iq.dataverse.UserNotificationServiceBean; import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.engine.command.CommandContext; @@ -19,14 +16,11 @@ import edu.harvard.iq.dataverse.workflow.internalspi.InternalWorkflowStepSP; import edu.harvard.iq.dataverse.workflow.step.Failure; import edu.harvard.iq.dataverse.workflow.step.Pending; -import edu.harvard.iq.dataverse.workflow.step.Success; import edu.harvard.iq.dataverse.workflow.step.WorkflowStep; import edu.harvard.iq.dataverse.workflow.step.WorkflowStepData; import edu.harvard.iq.dataverse.workflow.step.WorkflowStepResult; -import edu.harvard.iq.dataverse.workflows.WorkflowComment; -import java.sql.Timestamp; -import java.time.Instant; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -69,9 +63,6 @@ public class WorkflowServiceBean { @EJB SystemConfig systemConfig; - @EJB - UserNotificationServiceBean userNotificationService; - @EJB EjbDataverseEngine engine; @@ -106,9 +97,21 @@ public WorkflowServiceBean() { * @param ctxt the context in which the workflow is executed. * @throws CommandException If the dataset could not be locked. */ + //ToDo - should this be @Async? or just the forward() method? @Asynchronous public void start(Workflow wf, WorkflowContext ctxt) throws CommandException { + + // Since we are calling this asynchronously anyway - sleep here + // for a few seconds, just in case, to make sure the database update of + // the dataset initiated by the PublishDatasetCommand has finished, + // to avoid any concurrency/optimistic lock issues. + try { + Thread.sleep(1000); + } catch (Exception ex) { + logger.warning("Failed to sleep for a second."); + } ctxt = refresh(ctxt, retrieveRequestedSettings( wf.getRequiredSettings()), getCurrentApiToken(ctxt.getRequest().getAuthenticatedUser())); + lockDataset(ctxt, new DatasetLock(DatasetLock.Reason.Workflow, ctxt.getRequest().getAuthenticatedUser())); forward(wf, ctxt); } @@ -163,7 +166,6 @@ private Map retrieveRequestedSettings(Map requir @Asynchronous public void resume(PendingWorkflowInvocation pending, String body) { em.remove(em.merge(pending)); - em.flush(); doResume(pending, body); } @@ -182,17 +184,10 @@ private void doResume(PendingWorkflowInvocation pending, String body) { final WorkflowContext ctxt = refresh(newCtxt,retrieveRequestedSettings( wf.getRequiredSettings()), getCurrentApiToken(newCtxt.getRequest().getAuthenticatedUser())); WorkflowStepResult res = pendingStep.resume(ctxt, pending.getLocalData(), body); if (res instanceof Failure) { - userNotificationService.sendNotification(ctxt.getRequest().getAuthenticatedUser(), Timestamp.from(Instant.now()), UserNotification.Type.WORKFLOW_FAILURE, ctxt.getDataset().getLatestVersion().getId(), ((Failure) res).getMessage()); - //UserNotification isn't meant to be a long-term record and doesn't store the comment, so we'll also keep it as a workflow comment - WorkflowComment wfc = new WorkflowComment(ctxt.getDataset().getLatestVersion(), WorkflowComment.Type.WORKFLOW_FAILURE, ((Failure) res).getMessage(), ctxt.getRequest().getAuthenticatedUser()); - datasets.addWorkflowComment(wfc); rollback(wf, ctxt, (Failure) res, pending.getPendingStepIdx() - 1); } else if (res instanceof Pending) { pauseAndAwait(wf, ctxt, (Pending) res, pending.getPendingStepIdx()); } else { - if (res instanceof Success) { - - } executeSteps(wf, ctxt, pending.getPendingStepIdx() + 1); } } @@ -286,27 +281,41 @@ void rollbackStep( WorkflowStep step, WorkflowContext ctxt, Failure reason ) { } @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) - public void lockDataset( WorkflowContext ctxt ) throws CommandException { - DatasetLock datasetLock = new DatasetLock(DatasetLock.Reason.Workflow, ctxt.getRequest().getAuthenticatedUser()); - /* Note that this method directly adds a lock to the database rather than adding it via - * engine.submit(new AddLockCommand(ctxt.getRequest(), ctxt.getDataset(), datasetLock)); - * which would update the dataset's list of locks, etc. - * An em.find() for the dataset would get a Dataset that has an updated list of locks, but this copy would not have any changes - * made in a calling command (e.g. for a PostPublication workflow, the fact that the latest version is 'released' is not yet in the - * database. + void lockDataset(WorkflowContext ctxt, DatasetLock datasetLock) throws CommandException { + /* + * Note that this method directly adds a lock to the database rather than adding + * it via engine.submit(new AddLockCommand(ctxt.getRequest(), ctxt.getDataset(), + * datasetLock)); which would update the dataset's list of locks, etc. An + * em.find() for the dataset would get a Dataset that has an updated list of + * locks, but this copy would not have any changes made in a calling command + * (e.g. for a PostPublication workflow, the fact that the latest version is + * 'released' is not yet in the database. */ datasetLock.setDataset(ctxt.getDataset()); em.persist(datasetLock); //flush creates the id em.flush(); - ctxt.setLock(datasetLock); + ctxt.setLockId(datasetLock.getId()); } - + @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) - void unlockDataset( WorkflowContext ctxt ) throws CommandException { - - em.remove(em.merge(ctxt.getLock())); - + void unlockDataset(WorkflowContext ctxt) throws CommandException { + /* + * Since the lockDataset command above directly persists a lock to the database, + * the ctxt.getDataset() is not updated and its list of locks can't be used. + * Using the named query below will find the workflow lock and remove it + * (actually all workflow locks for this Dataset but only one workflow should be + * active). + */ + TypedQuery lockCounter = em.createNamedQuery("DatasetLock.getLocksByDatasetId", DatasetLock.class); + lockCounter.setParameter("datasetId", ctxt.getDataset().getId()); + List locks = lockCounter.getResultList(); + for (DatasetLock lock : locks) { + if (lock.getReason() == DatasetLock.Reason.Workflow) { + ctxt.getDataset().removeLock(lock); + em.remove(lock); + } + } em.flush(); } @@ -345,9 +354,11 @@ private void workflowCompleted(Workflow wf, WorkflowContext ctxt) { info += registerGlobalIdsForFiles ? "Registering PIDs for Datafiles; " : ""; info += validatePhysicalFiles ? "Validating Datafiles Asynchronously" : ""; lock.setInfo(info); - datasets.addDatasetLock(ctxt.getDataset(), lock); + lockDataset(ctxt, lock); + ctxt.getDataset().addLock(lock); + unlockDataset(ctxt); - ctxt.setLock(null); //the workflow lock + ctxt.setLockId(null); //the workflow lock //Refreshing merges the dataset ctxt = refresh(ctxt); //Then call Finalize @@ -466,7 +477,7 @@ private WorkflowContext refresh( WorkflowContext ctxt, Map setti */ WorkflowContext newCtxt =new WorkflowContext( ctxt.getRequest(), em.merge(ctxt.getDataset()), ctxt.getNextVersionNumber(), - ctxt.getNextMinorVersionNumber(), ctxt.getType(), settings, apiToken, ctxt.getDatasetExternallyReleased(), ctxt.getInvocationId(), ctxt.getLock()); + ctxt.getNextMinorVersionNumber(), ctxt.getType(), settings, apiToken, ctxt.getDatasetExternallyReleased(), ctxt.getInvocationId(), ctxt.getLockId()); return newCtxt; } diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/step/Failure.java b/src/main/java/edu/harvard/iq/dataverse/workflow/step/Failure.java index 950eed52245..0487313fd0d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/step/Failure.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/step/Failure.java @@ -11,7 +11,7 @@ public class Failure implements WorkflowStepResult { private final String message; public Failure( String reason ) { - this(reason, null); + this(reason, reason); } /** diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java b/src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java deleted file mode 100644 index 621afdc61f6..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java +++ /dev/null @@ -1,47 +0,0 @@ -package edu.harvard.iq.dataverse.workflow.step; - -/** - * Result returned when step execution succeeds. - */ -public class Success implements WorkflowStepResult { - - private final String reason; - private final String message; - - public Success( String reason ) { - this(reason, null); - } - - /** - * Constructs a new success message. - * @param reason Technical reason (for logs etc.). - * @param message Human readable reason. - */ - public Success(String reason, String message) { - this.reason = reason; - this.message = message; - } - - /** - * Holds the technical reason for the success, useful for debugging the problem. - * @return the technical reason for the problem. - */ - public String getReason() { - return reason; - } - - /** - * Holds the user-friendly message explaining the failure. - * @return user-friendly message for the success. - */ - public String getMessage() { - return message; - } - - @Override - public String toString() { - return "WorkflowStepResult.Success{" + "reason=" + reason + ", message=" + message + '}'; - } - - -} diff --git a/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowComment.java b/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowComment.java index 0da8ed223b4..26ddb9b1573 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowComment.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowComment.java @@ -1,7 +1,6 @@ package edu.harvard.iq.dataverse.workflows; import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import java.io.Serializable; import java.sql.Timestamp; @@ -23,9 +22,7 @@ public class WorkflowComment implements Serializable { More may be added in future releases, */ public enum Type { - RETURN_TO_AUTHOR, //, SUBMIT_FOR_REVIEW not available in this release but may be added in the future - WORKFLOW_SUCCESS, - WORKFLOW_FAILURE + RETURN_TO_AUTHOR //, SUBMIT_FOR_REVIEW not available in this release but may be added in the future }; @Id @@ -64,8 +61,6 @@ public enum Type { @Column(nullable = false) private Timestamp created; - private boolean toBeShown; - // TODO: Consider support editing in the GUI some day, like GitHub issue comments (show "Edited" in the UI). We won't send a second email, however. You only get one shot to prevent spam. // @Transient // private Timestamp modified; @@ -74,11 +69,12 @@ public enum Type { // private List notifications; public WorkflowComment(DatasetVersion version, WorkflowComment.Type type, String message, AuthenticatedUser authenticatedUser) { this.type = type; - this.datasetVersion = version; + if (this.type.equals(WorkflowComment.Type.RETURN_TO_AUTHOR)) { + this.datasetVersion = version; + } this.message = message; this.authenticatedUser = authenticatedUser; this.created = new Timestamp(new Date().getTime()); - this.setToBeShown(true); } /** @@ -127,12 +123,4 @@ public void setDatasetVersion(DatasetVersion dv) { datasetVersion=dv; } - public boolean isToBeShown() { - return toBeShown; - } - - public void setToBeShown(boolean toBeShown) { - this.toBeShown = toBeShown; - } - } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index cb62106a491..1fd7c300ab4 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -649,10 +649,6 @@ notification.email.submit.dataset.subject={0}: Your dataset has been submitted f notification.email.publish.dataset.subject={0}: Your dataset has been published notification.email.publishFailure.dataset.subject={0}: Failed to publish your dataset notification.email.returned.dataset.subject={0}: Your dataset has been returned -notification.email.workflow.success.subject={0}: Your dataset has been processed -notification.email.workflow.success=A workflow running on {0} (view at {1}) succeeded: {2} -notification.email.workflow.failure.subject={0}: Failed to process your dataset -notification.email.workflow.failure=A workflow running on {0} (view at {1}) failed: {2} notification.email.create.account.subject={0}: Your account has been created notification.email.assign.role.subject={0}: You have been assigned a role notification.email.revoke.role.subject={0}: Your role has been revoked From a2a3b513020972cc44adc135ee1c0555d37ddfc9 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 24 Feb 2021 14:15:39 -0500 Subject: [PATCH 365/576] docs for new step with authorization --- .../source/developers/workflows.rst | 52 +++++++++++++++++-- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/developers/workflows.rst b/doc/sphinx-guides/source/developers/workflows.rst index 78e87556a3f..43f3dfae598 100644 --- a/doc/sphinx-guides/source/developers/workflows.rst +++ b/doc/sphinx-guides/source/developers/workflows.rst @@ -1,5 +1,5 @@ Workflows -================ +========= The Dataverse Software has a flexible workflow mechanism that can be used to trigger actions before and after Dataset publication. @@ -12,7 +12,7 @@ Introduction The Dataverse Software can perform two sequences of actions when datasets are published: one prior to publishing (marked by a ``PrePublishDataset`` trigger), and one after the publication has succeeded (``PostPublishDataset``). The pre-publish workflow is useful for having an external system prepare a dataset for being publicly accessed (a possibly lengthy activity that requires moving files around, uploading videos to a streaming server, etc.), or to start an approval process. A post-publish workflow might be used for sending notifications about the newly published dataset. -Workflow steps are created using *step providers*. The Dataverse Software ships with an internal step provider that offers some basic functionality, and with the ability to load 3rd party step providers. This allows installations to implement functionality they need without changing the Dataverse Software source code. +Workflow steps are created using *step providers*. The Dataverse Software ships with an internal step provider that offers some basic functionality, and with the ability to load 3rd party step providers (currently disabled). This allows installations to implement functionality they need without changing the Dataverse Software source code. Steps can be internal (say, writing some data to the log) or external. External steps involve the Dataverse Software sending a request to an external system, and waiting for the system to reply. The wait period is arbitrary, and so allows the external system unbounded operation time. This is useful, e.g., for steps that require human intervention, such as manual approval of a dataset publication. @@ -22,8 +22,8 @@ If a step in a workflow fails, the Dataverse installation makes an effort to rol .. tip:: For invoking external systems using a REST api, the Dataverse Software's internal step - provider offers a step for sending and receiving customizable HTTP requests. - It's called *http/sr*, and is detailed below. + provider offers two steps for sending and receiving customizable HTTP requests. + *http/sr* and *http/authExt*, detailed below, with the latter able to use the API to make changes to the dataset being processed. Administration ~~~~~~~~~~~~~~ @@ -104,6 +104,50 @@ Available variables are: * ``majorVersion`` * ``releaseStatus`` +http/authext +++++++++++++ + +Similar to the *http/sr* step. A step that sends a HTTP request to an external system, and then waits for a response. The receiver can use the invocationId of the workflow in lieu of an api key to perform work on behlaf of the user launching the workflow. +The invocationId must be sent as an 'X-Dataverse-invocationId' HTTP Header or as an ?invocationId= query parameter. Note that any external process started using this step then has the ability to access a Dataverse instance via the API as the user. +Once this step completes and responds, the invocationId is invalidated and will not allow further access. + + + +The url, content type, and message body can use data from the workflow context, using a simple markup language. This step has specific parameters for rollback. +The workflow is restarted when the external system replies with a POST request to ``/api/workflows/{invocation-id}``. +The response has is expected to be a json object with three keys: +- "Status" - can be "Success" or "Failure" +- "Reason" - a message that will be logged +- "Message" - a message to send to the user (message sending is not yet implemented). + +.. code:: json + + { + "provider":":internal", + "stepType":"http/authext", + "parameters": { + "url":"http://localhost:5050/dump/${invocationId}", + "method":"POST", + "contentType":"text/plain", + "body":"START RELEASE ${dataset.id} as ${dataset.displayName}", + "rollbackUrl":"http://localhost:5050/dump/${invocationId}", + "rollbackMethod":"DELETE ${dataset.id}" + } + } + +Available variables are: + +* ``invocationId`` +* ``dataset.id`` +* ``dataset.identifier`` +* ``dataset.globalId`` +* ``dataset.displayName`` +* ``dataset.citation`` +* ``minorVersion`` +* ``majorVersion`` +* ``releaseStatus`` + + archiver ++++++++ From 55eb60fd103ff223922ae658c60102a9806f5753 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 24 Feb 2021 14:17:59 -0500 Subject: [PATCH 366/576] typo/style --- doc/sphinx-guides/source/developers/workflows.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/developers/workflows.rst b/doc/sphinx-guides/source/developers/workflows.rst index 43f3dfae598..d55ee6015b7 100644 --- a/doc/sphinx-guides/source/developers/workflows.rst +++ b/doc/sphinx-guides/source/developers/workflows.rst @@ -107,8 +107,8 @@ Available variables are: http/authext ++++++++++++ -Similar to the *http/sr* step. A step that sends a HTTP request to an external system, and then waits for a response. The receiver can use the invocationId of the workflow in lieu of an api key to perform work on behlaf of the user launching the workflow. -The invocationId must be sent as an 'X-Dataverse-invocationId' HTTP Header or as an ?invocationId= query parameter. Note that any external process started using this step then has the ability to access a Dataverse instance via the API as the user. +Similar to the *http/sr* step. A step that sends a HTTP request to an external system, and then waits for a response. The receiver can use the invocationId of the workflow in lieu of an api key to perform work on behalf of the user launching the workflow. +The invocationId must be sent as an 'X-Dataverse-invocationId' HTTP Header or as an ?invocationId= query parameter. *Note that any external process started using this step then has the ability to access a Dataverse instance via the API as the user.* Once this step completes and responds, the invocationId is invalidated and will not allow further access. From eb611e1d8a1adcfca543980af863d8deb83de82f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 24 Feb 2021 14:20:48 -0500 Subject: [PATCH 367/576] add note about locking. --- doc/sphinx-guides/source/developers/workflows.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/developers/workflows.rst b/doc/sphinx-guides/source/developers/workflows.rst index d55ee6015b7..9d519a52e5d 100644 --- a/doc/sphinx-guides/source/developers/workflows.rst +++ b/doc/sphinx-guides/source/developers/workflows.rst @@ -23,7 +23,7 @@ If a step in a workflow fails, the Dataverse installation makes an effort to rol .. tip:: For invoking external systems using a REST api, the Dataverse Software's internal step provider offers two steps for sending and receiving customizable HTTP requests. - *http/sr* and *http/authExt*, detailed below, with the latter able to use the API to make changes to the dataset being processed. + *http/sr* and *http/authExt*, detailed below, with the latter able to use the API to make changes to the dataset being processed. (Both lock the dataset to prevent other processes from changing the dataset between the time the step is launched to when the external process responds to the Dataverse instance.) Administration ~~~~~~~~~~~~~~ @@ -111,8 +111,6 @@ Similar to the *http/sr* step. A step that sends a HTTP request to an external s The invocationId must be sent as an 'X-Dataverse-invocationId' HTTP Header or as an ?invocationId= query parameter. *Note that any external process started using this step then has the ability to access a Dataverse instance via the API as the user.* Once this step completes and responds, the invocationId is invalidated and will not allow further access. - - The url, content type, and message body can use data from the workflow context, using a simple markup language. This step has specific parameters for rollback. The workflow is restarted when the external system replies with a POST request to ``/api/workflows/{invocation-id}``. The response has is expected to be a json object with three keys: From a849c8afe8a0dfd8a6f48c4de5e482aea5f46c2c Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 24 Feb 2021 14:24:46 -0500 Subject: [PATCH 368/576] missing class --- .../iq/dataverse/workflow/step/Success.java | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java b/src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java new file mode 100644 index 00000000000..621afdc61f6 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java @@ -0,0 +1,47 @@ +package edu.harvard.iq.dataverse.workflow.step; + +/** + * Result returned when step execution succeeds. + */ +public class Success implements WorkflowStepResult { + + private final String reason; + private final String message; + + public Success( String reason ) { + this(reason, null); + } + + /** + * Constructs a new success message. + * @param reason Technical reason (for logs etc.). + * @param message Human readable reason. + */ + public Success(String reason, String message) { + this.reason = reason; + this.message = message; + } + + /** + * Holds the technical reason for the success, useful for debugging the problem. + * @return the technical reason for the problem. + */ + public String getReason() { + return reason; + } + + /** + * Holds the user-friendly message explaining the failure. + * @return user-friendly message for the success. + */ + public String getMessage() { + return message; + } + + @Override + public String toString() { + return "WorkflowStepResult.Success{" + "reason=" + reason + ", message=" + message + '}'; + } + + +} From c5007b293299e87992ed9353cb1135d00253b185 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 24 Feb 2021 14:31:49 -0500 Subject: [PATCH 369/576] remove messaging related enums --- .../authorization/providers/builtin/DataverseUserPage.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java index 9199558765f..204d93b5b8f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java @@ -485,8 +485,6 @@ public void displayNotification() { case PUBLISHEDDS: case PUBLISHFAILED_PIDREG: case RETURNEDDS: - case WORKFLOW_SUCCESS: - case WORKFLOW_FAILURE: userNotification.setTheObject(datasetVersionService.find(userNotification.getObjectId())); break; From 35f5eab5ee0098dc3e1773d3047fda740d0a41fa Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 24 Feb 2021 16:13:14 -0500 Subject: [PATCH 370/576] pulling out messaging only for #7564 --- .../edu/harvard/iq/dataverse/DatasetPage.java | 24 ++- .../iq/dataverse/DatasetServiceBean.java | 6 + .../harvard/iq/dataverse/MailServiceBean.java | 15 ++ .../iq/dataverse/UserNotification.java | 5 +- .../providers/builtin/DataverseUserPage.java | 2 + .../harvard/iq/dataverse/util/MailUtil.java | 4 + .../workflow/WorkflowServiceBean.java | 23 ++- .../internalspi/AuthorizedExternalStep.java | 185 ++++++++++++++++++ .../internalspi/InternalWorkflowStepSP.java | 4 + .../internalspi/PauseWithMessageStep.java | 51 +++++ .../iq/dataverse/workflow/step/Failure.java | 2 +- .../iq/dataverse/workflow/step/Success.java | 47 +++++ .../dataverse/workflows/WorkflowComment.java | 20 +- src/main/java/propertyFiles/Bundle.properties | 4 + 14 files changed, 382 insertions(+), 10 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/AuthorizedExternalStep.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/PauseWithMessageStep.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 5f596d1b556..b07ac5015f3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -54,6 +54,8 @@ import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.workflows.WorkflowComment; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -1856,7 +1858,9 @@ private String init(boolean initFull) { MakeDataCountEntry entry = new MakeDataCountEntry(FacesContext.getCurrentInstance(), dvRequestService, workingVersion); mdcLogService.logEntry(entry); } - + displayWorkflowComments(); + + if (initFull) { // init the list of FileMetadatas if (workingVersion.isDraft() && canUpdateDataset()) { @@ -2016,6 +2020,22 @@ private String init(boolean initFull) { return null; } + private void displayWorkflowComments() { + List comments = workingVersion.getWorkflowComments(); + for (WorkflowComment wfc : comments) { + if (wfc.isToBeShown() && wfc.getDatasetVersion().equals(workingVersion) + && wfc.getAuthenticatedUser().equals(session.getUser())) { + if (wfc.getType() == WorkflowComment.Type.WORKFLOW_SUCCESS) { + JsfHelper.addSuccessMessage(wfc.getMessage()); + + } else if (wfc.getType() == WorkflowComment.Type.WORKFLOW_FAILURE) { + JsfHelper.addWarningMessage(wfc.getMessage()); + } + datasetService.markWorkflowCommentAsRead(wfc); + } + } + } + private void displayLockInfo(Dataset dataset) { // Various info messages, when the dataset is locked (for various reasons): if (dataset.isLocked() && canUpdateDataset()) { @@ -2703,6 +2723,8 @@ public String refresh() { } } + displayWorkflowComments(); + return ""; } diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index c1efe119fd2..71eef6d6c5f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -787,6 +787,12 @@ public WorkflowComment addWorkflowComment(WorkflowComment workflowComment) { return workflowComment; } + public void markWorkflowCommentAsRead(WorkflowComment workflowComment) { + workflowComment.setToBeShown(false); + em.merge(workflowComment); + } + + /** * This method used to throw CommandException, which was pretty pointless * seeing how it's called asynchronously. As of v5.0 any CommanExceptiom diff --git a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java index 13a92c9cd27..42feaf6d2c2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java @@ -496,6 +496,19 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio version.getDataset().getOwner().getDisplayName(), getDataverseLink(version.getDataset().getOwner()), optionalReturnReason}; messageText += MessageFormat.format(pattern, paramArrayReturnedDataset); return messageText; + + case WORKFLOW_SUCCESS: + version = (DatasetVersion) targetObject; + pattern = BundleUtil.getStringFromBundle("notification.email.workflow.success"); + String[] paramArrayWorkflowSuccess = {version.getDataset().getDisplayName(), getDatasetLink(version.getDataset()), comment}; + messageText += MessageFormat.format(pattern, paramArrayWorkflowSuccess); + return messageText; + case WORKFLOW_FAILURE: + version = (DatasetVersion) targetObject; + pattern = BundleUtil.getStringFromBundle("notification.email.workflow.failure"); + String[] paramArrayWorkflowFailure = {version.getDataset().getDisplayName(), getDatasetLink(version.getDataset())}; + messageText += MessageFormat.format(pattern, paramArrayWorkflowFailure); + return messageText; case CREATEACC: String rootDataverseName = dataverseService.findRootDataverse().getName(); InternetAddress systemAddress = getSystemAddress(); @@ -594,6 +607,8 @@ private Object getObjectOfNotification (UserNotification userNotification){ case PUBLISHEDDS: case PUBLISHFAILED_PIDREG: case RETURNEDDS: + case WORKFLOW_SUCCESS: + case WORKFLOW_FAILURE: return versionService.find(userNotification.getObjectId()); case CREATEACC: return userNotification.getUser(); diff --git a/src/main/java/edu/harvard/iq/dataverse/UserNotification.java b/src/main/java/edu/harvard/iq/dataverse/UserNotification.java index bac1df0f2ed..78f8f38206b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/UserNotification.java +++ b/src/main/java/edu/harvard/iq/dataverse/UserNotification.java @@ -27,7 +27,10 @@ public class UserNotification implements Serializable { public enum Type { - ASSIGNROLE, REVOKEROLE, CREATEDV, CREATEDS, CREATEACC, SUBMITTEDDS, RETURNEDDS, PUBLISHEDDS, REQUESTFILEACCESS, GRANTFILEACCESS, REJECTFILEACCESS, FILESYSTEMIMPORT, CHECKSUMIMPORT, CHECKSUMFAIL, CONFIRMEMAIL, APIGENERATED, INGESTCOMPLETED, INGESTCOMPLETEDWITHERRORS, PUBLISHFAILED_PIDREG + ASSIGNROLE, REVOKEROLE, CREATEDV, CREATEDS, CREATEACC, SUBMITTEDDS, RETURNEDDS, + PUBLISHEDDS, REQUESTFILEACCESS, GRANTFILEACCESS, REJECTFILEACCESS, FILESYSTEMIMPORT, + CHECKSUMIMPORT, CHECKSUMFAIL, CONFIRMEMAIL, APIGENERATED, INGESTCOMPLETED, INGESTCOMPLETEDWITHERRORS, + PUBLISHFAILED_PIDREG, WORKFLOW_SUCCESS, WORKFLOW_FAILURE }; private static final long serialVersionUID = 1L; diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java index 204d93b5b8f..9199558765f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java @@ -485,6 +485,8 @@ public void displayNotification() { case PUBLISHEDDS: case PUBLISHFAILED_PIDREG: case RETURNEDDS: + case WORKFLOW_SUCCESS: + case WORKFLOW_FAILURE: userNotification.setTheObject(datasetVersionService.find(userNotification.getObjectId())); break; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java index 37667d16b55..7d023ea38e2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java @@ -54,6 +54,10 @@ public static String getSubjectTextBasedOnNotification(UserNotification userNoti return BundleUtil.getStringFromBundle("notification.email.publishFailure.dataset.subject", rootDvNameAsList); case RETURNEDDS: return BundleUtil.getStringFromBundle("notification.email.returned.dataset.subject", rootDvNameAsList); + case WORKFLOW_SUCCESS: + return BundleUtil.getStringFromBundle("notification.email.workflow.success.subject", rootDvNameAsList); + case WORKFLOW_FAILURE: + return BundleUtil.getStringFromBundle("notification.email.workflow.failure.subject", rootDvNameAsList); case CREATEACC: return BundleUtil.getStringFromBundle("notification.email.create.account.subject", rootDvNameAsList); case CHECKSUMFAIL: diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java index 0538ab9851e..59a84fc75ea 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java @@ -5,22 +5,27 @@ import edu.harvard.iq.dataverse.DataverseRequestServiceBean; import edu.harvard.iq.dataverse.EjbDataverseEngine; import edu.harvard.iq.dataverse.RoleAssigneeServiceBean; +import edu.harvard.iq.dataverse.UserNotification; +import edu.harvard.iq.dataverse.UserNotification.Type; +import edu.harvard.iq.dataverse.UserNotificationServiceBean; import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.impl.FinalizeDatasetPublicationCommand; -import edu.harvard.iq.dataverse.engine.command.impl.RemoveLockCommand; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.workflow.WorkflowContext.TriggerType; import edu.harvard.iq.dataverse.workflow.internalspi.InternalWorkflowStepSP; import edu.harvard.iq.dataverse.workflow.step.Failure; import edu.harvard.iq.dataverse.workflow.step.Pending; +import edu.harvard.iq.dataverse.workflow.step.Success; import edu.harvard.iq.dataverse.workflow.step.WorkflowStep; import edu.harvard.iq.dataverse.workflow.step.WorkflowStepData; import edu.harvard.iq.dataverse.workflow.step.WorkflowStepResult; +import edu.harvard.iq.dataverse.workflows.WorkflowComment; -import java.util.Date; +import java.sql.Timestamp; +import java.time.Instant; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -35,7 +40,6 @@ import javax.inject.Inject; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; -import javax.persistence.Query; import javax.persistence.TypedQuery; /** @@ -61,6 +65,9 @@ public class WorkflowServiceBean { @EJB RoleAssigneeServiceBean roleAssignees; + @EJB + UserNotificationServiceBean userNotificationService; + @EJB EjbDataverseEngine engine; @@ -171,10 +178,20 @@ private void doResume(PendingWorkflowInvocation pending, String body) { final WorkflowContext ctxt = refresh(newCtxt,retrieveRequestedSettings( wf.getRequiredSettings()), getCurrentApiToken(newCtxt.getRequest().getAuthenticatedUser())); WorkflowStepResult res = pendingStep.resume(ctxt, pending.getLocalData(), body); if (res instanceof Failure) { + userNotificationService.sendNotification(ctxt.getRequest().getAuthenticatedUser(), Timestamp.from(Instant.now()), UserNotification.Type.WORKFLOW_FAILURE, ctxt.getDataset().getLatestVersion().getId(), ((Failure) res).getMessage()); + //UserNotification isn't meant to be a long-term record and doesn't store the comment, so we'll also keep it as a workflow comment + WorkflowComment wfc = new WorkflowComment(ctxt.getDataset().getLatestVersion(), WorkflowComment.Type.WORKFLOW_FAILURE, ((Failure) res).getMessage(), ctxt.getRequest().getAuthenticatedUser()); + datasets.addWorkflowComment(wfc); rollback(wf, ctxt, (Failure) res, pending.getPendingStepIdx() - 1); } else if (res instanceof Pending) { pauseAndAwait(wf, ctxt, (Pending) res, pending.getPendingStepIdx()); } else { + if (res instanceof Success) { + userNotificationService.sendNotification(ctxt.getRequest().getAuthenticatedUser(), Timestamp.from(Instant.now()), UserNotification.Type.WORKFLOW_SUCCESS, ctxt.getDataset().getLatestVersion().getId(), ((Success) res).getMessage()); + //UserNotification isn't meant to be a long-term record and doesn't store the comment, so we'll also keep it as a workflow comment + WorkflowComment wfc = new WorkflowComment(ctxt.getDataset().getLatestVersion(), WorkflowComment.Type.WORKFLOW_SUCCESS, ((Success) res).getMessage(), ctxt.getRequest().getAuthenticatedUser()); + datasets.addWorkflowComment(wfc); + } executeSteps(wf, ctxt, pending.getPendingStepIdx() + 1); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/AuthorizedExternalStep.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/AuthorizedExternalStep.java new file mode 100644 index 00000000000..414641ada5f --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/AuthorizedExternalStep.java @@ -0,0 +1,185 @@ +package edu.harvard.iq.dataverse.workflow.internalspi; + +import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.workflow.WorkflowContext; +import edu.harvard.iq.dataverse.workflow.WorkflowContext.TriggerType; +import edu.harvard.iq.dataverse.workflow.step.Failure; +import edu.harvard.iq.dataverse.workflow.step.Pending; +import edu.harvard.iq.dataverse.workflow.step.Success; +import edu.harvard.iq.dataverse.workflow.step.WorkflowStep; +import edu.harvard.iq.dataverse.workflow.step.WorkflowStepResult; +import static edu.harvard.iq.dataverse.workflow.step.WorkflowStepResult.OK; + +import java.io.StringReader; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Pattern; + +import javax.json.Json; +import javax.json.JsonObject; + +import org.apache.commons.httpclient.HttpClient; +import org.apache.commons.httpclient.HttpMethodBase; +import org.apache.commons.httpclient.URI; +import org.apache.commons.httpclient.URIException; +import org.apache.commons.httpclient.methods.DeleteMethod; +import org.apache.commons.httpclient.methods.EntityEnclosingMethod; +import org.apache.commons.httpclient.methods.GetMethod; +import org.apache.commons.httpclient.methods.PostMethod; +import org.apache.commons.httpclient.methods.PutMethod; +import org.apache.commons.httpclient.methods.StringRequestEntity; + +/** + * A workflow step that sends a HTTP request, and then pauses, waiting for an + * application/json response. This step is allowed to act as the user causing + * the workflow launch and can make changes to the dataset it is associated + * with. To do so, it must send the invocationId in a manner analogous to + * sending an api key. See guides. + * + */ +public class AuthorizedExternalStep implements WorkflowStep { + private static final Logger logger = Logger.getLogger(AuthorizedExternalStep.class.getName()); + private final Map params; + + public AuthorizedExternalStep(Map paramSet) { + params = new HashMap<>(paramSet); + } + + @Override + public WorkflowStepResult run(WorkflowContext context) { + HttpClient client = new HttpClient(); + + try { + // build method + HttpMethodBase mtd = buildMethod(false, context); + // execute + int responseStatus = client.executeMethod(mtd); + if (responseStatus>=200 && responseStatus<300 ) { + // HTTP OK range + Map data = new HashMap (); + data.put("authorized", "true"); + return new Pending(data); + } else { + String responseBody = mtd.getResponseBodyAsString(); + return new Failure("Error communicating with server. Server response: " + responseBody + " (" + responseStatus + ")."); + } + + } catch (Exception ex) { + logger.log(Level.SEVERE, "Error communicating with remote server: " + ex.getMessage(), ex); + return new Failure("Error executing request: " + ex.getLocalizedMessage(), "Cannot communicate with remote server."); + } + } + + @Override + public WorkflowStepResult resume(WorkflowContext context, Map internalData, String externalData) { + try (StringReader reader = new StringReader(externalData)) { + JsonObject response = Json.createReader(reader).readObject(); + String status = response.getString("Status"); + String reason = null; + String message = null; + if (response.containsKey("Reason")) { + reason = response.getString("Reason"); + } + if (response.containsKey("Message")) { + message = response.getString("Message"); + } + switch (status) { + case "Success": + logger.log(Level.FINE, "AuthExt Worfklow Step Succeeded: " + reason); + return new Success(reason, message); + case "Failure": + logger.log(Level.WARNING, "Remote system indicates workflow failed: {0}", reason); + return new Failure(reason, message); + } + } catch (Exception e) { + logger.log(Level.WARNING, "Remote system returned a bad reposonse: {0}", externalData); + } + return new Failure("Workflow failure: Response from remote server could not be parsed:" + externalData, null); + } + + @Override + public void rollback(WorkflowContext context, Failure reason) { + HttpClient client = new HttpClient(); + + try { + // build method + HttpMethodBase mtd = buildMethod(true, context); + + // execute + int responseStatus = client.executeMethod(mtd); + if (responseStatus<200 || responseStatus>=300 ) { + // out of HTTP OK range + String responseBody = mtd.getResponseBodyAsString(); + Logger.getLogger(AuthorizedExternalStep.class.getName()).log(Level.WARNING, + "Bad response from remote server while rolling back step: {0}", responseBody); + } + + } catch (Exception ex) { + Logger.getLogger(AuthorizedExternalStep.class.getName()).log(Level.WARNING, "IO error rolling back step: " + ex.getMessage(), ex); + } + } + + HttpMethodBase buildMethod(boolean rollback, WorkflowContext ctxt) throws Exception { + String methodName = params.getOrDefault("method" + (rollback ? "-rollback":""), "GET").trim().toUpperCase(); + HttpMethodBase m = null; + switch (methodName) { + case "GET": m = new GetMethod(); m.setFollowRedirects(true); break; + case "POST": m = new PostMethod(); break; + case "PUT": m = new PutMethod(); break; + case "DELETE": m = new DeleteMethod(); m.setFollowRedirects(true); break; + default: throw new IllegalStateException("Unsupported HTTP method: '" + methodName + "'"); + } + + + Map templateParams = new HashMap<>(); + templateParams.put( "invocationId", ctxt.getInvocationId() ); + templateParams.put( "dataset.id", Long.toString(ctxt.getDataset().getId()) ); + templateParams.put( "dataset.identifier", ctxt.getDataset().getIdentifier() ); + templateParams.put( "dataset.globalId", ctxt.getDataset().getGlobalId().toString() ); + templateParams.put( "dataset.displayName", ctxt.getDataset().getDisplayName() ); + templateParams.put( "dataset.citation", ctxt.getDataset().getCitation() ); + templateParams.put( "minorVersion", Long.toString(ctxt.getNextMinorVersionNumber()) ); + templateParams.put( "majorVersion", Long.toString(ctxt.getNextVersionNumber()) ); + templateParams.put( "releaseStatus", (ctxt.getType()==TriggerType.PostPublishDataset) ? "done":"in-progress" ); + templateParams.put( "language", BundleUtil.getDefaultLocale().getISO3Language()); + + + m.addRequestHeader("Content-Type", params.getOrDefault("contentType", "text/plain")); + + String urlKey = rollback ? "rollbackUrl":"url"; + String url = params.get(urlKey); + try { + m.setURI(new URI(process(url,templateParams), true) ); + } catch (URIException ex) { + throw new IllegalStateException("Illegal URL: '" + url + "'"); + } + + String bodyKey = (rollback ? "rollbackBody" : "body"); + if ( params.containsKey(bodyKey) && m instanceof EntityEnclosingMethod ) { + String body = params.get(bodyKey); + ((EntityEnclosingMethod)m).setRequestEntity(new StringRequestEntity(process( body, templateParams), params.getOrDefault("contentType", "text/plain"), StandardCharsets.UTF_8.name())); + } + + return m; + } + + String process(String template, Map values ) { + String curValue = template; + for ( Map.Entry ent : values.entrySet() ) { + String val = ent.getValue(); + if ( val == null ) { + val = "" ; + } + String varRef = "${" + ent.getKey() + "}"; + while ( curValue.contains(varRef) ) { + curValue = curValue.replace(varRef, val); + } + } + + return curValue; + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/InternalWorkflowStepSP.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/InternalWorkflowStepSP.java index 76b7bcd7d44..ef11d306cd3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/InternalWorkflowStepSP.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/InternalWorkflowStepSP.java @@ -17,8 +17,12 @@ public WorkflowStep getStep(String stepType, Map stepParameters) return new LoggingWorkflowStep(stepParameters); case "pause": return new PauseStep(stepParameters); + case "pause/message": + return new PauseWithMessageStep(stepParameters); case "http/sr": return new HttpSendReceiveClientStep(stepParameters); + case "http/authExt": + return new AuthorizedExternalStep(stepParameters); case "archiver": return new ArchivalSubmissionWorkflowStep(stepParameters); default: diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/PauseWithMessageStep.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/PauseWithMessageStep.java new file mode 100644 index 00000000000..28a660bad76 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/PauseWithMessageStep.java @@ -0,0 +1,51 @@ +package edu.harvard.iq.dataverse.workflow.internalspi; + +import edu.harvard.iq.dataverse.workflow.WorkflowContext; +import edu.harvard.iq.dataverse.workflow.step.Failure; +import edu.harvard.iq.dataverse.workflow.step.Pending; +import edu.harvard.iq.dataverse.workflow.step.Success; +import edu.harvard.iq.dataverse.workflow.step.WorkflowStep; +import edu.harvard.iq.dataverse.workflow.step.WorkflowStepResult; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * A sample step that pauses the workflow. + * + * @author michael + */ +public class PauseWithMessageStep implements WorkflowStep { + + private static final Logger logger = Logger.getLogger(PauseWithMessageStep.class.getName()); + + /** Constant used by testing to simulate a failed step. */ + public static final String FAILURE_RESPONSE="fail"; + + private final Map params = new HashMap<>(); + + public PauseWithMessageStep( Map paramSet ) { + params.putAll(paramSet); + } + + @Override + public WorkflowStepResult run(WorkflowContext context) { + final Pending result = new Pending(); + result.getData().putAll(params); + return result; + } + + @Override + public WorkflowStepResult resume(WorkflowContext context, Map internalData, String externalData) { + logger.log(Level.INFO, "local parameters match: {0}", internalData.equals(params)); + logger.log(Level.INFO, "externalData: \"{0}\"", externalData); + return externalData.trim().equals(FAILURE_RESPONSE) ? new Failure("Simulated fail", "SimulatedWF Failure") : new Success("Hello Log, the step succeeded!", "A workflow was successfully run on your dataset"); + } + + @Override + public void rollback(WorkflowContext context, Failure reason) { + // nothing to roll back + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/step/Failure.java b/src/main/java/edu/harvard/iq/dataverse/workflow/step/Failure.java index 0487313fd0d..950eed52245 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/step/Failure.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/step/Failure.java @@ -11,7 +11,7 @@ public class Failure implements WorkflowStepResult { private final String message; public Failure( String reason ) { - this(reason, reason); + this(reason, null); } /** diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java b/src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java new file mode 100644 index 00000000000..621afdc61f6 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java @@ -0,0 +1,47 @@ +package edu.harvard.iq.dataverse.workflow.step; + +/** + * Result returned when step execution succeeds. + */ +public class Success implements WorkflowStepResult { + + private final String reason; + private final String message; + + public Success( String reason ) { + this(reason, null); + } + + /** + * Constructs a new success message. + * @param reason Technical reason (for logs etc.). + * @param message Human readable reason. + */ + public Success(String reason, String message) { + this.reason = reason; + this.message = message; + } + + /** + * Holds the technical reason for the success, useful for debugging the problem. + * @return the technical reason for the problem. + */ + public String getReason() { + return reason; + } + + /** + * Holds the user-friendly message explaining the failure. + * @return user-friendly message for the success. + */ + public String getMessage() { + return message; + } + + @Override + public String toString() { + return "WorkflowStepResult.Success{" + "reason=" + reason + ", message=" + message + '}'; + } + + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowComment.java b/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowComment.java index 26ddb9b1573..d03afcaa91a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowComment.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowComment.java @@ -22,7 +22,9 @@ public class WorkflowComment implements Serializable { More may be added in future releases, */ public enum Type { - RETURN_TO_AUTHOR //, SUBMIT_FOR_REVIEW not available in this release but may be added in the future + RETURN_TO_AUTHOR, //, SUBMIT_FOR_REVIEW not available in this release but may be added in the future + WORKFLOW_SUCCESS, + WORKFLOW_FAILURE }; @Id @@ -61,20 +63,22 @@ public enum Type { @Column(nullable = false) private Timestamp created; + private boolean toBeShown; + // TODO: Consider support editing in the GUI some day, like GitHub issue comments (show "Edited" in the UI). We won't send a second email, however. You only get one shot to prevent spam. // @Transient // private Timestamp modified; // TODO: How should we best associate these entries to notifications, which can go to multiple authors and curators? + //FWIW: Workflow success/failure messages get shown to the user running the workflow if/when on the relevant dataset version page // @Transient // private List notifications; public WorkflowComment(DatasetVersion version, WorkflowComment.Type type, String message, AuthenticatedUser authenticatedUser) { this.type = type; - if (this.type.equals(WorkflowComment.Type.RETURN_TO_AUTHOR)) { - this.datasetVersion = version; - } + this.datasetVersion = version; this.message = message; this.authenticatedUser = authenticatedUser; this.created = new Timestamp(new Date().getTime()); + this.setToBeShown(true); } /** @@ -123,4 +127,12 @@ public void setDatasetVersion(DatasetVersion dv) { datasetVersion=dv; } + public boolean isToBeShown() { + return toBeShown; + } + + public void setToBeShown(boolean toBeShown) { + this.toBeShown = toBeShown; + } + } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 29ba794821b..5bda5b6ee3f 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -650,6 +650,10 @@ notification.email.submit.dataset.subject={0}: Your dataset has been submitted f notification.email.publish.dataset.subject={0}: Your dataset has been published notification.email.publishFailure.dataset.subject={0}: Failed to publish your dataset notification.email.returned.dataset.subject={0}: Your dataset has been returned +notification.email.workflow.success.subject={0}: Your dataset has been processed +notification.email.workflow.success=A workflow running on {0} (view at {1}) succeeded: {2} +notification.email.workflow.failure.subject={0}: Failed to process your dataset +notification.email.workflow.failure=A workflow running on {0} (view at {1}) failed: {2} notification.email.create.account.subject={0}: Your account has been created notification.email.assign.role.subject={0}: You have been assigned a role notification.email.revoke.role.subject={0}: Your role has been revoked From 26e7ecc88ba9164e8f9eef558e32996d1c9188d8 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 24 Feb 2021 16:38:46 -0500 Subject: [PATCH 371/576] update pause/message step to accept the same json as http/extauth --- .../internalspi/PauseWithMessageStep.java | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/PauseWithMessageStep.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/PauseWithMessageStep.java index 28a660bad76..c19de39f2ab 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/PauseWithMessageStep.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/PauseWithMessageStep.java @@ -6,11 +6,16 @@ import edu.harvard.iq.dataverse.workflow.step.Success; import edu.harvard.iq.dataverse.workflow.step.WorkflowStep; import edu.harvard.iq.dataverse.workflow.step.WorkflowStepResult; + +import java.io.StringReader; import java.util.HashMap; import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; +import javax.json.Json; +import javax.json.JsonObject; + /** * A sample step that pauses the workflow. * @@ -38,9 +43,29 @@ public WorkflowStepResult run(WorkflowContext context) { @Override public WorkflowStepResult resume(WorkflowContext context, Map internalData, String externalData) { - logger.log(Level.INFO, "local parameters match: {0}", internalData.equals(params)); - logger.log(Level.INFO, "externalData: \"{0}\"", externalData); - return externalData.trim().equals(FAILURE_RESPONSE) ? new Failure("Simulated fail", "SimulatedWF Failure") : new Success("Hello Log, the step succeeded!", "A workflow was successfully run on your dataset"); + try (StringReader reader = new StringReader(externalData)) { + JsonObject response = Json.createReader(reader).readObject(); + String status = response.getString("Status"); + String reason = null; + String message = null; + if (response.containsKey("Reason")) { + reason = response.getString("Reason"); + } + if (response.containsKey("Message")) { + message = response.getString("Message"); + } + switch (status) { + case "Success": + logger.log(Level.FINE, "AuthExt Worfklow Step Succeeded: " + reason); + return new Success(reason, message); + case "Failure": + logger.log(Level.WARNING, "Remote system indicates workflow failed: {0}", reason); + return new Failure(reason, message); + } + } catch (Exception e) { + logger.log(Level.WARNING, "Remote system returned a bad reposonse: {0}", externalData); + } + return new Failure("Workflow failure: Response from remote server could not be parsed:" + externalData, null); } @Override From f50b1d71ccc882779a333cef8c6eea3c6649637e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 24 Feb 2021 16:38:54 -0500 Subject: [PATCH 372/576] update docs --- .../source/developers/workflows.rst | 68 ++++++++++++++++++- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/developers/workflows.rst b/doc/sphinx-guides/source/developers/workflows.rst index 78e87556a3f..8684296c589 100644 --- a/doc/sphinx-guides/source/developers/workflows.rst +++ b/doc/sphinx-guides/source/developers/workflows.rst @@ -1,5 +1,5 @@ Workflows -================ +========= The Dataverse Software has a flexible workflow mechanism that can be used to trigger actions before and after Dataset publication. @@ -18,12 +18,14 @@ Steps can be internal (say, writing some data to the log) or external. External The external system reports the step result back to the Dataverse installation, by sending a HTTP ``POST`` command to ``api/workflows/{invocation-id}`` with Content-Type: text/plain. The body of the request is passed to the paused step for further processing. +Steps can define messages to send to the log and to users. If defined, the message to users is sent as a user notification (creating an email and showing in the user notification tab) and will show once for the given user if/when they view the relevant dataset page. The latter provides a means for the asynchronous workflow execution to report success or failure anologous to the way the publication and other processes report on the page. + If a step in a workflow fails, the Dataverse installation makes an effort to roll back all the steps that preceded it. Some actions, such as writing to the log, cannot be rolled back. If such an action has a public external effect (e.g. send an EMail to a mailing list) it is advisable to put it in the post-release workflow. .. tip:: For invoking external systems using a REST api, the Dataverse Software's internal step - provider offers a step for sending and receiving customizable HTTP requests. - It's called *http/sr*, and is detailed below. + provider offers two steps for sending and receiving customizable HTTP requests. + *http/sr* and *http/authExt*, detailed below, with the latter able to use the API to make changes to the dataset being processed. (Both lock the dataset to prevent other processes from changing the dataset between the time the step is launched to when the external process responds to the Dataverse instance.) Administration ~~~~~~~~~~~~~~ @@ -69,6 +71,23 @@ A step that pauses the workflow. The workflow is paused until a POST request is "stepType":"pause" } +pause/message ++++++++++++++ + +A variant of the pause step that pauses the workflow and allows the external process to send a success/failure message. The workflow is paused until a POST request is sent to ``/api/workflows/{invocation-id}``. +The response in the POST body (Content-type:application/json) should be a json object (the same as for the http/extauth step) containing: +- "Status" - can be "Success" or "Failure" +- "Reason" - a message that will be logged +- "Message" - a message to send to the user that will be sent as a notification and as a banner on the relevant dataset page. +An unparsable reponse will be considered a Failure that will be logged with no user message. + +.. code:: json + + { + "provider":":internal", + "stepType":"pause/message" + } + http/sr +++++++ @@ -104,6 +123,49 @@ Available variables are: * ``majorVersion`` * ``releaseStatus`` +http/authext +++++++++++++ + +Similar to the *http/sr* step. A step that sends a HTTP request to an external system, and then waits for a response. The receiver can use the invocationId of the workflow in lieu of an api key to perform work on behalf of the user launching the workflow. +The invocationId must be sent as an 'X-Dataverse-invocationId' HTTP Header or as an ?invocationId= query parameter. *Note that any external process started using this step then has the ability to access a Dataverse instance via the API as the user.* +Once this step completes and responds, the invocationId is invalidated and will not allow further access. + +The url, content type, and message body can use data from the workflow context, using a simple markup language. This step has specific parameters for rollback. +The workflow is restarted when the external system replies with a POST request to ``/api/workflows/{invocation-id}`` (Content-Type: application/json). + +The response has is expected to be a json object with three keys: +- "Status" - can be "Success" or "Failure" +- "Reason" - a message that will be logged +- "Message" - a message to send to the user that will be sent as a notification and as a banner on the relevant dataset page. + +.. code:: json + + { + "provider":":internal", + "stepType":"http/authext", + "parameters": { + "url":"http://localhost:5050/dump/${invocationId}", + "method":"POST", + "contentType":"text/plain", + "body":"START RELEASE ${dataset.id} as ${dataset.displayName}", + "rollbackUrl":"http://localhost:5050/dump/${invocationId}", + "rollbackMethod":"DELETE ${dataset.id}" + } + } + +Available variables are: + +* ``invocationId`` +* ``dataset.id`` +* ``dataset.identifier`` +* ``dataset.globalId`` +* ``dataset.displayName`` +* ``dataset.citation`` +* ``minorVersion`` +* ``majorVersion`` +* ``releaseStatus`` + + archiver ++++++++ From 6f77d98ce9cb7e764eef05c409757dbf57ae0248 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 24 Feb 2021 16:39:08 -0500 Subject: [PATCH 373/576] flyway sript --- src/main/resources/db/migration/V5.5.0.2__7564-workflow.sql | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 src/main/resources/db/migration/V5.5.0.2__7564-workflow.sql diff --git a/src/main/resources/db/migration/V5.5.0.2__7564-workflow.sql b/src/main/resources/db/migration/V5.5.0.2__7564-workflow.sql new file mode 100644 index 00000000000..d2a8d48726e --- /dev/null +++ b/src/main/resources/db/migration/V5.5.0.2__7564-workflow.sql @@ -0,0 +1,2 @@ +ALTER TABLE workflowcomment +ADD COLUMN IF NOT EXISTS tobeshown boolean; From 65b19931976efbaa348a9acb587d61372a175dea Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 24 Feb 2021 16:40:22 -0500 Subject: [PATCH 374/576] rename flyway script --- .../{V5.4.0.1__7563-workflow.sql => V5.5.0.1__7563-workflow.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V5.4.0.1__7563-workflow.sql => V5.5.0.1__7563-workflow.sql} (100%) diff --git a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql b/src/main/resources/db/migration/V5.5.0.1__7563-workflow.sql similarity index 100% rename from src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql rename to src/main/resources/db/migration/V5.5.0.1__7563-workflow.sql From f7c35fe67595ed45a90d427359d07a42351280a7 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 25 Feb 2021 12:17:21 -0500 Subject: [PATCH 375/576] allow publish conditional on indexing being done, add /timestamps --- .../harvard/iq/dataverse/api/Datasets.java | 102 +++++++++++++++++- 1 file changed, 99 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 4ea4144fbf9..69769b8172c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -112,6 +112,8 @@ import java.io.StringReader; import java.sql.Timestamp; import java.text.MessageFormat; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; @@ -969,12 +971,12 @@ private String validateDatasetFieldValues(List fields) { @Deprecated public Response publishDataseUsingGetDeprecated( @PathParam("id") String id, @QueryParam("type") String type ) { logger.info("publishDataseUsingGetDeprecated called on id " + id + ". Encourage use of POST rather than GET, which is deprecated."); - return publishDataset(id, type); + return publishDataset(id, type, false); } @POST @Path("{id}/actions/:publish") - public Response publishDataset(@PathParam("id") String id, @QueryParam("type") String type) { + public Response publishDataset(@PathParam("id") String id, @QueryParam("type") String type, @QueryParam("assureIsIndexed") boolean mustBeIndexed) { try { if (type == null) { return error(Response.Status.BAD_REQUEST, "Missing 'type' parameter (either 'major','minor', or 'updatecurrent')."); @@ -1002,6 +1004,29 @@ public Response publishDataset(@PathParam("id") String id, @QueryParam("type") S } Dataset ds = findDatasetOrDie(id); + if (mustBeIndexed) { + logger.fine("IT: " + ds.getIndexTime()); + logger.fine("MT: " + ds.getModificationTime()); + logger.fine("PIT: " + ds.getPermissionIndexTime()); + logger.fine("PMT: " + ds.getPermissionModificationTime()); + if (ds.getIndexTime() != null && ds.getModificationTime() != null) { + logger.fine("ITMT: " + (ds.getIndexTime().compareTo(ds.getModificationTime()) <= 0)); + } + /* + * Some calls, such as the /datasets/actions/:import* commands do not set the + * modification or permission modification times. The checks here are trying to + * see if indexing or permissionindexing could be pending, so they check to see + * if the relevant modification time is set and if so, whether the index is also + * set and if so, if it after the modification time. If the modification time is + * set and the index time is null or is before the mod time, the 409/conflict + * error is returned. + * + */ + if ((ds.getModificationTime()!=null && (ds.getIndexTime() == null || (ds.getIndexTime().compareTo(ds.getModificationTime()) <= 0))) || + (ds.getPermissionModificationTime()!=null && (ds.getPermissionIndexTime() == null || (ds.getPermissionIndexTime().compareTo(ds.getPermissionModificationTime()) <= 0)))) { + return error(Response.Status.CONFLICT, "Dataset is awaiting indexing"); + } + } if (updateCurrent) { /* * Note: The code here mirrors that in the @@ -2324,5 +2349,76 @@ public Response resetFileStore(@PathParam("identifier") String dvIdtf, datasetService.merge(dataset); return ok("Storage reset to default: " + DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER); } -} + @GET + @Path("{identifier}/timestamps") + @Produces(MediaType.APPLICATION_JSON) + public Response getTimestamps(@PathParam("identifier") String id) { + + Dataset dataset = null; + DateTimeFormatter formatter = DateTimeFormatter.ISO_LOCAL_DATE_TIME; + try { + dataset = findDatasetOrDie(id); + User u = findUserOrDie(); + Set perms = new HashSet(); + perms.add(Permission.ViewUnpublishedDataset); + boolean canSeeDraft = permissionSvc.hasPermissionsFor(u, dataset, perms); + JsonObjectBuilder timestamps = Json.createObjectBuilder(); + logger.fine("CSD: " + canSeeDraft); + logger.fine("IT: " + dataset.getIndexTime()); + logger.fine("MT: " + dataset.getModificationTime()); + logger.fine("PIT: " + dataset.getPermissionIndexTime()); + logger.fine("PMT: " + dataset.getPermissionModificationTime()); + // Basic info if it's released + if (dataset.isReleased() || canSeeDraft) { + timestamps.add("createTime", formatter.format(dataset.getCreateDate().toLocalDateTime())); + if (dataset.getPublicationDate() != null) { + timestamps.add("publicationTime", formatter.format(dataset.getPublicationDate().toLocalDateTime())); + } + + if (dataset.getLastExportTime() != null) { + timestamps.add("lastMetadataExportTime", + formatter.format(dataset.getLastExportTime().toInstant().atZone(ZoneId.systemDefault()))); + } + + if (dataset.getMostRecentMajorVersionReleaseDate() != null) { + timestamps.add("lastMajorVersionReleaseTime", formatter.format( + dataset.getMostRecentMajorVersionReleaseDate().toInstant().atZone(ZoneId.systemDefault()))); + } + // If the modification/permissionmodification time is + // set and the index time is null or is before the mod time, the relevant index is stale + timestamps.add("hasStaleIndex", + (dataset.getModificationTime() != null && (dataset.getIndexTime() == null + || (dataset.getIndexTime().compareTo(dataset.getModificationTime()) <= 0))) ? true + : false); + timestamps.add("hasStalePermissionIndex", + (dataset.getPermissionModificationTime() != null && (dataset.getIndexTime() == null + || (dataset.getIndexTime().compareTo(dataset.getModificationTime()) <= 0))) ? true + : false); + } + // More detail if you can see a draft + if (canSeeDraft) { + timestamps.add("lastUpdateTime", formatter.format(dataset.getModificationTime().toLocalDateTime())); + if (dataset.getIndexTime() != null) { + timestamps.add("lastIndexTime", formatter.format(dataset.getIndexTime().toLocalDateTime())); + } + if (dataset.getPermissionModificationTime() != null) { + timestamps.add("lastPermissionUpdateTime", + formatter.format(dataset.getPermissionModificationTime().toLocalDateTime())); + } + if (dataset.getPermissionIndexTime() != null) { + timestamps.add("lastPermissionIndexTime", + formatter.format(dataset.getPermissionIndexTime().toLocalDateTime())); + } + if (dataset.getGlobalIdCreateTime() != null) { + timestamps.add("globalIdCreateTime", formatter + .format(dataset.getGlobalIdCreateTime().toInstant().atZone(ZoneId.systemDefault()))); + } + + } + return ok(timestamps); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + } +} From 788605ea6d2d6df02dce5d026975dfcd3c5989d1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 25 Feb 2021 12:41:26 -0500 Subject: [PATCH 376/576] properly report error --- .../java/edu/harvard/iq/dataverse/api/Datasets.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 4ea4144fbf9..0e69fc6b942 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -805,11 +805,16 @@ private String getCompoundDisplayValue (DatasetFieldCompoundValue dscv){ @PUT @Path("{id}/editMetadata") - public Response editVersionMetadata(String jsonBody, @PathParam("id") String id, @QueryParam("replace") Boolean replace) throws WrappedResponse{ + public Response editVersionMetadata(String jsonBody, @PathParam("id") String id, @QueryParam("replace") Boolean replace) { Boolean replaceData = replace != null; - - DataverseRequest req = createDataverseRequest(findUserOrDie()); + DataverseRequest req = null; + try { + req = createDataverseRequest(findUserOrDie()); + } catch (WrappedResponse ex) { + logger.log(Level.SEVERE, "Edit metdata error: " + ex.getMessage(), ex); + return ex.getResponse(); + } return processDatasetUpdate(jsonBody, id, req, replaceData); } From 89c4a26f14821575d06fc1877343bd61d0c5c192 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 25 Feb 2021 14:17:22 -0500 Subject: [PATCH 377/576] minor text edit --- doc/sphinx-guides/source/developers/workflows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/workflows.rst b/doc/sphinx-guides/source/developers/workflows.rst index 9d519a52e5d..36711488bf3 100644 --- a/doc/sphinx-guides/source/developers/workflows.rst +++ b/doc/sphinx-guides/source/developers/workflows.rst @@ -23,7 +23,7 @@ If a step in a workflow fails, the Dataverse installation makes an effort to rol .. tip:: For invoking external systems using a REST api, the Dataverse Software's internal step provider offers two steps for sending and receiving customizable HTTP requests. - *http/sr* and *http/authExt*, detailed below, with the latter able to use the API to make changes to the dataset being processed. (Both lock the dataset to prevent other processes from changing the dataset between the time the step is launched to when the external process responds to the Dataverse instance.) + *http/sr* and *http/authExt*, detailed below, with the latter able to use the API to make changes to the dataset being processed. Both lock the dataset to prevent other processes from changing the dataset between the time the step is launched to when the external process responds to the Dataverse instance. Administration ~~~~~~~~~~~~~~ From 86dfba1a41633620bcf2ee33b11e42e888b20d76 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 26 Feb 2021 10:20:53 -0500 Subject: [PATCH 378/576] use constant, add comment --- .../workflow/internalspi/AuthorizedExternalStep.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/AuthorizedExternalStep.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/AuthorizedExternalStep.java index 414641ada5f..244abdb017c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/AuthorizedExternalStep.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/AuthorizedExternalStep.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.workflow.internalspi; import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.workflow.PendingWorkflowInvocation; import edu.harvard.iq.dataverse.workflow.WorkflowContext; import edu.harvard.iq.dataverse.workflow.WorkflowContext.TriggerType; import edu.harvard.iq.dataverse.workflow.step.Failure; @@ -60,7 +61,8 @@ public WorkflowStepResult run(WorkflowContext context) { if (responseStatus>=200 && responseStatus<300 ) { // HTTP OK range Map data = new HashMap (); - data.put("authorized", "true"); + //Allow external client to use invocationId as a key to act on the user's behalf + data.put(PendingWorkflowInvocation.AUTHORIZED, "true"); return new Pending(data); } else { String responseBody = mtd.getResponseBodyAsString(); From afdfe8df47d0dd14ff326311e3d0cfc1ae236be1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 26 Feb 2021 10:32:16 -0500 Subject: [PATCH 379/576] note pause step is primarily for testing --- doc/sphinx-guides/source/developers/workflows.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/workflows.rst b/doc/sphinx-guides/source/developers/workflows.rst index 9d519a52e5d..3f866a08b99 100644 --- a/doc/sphinx-guides/source/developers/workflows.rst +++ b/doc/sphinx-guides/source/developers/workflows.rst @@ -60,7 +60,8 @@ A step that writes data about the current workflow invocation to the instance lo pause +++++ -A step that pauses the workflow. The workflow is paused until a POST request is sent to ``/api/workflows/{invocation-id}``. Sending 'fail' in the POST body (Content-type:text/plain) will trigger a failure and workflow rollback. All other responses are considered as successes. +A step that pauses the workflow. The workflow is paused until a POST request is sent to ``/api/workflows/{invocation-id}``. Sending 'fail' in the POST body (Content-type:text/plain) will trigger a failure and workflow rollback. All other responses are considered as successes. +The pause step is intended for testing - the invocationId required to end the pause is only available in the log (and database). Adding a parameter (see log step) with the key/value "authorized":"true" will allow the invocationId to be used as a credential as with the http/authext step below. .. code:: json From 80552f7f99bf6aff3ca0f01893084a2a36437f2e Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 26 Feb 2021 11:03:55 -0500 Subject: [PATCH 380/576] #7373 copy files to current solr directory --- conf/solr/8.8.1/readme.md | 1 + conf/solr/8.8.1/schema.xml | 1198 ++++++++++++++++++ conf/solr/8.8.1/schema_dv_mdb_copies.xml | 161 +++ conf/solr/8.8.1/schema_dv_mdb_fields.xml | 161 +++ conf/solr/8.8.1/solrconfig.xml | 1410 ++++++++++++++++++++++ conf/solr/8.8.1/updateSchemaMDB.sh | 85 ++ 6 files changed, 3016 insertions(+) create mode 100644 conf/solr/8.8.1/readme.md create mode 100644 conf/solr/8.8.1/schema.xml create mode 100644 conf/solr/8.8.1/schema_dv_mdb_copies.xml create mode 100644 conf/solr/8.8.1/schema_dv_mdb_fields.xml create mode 100644 conf/solr/8.8.1/solrconfig.xml create mode 100644 conf/solr/8.8.1/updateSchemaMDB.sh diff --git a/conf/solr/8.8.1/readme.md b/conf/solr/8.8.1/readme.md new file mode 100644 index 00000000000..4457cf9a7df --- /dev/null +++ b/conf/solr/8.8.1/readme.md @@ -0,0 +1 @@ +Please see the dev guide for what to do with Solr config files. \ No newline at end of file diff --git a/conf/solr/8.8.1/schema.xml b/conf/solr/8.8.1/schema.xml new file mode 100644 index 00000000000..da40a8e99fa --- /dev/null +++ b/conf/solr/8.8.1/schema.xml @@ -0,0 +1,1198 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/solr/8.8.1/schema_dv_mdb_copies.xml b/conf/solr/8.8.1/schema_dv_mdb_copies.xml new file mode 100644 index 00000000000..080cc71ef50 --- /dev/null +++ b/conf/solr/8.8.1/schema_dv_mdb_copies.xml @@ -0,0 +1,161 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/solr/8.8.1/schema_dv_mdb_fields.xml b/conf/solr/8.8.1/schema_dv_mdb_fields.xml new file mode 100644 index 00000000000..3f844c6183c --- /dev/null +++ b/conf/solr/8.8.1/schema_dv_mdb_fields.xml @@ -0,0 +1,161 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/solr/8.8.1/solrconfig.xml b/conf/solr/8.8.1/solrconfig.xml new file mode 100644 index 00000000000..51100af7d6e --- /dev/null +++ b/conf/solr/8.8.1/solrconfig.xml @@ -0,0 +1,1410 @@ + + + + + + + + + 7.3.0 + + + + + + + + + + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.lock.type:native} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + + + ${solr.autoCommit.maxTime:15000} + false + + + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + + + + + + + + 1024 + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + + + + + false + + + + + + + + + + + + + + + + + + + + + + explicit + 10 + edismax + 0.075 + + dvName^400 + authorName^180 + dvSubject^190 + dvDescription^180 + dvAffiliation^170 + title^130 + subject^120 + keyword^110 + topicClassValue^100 + dsDescriptionValue^90 + authorAffiliation^80 + publicationCitation^60 + producerName^50 + fileName^30 + fileDescription^30 + variableLabel^20 + variableName^10 + _text_^1.0 + + + dvName^200 + authorName^100 + dvSubject^100 + dvDescription^100 + dvAffiliation^100 + title^75 + subject^75 + keyword^75 + topicClassValue^75 + dsDescriptionValue^75 + authorAffiliation^75 + publicationCitation^75 + producerName^75 + + + + isHarvested:false^25000 + + + + + + + + + + + + + + + + + + explicit + json + true + + + + + + + + explicit + + + + + + _text_ + + + + + + + true + ignored_ + _text_ + + + + + + + + + text_general + + + + + + default + _text_ + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + + + + + + + default + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + + + + + true + + + tvComponent + + + + + + + + + + + + true + false + + + terms + + + + + + + + string + + + + + + explicit + + + elevator + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + + + + + [^\w-\.] + _ + + + + + + + yyyy-MM-dd'T'HH:mm:ss.SSSZ + yyyy-MM-dd'T'HH:mm:ss,SSSZ + yyyy-MM-dd'T'HH:mm:ss.SSS + yyyy-MM-dd'T'HH:mm:ss,SSS + yyyy-MM-dd'T'HH:mm:ssZ + yyyy-MM-dd'T'HH:mm:ss + yyyy-MM-dd'T'HH:mmZ + yyyy-MM-dd'T'HH:mm + yyyy-MM-dd HH:mm:ss.SSSZ + yyyy-MM-dd HH:mm:ss,SSSZ + yyyy-MM-dd HH:mm:ss.SSS + yyyy-MM-dd HH:mm:ss,SSS + yyyy-MM-dd HH:mm:ssZ + yyyy-MM-dd HH:mm:ss + yyyy-MM-dd HH:mmZ + yyyy-MM-dd HH:mm + yyyy-MM-dd + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/plain; charset=UTF-8 + + + + + ${velocity.template.base.dir:} + ${velocity.solr.resource.loader.enabled:true} + ${velocity.params.resource.loader.enabled:false} + + + + + 5 + + + + + + + + + + + + + + diff --git a/conf/solr/8.8.1/updateSchemaMDB.sh b/conf/solr/8.8.1/updateSchemaMDB.sh new file mode 100644 index 00000000000..0044f15c7cd --- /dev/null +++ b/conf/solr/8.8.1/updateSchemaMDB.sh @@ -0,0 +1,85 @@ +#!/bin/bash +set -euo pipefail + +# This script updates the and schema configuration necessary to properly +# index custom metadata fields in Solr. +# 1. Retrieve from Dataverse API endpoint +# 2. Parse and write Solr schema files (which might replace the included files) +# 3. Reload Solr +# +# List of variables: +# ${DATAVERSE_URL}: URL to Dataverse. Defaults to http://localhost:8080 +# ${SOLR_URL}: URL to Solr. Defaults to http://localhost:8983 +# ${UNBLOCK_KEY}: File path to secret or unblock key as string. Only necessary on k8s or when you secured your installation. +# ${TARGET}: Directory where to write the XML files. Defaults to /tmp +# +# Programs used (need to be available on your PATH): +# coreutils: mktemp, csplit +# curl + +usage() { + echo "usage: updateSchemaMDB.sh [options]" + echo "options:" + echo " -d Dataverse URL, defaults to http://localhost:8080" + echo " -h Show this help text" + echo " -s Solr URL, defaults to http://localhost:8983" + echo " -t Directory where to write the XML files. Defaults to /tmp" + echo " -u Dataverse unblock key either as key string or path to keyfile" +} + +### Init (with sane defaults) +DATAVERSE_URL=${DATAVERSE_URL:-"http://localhost:8080"} +SOLR_URL=${SOLR_URL:-"http://localhost:8983"} +TARGET=${TARGET:-"/tmp"} +UNBLOCK_KEY=${UNBLOCK_KEY:-""} + +# if cmdline args are given, override any env var setting (or defaults) +while getopts ":d:hs:t:u:" opt +do + case $opt in + d) DATAVERSE_URL=${OPTARG};; + h) usage; exit 0;; + s) SOLR_URL=${OPTARG};; + t) TARGET=${OPTARG};; + u) UNBLOCK_KEY=${OPTARG};; + :) echo "Missing option argument for -${OPTARG}. Use -h for help." >&2; exit 1;; + \?) echo "Unknown option -${OPTARG}." >&2; usage; exit 1;; + esac +done + +# Special handling of unblock key depending on referencing a secret file or key in var +if [ ! -z "${UNBLOCK_KEY}" ]; then + if [ -f "${UNBLOCK_KEY}" ]; then + UNBLOCK_KEY="?unblock-key=$(cat ${UNBLOCK_KEY})" + else + UNBLOCK_KEY="?unblock-key=${UNBLOCK_KEY}" + fi +fi + +### Retrieval +echo "Retrieve schema data from ${DATAVERSE_URL}/api/admin/index/solr/schema" +TMPFILE=`mktemp` +curl -f -sS "${DATAVERSE_URL}/api/admin/index/solr/schema${UNBLOCK_KEY}" > $TMPFILE + +### Fail gracefull if Dataverse is not ready yet. +if [[ "`wc -l ${TMPFILE}`" < "3" ]]; then + echo "Dataverse responded with empty file. When running on K8s: did you bootstrap yet?" + exit 123 +fi + +### Processing +echo "Writing ${TARGET}/schema_dv_mdb_fields.xml" +echo "" > ${TARGET}/schema_dv_mdb_fields.xml +cat ${TMPFILE} | grep ".*> ${TARGET}/schema_dv_mdb_fields.xml +echo "" >> ${TARGET}/schema_dv_mdb_fields.xml + +echo "Writing ${TARGET}/schema_dv_mdb_copies.xml" +echo "" > ${TARGET}/schema_dv_mdb_copies.xml +cat ${TMPFILE} | grep ".*> ${TARGET}/schema_dv_mdb_copies.xml +echo "" >> ${TARGET}/schema_dv_mdb_copies.xml + +rm ${TMPFILE}* + +### Reloading +echo "Triggering Solr RELOAD at ${SOLR_URL}/solr/admin/cores?action=RELOAD&core=collection1" +curl -f -sS "${SOLR_URL}/solr/admin/cores?action=RELOAD&core=collection1" From be5ff941b73e0780ccf7c1613608c13a0b80dae3 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 26 Feb 2021 11:42:22 -0500 Subject: [PATCH 381/576] restricted summary stats #7619 - At the dataset level, DDI exports no longer show "dataDscr" information for restricted files. There is only one version of this export and it is the version that's suitable for public consumption with the "dataDscr" information hidden for restricted files. - Similarly, at the dataset level, the DDI HTML Codebook no longer shows "dataDscr" information for restricted files. - At the file level, "dataDscr" information is no longer publicly available for restricted files. In practice, it was only possible to get this publicly via API (the download/access button was hidden). - At the file level, "dataDscr" (variable metadata) information can still be downloaded for restricted files if you have access to download the file. --- .../7619-restricted-summary-starts.md | 14 ++ .../source/user/dataset-management.rst | 15 --- .../edu/harvard/iq/dataverse/api/Access.java | 18 +++ .../WebApplicationExceptionHandler.java | 3 + .../dataverse/export/ddi/DdiExportUtil.java | 11 ++ .../harvard/iq/dataverse/api/DatasetsIT.java | 123 ++++++++++++++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 28 +++- 7 files changed, 195 insertions(+), 17 deletions(-) create mode 100644 doc/release-notes/7619-restricted-summary-starts.md diff --git a/doc/release-notes/7619-restricted-summary-starts.md b/doc/release-notes/7619-restricted-summary-starts.md new file mode 100644 index 00000000000..e1c20f3bde2 --- /dev/null +++ b/doc/release-notes/7619-restricted-summary-starts.md @@ -0,0 +1,14 @@ +Restricted Files and DDI "dataDscr" Information (Summary Statistics, Variable Names, Variable Labels) + +In previous releases, DDI "dataDscr" information (summary statistics, variable names, and variable labels, sometimes known as "variable metadata") for tabular files that were ingested successfully were available even if files were restricted. This has been changed in the following ways: + +- At the dataset level, DDI exports no longer show "dataDscr" information for restricted files. There is only one version of this export and it is the version that's suitable for public consumption with the "dataDscr" information hidden for restricted files. +- Similarly, at the dataset level, the DDI HTML Codebook no longer shows "dataDscr" information for restricted files. +- At the file level, "dataDscr" information is no longer publicly available for restricted files. In practice, it was only possible to get this publicly via API (the download/access button was hidden). +- At the file level, "dataDscr" (variable metadata) information can still be downloaded for restricted files if you have access to download the file. + +After upgrading, you should re-export to replace cached DDI exports with restricted summary stats with DDI exports fit for public consumption: + + curl http://localhost:8080/api/admin/metadata/reExportAll + +For details on this operation, see https://guides.dataverse.org/en/5.4/admin/metadataexport.html diff --git a/doc/sphinx-guides/source/user/dataset-management.rst b/doc/sphinx-guides/source/user/dataset-management.rst index 8a853449488..1d5f1344c6c 100755 --- a/doc/sphinx-guides/source/user/dataset-management.rst +++ b/doc/sphinx-guides/source/user/dataset-management.rst @@ -179,8 +179,6 @@ Additional download options available for tabular data (found in the same drop-d - Data File Citation (currently in either RIS, EndNote XML, or BibTeX format); - All of the above, as a zipped bundle. -See also :ref:`restricted-tabular-files`. - Astronomy (FITS) ---------------- @@ -214,19 +212,6 @@ When you restrict a file it cannot be downloaded unless permission has been gran See also :ref:`terms-of-access` and :ref:`permissions`. -.. _restricted-tabular-files: - -Restricted Tabular Files ------------------------- - -Restricted tabular files are treated differently than other file types in that the following information is exposed when the files are published: - -- The name of columns (variables). -- The "label" (description) of columns. -- Summary statistics (max, min, mean, etc.) of columns. - -This information can been seen from the file landing page if you click "Export Metada" and then "DDI". Depending on your installation, the information above may also be available from :doc:`/admin/external-tools`. - Edit Files ========== diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 281f0a76ce1..2cd7e0dd122 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -450,6 +450,24 @@ public String tabularDatafileMetadataDDI(@PathParam("fileId") String fileId, @Q throw new BadRequestException("tabular data required"); } + if (dataFile.isRestricted()) { + boolean hasPermissionToDownloadFile = false; + DataverseRequest dataverseRequest; + try { + dataverseRequest = createDataverseRequest(findUserOrDie()); + } catch (WrappedResponse ex) { + throw new BadRequestException("cannot find user"); + } + if (dataverseRequest != null && dataverseRequest.getUser() instanceof GuestUser) { + // We must be in the UI. Try to get a non-GuestUser from the session. + dataverseRequest = dvRequestService.getDataverseRequest(); + } + hasPermissionToDownloadFile = permissionService.requestOn(dataverseRequest, dataFile).has(Permission.DownloadFile); + if (!hasPermissionToDownloadFile) { + throw new BadRequestException("no permission to download file"); + } + } + response.setHeader("Content-disposition", "attachment; filename=\"dataverse_files.zip\""); FileMetadata fm = null; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java index b6229e58192..aa8bb2b3772 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java @@ -43,8 +43,11 @@ public Response toResponse(WebApplicationException ex) { switch (ex.getResponse().getStatus()) { // BadRequest case 400: + // It's strange to have these "startsWith" conditionals here. They both come from Access.java. if ( (ex.getMessage()+"").toLowerCase().startsWith("tabular data required")) { jrb.message(BundleUtil.getStringFromBundle("access.api.exception.metadata.not.available.for.nontabular.file")); + } else if ((ex.getMessage() + "").toLowerCase().startsWith("no permission to download file")) { + jrb.message("You do not have permission to download this file."); } else { jrb.message("Bad Request. The API request cannot be completed with the parameters supplied. Please check your code for typos, or consult our API guide at http://guides.dataverse.org."); jrb.request(request); diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index dd797a55539..1ce1a2119a7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -1388,6 +1388,17 @@ public static void createDataDscr(XMLStreamWriter xmlw, DatasetVersion datasetVe for (FileMetadata fileMetadata : datasetVersion.getFileMetadatas()) { DataFile dataFile = fileMetadata.getDataFile(); + /** + * Previously (in Dataverse 5.3 and below) the dataDscr section was + * included for restricted files but that meant that summary + * statistics were exposed. (To get at these statistics, API users + * should instead use the "Data Variable Metadata Access" endpoint.) + * These days we return early to avoid this exposure. + */ + if (dataFile.isRestricted()) { + return; + } + if (dataFile != null && dataFile.isTabularData()) { if (!tabularData) { xmlw.writeStartElement("dataDscr"); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index fb7914561bc..04d7c77d4fb 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -36,7 +36,10 @@ import edu.harvard.iq.dataverse.authorization.groups.impl.builtin.AuthenticatedUsers; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.SystemConfig; +import java.io.File; import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; import javax.json.Json; @@ -53,6 +56,7 @@ import static javax.ws.rs.core.Response.Status.OK; import static javax.ws.rs.core.Response.Status.UNAUTHORIZED; import static junit.framework.Assert.assertEquals; +import org.hamcrest.CoreMatchers; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.nullValue; import org.junit.After; @@ -2041,4 +2045,123 @@ public void testLinkingDatasets() { */ } + /** + * In this test we are restricting a file and testing "export DDI" at the + * dataset level as well as getting the DDI at the file level. + * + * Export at the dataset level is always the public version. + * + * At the file level, you can still get summary statistics (and friends, + * "dataDscr") if you have access to download the file. If you don't have + * access, you get an error. + */ + @Test + public void testRestrictFileExportDdi() throws IOException { + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String authorUsername = UtilIT.getUsernameFromResponse(createUser); + String authorApiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverse = UtilIT.createRandomDataverse(authorApiToken); + createDataverse.prettyPrint(); + createDataverse.then().assertThat() + .statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverse); + + Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, authorApiToken); + createDataset.prettyPrint(); + createDataset.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset); + String datasetPid = JsonPath.from(createDataset.asString()).getString("data.persistentId"); + + Path pathToFile = Paths.get(java.nio.file.Files.createTempDirectory(null) + File.separator + "data.csv"); + String contentOfCsv = "" + + "name,pounds,species\n" + + "Marshall,40,dog\n" + + "Tiger,17,cat\n" + + "Panther,21,cat\n"; + java.nio.file.Files.write(pathToFile, contentOfCsv.getBytes()); + + Response uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile.toString(), authorApiToken); + uploadFile.prettyPrint(); + uploadFile.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.files[0].label", equalTo("data.csv")); + + String fileId = JsonPath.from(uploadFile.body().asString()).getString("data.files[0].dataFile.id"); + + assertTrue("Failed test if Ingest Lock exceeds max duration " + pathToFile, UtilIT.sleepForLock(datasetId.longValue(), "Ingest", authorApiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION)); + + Response restrictFile = UtilIT.restrictFile(fileId, true, authorApiToken); + restrictFile.prettyPrint(); + restrictFile.then().assertThat().statusCode(OK.getStatusCode()); + + Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, authorApiToken); + publishDataverse.then().assertThat().statusCode(OK.getStatusCode()); + Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetPid, "major", authorApiToken); + publishDataset.then().assertThat().statusCode(OK.getStatusCode()); + + // We're testing export here, which is at dataset level. + // Guest/public version + Response exportByGuest = UtilIT.exportDataset(datasetPid, "ddi"); + exportByGuest.prettyPrint(); + exportByGuest.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("codeBook.fileDscr.fileTxt.fileName", equalTo("data.tab")); + + // Here we are asserting that dataDscr is empty. TODO: Do this in REST Assured. + String dataDscrForGuest = XmlPath.from(exportByGuest.asString()).getString("codeBook.dataDscr"); + Assert.assertEquals("", dataDscrForGuest); + + // Author export (has access) + Response exportByAuthor = UtilIT.exportDataset(datasetPid, "ddi", authorApiToken); + exportByAuthor.prettyPrint(); + exportByAuthor.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("codeBook.fileDscr.fileTxt.fileName", equalTo("data.tab")); + + // Here we are asserting that dataDscr is empty. TODO: Do this in REST Assured. + String dataDscrForAuthor = XmlPath.from(exportByAuthor.asString()).getString("codeBook.dataDscr"); + Assert.assertEquals("", dataDscrForAuthor); + + // Now we are testing file-level retrieval. + // The author has access to a restricted file and gets all the metadata. + Response fileMetadataDdiAuthor = UtilIT.getFileMetadata(fileId, "ddi", authorApiToken); + fileMetadataDdiAuthor.prettyPrint(); + fileMetadataDdiAuthor.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("codeBook.fileDscr.fileTxt.fileName", equalTo("data.tab")) + // .body("codeBook", containsString("dataDscr")) + // The names of all these variables (name, pounds, species) should be visible. + .body("codeBook.dataDscr", CoreMatchers.not(equalTo(null))) + // .body("codeBook.dataDscr", equalTo(null)) + .body("codeBook.dataDscr.var[0].@name", equalTo("name")) + .body("codeBook.dataDscr.var[1].@name", equalTo("pounds")) + // This is an example of a summary stat (max) that should be visible. + .body("codeBook.dataDscr.var[1].sumStat.find { it.@type == 'max' }", equalTo("40.0")) + .body("codeBook.dataDscr.var[2].@name", equalTo("species")); + + Response createUserNoAuth = UtilIT.createRandomUser(); + createUserNoAuth.prettyPrint(); + String usernameNoAuth = UtilIT.getUsernameFromResponse(createUserNoAuth); + String apiTokenNoAuth = UtilIT.getApiTokenFromResponse(createUserNoAuth); + + // Users with no access to the restricted file are blocked. + Response fileMetadataDdiUserNoAuth = UtilIT.getFileMetadata(fileId, "ddi", apiTokenNoAuth); + fileMetadataDdiUserNoAuth.prettyPrint(); + fileMetadataDdiUserNoAuth.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo("You do not have permission to download this file.")); + + // Guest users (not logged in) are also blocked. + Response fileMetadataDdiGuest = UtilIT.getFileMetadata(fileId, "ddi"); + fileMetadataDdiGuest.prettyPrint(); + fileMetadataDdiGuest.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo("You do not have permission to download this file.")); + } + } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 2b6bbaf0caa..f3ff8f8fae4 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -834,6 +834,22 @@ static Response getFileMetadata(String fileIdOrPersistentId, String optionalForm .get("/api/access/datafile/" + idInPath + "/metadata" + optionalFormatInPath + "?key=" + apiToken + optionalQueryParam); } + static Response getFileMetadata(String fileIdOrPersistentId, String optionalFormat) { + String idInPath = fileIdOrPersistentId; // Assume it's a number. + String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path. + if (!NumberUtils.isNumber(fileIdOrPersistentId)) { + idInPath = ":persistentId"; + optionalQueryParam = "?persistentId=" + fileIdOrPersistentId; + } + String optionalFormatInPath = ""; + if (optionalFormat != null) { + optionalFormatInPath = "/" + optionalFormat; + } + return given() + .urlEncodingEnabled(false) + .get("/api/access/datafile/" + idInPath + "/metadata" + optionalFormatInPath + optionalQueryParam); + } + static Response testIngest(String fileName, String fileType) { return given() .get("/api/ingest/test/file?fileName=" + fileName + "&fileType=" + fileType); @@ -1634,11 +1650,19 @@ static Response uploadProvFreeForm(String idOrPersistentId, JsonObject jsonObjec // } // return requestSpecification.delete("/api/files/" + idInPath + "/prov-freeform" + optionalQueryParam); // } + static Response exportDataset(String datasetPersistentId, String exporter) { + return exportDataset(datasetPersistentId, exporter, null); + } static Response exportDataset(String datasetPersistentId, String exporter, String apiToken) { // http://localhost:8080/api/datasets/export?exporter=dataverse_json&persistentId=doi%3A10.5072/FK2/W6WIMQ - return given() - .header(API_TOKEN_HTTP_HEADER, apiToken) + RequestSpecification requestSpecification = given(); + if (apiToken != null) { + requestSpecification = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken); + } + return requestSpecification + // .header(API_TOKEN_HTTP_HEADER, apiToken) // .get("/api/datasets/:persistentId/export" + "?persistentId=" + datasetPersistentId + "&exporter=" + exporter); .get("/api/datasets/export" + "?persistentId=" + datasetPersistentId + "&exporter=" + exporter); } From 26a83c4f539e524169b8159500493a8fe40fe7e6 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 26 Feb 2021 14:02:31 -0500 Subject: [PATCH 382/576] change flyway number --- .../{V5.5.0.1__7563-workflow.sql => V5.3.0.4__7563-workflow.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V5.5.0.1__7563-workflow.sql => V5.3.0.4__7563-workflow.sql} (100%) diff --git a/src/main/resources/db/migration/V5.5.0.1__7563-workflow.sql b/src/main/resources/db/migration/V5.3.0.4__7563-workflow.sql similarity index 100% rename from src/main/resources/db/migration/V5.5.0.1__7563-workflow.sql rename to src/main/resources/db/migration/V5.3.0.4__7563-workflow.sql From 9813c77eecbe4b2e3d62a5cd29a4e522d6a6ffad Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 26 Feb 2021 14:28:27 -0500 Subject: [PATCH 383/576] put message in bundle #7619 --- .../api/errorhandlers/WebApplicationExceptionHandler.java | 2 +- src/main/java/propertyFiles/Bundle.properties | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java index aa8bb2b3772..5f28bfd0afc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java @@ -47,7 +47,7 @@ public Response toResponse(WebApplicationException ex) { if ( (ex.getMessage()+"").toLowerCase().startsWith("tabular data required")) { jrb.message(BundleUtil.getStringFromBundle("access.api.exception.metadata.not.available.for.nontabular.file")); } else if ((ex.getMessage() + "").toLowerCase().startsWith("no permission to download file")) { - jrb.message("You do not have permission to download this file."); + jrb.message(BundleUtil.getStringFromBundle("access.api.exception.metadata.restricted.no.permission")); } else { jrb.message("Bad Request. The API request cannot be completed with the parameters supplied. Please check your code for typos, or consult our API guide at http://guides.dataverse.org."); jrb.request(request); diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 4d95322caf2..8352e66c92d 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2354,6 +2354,7 @@ access.api.requestList.fileNotFound=Could not find datafile with id {0}. access.api.requestList.noKey=You must provide a key to get list of access requests for a file. access.api.requestList.noRequestsFound=There are no access requests for this file {0}. access.api.exception.metadata.not.available.for.nontabular.file=This type of metadata is only available for tabular files. +access.api.exception.metadata.restricted.no.permission=You do not have permission to download this file. access.api.exception.version.not.found=Could not find requested dataset version. #permission From 8f08f7b31f268f9a649db93383824c7aab629e61 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 26 Feb 2021 16:02:13 -0500 Subject: [PATCH 384/576] disabled users should never get notifications #2419 #4475 That means it's safe not to have this check. --- .../edu/harvard/iq/dataverse/UserNotificationServiceBean.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java index f02962498c8..071805d3d26 100644 --- a/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java @@ -103,10 +103,6 @@ public void sendNotification(AuthenticatedUser dataverseUser, Timestamp sendDate } public void sendNotification(AuthenticatedUser dataverseUser, Timestamp sendDate, Type type, Long objectId, String comment, AuthenticatedUser requestor, boolean isHtmlContent) { - if (dataverseUser.isDisabled()) { - logger.info("An attempt was made to send a " + type + " notification to a disabled user: " + dataverseUser); - return; - } UserNotification userNotification = new UserNotification(); userNotification.setUser(dataverseUser); userNotification.setSendDate(sendDate); From 698c9b718f0d48394c4320c4e03d8ad54c8805fc Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Fri, 26 Feb 2021 16:43:57 -0500 Subject: [PATCH 385/576] Add truncation plugin to apply to html+text desc blocks [ref #6685] --- src/main/webapp/dataverse_template.xhtml | 3 +- src/main/webapp/resources/css/structure.css | 26 ++++- .../resources/js/dv_rebind_bootstrap_ui.js | 64 ++++++++++- .../webapp/resources/js/jquery.truncate.js | 103 ++++++++++++++++++ 4 files changed, 191 insertions(+), 5 deletions(-) create mode 100644 src/main/webapp/resources/js/jquery.truncate.js diff --git a/src/main/webapp/dataverse_template.xhtml b/src/main/webapp/dataverse_template.xhtml index e461e853955..61487c0d1c0 100644 --- a/src/main/webapp/dataverse_template.xhtml +++ b/src/main/webapp/dataverse_template.xhtml @@ -84,7 +84,8 @@ - + + diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index 2603fd6ba0b..f4dea7efe7c 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -627,13 +627,37 @@ div.edit-field div.ui-message {margin:6px 0;} .metadata-container table.metadata {border-collapse:separate;margin:0 4px;width:100%;} .metadata-container table.metadata th {width:25%;vertical-align:top;padding:6px 12px;} .metadata-container table.metadata td {padding:6px 12px;vertical-align:top;} -@media(max-width:767px){ + +/* TO-DO FIX max-width:1199px FOR SUMMARY BLOCK ONLY, NOT OTHER PAGES */ +@media(max-width:1199px){ .metadata-container table.metadata th, .metadata-container table.metadata td {display:block;width:100%;} .metadata-container table.metadata th {padding-bottom:0;} .metadata-container table.metadata td {padding-left:30px;} } .panel-body.metadata-container {padding:15px 0;} .metadata-logo {max-width:180px;max-height:40px;} +.metadata-container .truncate-more-link, .metadata-container .collapse-less-link {color: #337AB7; cursor: pointer; cursor: hand;} + +/* HTML METADATA OVERWRITE */ +.metadata-container table.metadata td > div { + /* https://stackoverflow.com/a/24158056 */ + /*-moz-white-space: pre-wrap; + white-space: pre-wrap; + hyphens: auto; + -ms-word-break: break-all; + -ms-word-wrap: break-all; + -webkit-word-break: break-word; + -webkit-word-wrap: break-word; + word-break: break-word; + word-wrap: break-word; + -webkit-hyphens: auto; + -moz-hyphens: auto; + -ms-hyphens: auto; + hyphens: auto;*/ +} +.metadata-container pre { + white-space: pre-wrap !important; +} /* Primefaces vs Bootstrap selectOneMenu fix */ div.ui-selectonemenu.form-control {padding:0px;} diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index 44e86831c69..608b61ce45f 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -22,6 +22,12 @@ function bind_bsui_components(){ // Hide open tooltips + popovers $('.bootstrap-button-tooltip, [data-toggle="tooltip"]').tooltip("hide"); $("[data-toggle='popover']").popover("hide"); + + // Truncate content + contentTruncate(); + + // Truncate checksums + checksumTruncate(); // Tooltips + popovers bind_tooltip_popover(); @@ -29,9 +35,6 @@ function bind_bsui_components(){ // Disabled pagination links disabledLinks(); - // Truncate checksums - checksumTruncate(); - // Sharrre sharrre(); @@ -157,7 +160,62 @@ function sharrre(){ } }); } + +/* + * Truncate any metadata content + */ +function contentTruncate(){ + // jquery.truncate.js + // HTML-safe truncation + // https://github.com/pathable/truncate + + // WHAT ARE WE TRUNCATING + // TO-DO IS THERE A MORE SCALABLE SELECTOR METHOD THAN THIS? ADD TRUNCATE CLASS TO OBJECTS? e.g. span.checksum-truncate + $('#dsDescription td div').each(function () { + + // GET THE DESC TEXT AND HTML + var truncateThis = $(this).html(); + + // TO-DO TRUNCATE BTN LABEL TEXT FROM BUNDLE + var readMoreBtn = '... '; + var readLessBtn = ''; + + if ($(this).hasClass('thisWasTruncated')) { + // DO NOTHING + } + else { + + // TO-DO PUT THE ORIGINAL FULL DESCRIPTION SOMEWHERE + // $('#dsDescription td div').attr({'data-html':'true', 'data-original-text':truncateThis}); + + var thisWasTruncated = jQuery.truncate(truncateThis, { + length: 500, + words: true, + keepFirstWord: true, + ellipsis: readMoreBtn, + finishBlock: true + }); + + // ... add responsive img class ++ link target ... DO THIS ANYWAY SOMEWHERE ELSE!! + $(this).toggleClass('thisWasTruncated').html(thisWasTruncated).find('img').attr('class', 'img-responsive'); + + // Return full description on Read More link click ++ add responsive img class ++ link target ... DO THIS ANYWAY SOMEWHERE ELSE!! + $(document).on('click', 'button.truncate-more-link', function() { + $(this).tooltip('hide').parent('div').toggleClass('thisWasTruncated').html(truncateThis).append(readLessBtn).find('img').attr('class', 'img-responsive'); + $('button.collapse-less-link').tooltip(); + }); + // ... add responsive img class ++ link target ... DO THIS ANYWAY SOMEWHERE ELSE!! + $(document).on('click', 'button.collapse-less-link', function() { + $(this).tooltip('hide').parent('div').toggleClass('thisWasTruncated').html(thisWasTruncated).find('img').attr('class', 'img-responsive'); + $('button.truncate-more-link').tooltip(); + }); + + } + + }); +} + /* * Truncate file checksums */ diff --git a/src/main/webapp/resources/js/jquery.truncate.js b/src/main/webapp/resources/js/jquery.truncate.js new file mode 100644 index 00000000000..e43d4134ae1 --- /dev/null +++ b/src/main/webapp/resources/js/jquery.truncate.js @@ -0,0 +1,103 @@ +(function($) { + + // Matches trailing non-space characters. + var chop = /(\s*\S+|\s)$/; + + // Matches the first word in the string. + var start = /^(\S*)/; + + // Return a truncated html string. Delegates to $.fn.truncate. + $.truncate = function(html, options) { + return $('
    ').append(html).truncate(options).html(); + }; + + // Truncate the contents of an element in place. + $.fn.truncate = function(options) { + if ($.isNumeric(options)) options = {length: options}; + var o = $.extend({}, $.truncate.defaults, options); + + return this.each(function() { + var self = $(this); + + if (o.noBreaks) self.find('br').replaceWith(' '); + + var text = self.text(); + var excess = text.length - o.length; + + if (o.stripTags) self.text(text); + + // Chop off any partial words if appropriate. + if (o.words && excess > 0) { + var truncated = text.slice(0, o.length).replace(chop, '').length; + + if (o.keepFirstWord && truncated === 0) { + excess = text.length - start.exec(text)[0].length - 1; + } else { + excess = text.length - truncated - 1; + } + } + + if (excess < 0 || !excess && !o.truncated) return; + + // Iterate over each child node in reverse, removing excess text. + $.each(self.contents().get().reverse(), function(i, el) { + var $el = $(el); + var text = $el.text(); + var length = text.length; + + // If the text is longer than the excess, remove the node and continue. + if (length <= excess) { + o.truncated = true; + excess -= length; + $el.remove(); + return; + } + + // Remove the excess text and append the ellipsis. + if (el.nodeType === 3) { + // should we finish the block anyway? + if (o.finishBlock) { + $(el.splitText(length)).replaceWith(o.ellipsis); + } else { + $(el.splitText(length - excess - 1)).replaceWith(o.ellipsis); + } + return false; + } + + // Recursively truncate child nodes. + $el.truncate($.extend(o, {length: length - excess})); + return false; + }); + }); + }; + + $.truncate.defaults = { + + // Strip all html elements, leaving only plain text. + stripTags: false, + + // Only truncate at word boundaries. + words: false, + + // When 'words' is active, keeps the first word in the string + // even if it's longer than a target length. + keepFirstWord: false, + + // Replace instances of
    with a single space. + noBreaks: false, + + // if true always truncate the content at the end of the block. + finishBlock: false, + + // The maximum length of the truncated html. + length: Infinity, + + // The character to use as the ellipsis. The word joiner (U+2060) can be + // used to prevent a hanging ellipsis, but displays incorrectly in Chrome + // on Windows 7. + // http://code.google.com/p/chromium/issues/detail?id=68323 + ellipsis: '\u2026' // '\u2060\u2026' + + }; + +})(jQuery); From 0f7b3aa433dbf5179e16b7e47bcd4de21f6813d2 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 1 Mar 2021 10:07:10 -0500 Subject: [PATCH 386/576] #7373 update scripts --- conf/docker-aio/1prep.sh | 6 +++--- conf/docker-aio/c8.dockerfile | 8 ++++---- conf/docker-aio/entrypoint.bash | 2 +- .../installation/files/etc/systemd/solr.service | 6 +++--- downloads/download.sh | 2 +- scripts/installer/README.txt | 2 +- scripts/vagrant/setup-solr.sh | 12 ++++++------ scripts/vagrant/setup.sh | 2 +- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/conf/docker-aio/1prep.sh b/conf/docker-aio/1prep.sh index c5b2595925e..98cbc2eec93 100755 --- a/conf/docker-aio/1prep.sh +++ b/conf/docker-aio/1prep.sh @@ -4,9 +4,9 @@ # this was based off the phoenix deployment; and is likely uglier and bulkier than necessary in a perfect world mkdir -p testdata/doc/sphinx-guides/source/_static/util/ -cp ../solr/7.7.2/schema*.xml testdata/ -cp ../solr/7.7.2/solrconfig.xml testdata/ -cp ../solr/7.7.2/updateSchemaMDB.sh testdata/ +cp ../solr/8.8.1/schema*.xml testdata/ +cp ../solr/8.8.1/solrconfig.xml testdata/ +cp ../solr/8.8.1/updateSchemaMDB.sh testdata/ cp ../jhove/jhove.conf testdata/ cp ../jhove/jhoveConfig.xsd testdata/ cd ../../ diff --git a/conf/docker-aio/c8.dockerfile b/conf/docker-aio/c8.dockerfile index c505211cb75..92c3dc24d56 100644 --- a/conf/docker-aio/c8.dockerfile +++ b/conf/docker-aio/c8.dockerfile @@ -17,7 +17,7 @@ COPY testdata/sushi_sample_logs.json /tmp/ COPY disableipv6.conf /etc/sysctl.d/ RUN rm /etc/httpd/conf/* COPY httpd.conf /etc/httpd/conf -RUN cd /opt ; tar zxf /tmp/dv/deps/solr-7.7.2dv.tgz +RUN cd /opt ; tar zxf /tmp/dv/deps/solr-8.8.1dv.tgz RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2020.6.zip ; ln -s /opt/payara5 /opt/glassfish4 # this copy of domain.xml is the result of running `asadmin set server.monitoring-service.module-monitoring-levels.jvm=LOW` on a default glassfish installation (aka - enable the glassfish REST monitir endpoint for the jvm` @@ -28,9 +28,9 @@ RUN sudo -u postgres /usr/bin/initdb /var/lib/pgsql/data # copy configuration related files RUN cp /tmp/dv/pg_hba.conf /var/lib/pgsql/data/ -RUN cp -r /opt/solr-7.7.2/server/solr/configsets/_default /opt/solr-7.7.2/server/solr/collection1 -RUN cp /tmp/dv/schema*.xml /opt/solr-7.7.2/server/solr/collection1/conf/ -RUN cp /tmp/dv/solrconfig.xml /opt/solr-7.7.2/server/solr/collection1/conf/solrconfig.xml +RUN cp -r /opt/solr-8.8.1/server/solr/configsets/_default /opt/solr-8.8.1/server/solr/collection1 +RUN cp /tmp/dv/schema*.xml /opt/solr-8.8.1/server/solr/collection1/conf/ +RUN cp /tmp/dv/solrconfig.xml /opt/solr-8.8.1/server/solr/collection1/conf/solrconfig.xml # skipping payara user and solr user (run both as root) diff --git a/conf/docker-aio/entrypoint.bash b/conf/docker-aio/entrypoint.bash index 7ff0c527a28..6cb5463cd1a 100755 --- a/conf/docker-aio/entrypoint.bash +++ b/conf/docker-aio/entrypoint.bash @@ -1,7 +1,7 @@ #!/usr/bin/env bash export LANG=en_US.UTF-8 sudo -u postgres /usr/bin/pg_ctl start -D /var/lib/pgsql/data & -cd /opt/solr-7.7.2/ +cd /opt/solr-8.8.1/ # TODO: Run Solr as non-root and remove "-force". bin/solr start -force bin/solr create_core -c collection1 -d server/solr/collection1/conf -force diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service index 06eacc68ca2..96960793938 100644 --- a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service +++ b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service @@ -5,9 +5,9 @@ After = syslog.target network.target remote-fs.target nss-lookup.target [Service] User = solr Type = forking -WorkingDirectory = /usr/local/solr/solr-7.7.2 -ExecStart = /usr/local/solr/solr-7.7.2/bin/solr start -m 1g -j "jetty.host=127.0.0.1" -ExecStop = /usr/local/solr/solr-7.7.2/bin/solr stop +WorkingDirectory = /usr/local/solr/solr-8.8.1 +ExecStart = /usr/local/solr/solr-8.8.1/bin/solr start -m 1g -j "jetty.host=127.0.0.1" +ExecStop = /usr/local/solr/solr-8.8.1/bin/solr stop LimitNOFILE=65000 LimitNPROC=65000 Restart=on-failure diff --git a/downloads/download.sh b/downloads/download.sh index 59bbcef9dc8..8c2b51dd4c7 100755 --- a/downloads/download.sh +++ b/downloads/download.sh @@ -1,5 +1,5 @@ #!/bin/sh curl -L -O https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2020.6/payara-5.2020.6.zip -curl -L -O https://archive.apache.org/dist/lucene/solr/7.7.2/solr-7.7.2.tgz +curl -L -O https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz curl -L -O https://search.maven.org/remotecontent?filepath=org/jboss/weld/weld-osgi-bundle/2.2.10.Final/weld-osgi-bundle-2.2.10.Final-glassfish4.jar curl -s -L http://sourceforge.net/projects/schemaspy/files/schemaspy/SchemaSpy%205.0.0/schemaSpy_5.0.0.jar/download > schemaSpy_5.0.0.jar diff --git a/scripts/installer/README.txt b/scripts/installer/README.txt index 35dbaadc3bf..70f08550a75 100644 --- a/scripts/installer/README.txt +++ b/scripts/installer/README.txt @@ -35,7 +35,7 @@ from conf/jhove: jhove.conf -SOLR schema and config files, from conf/solr/7.7.2: +SOLR schema and config files, from conf/solr/8.8.1: schema.xml schema_dv_cmb_copies.xml diff --git a/scripts/vagrant/setup-solr.sh b/scripts/vagrant/setup-solr.sh index 02654dec1d0..9c1ba50ee49 100755 --- a/scripts/vagrant/setup-solr.sh +++ b/scripts/vagrant/setup-solr.sh @@ -5,12 +5,12 @@ SOLR_USER=solr SOLR_HOME=/usr/local/solr mkdir $SOLR_HOME chown $SOLR_USER:$SOLR_USER $SOLR_HOME -su $SOLR_USER -s /bin/sh -c "cp /dataverse/downloads/solr-7.7.2.tgz $SOLR_HOME" -su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-7.7.2.tgz" -su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.7.2/server/solr && cp -r configsets/_default . && mv _default collection1" -su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/7.7.2/schema*.xml $SOLR_HOME/solr-7.7.2/server/solr/collection1/conf/" -su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/7.7.2/solrconfig.xml $SOLR_HOME/solr-7.7.2/server/solr/collection1/conf/solrconfig.xml" -su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.7.2 && bin/solr start && bin/solr create_core -c collection1 -d server/solr/collection1/conf/" +su $SOLR_USER -s /bin/sh -c "cp /dataverse/downloads/solr-8.8.1.tgz $SOLR_HOME" +su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-78.8.1.tgz" +su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-8.8.1/server/solr && cp -r configsets/_default . && mv _default collection1" +su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/8.8.1/schema*.xml $SOLR_HOME/solr-8.8.1/server/solr/collection1/conf/" +su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/8.8.1/solrconfig.xml $SOLR_HOME/solr-8.8.1/server/solr/collection1/conf/solrconfig.xml" +su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-8.8.1 && bin/solr start && bin/solr create_core -c collection1 -d server/solr/collection1/conf/" cp /dataverse/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr /etc/init.d/solr chmod 755 /etc/init.d/solr /etc/init.d/solr stop diff --git a/scripts/vagrant/setup.sh b/scripts/vagrant/setup.sh index 7e62fefd0ae..14f12cea692 100644 --- a/scripts/vagrant/setup.sh +++ b/scripts/vagrant/setup.sh @@ -54,7 +54,7 @@ echo "Ensuring Unix user '$SOLR_USER' exists" useradd $SOLR_USER || : DOWNLOAD_DIR='/dataverse/downloads' PAYARA_ZIP="$DOWNLOAD_DIR/payara-5.2020.6.zip" -SOLR_TGZ="$DOWNLOAD_DIR/solr-7.7.2.tgz" +SOLR_TGZ="$DOWNLOAD_DIR/solr-8.8.1.tgz" if [ ! -f $PAYARA_ZIP ] || [ ! -f $SOLR_TGZ ]; then echo "Couldn't find $PAYARA_ZIP or $SOLR_TGZ! Running download script...." cd $DOWNLOAD_DIR && ./download.sh && cd From 45b4cc092e9ed2988875c7d3419d3f687a7989c1 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 1 Mar 2021 10:40:07 -0500 Subject: [PATCH 387/576] #7373 add release notes --- doc/release-notes/7373-solr-upgrade.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/release-notes/7373-solr-upgrade.md diff --git a/doc/release-notes/7373-solr-upgrade.md b/doc/release-notes/7373-solr-upgrade.md new file mode 100644 index 00000000000..c6c084961bf --- /dev/null +++ b/doc/release-notes/7373-solr-upgrade.md @@ -0,0 +1,11 @@ +### Solr Update + +With this release we upgrade to the latest available stable release in the Solr 8.x branch. We recommend a fresh installation of Solr 8.8.1 (the index will be empty) +followed by an "index all". + +Before you start the "index all", Dataverse will appear to be empty because +the search results come from Solr. As indexing progresses, partial results will +appear until indexing is complete. + + +See http://guides.dataverse.org/installation/prerequisites.html#installing-solr From 663652d6f472e81ac666a4aa9f5eaeae111b03f6 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 1 Mar 2021 10:49:20 -0500 Subject: [PATCH 388/576] #7373 update script --- conf/docker-aio/0prep_deps.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/conf/docker-aio/0prep_deps.sh b/conf/docker-aio/0prep_deps.sh index 71398f03a42..50294dd3772 100755 --- a/conf/docker-aio/0prep_deps.sh +++ b/conf/docker-aio/0prep_deps.sh @@ -10,12 +10,11 @@ if [ ! -e dv/deps/payara-5.2020.6.zip ]; then wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2020.6/payara-5.2020.6.zip -O dv/deps/payara-5.2020.6.zip fi -if [ ! -e dv/deps/solr-7.7.2dv.tgz ]; then +if [ ! -e dv/deps/solr-8.8.1dv.tgz ]; then echo "solr dependency prep" # schema changes *should* be the only ones... - cd dv/deps/ - #wget https://archive.apache.org/dist/lucene/solr/7.3.0/solr-7.3.0.tgz -O solr-7.3.0dv.tgz - wget https://archive.apache.org/dist/lucene/solr/7.7.2/solr-7.7.2.tgz -O solr-7.7.2dv.tgz + cd dv/deps/ + wget https://archive.apache.org/dist/lucene/solr/8.8.2/solr-8.8.1.tgz -O solr-8.8.1dv.tgz cd ../../ fi From 2064fec9c8264ecb1f36fea325e7065cddcab66a Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Mon, 1 Mar 2021 15:03:22 -0500 Subject: [PATCH 389/576] Truncation clean up, moved text to bundle [ref #6685] --- src/main/java/propertyFiles/Bundle.properties | 6 ++++ src/main/webapp/dataset.xhtml | 3 ++ src/main/webapp/resources/css/structure.css | 18 ++--------- .../resources/js/dv_rebind_bootstrap_ui.js | 30 ++++++++----------- 4 files changed, 24 insertions(+), 33 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 29ba794821b..fdaa68d3650 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -59,6 +59,12 @@ apply=Apply add=Add delete=Delete copyClipboard=Copy to Clipboard + +truncateMoreBtn=Read More +truncateMoreTip=Click to read the full +truncateLessBtn=Read Less +truncateLessTip=Click to collapse the + yes=Yes no=No previous=Previous diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 4f23d4c7b44..3480e83ba57 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1655,6 +1655,9 @@ // div { - /* https://stackoverflow.com/a/24158056 */ - /*-moz-white-space: pre-wrap; - white-space: pre-wrap; - hyphens: auto; - -ms-word-break: break-all; - -ms-word-wrap: break-all; - -webkit-word-break: break-word; - -webkit-word-wrap: break-word; - word-break: break-word; - word-wrap: break-word; - -webkit-hyphens: auto; - -moz-hyphens: auto; - -ms-hyphens: auto; - hyphens: auto;*/ + /* TO-DO #6685 #6578 #1634 #1249 */ } +/* METADATA HTML */ .metadata-container pre { white-space: pre-wrap !important; } diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index 608b61ce45f..16a5584ef01 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -164,55 +164,49 @@ function sharrre(){ /* * Truncate any metadata content */ -function contentTruncate(){ +function contentTruncate(truncSelector, truncMetaLabel, truncMoreBtn, truncMoreTip, truncLessBtn, truncLessTip){ // jquery.truncate.js // HTML-safe truncation // https://github.com/pathable/truncate // WHAT ARE WE TRUNCATING - // TO-DO IS THERE A MORE SCALABLE SELECTOR METHOD THAN THIS? ADD TRUNCATE CLASS TO OBJECTS? e.g. span.checksum-truncate - $('#dsDescription td div').each(function () { - + // SELECTOR ID FROM PARAMETERS + $('#' + truncSelector + ' td div').each(function () { // GET THE DESC TEXT AND HTML var truncateThis = $(this).html(); - // TO-DO TRUNCATE BTN LABEL TEXT FROM BUNDLE - var readMoreBtn = '... '; - var readLessBtn = ''; + // BTN LABEL TEXT, DYNAMIC ARIA ATTR'S, FROM BUNDLE VIA PARAMETERS + var readMoreBtn = '... '; + var readLessBtn = ''; if ($(this).hasClass('thisWasTruncated')) { - // DO NOTHING + // ALREADY TRUNCATED, DO NOTHING } else { - - // TO-DO PUT THE ORIGINAL FULL DESCRIPTION SOMEWHERE - // $('#dsDescription td div').attr({'data-html':'true', 'data-original-text':truncateThis}); - var thisWasTruncated = jQuery.truncate(truncateThis, { length: 500, words: true, keepFirstWord: true, ellipsis: readMoreBtn, - finishBlock: true + // REMOVED FINISH BLOCK TO TRUNCATE MID CONENT + // finishBlock: true }); - // ... add responsive img class ++ link target ... DO THIS ANYWAY SOMEWHERE ELSE!! + // ... add responsive img class ++ link target $(this).toggleClass('thisWasTruncated').html(thisWasTruncated).find('img').attr('class', 'img-responsive'); - // Return full description on Read More link click ++ add responsive img class ++ link target ... DO THIS ANYWAY SOMEWHERE ELSE!! + // ... add full description to summary block on Read More link click ++ add responsive img class ++ link target $(document).on('click', 'button.truncate-more-link', function() { $(this).tooltip('hide').parent('div').toggleClass('thisWasTruncated').html(truncateThis).append(readLessBtn).find('img').attr('class', 'img-responsive'); $('button.collapse-less-link').tooltip(); }); - // ... add responsive img class ++ link target ... DO THIS ANYWAY SOMEWHERE ELSE!! + // ... add responsive img class ++ link target $(document).on('click', 'button.collapse-less-link', function() { $(this).tooltip('hide').parent('div').toggleClass('thisWasTruncated').html(thisWasTruncated).find('img').attr('class', 'img-responsive'); $('button.truncate-more-link').tooltip(); }); - } - }); } From 6633943a2988ddce4322d495b168b6a47bc9116f Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 1 Mar 2021 15:09:11 -0500 Subject: [PATCH 390/576] #7373 update dev environment doc --- .../source/developers/dev-environment.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 63f482b86ee..b6177e2cd9f 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -117,7 +117,7 @@ On Linux, you should just install PostgreSQL from your package manager without w Install Solr ~~~~~~~~~~~~ -`Solr `_ 7.7.2 is required. +`Solr `_ 8.8.1 is required. To install Solr, execute the following commands: @@ -127,27 +127,27 @@ To install Solr, execute the following commands: ``cd /usr/local/solr`` -``curl -O http://archive.apache.org/dist/lucene/solr/7.7.2/solr-7.7.2.tgz`` +``curl -O http://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz`` -``tar xvfz solr-7.7.2.tgz`` +``tar xvfz solr-8.8.1.tgz`` -``cd solr-7.7.2/server/solr`` +``cd solr-8.8.1/server/solr`` ``cp -r configsets/_default collection1`` -``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/7.7.2/schema.xml`` +``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.8.1/schema.xml`` -``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/7.7.2/schema_dv_mdb_fields.xml`` +``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.8.1/schema_dv_mdb_fields.xml`` -``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/7.7.2/schema_dv_mdb_copies.xml`` +``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.8.1/schema_dv_mdb_copies.xml`` ``mv schema*.xml collection1/conf`` -``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/7.7.2/solrconfig.xml`` +``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.8.1/solrconfig.xml`` ``mv solrconfig.xml collection1/conf/solrconfig.xml`` -``cd /usr/local/solr/solr-7.7.2`` +``cd /usr/local/solr/solr-8.8.1`` (Please note that the extra jetty argument below is a security measure to limit connections to Solr to only your computer. For extra security, run a firewall.) From 924087a719ee5b75d7af01fc48b07098c13bf861 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 1 Mar 2021 15:31:15 -0500 Subject: [PATCH 391/576] #7373 update prerequisites doc --- .../source/installation/prerequisites.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index b1643fabf40..bc96680b8de 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -166,7 +166,7 @@ The Dataverse Software search index is powered by Solr. Supported Versions ================== -The Dataverse Software has been tested with Solr version 7.7.2. Future releases in the 7.x series are likely to be compatible; however, this cannot be confirmed until they are officially tested. Major releases above 7.x (e.g. 8.x) are not supported. +The Dataverse Software has been tested with Solr version 8.8.1. Future releases in the 8.x series are likely to be compatible; however, this cannot be confirmed until they are officially tested. Major releases above 8.x (e.g. 9.x) are not supported. Installing Solr =============== @@ -181,19 +181,19 @@ Become the ``solr`` user and then download and configure Solr:: su - solr cd /usr/local/solr - wget https://archive.apache.org/dist/lucene/solr/7.7.2/solr-7.7.2.tgz - tar xvzf solr-7.7.2.tgz - cd solr-7.7.2 + wget https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz + tar xvzf solr-8.8.1.tgz + cd solr-8.8.1 cp -r server/solr/configsets/_default server/solr/collection1 You should already have a "dvinstall.zip" file that you downloaded from https://github.com/IQSS/dataverse/releases . Unzip it into ``/tmp``. Then copy the files into place:: - cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-7.7.2/server/solr/collection1/conf - cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-7.7.2/server/solr/collection1/conf + cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-8.8.1/server/solr/collection1/conf + cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-8.8.1/server/solr/collection1/conf Note: The Dataverse Project team has customized Solr to boost results that come from certain indexed elements inside the Dataverse installation, for example prioritizing results from Dataverse collections over Datasets. If you would like to remove this, edit your ``solrconfig.xml`` and remove the ```` element and its contents. If you have ideas about how this boosting could be improved, feel free to contact us through our Google Group https://groups.google.com/forum/#!forum/dataverse-dev . -A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-7.7.2/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400`` +A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-8.8.1/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400`` Solr will warn about needing to increase the number of file descriptors and max processes in a production environment but will still run with defaults. We have increased these values to the recommended levels by adding ulimit -n 65000 to the init script, and the following to ``/etc/security/limits.conf``:: @@ -212,7 +212,7 @@ Solr launches asynchronously and attempts to use the ``lsof`` binary to watch fo Finally, you need to tell Solr to create the core "collection1" on startup:: - echo "name=collection1" > /usr/local/solr/solr-7.7.2/server/solr/collection1/core.properties + echo "name=collection1" > /usr/local/solr/solr-8.8.1/server/solr/collection1/core.properties Solr Init Script ================ From 273133fc7890e9dcc288e5586d7be74f2765a7a2 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 1 Mar 2021 15:54:48 -0500 Subject: [PATCH 392/576] #7373 update metadata customization --- doc/sphinx-guides/source/admin/metadatacustomization.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index fdbac996108..923f9ff6e3d 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -644,7 +644,7 @@ configuration, including any enabled metadata schemas: ``curl http://localhost:8080/api/admin/index/solr/schema`` -For convenience and automation you can download and consider running :download:`updateSchemaMDB.sh <../../../../conf/solr/7.7.2/updateSchemaMDB.sh>`. It uses the API endpoint above and writes schema files to the filesystem (so be sure to run it on the Solr server itself as the Unix user who owns the Solr files) and then triggers a Solr reload. +For convenience and automation you can download and consider running :download:`updateSchemaMDB.sh <../../../../conf/solr/8.8.1/updateSchemaMDB.sh>`. It uses the API endpoint above and writes schema files to the filesystem (so be sure to run it on the Solr server itself as the Unix user who owns the Solr files) and then triggers a Solr reload. By default, it will download from your Dataverse installation at `http://localhost:8080` and reload Solr at `http://localhost:8983`. You may use the following environment variables with this script or mix'n'match with options: @@ -657,13 +657,13 @@ Environment variable Option Description E `UNBLOCK_KEY` `-u` If your installation has a blocked admin API *xyz* or */secrets/unblock.key* endpoint, you can provide either the key itself or a path to a keyfile -`TARGET` `-t` Provide the config directory of your Solr core */usr/local/solr/solr-7.7.2/server/solr/collection1/conf* +`TARGET` `-t` Provide the config directory of your Solr core */usr/local/solr/solr-8.8.1/server/solr/collection1/conf* "collection1" ==================== ====== =============================================== ========================================================= See the :doc:`/installation/prerequisites/` section of the Installation Guide for a suggested location on disk for the Solr schema file. -Please note that if you are going to make a pull request updating ``conf/solr/7.7.2/schema.xml`` with fields you have added, you should first load all the custom metadata blocks in ``scripts/api/data/metadatablocks`` (including ones you don't care about) to create a complete list of fields. +Please note that if you are going to make a pull request updating ``conf/solr/8.8.1/schema.xml`` with fields you have added, you should first load all the custom metadata blocks in ``scripts/api/data/metadatablocks`` (including ones you don't care about) to create a complete list of fields. Reloading a Metadata Block -------------------------- From c498f6a47ffe4352aa374911a62e5deb196cb851 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 2 Mar 2021 09:46:00 -0500 Subject: [PATCH 393/576] #7373 fix directory typo --- conf/docker-aio/0prep_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/docker-aio/0prep_deps.sh b/conf/docker-aio/0prep_deps.sh index 50294dd3772..5bf44bfbe82 100755 --- a/conf/docker-aio/0prep_deps.sh +++ b/conf/docker-aio/0prep_deps.sh @@ -14,7 +14,7 @@ if [ ! -e dv/deps/solr-8.8.1dv.tgz ]; then echo "solr dependency prep" # schema changes *should* be the only ones... cd dv/deps/ - wget https://archive.apache.org/dist/lucene/solr/8.8.2/solr-8.8.1.tgz -O solr-8.8.1dv.tgz + wget https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz -O solr-8.8.1dv.tgz cd ../../ fi From cb74b81012d634acb1b1b36e586d2c4fd8a5dd9f Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 2 Mar 2021 10:10:24 -0500 Subject: [PATCH 394/576] #7373 another typo --- scripts/vagrant/setup-solr.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/vagrant/setup-solr.sh b/scripts/vagrant/setup-solr.sh index 9c1ba50ee49..41635994bc5 100755 --- a/scripts/vagrant/setup-solr.sh +++ b/scripts/vagrant/setup-solr.sh @@ -6,7 +6,7 @@ SOLR_HOME=/usr/local/solr mkdir $SOLR_HOME chown $SOLR_USER:$SOLR_USER $SOLR_HOME su $SOLR_USER -s /bin/sh -c "cp /dataverse/downloads/solr-8.8.1.tgz $SOLR_HOME" -su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-78.8.1.tgz" +su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-8.8.1.tgz" su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-8.8.1/server/solr && cp -r configsets/_default . && mv _default collection1" su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/8.8.1/schema*.xml $SOLR_HOME/solr-8.8.1/server/solr/collection1/conf/" su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/8.8.1/solrconfig.xml $SOLR_HOME/solr-8.8.1/server/solr/collection1/conf/solrconfig.xml" From d3cbab31d8fbb2bdb5861a95c9371b09e368864f Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 2 Mar 2021 10:15:24 -0500 Subject: [PATCH 395/576] #7373 update pom --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index de8ff91863a..717c0f6afa1 100644 --- a/pom.xml +++ b/pom.xml @@ -325,7 +325,7 @@ org.apache.solr solr-solrj - 7.7.2 + 8.8.1 org.apache.commons From 23b8bf6384eb50c9596f04a8caf4f633b4937596 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 2 Mar 2021 14:38:17 -0500 Subject: [PATCH 396/576] use default methods for isDisabled on User #2419 #4475 --- .../harvard/iq/dataverse/authorization/users/GuestUser.java | 5 ----- .../iq/dataverse/authorization/users/PrivateUrlUser.java | 5 ----- .../edu/harvard/iq/dataverse/authorization/users/User.java | 4 +++- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/users/GuestUser.java b/src/main/java/edu/harvard/iq/dataverse/authorization/users/GuestUser.java index 6c4d45d6674..16de1b2eaff 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/users/GuestUser.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/users/GuestUser.java @@ -33,11 +33,6 @@ public boolean isSuperuser() { return false; } - @Override - public boolean isDisabled() { - return false; - } - @Override public boolean equals( Object o ) { return (o instanceof GuestUser); diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/users/PrivateUrlUser.java b/src/main/java/edu/harvard/iq/dataverse/authorization/users/PrivateUrlUser.java index e318cf5ce8c..59c3240fdfa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/users/PrivateUrlUser.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/users/PrivateUrlUser.java @@ -47,11 +47,6 @@ public boolean isSuperuser() { return false; } - @Override - public boolean isDisabled() { - return false; - } - @Override public String getIdentifier() { return PREFIX + datasetId; diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/users/User.java b/src/main/java/edu/harvard/iq/dataverse/authorization/users/User.java index f3291a2a11f..baaf0e5f627 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/users/User.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/users/User.java @@ -14,6 +14,8 @@ public interface User extends RoleAssignee, Serializable { public boolean isSuperuser(); - public boolean isDisabled(); + default boolean isDisabled() { + return false; + } } From d46f6a783ae8990c76b0d92160f7663fb32b3f06 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Tue, 2 Mar 2021 15:54:40 -0500 Subject: [PATCH 397/576] Removed truncation plugin and attempted hacky css solution WIP [ref #6685] --- src/main/java/propertyFiles/Bundle.properties | 4 +- .../resources/js/dv_rebind_bootstrap_ui.js | 51 +++------ .../webapp/resources/js/jquery.truncate.js | 103 ------------------ 3 files changed, 18 insertions(+), 140 deletions(-) delete mode 100644 src/main/webapp/resources/js/jquery.truncate.js diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index fdaa68d3650..ce45c768d9b 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -59,12 +59,10 @@ apply=Apply add=Add delete=Delete copyClipboard=Copy to Clipboard - truncateMoreBtn=Read More truncateMoreTip=Click to read the full truncateLessBtn=Read Less -truncateLessTip=Click to collapse the - +truncateLessTip=Click to collapse the yes=Yes no=No previous=Previous diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index 16a5584ef01..1e1c6d5d6e3 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -166,47 +166,30 @@ function sharrre(){ */ function contentTruncate(truncSelector, truncMetaLabel, truncMoreBtn, truncMoreTip, truncLessBtn, truncLessTip){ - // jquery.truncate.js - // HTML-safe truncation - // https://github.com/pathable/truncate + // I MADE IT MYSELF... // WHAT ARE WE TRUNCATING // SELECTOR ID FROM PARAMETERS $('#' + truncSelector + ' td div').each(function () { - // GET THE DESC TEXT AND HTML - var truncateThis = $(this).html(); - + + // ADD A MAX-HEIGHT TO CONTAINER + $(this).css({'max-height':'250px','overflow-y':'hidden','position':'relative'}); + // BTN LABEL TEXT, DYNAMIC ARIA ATTR'S, FROM BUNDLE VIA PARAMETERS - var readMoreBtn = '... '; - var readLessBtn = ''; + var readMoreBtn = ''; + var moreFade = '
    ' + readMoreBtn + '
    '; - if ($(this).hasClass('thisWasTruncated')) { - // ALREADY TRUNCATED, DO NOTHING - } - else { - var thisWasTruncated = jQuery.truncate(truncateThis, { - length: 500, - words: true, - keepFirstWord: true, - ellipsis: readMoreBtn, - // REMOVED FINISH BLOCK TO TRUNCATE MID CONENT - // finishBlock: true - }); - - // ... add responsive img class ++ link target - $(this).toggleClass('thisWasTruncated').html(thisWasTruncated).find('img').attr('class', 'img-responsive'); + // add responsive img class + $(this).find('img').attr('class', 'img-responsive'); + + // add Read More + button background fade + $(this).append(moreFade); - // ... add full description to summary block on Read More link click ++ add responsive img class ++ link target - $(document).on('click', 'button.truncate-more-link', function() { - $(this).tooltip('hide').parent('div').toggleClass('thisWasTruncated').html(truncateThis).append(readLessBtn).find('img').attr('class', 'img-responsive'); - $('button.collapse-less-link').tooltip(); - }); - // ... add responsive img class ++ link target - $(document).on('click', 'button.collapse-less-link', function() { - $(this).tooltip('hide').parent('div').toggleClass('thisWasTruncated').html(thisWasTruncated).find('img').attr('class', 'img-responsive'); - $('button.truncate-more-link').tooltip(); - }); - } + // ... add full description to summary block on Read More link click ++ add responsive img class + $(document).on('click', 'button.desc-more-link', function() { + $(this).tooltip('hide').parent('div').parent('div').css({'max-height':'none','overflow-y':'visible','position':'relative'}); + $('#more-fade-block').remove(); + }); }); } diff --git a/src/main/webapp/resources/js/jquery.truncate.js b/src/main/webapp/resources/js/jquery.truncate.js deleted file mode 100644 index e43d4134ae1..00000000000 --- a/src/main/webapp/resources/js/jquery.truncate.js +++ /dev/null @@ -1,103 +0,0 @@ -(function($) { - - // Matches trailing non-space characters. - var chop = /(\s*\S+|\s)$/; - - // Matches the first word in the string. - var start = /^(\S*)/; - - // Return a truncated html string. Delegates to $.fn.truncate. - $.truncate = function(html, options) { - return $('
    ').append(html).truncate(options).html(); - }; - - // Truncate the contents of an element in place. - $.fn.truncate = function(options) { - if ($.isNumeric(options)) options = {length: options}; - var o = $.extend({}, $.truncate.defaults, options); - - return this.each(function() { - var self = $(this); - - if (o.noBreaks) self.find('br').replaceWith(' '); - - var text = self.text(); - var excess = text.length - o.length; - - if (o.stripTags) self.text(text); - - // Chop off any partial words if appropriate. - if (o.words && excess > 0) { - var truncated = text.slice(0, o.length).replace(chop, '').length; - - if (o.keepFirstWord && truncated === 0) { - excess = text.length - start.exec(text)[0].length - 1; - } else { - excess = text.length - truncated - 1; - } - } - - if (excess < 0 || !excess && !o.truncated) return; - - // Iterate over each child node in reverse, removing excess text. - $.each(self.contents().get().reverse(), function(i, el) { - var $el = $(el); - var text = $el.text(); - var length = text.length; - - // If the text is longer than the excess, remove the node and continue. - if (length <= excess) { - o.truncated = true; - excess -= length; - $el.remove(); - return; - } - - // Remove the excess text and append the ellipsis. - if (el.nodeType === 3) { - // should we finish the block anyway? - if (o.finishBlock) { - $(el.splitText(length)).replaceWith(o.ellipsis); - } else { - $(el.splitText(length - excess - 1)).replaceWith(o.ellipsis); - } - return false; - } - - // Recursively truncate child nodes. - $el.truncate($.extend(o, {length: length - excess})); - return false; - }); - }); - }; - - $.truncate.defaults = { - - // Strip all html elements, leaving only plain text. - stripTags: false, - - // Only truncate at word boundaries. - words: false, - - // When 'words' is active, keeps the first word in the string - // even if it's longer than a target length. - keepFirstWord: false, - - // Replace instances of
    with a single space. - noBreaks: false, - - // if true always truncate the content at the end of the block. - finishBlock: false, - - // The maximum length of the truncated html. - length: Infinity, - - // The character to use as the ellipsis. The word joiner (U+2060) can be - // used to prevent a hanging ellipsis, but displays incorrectly in Chrome - // on Windows 7. - // http://code.google.com/p/chromium/issues/detail?id=68323 - ellipsis: '\u2026' // '\u2060\u2026' - - }; - -})(jQuery); From e3e5cbe1468221b26180d11f3ce89df6111926b1 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Tue, 2 Mar 2021 16:22:12 -0500 Subject: [PATCH 398/576] Few minor code clean up edits for desc truncation [ref #6685] --- src/main/java/propertyFiles/Bundle.properties | 2 -- src/main/webapp/dataset.xhtml | 6 ++++-- src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js | 10 ++-------- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index ce45c768d9b..7ab054e18fc 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -61,8 +61,6 @@ delete=Delete copyClipboard=Copy to Clipboard truncateMoreBtn=Read More truncateMoreTip=Click to read the full -truncateLessBtn=Read Less -truncateLessTip=Click to collapse the yes=Yes no=No previous=Previous diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 3480e83ba57..1da3c2a69eb 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1656,8 +1656,10 @@ $(document).ready(function () { popoverHTML('#{bundle.htmlAllowedTitle}', '#{bundle.htmlAllowedTags}'); - // TO-DO MAKE DYNAMIC - contentTruncate('dsDescription', 'Description', '#{bundle.truncateMoreBtn}', '#{bundle.truncateMoreTip}', '#{bundle.truncateLessBtn}', '#{bundle.truncateLessTip}'); + // truncate summary description, contentTruncate function in dv_rebind_bootstrap_ui.js + var = 'dsDescription' + var descLabel = $('#' + descId + ' th').text(); + contentTruncate(descId, descLabel, '#{bundle.truncateMoreBtn}', '#{bundle.truncateMoreTip}'); }); function openDialog() { PF('details').show(); diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index 1e1c6d5d6e3..e0acdc77d4c 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -23,9 +23,6 @@ function bind_bsui_components(){ $('.bootstrap-button-tooltip, [data-toggle="tooltip"]').tooltip("hide"); $("[data-toggle='popover']").popover("hide"); - // Truncate content - contentTruncate(); - // Truncate checksums checksumTruncate(); @@ -162,13 +159,10 @@ function sharrre(){ } /* - * Truncate any metadata content + * Truncate dataset description content */ -function contentTruncate(truncSelector, truncMetaLabel, truncMoreBtn, truncMoreTip, truncLessBtn, truncLessTip){ - - // I MADE IT MYSELF... +function contentTruncate(truncSelector, truncMetaLabel, truncMoreBtn, truncMoreTip){ - // WHAT ARE WE TRUNCATING // SELECTOR ID FROM PARAMETERS $('#' + truncSelector + ' td div').each(function () { From d6e11732cb7cacd67ccd77e6f6ffb38eb0de8515 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 2 Mar 2021 17:05:33 -0500 Subject: [PATCH 399/576] run disable user command without superuser API token #2419 #4475 --- doc/sphinx-guides/source/api/native-api.rst | 2 +- .../java/edu/harvard/iq/dataverse/api/Admin.java | 6 +++++- .../harvard/iq/dataverse/api/DisableUsersIT.java | 16 ++++++---------- .../edu/harvard/iq/dataverse/api/UtilIT.java | 8 ++------ 4 files changed, 14 insertions(+), 18 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 9c4f5ffb70d..c99b903ff38 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3086,7 +3086,7 @@ Note: If the user has performed certain actions such as creating or contributing Disable a User ~~~~~~~~~~~~~~ -Disables a user. +Disables a user. A superuser API token is not required but the command will operate using the first superuser it finds. .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below. diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 4c39103ce3e..6e8ed301a68 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -404,8 +404,12 @@ public Response disableAuthenticatedUserById(@PathParam("id") Long id) { } private Response disableAuthenticatedUser(AuthenticatedUser userToDisable) { + AuthenticatedUser superuser = authSvc.getAdminUser(); + if (superuser == null) { + return error(Response.Status.INTERNAL_SERVER_ERROR, "Cannot find superuser to execute DisableUserCommand."); + } try { - execCommand(new DisableUserCommand(createDataverseRequest(findUserOrDie()), userToDisable)); + execCommand(new DisableUserCommand(createDataverseRequest(superuser), userToDisable)); return ok("User " + userToDisable.getIdentifier() + " disabled."); } catch (WrappedResponse ex) { return ex.getResponse(); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DisableUsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DisableUsersIT.java index bfba9388b93..d9c23b100e0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DisableUsersIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DisableUsersIT.java @@ -83,11 +83,7 @@ public void testDisableUser() { .body("data.traces.roleAssignments.items[0].definitionPointId", equalTo(dataverseId)) .body("data.traces.explicitGroups.items[0].name", equalTo("Group for " + dataverseAlias)); - Response failToDisableUserWithoutSuperuserApiToken = UtilIT.disableUser(username, apiToken); - failToDisableUserWithoutSuperuserApiToken.prettyPrint(); - failToDisableUserWithoutSuperuserApiToken.then().assertThat().statusCode(UNAUTHORIZED.getStatusCode()); - - Response disableUser = UtilIT.disableUser(username, superuserApiToken); + Response disableUser = UtilIT.disableUser(username); disableUser.prettyPrint(); disableUser.then().assertThat().statusCode(OK.getStatusCode()); @@ -152,7 +148,7 @@ public void testDisableUserById() { Long userId = JsonPath.from(createUser.body().asString()).getLong("data.user.id"); String apiToken = UtilIT.getApiTokenFromResponse(createUser); - Response disableUser = UtilIT.disableUser(userId, superuserApiToken); + Response disableUser = UtilIT.disableUser(userId); disableUser.prettyPrint(); disableUser.then().assertThat().statusCode(OK.getStatusCode()); } @@ -175,7 +171,7 @@ public void testMergeDisabledIntoEnabledUser() { createUserToMerge.prettyPrint(); String usernameToMerge = UtilIT.getUsernameFromResponse(createUserToMerge); - Response disableUser = UtilIT.disableUser(usernameToMerge, superuserApiToken); + Response disableUser = UtilIT.disableUser(usernameToMerge); disableUser.prettyPrint(); disableUser.then().assertThat().statusCode(OK.getStatusCode()); @@ -203,7 +199,7 @@ public void testMergeEnabledIntoDisabledUser() { createUserToMerge.prettyPrint(); String usernameToMerge = UtilIT.getUsernameFromResponse(createUserToMerge); - Response disableUser = UtilIT.disableUser(usernameMergeTarget, superuserApiToken); + Response disableUser = UtilIT.disableUser(usernameMergeTarget); disableUser.prettyPrint(); disableUser.then().assertThat().statusCode(OK.getStatusCode()); @@ -231,11 +227,11 @@ public void testMergeDisabledIntoDisabledUser() { createUserToMerge.prettyPrint(); String usernameToMerge = UtilIT.getUsernameFromResponse(createUserToMerge); - Response disableUserMergeTarget = UtilIT.disableUser(usernameMergeTarget, superuserApiToken); + Response disableUserMergeTarget = UtilIT.disableUser(usernameMergeTarget); disableUserMergeTarget.prettyPrint(); disableUserMergeTarget.then().assertThat().statusCode(OK.getStatusCode()); - Response disableUserToMerge = UtilIT.disableUser(usernameToMerge, superuserApiToken); + Response disableUserToMerge = UtilIT.disableUser(usernameToMerge); disableUserToMerge.prettyPrint(); disableUserToMerge.then().assertThat().statusCode(OK.getStatusCode()); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 54ce10640dc..db9f94bf28d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -961,18 +961,14 @@ static public Response grantRoleOnDataverse(String definitionPoint, String role, .post("api/dataverses/" + definitionPoint + "/assignments?key=" + apiToken); } - public static Response disableUser(String username, String apiToken) { -// public static Response disableUser(String username, String apiToken) { + public static Response disableUser(String username) { Response disableUserResponse = given() - .header(API_TOKEN_HTTP_HEADER, apiToken) -// .post("/api/users/" + username + "/disable"); .post("/api/admin/authenticatedUsers/" + username + "/disable"); return disableUserResponse; } - public static Response disableUser(Long userId, String apiToken) { + public static Response disableUser(Long userId) { Response disableUserResponse = given() - .header(API_TOKEN_HTTP_HEADER, apiToken) .post("/api/admin/authenticatedUsers/id/" + userId + "/disable"); return disableUserResponse; } From f9e94835939a94898bb2595b84a12ce362be8503 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Wed, 3 Mar 2021 09:58:14 -0500 Subject: [PATCH 400/576] Fixed broken var in truncation code on dataset pg, other code clean up [ref #6685] --- src/main/webapp/dataset.xhtml | 2 +- src/main/webapp/resources/css/structure.css | 17 +++++++++++++---- .../resources/js/dv_rebind_bootstrap_ui.js | 12 ++++++------ 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 1da3c2a69eb..2ff088c5220 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1657,7 +1657,7 @@ popoverHTML('#{bundle.htmlAllowedTitle}', '#{bundle.htmlAllowedTags}'); // truncate summary description, contentTruncate function in dv_rebind_bootstrap_ui.js - var = 'dsDescription' + var descId = 'dsDescription'; var descLabel = $('#' + descId + ' th').text(); contentTruncate(descId, descLabel, '#{bundle.truncateMoreBtn}', '#{bundle.truncateMoreTip}'); }); diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index c531950d19a..cd6785679ae 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -628,11 +628,17 @@ div.edit-field div.ui-message {margin:6px 0;} .metadata-container table.metadata th {width:25%;vertical-align:top;padding:6px 12px;} .metadata-container table.metadata td {padding:6px 12px;vertical-align:top;} -/* TO-DO FIX max-width:1199px FOR SUMMARY BLOCK ONLY, NOT OTHER PAGES */ +/* STACK METADATA TABLE LABELS + VALUES FOR SM, XS */ +@media(max-width:767px){ + .metadata-container:not(#dataset-summary-metadata) table.metadata th, .metadata-container:not(#dataset-summary-metadata) table.metadata td {display:block;width:100%;} + .metadata-container:not(#dataset-summary-metadata) table.metadata th {padding-bottom:0;} + .metadata-container:not(#dataset-summary-metadata) table.metadata td {padding-left:30px;} +} +/* STACK METADATA TABLE LABELS + VALUES FOR MD, SM, XS ... DATASET-SUMMARY-BLOCK ONLY */ @media(max-width:1199px){ - .metadata-container table.metadata th, .metadata-container table.metadata td {display:block;width:100%;} - .metadata-container table.metadata th {padding-bottom:0;} - .metadata-container table.metadata td {padding-left:30px;} + #dataset-summary-metadata.metadata-container table.metadata th, #dataset-summary-metadata.metadata-container table.metadata td {display:block;width:100%;} + #dataset-summary-metadata.metadata-container table.metadata th {padding-bottom:0;} + #dataset-summary-metadata.metadata-container table.metadata td {padding-left:30px;} } .panel-body.metadata-container {padding:15px 0;} .metadata-logo {max-width:180px;max-height:40px;} @@ -646,6 +652,9 @@ div.edit-field div.ui-message {margin:6px 0;} .metadata-container pre { white-space: pre-wrap !important; } +/* TRUNCATION */ +div.more-fade-block {text-align:center; padding-top:250px; width:100%; position:absolute; bottom:0; background:linear-gradient(180deg,hsla(0,0%,100%,0),#fff 80%);} +button.desc-more-link {margin:0; padding:0;} /* Primefaces vs Bootstrap selectOneMenu fix */ div.ui-selectonemenu.form-control {padding:0px;} diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index e0acdc77d4c..6d9d42c276c 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -22,9 +22,6 @@ function bind_bsui_components(){ // Hide open tooltips + popovers $('.bootstrap-button-tooltip, [data-toggle="tooltip"]').tooltip("hide"); $("[data-toggle='popover']").popover("hide"); - - // Truncate checksums - checksumTruncate(); // Tooltips + popovers bind_tooltip_popover(); @@ -32,6 +29,9 @@ function bind_bsui_components(){ // Disabled pagination links disabledLinks(); + // Truncate checksums + checksumTruncate(); + // Sharrre sharrre(); @@ -170,8 +170,8 @@ function contentTruncate(truncSelector, truncMetaLabel, truncMoreBtn, truncMoreT $(this).css({'max-height':'250px','overflow-y':'hidden','position':'relative'}); // BTN LABEL TEXT, DYNAMIC ARIA ATTR'S, FROM BUNDLE VIA PARAMETERS - var readMoreBtn = ''; - var moreFade = '
    ' + readMoreBtn + '
    '; + var readMoreBtn = ''; + var moreFade = '
    ' + readMoreBtn + '
    '; // add responsive img class $(this).find('img').attr('class', 'img-responsive'); @@ -182,7 +182,7 @@ function contentTruncate(truncSelector, truncMetaLabel, truncMoreBtn, truncMoreT // ... add full description to summary block on Read More link click ++ add responsive img class $(document).on('click', 'button.desc-more-link', function() { $(this).tooltip('hide').parent('div').parent('div').css({'max-height':'none','overflow-y':'visible','position':'relative'}); - $('#more-fade-block').remove(); + $(this).parent('div.more-fade-block').remove(); }); }); } From fb406a726fbf961ed2c1cbacbf6ee0c635556e63 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Wed, 3 Mar 2021 10:14:06 -0500 Subject: [PATCH 401/576] Fixed truncation function to only apply read more button for desc greater than 250px in height [ref #6685] --- .../resources/js/dv_rebind_bootstrap_ui.js | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index 6d9d42c276c..2b32be6de0b 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -166,24 +166,30 @@ function contentTruncate(truncSelector, truncMetaLabel, truncMoreBtn, truncMoreT // SELECTOR ID FROM PARAMETERS $('#' + truncSelector + ' td div').each(function () { - // ADD A MAX-HEIGHT TO CONTAINER - $(this).css({'max-height':'250px','overflow-y':'hidden','position':'relative'}); - - // BTN LABEL TEXT, DYNAMIC ARIA ATTR'S, FROM BUNDLE VIA PARAMETERS - var readMoreBtn = ''; - var moreFade = '
    ' + readMoreBtn + '
    '; - - // add responsive img class + // add responsive img class to limit width to that of container $(this).find('img').attr('class', 'img-responsive'); - // add Read More + button background fade - $(this).append(moreFade); + // find container height + var containerHeight = $(this).outerHeight(); + + if (containerHeight > 250) { + // ADD A MAX-HEIGHT TO CONTAINER + $(this).css({'max-height':'250px','overflow-y':'hidden','position':'relative'}); - // ... add full description to summary block on Read More link click ++ add responsive img class - $(document).on('click', 'button.desc-more-link', function() { - $(this).tooltip('hide').parent('div').parent('div').css({'max-height':'none','overflow-y':'visible','position':'relative'}); - $(this).parent('div.more-fade-block').remove(); - }); + // BTN LABEL TEXT, DYNAMIC ARIA ATTR'S, FROM BUNDLE VIA PARAMETERS + var readMoreBtn = ''; + var moreFade = '
    ' + readMoreBtn + '
    '; + + // add Read More + button background fade + $(this).append(moreFade); + + // ... add full description to summary block on Read More link click ++ add responsive img class + $(document).on('click', 'button.desc-more-link', function() { + $(this).tooltip('hide').parent('div').parent('div').css({'max-height':'none','overflow-y':'visible','position':'relative'}); + $(this).parent('div.more-fade-block').remove(); + }); + } + }); } From 2a2f7bfc004f46da88287d046a409c72f87f0656 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Wed, 3 Mar 2021 11:35:35 -0500 Subject: [PATCH 402/576] Improved truncation btn label text and tooltip by passing dynamic label text [ref #6685] --- src/main/java/propertyFiles/Bundle.properties | 4 ++-- src/main/webapp/dataset.xhtml | 11 ++++++++--- .../webapp/resources/js/dv_rebind_bootstrap_ui.js | 7 ++----- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 7ab054e18fc..b386e8084f4 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -59,8 +59,8 @@ apply=Apply add=Add delete=Delete copyClipboard=Copy to Clipboard -truncateMoreBtn=Read More -truncateMoreTip=Click to read the full +truncateMoreBtn=Read full {0} [+] +truncateMoreTip=Click to read the full {0}. yes=Yes no=No previous=Previous diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 2ff088c5220..949c97ceb4d 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -9,7 +9,8 @@ xmlns:jsf="http://xmlns.jcp.org/jsf" xmlns:pt="http://java.sun.com/jsf/passthrough" xmlns:cc="http://java.sun.com/jsf/composite" - xmlns:o="http://omnifaces.org/ui"> + xmlns:o="http://omnifaces.org/ui" + xmlns:of="http://omnifaces.org/functions"> @@ -1657,9 +1658,13 @@ popoverHTML('#{bundle.htmlAllowedTitle}', '#{bundle.htmlAllowedTags}'); // truncate summary description, contentTruncate function in dv_rebind_bootstrap_ui.js + // add summary desc row id to pass as function selector var descId = 'dsDescription'; - var descLabel = $('#' + descId + ' th').text(); - contentTruncate(descId, descLabel, '#{bundle.truncateMoreBtn}', '#{bundle.truncateMoreTip}'); + // add metadata dynamic label text to button and tooltip text from bundle + var truncMoreBtn = '#{of:format1(bundle.truncateMoreBtn, DatasetPage.datasetVersionUI.description.datasetFieldType.localeTitle)}'; + var truncMoreTip = '#{of:format1(bundle.truncateMoreTip, DatasetPage.datasetVersionUI.description.datasetFieldType.localeTitle)}'; + // pass text params to function + contentTruncate(descId, truncMoreBtn, truncMoreTip); }); function openDialog() { PF('details').show(); diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index 2b32be6de0b..a5a77009782 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -161,11 +161,9 @@ function sharrre(){ /* * Truncate dataset description content */ -function contentTruncate(truncSelector, truncMetaLabel, truncMoreBtn, truncMoreTip){ - +function contentTruncate(truncSelector, truncMoreBtn, truncMoreTip){ // SELECTOR ID FROM PARAMETERS $('#' + truncSelector + ' td div').each(function () { - // add responsive img class to limit width to that of container $(this).find('img').attr('class', 'img-responsive'); @@ -177,7 +175,7 @@ function contentTruncate(truncSelector, truncMetaLabel, truncMoreBtn, truncMoreT $(this).css({'max-height':'250px','overflow-y':'hidden','position':'relative'}); // BTN LABEL TEXT, DYNAMIC ARIA ATTR'S, FROM BUNDLE VIA PARAMETERS - var readMoreBtn = ''; + var readMoreBtn = ''; var moreFade = '
    ' + readMoreBtn + '
    '; // add Read More + button background fade @@ -189,7 +187,6 @@ function contentTruncate(truncSelector, truncMetaLabel, truncMoreBtn, truncMoreT $(this).parent('div.more-fade-block').remove(); }); } - }); } From 6e61841cb6a5d870cdd6c16ae718c1fdf1485f3c Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 3 Mar 2021 15:45:24 -0500 Subject: [PATCH 403/576] remove password reset data with cascade #7575 --- .../authorization/providers/builtin/BuiltinUser.java | 7 +++++++ .../providers/builtin/BuiltinUserServiceBean.java | 2 -- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/BuiltinUser.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/BuiltinUser.java index fd7231e827c..9ae4e4b0e87 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/BuiltinUser.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/BuiltinUser.java @@ -3,8 +3,10 @@ import edu.harvard.iq.dataverse.ValidateEmail; import edu.harvard.iq.dataverse.ValidateUserName; import edu.harvard.iq.dataverse.authorization.AuthenticatedUserDisplayInfo; +import edu.harvard.iq.dataverse.passwordreset.PasswordResetData; import static edu.harvard.iq.dataverse.util.StringUtil.nonEmpty; import java.io.Serializable; +import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.GeneratedValue; @@ -13,6 +15,7 @@ import javax.persistence.Index; import javax.persistence.NamedQueries; import javax.persistence.NamedQuery; +import javax.persistence.OneToOne; import javax.persistence.Table; import javax.persistence.Transient; import javax.validation.constraints.Size; @@ -47,6 +50,10 @@ public class BuiltinUser implements Serializable { private String userName; private int passwordEncryptionVersion; + + @OneToOne(mappedBy = "builtinUser", cascade = {CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST}) + private PasswordResetData passwordResetData; + private String encryptedPassword; /** diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/BuiltinUserServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/BuiltinUserServiceBean.java index 7804716561b..c39c7cb2985 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/BuiltinUserServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/BuiltinUserServiceBean.java @@ -79,8 +79,6 @@ public BuiltinUser find(Long pk) { public void removeUser( String userName ) { final BuiltinUser user = findByUserName(userName); if ( user != null ) { - // TODO: Consider adding a cascade delete instead. - passwordResetService.deleteResetDataByDataverseUser(user); em.remove(user); } } From ed6f57583f10879659ab8ab9079b1d345d2957f8 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 3 Mar 2021 16:08:12 -0500 Subject: [PATCH 404/576] #7373 remove old version --- conf/solr/7.7.2/readme.md | 1 - conf/solr/7.7.2/schema.xml | 1198 ------------------ conf/solr/7.7.2/schema_dv_mdb_copies.xml | 161 --- conf/solr/7.7.2/schema_dv_mdb_fields.xml | 161 --- conf/solr/7.7.2/solrconfig.xml | 1410 ---------------------- conf/solr/7.7.2/updateSchemaMDB.sh | 85 -- 6 files changed, 3016 deletions(-) delete mode 100644 conf/solr/7.7.2/readme.md delete mode 100644 conf/solr/7.7.2/schema.xml delete mode 100644 conf/solr/7.7.2/schema_dv_mdb_copies.xml delete mode 100644 conf/solr/7.7.2/schema_dv_mdb_fields.xml delete mode 100644 conf/solr/7.7.2/solrconfig.xml delete mode 100755 conf/solr/7.7.2/updateSchemaMDB.sh diff --git a/conf/solr/7.7.2/readme.md b/conf/solr/7.7.2/readme.md deleted file mode 100644 index 4457cf9a7df..00000000000 --- a/conf/solr/7.7.2/readme.md +++ /dev/null @@ -1 +0,0 @@ -Please see the dev guide for what to do with Solr config files. \ No newline at end of file diff --git a/conf/solr/7.7.2/schema.xml b/conf/solr/7.7.2/schema.xml deleted file mode 100644 index da40a8e99fa..00000000000 --- a/conf/solr/7.7.2/schema.xml +++ /dev/null @@ -1,1198 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/conf/solr/7.7.2/schema_dv_mdb_copies.xml b/conf/solr/7.7.2/schema_dv_mdb_copies.xml deleted file mode 100644 index 080cc71ef50..00000000000 --- a/conf/solr/7.7.2/schema_dv_mdb_copies.xml +++ /dev/null @@ -1,161 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/conf/solr/7.7.2/schema_dv_mdb_fields.xml b/conf/solr/7.7.2/schema_dv_mdb_fields.xml deleted file mode 100644 index 3f844c6183c..00000000000 --- a/conf/solr/7.7.2/schema_dv_mdb_fields.xml +++ /dev/null @@ -1,161 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/conf/solr/7.7.2/solrconfig.xml b/conf/solr/7.7.2/solrconfig.xml deleted file mode 100644 index 51100af7d6e..00000000000 --- a/conf/solr/7.7.2/solrconfig.xml +++ /dev/null @@ -1,1410 +0,0 @@ - - - - - - - - - 7.3.0 - - - - - - - - - - - - - - - - - - - - ${solr.data.dir:} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.lock.type:native} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} - - - - - ${solr.autoCommit.maxTime:15000} - false - - - - - - ${solr.autoSoftCommit.maxTime:-1} - - - - - - - - - - - - - - 1024 - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - 20 - - - 200 - - - - - - - - - - - - - - - - false - - - - - - - - - - - - - - - - - - - - - - explicit - 10 - edismax - 0.075 - - dvName^400 - authorName^180 - dvSubject^190 - dvDescription^180 - dvAffiliation^170 - title^130 - subject^120 - keyword^110 - topicClassValue^100 - dsDescriptionValue^90 - authorAffiliation^80 - publicationCitation^60 - producerName^50 - fileName^30 - fileDescription^30 - variableLabel^20 - variableName^10 - _text_^1.0 - - - dvName^200 - authorName^100 - dvSubject^100 - dvDescription^100 - dvAffiliation^100 - title^75 - subject^75 - keyword^75 - topicClassValue^75 - dsDescriptionValue^75 - authorAffiliation^75 - publicationCitation^75 - producerName^75 - - - - isHarvested:false^25000 - - - - - - - - - - - - - - - - - - explicit - json - true - - - - - - - - explicit - - - - - - _text_ - - - - - - - true - ignored_ - _text_ - - - - - - - - - text_general - - - - - - default - _text_ - solr.DirectSolrSpellChecker - - internal - - 0.5 - - 2 - - 1 - - 5 - - 4 - - 0.01 - - - - - - - - - - - - default - on - true - 10 - 5 - 5 - true - true - 10 - 5 - - - spellcheck - - - - - - - - - - true - - - tvComponent - - - - - - - - - - - - true - false - - - terms - - - - - - - - string - - - - - - explicit - - - elevator - - - - - - - - - - - 100 - - - - - - - - 70 - - 0.5 - - [-\w ,/\n\"']{20,200} - - - - - - - ]]> - ]]> - - - - - - - - - - - - - - - - - - - - - - - - ,, - ,, - ,, - ,, - ,]]> - ]]> - - - - - - 10 - .,!? - - - - - - - WORD - - - en - US - - - - - - - - - - - - - - [^\w-\.] - _ - - - - - - - yyyy-MM-dd'T'HH:mm:ss.SSSZ - yyyy-MM-dd'T'HH:mm:ss,SSSZ - yyyy-MM-dd'T'HH:mm:ss.SSS - yyyy-MM-dd'T'HH:mm:ss,SSS - yyyy-MM-dd'T'HH:mm:ssZ - yyyy-MM-dd'T'HH:mm:ss - yyyy-MM-dd'T'HH:mmZ - yyyy-MM-dd'T'HH:mm - yyyy-MM-dd HH:mm:ss.SSSZ - yyyy-MM-dd HH:mm:ss,SSSZ - yyyy-MM-dd HH:mm:ss.SSS - yyyy-MM-dd HH:mm:ss,SSS - yyyy-MM-dd HH:mm:ssZ - yyyy-MM-dd HH:mm:ss - yyyy-MM-dd HH:mmZ - yyyy-MM-dd HH:mm - yyyy-MM-dd - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/plain; charset=UTF-8 - - - - - ${velocity.template.base.dir:} - ${velocity.solr.resource.loader.enabled:true} - ${velocity.params.resource.loader.enabled:false} - - - - - 5 - - - - - - - - - - - - - - diff --git a/conf/solr/7.7.2/updateSchemaMDB.sh b/conf/solr/7.7.2/updateSchemaMDB.sh deleted file mode 100755 index 0044f15c7cd..00000000000 --- a/conf/solr/7.7.2/updateSchemaMDB.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# This script updates the and schema configuration necessary to properly -# index custom metadata fields in Solr. -# 1. Retrieve from Dataverse API endpoint -# 2. Parse and write Solr schema files (which might replace the included files) -# 3. Reload Solr -# -# List of variables: -# ${DATAVERSE_URL}: URL to Dataverse. Defaults to http://localhost:8080 -# ${SOLR_URL}: URL to Solr. Defaults to http://localhost:8983 -# ${UNBLOCK_KEY}: File path to secret or unblock key as string. Only necessary on k8s or when you secured your installation. -# ${TARGET}: Directory where to write the XML files. Defaults to /tmp -# -# Programs used (need to be available on your PATH): -# coreutils: mktemp, csplit -# curl - -usage() { - echo "usage: updateSchemaMDB.sh [options]" - echo "options:" - echo " -d Dataverse URL, defaults to http://localhost:8080" - echo " -h Show this help text" - echo " -s Solr URL, defaults to http://localhost:8983" - echo " -t Directory where to write the XML files. Defaults to /tmp" - echo " -u Dataverse unblock key either as key string or path to keyfile" -} - -### Init (with sane defaults) -DATAVERSE_URL=${DATAVERSE_URL:-"http://localhost:8080"} -SOLR_URL=${SOLR_URL:-"http://localhost:8983"} -TARGET=${TARGET:-"/tmp"} -UNBLOCK_KEY=${UNBLOCK_KEY:-""} - -# if cmdline args are given, override any env var setting (or defaults) -while getopts ":d:hs:t:u:" opt -do - case $opt in - d) DATAVERSE_URL=${OPTARG};; - h) usage; exit 0;; - s) SOLR_URL=${OPTARG};; - t) TARGET=${OPTARG};; - u) UNBLOCK_KEY=${OPTARG};; - :) echo "Missing option argument for -${OPTARG}. Use -h for help." >&2; exit 1;; - \?) echo "Unknown option -${OPTARG}." >&2; usage; exit 1;; - esac -done - -# Special handling of unblock key depending on referencing a secret file or key in var -if [ ! -z "${UNBLOCK_KEY}" ]; then - if [ -f "${UNBLOCK_KEY}" ]; then - UNBLOCK_KEY="?unblock-key=$(cat ${UNBLOCK_KEY})" - else - UNBLOCK_KEY="?unblock-key=${UNBLOCK_KEY}" - fi -fi - -### Retrieval -echo "Retrieve schema data from ${DATAVERSE_URL}/api/admin/index/solr/schema" -TMPFILE=`mktemp` -curl -f -sS "${DATAVERSE_URL}/api/admin/index/solr/schema${UNBLOCK_KEY}" > $TMPFILE - -### Fail gracefull if Dataverse is not ready yet. -if [[ "`wc -l ${TMPFILE}`" < "3" ]]; then - echo "Dataverse responded with empty file. When running on K8s: did you bootstrap yet?" - exit 123 -fi - -### Processing -echo "Writing ${TARGET}/schema_dv_mdb_fields.xml" -echo "" > ${TARGET}/schema_dv_mdb_fields.xml -cat ${TMPFILE} | grep ".*> ${TARGET}/schema_dv_mdb_fields.xml -echo "" >> ${TARGET}/schema_dv_mdb_fields.xml - -echo "Writing ${TARGET}/schema_dv_mdb_copies.xml" -echo "" > ${TARGET}/schema_dv_mdb_copies.xml -cat ${TMPFILE} | grep ".*> ${TARGET}/schema_dv_mdb_copies.xml -echo "" >> ${TARGET}/schema_dv_mdb_copies.xml - -rm ${TMPFILE}* - -### Reloading -echo "Triggering Solr RELOAD at ${SOLR_URL}/solr/admin/cores?action=RELOAD&core=collection1" -curl -f -sS "${SOLR_URL}/solr/admin/cores?action=RELOAD&core=collection1" From 8afbf5e929de840f3e1683ee07176e18541c54d8 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 3 Mar 2021 16:32:23 -0500 Subject: [PATCH 405/576] remove oauth2tokendata with a cascade #2419 #4475 --- .../dataverse/authorization/AuthenticationServiceBean.java | 1 - .../iq/dataverse/authorization/users/AuthenticatedUser.java | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java index c34015a5dda..5b11dd1b1cb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java @@ -220,7 +220,6 @@ public void deleteAuthenticatedUser(Object pk) { em.remove(confirmEmailData); } userNotificationService.findByUser(user.getId()).forEach(userNotificationService::delete); - em.createNativeQuery("delete from OAuth2TokenData where user_id =" + user.getId()).executeUpdate(); AuthenticationProvider prv = lookupProvider(user); if ( prv != null && prv.isUserDeletionAllowed() ) { diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java index ae485103be6..03069536287 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java @@ -7,6 +7,7 @@ import edu.harvard.iq.dataverse.authorization.AccessRequest; import edu.harvard.iq.dataverse.authorization.AuthenticatedUserDisplayInfo; import edu.harvard.iq.dataverse.authorization.AuthenticatedUserLookup; +import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2TokenData; import edu.harvard.iq.dataverse.userdata.UserUtil; import edu.harvard.iq.dataverse.authorization.providers.oauth2.impl.OrcidOAuth2AP; import edu.harvard.iq.dataverse.util.BundleUtil; @@ -166,7 +167,10 @@ public List getDatasetLocks() { public void setDatasetLocks(List datasetLocks) { this.datasetLocks = datasetLocks; } - + + @OneToMany(mappedBy = "user", cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST}) + private List oAuth2TokenDatas; + @Override public AuthenticatedUserDisplayInfo getDisplayInfo() { return new AuthenticatedUserDisplayInfo(firstName, lastName, email, affiliation, position); From 610e9ca32e4c0d883f3d160625bfac40d7ab34ef Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 4 Mar 2021 11:11:51 -0500 Subject: [PATCH 406/576] merge configureSessionTimeout into setUser #2419 #4475 --- .../iq/dataverse/DataverseSession.java | 23 ++++++++----------- .../edu/harvard/iq/dataverse/LoginPage.java | 1 - .../java/edu/harvard/iq/dataverse/Shib.java | 1 - .../providers/builtin/DataverseUserPage.java | 1 - .../oauth2/OAuth2FirstLoginPage.java | 2 -- .../oauth2/OAuth2LoginBackingBean.java | 1 - .../confirmemail/ConfirmEmailPage.java | 1 - .../passwordreset/PasswordResetPage.java | 1 - .../dataverse/privateurl/PrivateUrlPage.java | 1 - 9 files changed, 10 insertions(+), 22 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java b/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java index fcf8da22f7e..5093cd99b3f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java @@ -93,6 +93,9 @@ public User getUser() { return user; } + /** + * Sets the user and configures the session timeout. + */ public void setUser(User aUser) { // We check for disabled status here in "setUser" to ensure a common user // experience across Builtin, Shib, OAuth, and OIDC users. @@ -108,6 +111,7 @@ public void setUser(User aUser) { // Log the login/logout and Change the session id if we're using the UI and have // a session, versus an API call with no session - (i.e. /admin/submitToArchive() // which sets the user in the session to pass it through to the underlying command) + // TODO: reformat to remove tabs etc. if(context != null) { logSvc.log( new ActionLogRecord(ActionLogRecord.ActionType.SessionManagement,(aUser==null) ? "logout" : "login") @@ -115,6 +119,12 @@ public void setUser(User aUser) { //#3254 - change session id when user changes SessionUtil.changeSessionId((HttpServletRequest) context.getExternalContext().getRequest()); + HttpSession httpSession = (HttpSession) context.getExternalContext().getSession(false); + if (httpSession != null) { + // Configure session timeout. + logger.fine("jsession: " + httpSession.getId() + " setting the lifespan of the session to " + systemConfig.getLoginSessionTimeout() + " minutes"); + httpSession.setMaxInactiveInterval(systemConfig.getLoginSessionTimeout() * 60); // session timeout, in seconds + } } this.user = aUser; } @@ -219,18 +229,5 @@ public void dismissMessage(BannerMessage message){ } } - - public void configureSessionTimeout() { - if (user instanceof GuestUser) { - return; - } - HttpSession httpSession = (HttpSession) FacesContext.getCurrentInstance().getExternalContext().getSession(false); - - if (httpSession != null) { - logger.fine("jsession: "+httpSession.getId()+" setting the lifespan of the session to " + systemConfig.getLoginSessionTimeout() + " minutes"); - httpSession.setMaxInactiveInterval(systemConfig.getLoginSessionTimeout() * 60); // session timeout, in seconds - } - - } } diff --git a/src/main/java/edu/harvard/iq/dataverse/LoginPage.java b/src/main/java/edu/harvard/iq/dataverse/LoginPage.java index 38376fa84c0..166c0c081d0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/LoginPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/LoginPage.java @@ -171,7 +171,6 @@ public String login() { AuthenticatedUser r = authSvc.getUpdateAuthenticatedUser(credentialsAuthProviderId, authReq); logger.log(Level.FINE, "User authenticated: {0}", r.getEmail()); session.setUser(r); - session.configureSessionTimeout(); if ("dataverse.xhtml".equals(redirectPage)) { redirectPage = redirectToRoot(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/Shib.java b/src/main/java/edu/harvard/iq/dataverse/Shib.java index 4bd38d2377d..cd3b4a30f52 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Shib.java +++ b/src/main/java/edu/harvard/iq/dataverse/Shib.java @@ -360,7 +360,6 @@ private void logInUserAndSetShibAttributes(AuthenticatedUser au) { au.setShibIdentityProvider(shibIdp); // setUser checks for disabled users. session.setUser(au); - session.configureSessionTimeout(); logger.fine("Groups for user " + au.getId() + " (" + au.getIdentifier() + "): " + getGroups(au)); } diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java index 204d93b5b8f..663cdf4b4b2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java @@ -327,7 +327,6 @@ public String save() { // Authenticated user registered. Save the new bulitin, and log in. builtinUserService.save(builtinUser); session.setUser(au); - session.configureSessionTimeout(); /** * @todo Move this to * AuthenticationServiceBean.createAuthenticatedUser diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2FirstLoginPage.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2FirstLoginPage.java index a3ce3c5bdf7..bd65e296d52 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2FirstLoginPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2FirstLoginPage.java @@ -183,7 +183,6 @@ public String createNewAccount() { newUser.getDisplayInfo().getPosition()); final AuthenticatedUser user = authenticationSvc.createAuthenticatedUser(newUser.getUserRecordIdentifier(), getUsername(), newAud, true); session.setUser(user); - session.configureSessionTimeout(); /** * @todo Move this to AuthenticationServiceBean.createAuthenticatedUser */ @@ -213,7 +212,6 @@ public String convertExistingAccount() { builtinUserSvc.removeUser(existingUser.getUserIdentifier()); session.setUser(existingUser); - session.configureSessionTimeout(); AuthenticationProvider newUserAuthProvider = authenticationSvc.getAuthenticationProvider(newUser.getServiceId()); JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("oauth2.convertAccount.success", Arrays.asList(newUserAuthProvider.getInfo().getTitle()))); diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java index c034f7ebc8b..8daa6b2c17c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java @@ -108,7 +108,6 @@ public void exchangeCodeForToken() throws IOException { // login the user and redirect to HOME of intended page (if any). // setUser checks for disabled users. session.setUser(dvUser); - session.configureSessionTimeout(); final OAuth2TokenData tokenData = oauthUser.getTokenData(); if (tokenData != null) { tokenData.setUser(dvUser); diff --git a/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailPage.java b/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailPage.java index 45a04ba4185..823d2c111f2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailPage.java @@ -55,7 +55,6 @@ public String init() { if (confirmEmailData != null) { user = confirmEmailData.getAuthenticatedUser(); session.setUser(user); - session.configureSessionTimeout(); // TODO: is this needed here? (it can't hurt, but still) JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("confirmEmail.details.success")); return "/dataverse.xhtml?faces-redirect=true"; } diff --git a/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetPage.java b/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetPage.java index b5cef3731f5..f27621cef74 100644 --- a/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetPage.java @@ -148,7 +148,6 @@ public String resetPassword() { String builtinAuthProviderId = BuiltinAuthenticationProvider.PROVIDER_ID; AuthenticatedUser au = authSvc.lookupUser(builtinAuthProviderId, user.getUserName()); session.setUser(au); - session.configureSessionTimeout(); return "/dataverse.xhtml?alias=" + dataverseService.findRootDataverse().getAlias() + "faces-redirect=true"; } else { FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_ERROR, response.getMessageSummary(), response.getMessageDetail())); diff --git a/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlPage.java b/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlPage.java index e8bc9fc3da7..b0658f10b34 100644 --- a/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlPage.java @@ -35,7 +35,6 @@ public String init() { String draftDatasetPageToBeRedirectedTo = privateUrlRedirectData.getDraftDatasetPageToBeRedirectedTo() + "&faces-redirect=true"; PrivateUrlUser privateUrlUser = privateUrlRedirectData.getPrivateUrlUser(); session.setUser(privateUrlUser); - session.configureSessionTimeout(); logger.info("Redirecting PrivateUrlUser '" + privateUrlUser.getIdentifier() + "' to " + draftDatasetPageToBeRedirectedTo); return draftDatasetPageToBeRedirectedTo; } catch (Exception ex) { From 17b85ea3755cc73325fc3c17c6b2fa1b87f3cfb6 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 4 Mar 2021 11:17:22 -0500 Subject: [PATCH 407/576] Changing the language on required postgres versions/announcing support for later versions. (#7374) --- doc/release-notes/7374-postgresql-update | 9 ++++++++ .../source/installation/prerequisites.rst | 23 ++++++------------- pom.xml | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) create mode 100644 doc/release-notes/7374-postgresql-update diff --git a/doc/release-notes/7374-postgresql-update b/doc/release-notes/7374-postgresql-update new file mode 100644 index 00000000000..332ffda0060 --- /dev/null +++ b/doc/release-notes/7374-postgresql-update @@ -0,0 +1,9 @@ +## Notes for Dataverse Administrators + +Up until this release our installation guide "strongly recommended" to install PostgreSQL v. 9.6. While that version is known to be very stable, it is nearing its end-of-life (in Nov. 2021). Dataverse Software has now been tested with versions up to 13. If you decide to upgrade PostgreSQL, the tested and recommended way of doing that is as follows: + +- Export your current database with ``pg_dumpall``; +- Install the new version of PostgreSQL; (make sure it's running on the same port, etc. so that no changes are needed in the Payara configuration) +- Re-import the database with ``psql``, as the postgres user. + +Consult the PostgreSQL upgrde documentation for more information, for example https://www.postgresql.org/docs/13/upgrading.html#UPGRADING-VIA-PGDUMPALL. diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index b1643fabf40..2606ad5fbb5 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -98,25 +98,16 @@ PostgreSQL Installing PostgreSQL ======================= -Version 9.6 is strongly recommended because it is the version developers and QA test with:: +The application has been tested with PostgreSQL versions up to 13. We recommend installing the latest version that is available for your OS distribution. *For example*, to install PostgreSQL 13 under RHEL/CentOS 7:: # yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm # yum makecache fast - # yum install -y postgresql96-server - # /usr/pgsql-9.6/bin/postgresql96-setup initdb - # /usr/bin/systemctl start postgresql-9.6 - # /usr/bin/systemctl enable postgresql-9.6 + # yum install -y postgresql13-server + # /usr/pgsql-13/bin/postgresql-13-setup initdb + # /usr/bin/systemctl start postgresql-13 + # /usr/bin/systemctl enable postgresql-13 -The above steps are specific to RHEL/CentOS 7. For RHEL/CentOS 8 you must install Postgres 10 or higher:: - - # yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm - # yum makecache fast - # yum install -y postgresql10-server - # /usr/pgsql-10/bin/postgresql-10-setup initdb - # systemctl start postgresql-10 - # systemctl enable postgresql-10 - -Note that the Dataverse Software installer includes its own Postgres JDBC driver. If you choose to install the newest version of Postgres (12 as of this writing), you may need to grab a current JDBC driver from https://jdbc.postgresql.org/download.html before launching the install script. +For RHEL/CentOS 8 the process would be identical, except for the very first command - you would need to install the "EL-8" yum repository configuration instead. Configuring Database Access for the Dataverse Installation (and the Dataverse Software Installer) ================================================================================================= @@ -152,7 +143,7 @@ Configuring Database Access for the Dataverse Installation (and the Dataverse So - **Important: PostgreSQL must be restarted** for the configuration changes to take effect! On RHEL/CentOS 7 and similar (provided you installed Postgres as instructed above):: - # systemctl restart postgresql-9.6 + # systemctl restart postgresql-13 On MacOS X a "Reload Configuration" icon is usually supplied in the PostgreSQL application folder. Or you could look up the process id of the PostgreSQL postmaster process, and send it the SIGHUP signal:: diff --git a/pom.xml b/pom.xml index de8ff91863a..790e0649d39 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ 8.0.0 5.2020.6 - 42.2.18 + 42.2.19 1.11.762 1.2 4.5.5 From f00b6378a1d995c29a3d6d01db1b909fd57644c3 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 4 Mar 2021 11:23:46 -0500 Subject: [PATCH 408/576] typo. (#7374) --- doc/release-notes/7374-postgresql-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/7374-postgresql-update b/doc/release-notes/7374-postgresql-update index 332ffda0060..3ac74ad70a1 100644 --- a/doc/release-notes/7374-postgresql-update +++ b/doc/release-notes/7374-postgresql-update @@ -6,4 +6,4 @@ Up until this release our installation guide "strongly recommended" to install P - Install the new version of PostgreSQL; (make sure it's running on the same port, etc. so that no changes are needed in the Payara configuration) - Re-import the database with ``psql``, as the postgres user. -Consult the PostgreSQL upgrde documentation for more information, for example https://www.postgresql.org/docs/13/upgrading.html#UPGRADING-VIA-PGDUMPALL. +Consult the PostgreSQL upgrade documentation for more information, for example https://www.postgresql.org/docs/13/upgrading.html#UPGRADING-VIA-PGDUMPALL. From 28579e2699338ec9e37b3a75356024966a4c2739 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 4 Mar 2021 11:25:08 -0500 Subject: [PATCH 409/576] rename SQL script #2419 #4475 --- ...4__4475-disable-users.sql => V5.3.0.5__4475-disable-users.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V5.3.0.4__4475-disable-users.sql => V5.3.0.5__4475-disable-users.sql} (100%) diff --git a/src/main/resources/db/migration/V5.3.0.4__4475-disable-users.sql b/src/main/resources/db/migration/V5.3.0.5__4475-disable-users.sql similarity index 100% rename from src/main/resources/db/migration/V5.3.0.4__4475-disable-users.sql rename to src/main/resources/db/migration/V5.3.0.5__4475-disable-users.sql From 35aa097b5500fc6bfee572ac897d64cddf96877c Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 4 Mar 2021 11:35:32 -0500 Subject: [PATCH 410/576] #7633 fix named q so delete role works --- src/main/java/edu/harvard/iq/dataverse/RoleAssignment.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/RoleAssignment.java b/src/main/java/edu/harvard/iq/dataverse/RoleAssignment.java index 6241f120f80..22d2679efb6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/RoleAssignment.java +++ b/src/main/java/edu/harvard/iq/dataverse/RoleAssignment.java @@ -42,7 +42,7 @@ @NamedQuery( name = "RoleAssignment.listByDefinitionPointId", query = "SELECT r FROM RoleAssignment r WHERE r.definitionPoint.id=:definitionPointId" ), @NamedQuery( name = "RoleAssignment.listByRoleId", - query = "SELECT r FROM RoleAssignment r WHERE r.role=:roleId" ), + query = "SELECT r FROM RoleAssignment r WHERE r.role.id=:roleId" ), @NamedQuery( name = "RoleAssignment.listByPrivateUrlToken", query = "SELECT r FROM RoleAssignment r WHERE r.privateUrlToken=:privateUrlToken" ), @NamedQuery( name = "RoleAssignment.deleteByAssigneeIdentifier_RoleIdDefinition_PointId", From 0565ff90bb7c3912219caab3b2db98f5effe688e Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 4 Mar 2021 12:49:38 -0500 Subject: [PATCH 411/576] similar changes for the dev. guide. (#7374) --- doc/sphinx-guides/source/developers/dev-environment.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 63f482b86ee..e5cab7df0d1 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -94,9 +94,9 @@ To install Payara, run the following commands: Install PostgreSQL ~~~~~~~~~~~~~~~~~~ -PostgreSQL 9.6 is recommended to match the version in the Installation Guide. +For the past few release cycles much of the development has been done under PostgreSQL 9.6. While that version is known to be very stable, it is nearing its end-of-life (in Nov. 2021). The Dataverse Software has now been tested with versions up to 13 (13.2 is the latest released version as of writing this). -On Mac, go to https://www.postgresql.org/download/macosx/ and choose "Interactive installer by EDB" option. We've tested version 9.6.19. When prompted to set a password for the "database superuser (postgres)" just enter "password". +On Mac, go to https://www.postgresql.org/download/macosx/ and choose "Interactive installer by EDB" option. Note that version 9.6 is used in the command line examples below, but the process will be identical for any version up to 13. When prompted to set a password for the "database superuser (postgres)" just enter "password". After installation is complete, make a backup of the ``pg_hba.conf`` file like this: @@ -112,7 +112,7 @@ In the Finder, click "Applications" then "PostgreSQL 9.6" and launch the "Reload Next, to confirm the edit worked, launch the "pgAdmin" application from the same folder. Under "Browser", expand "Servers" and double click "PostgreSQL 9.6". When you are prompted for a password, leave it blank and click "OK". If you have successfully edited "pg_hba.conf", you can get in without a password. -On Linux, you should just install PostgreSQL from your package manager without worrying about the version as long as it's 9.x. Find ``pg_hba.conf`` and set the authentication method to "trust" and restart PostgreSQL. +On Linux, you should just install PostgreSQL using your favorite package manager, such as ``yum``. (Consult the PostgreSQL section of :doc:`/installation/prerequisites` in the main Installation guide for more info and command line examples). Find ``pg_hba.conf`` and set the authentication method to "trust" and restart PostgreSQL. Install Solr ~~~~~~~~~~~~ From 34453f533e2bc911f0532aa4b9f67e3bb7f6f05f Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 4 Mar 2021 13:12:40 -0500 Subject: [PATCH 412/576] renamed the release note (#7374) --- .../{7374-postgresql-update => 7374-postgresql-update.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename doc/release-notes/{7374-postgresql-update => 7374-postgresql-update.md} (100%) diff --git a/doc/release-notes/7374-postgresql-update b/doc/release-notes/7374-postgresql-update.md similarity index 100% rename from doc/release-notes/7374-postgresql-update rename to doc/release-notes/7374-postgresql-update.md From 4a3df0a25a9c5a8956c92877e9435464a7f39883 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 4 Mar 2021 15:23:29 -0500 Subject: [PATCH 413/576] use helper for BrandingUtil --- .../iq/dataverse/DataverseServiceBean.java | 11 ++++++ .../iq/dataverse/branding/BrandingUtil.java | 12 +++++- .../branding/BrandingUtilHelper.java | 28 ++++++++++++++ .../source/DbSettingConfigSource.java | 2 +- .../dataverse/branding/BrandingUtilTest.java | 38 +++++++++++++++++-- 5 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtilHelper.java diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index 96963bb9cb4..02bd7c841ab 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -18,6 +18,7 @@ import edu.harvard.iq.dataverse.search.IndexServiceBean; import edu.harvard.iq.dataverse.search.SolrIndexServiceBean; import edu.harvard.iq.dataverse.search.SolrSearchResult; +import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import java.io.File; import java.io.IOException; @@ -195,6 +196,16 @@ public Dataverse findRootDataverse() { return em.createNamedQuery("Dataverse.findRoot", Dataverse.class).getSingleResult(); } + public String getRootDataverseName() { + Dataverse root = findRootDataverse(); + if(root==null) { + logger.warning("Failed to find root dataverse!"); + return ""; + } + String rootDataverseName=root.getName(); + return StringUtil.isEmpty(rootDataverseName) ? "" : rootDataverseName; + } + public List findAllPublishedByOwnerId(Long ownerId) { List retVal = new ArrayList<>(); List previousLevel = findPublishedByOwnerId(ownerId); diff --git a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java index ea83fc15ebc..abae2f53737 100644 --- a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java @@ -1,13 +1,20 @@ package edu.harvard.iq.dataverse.branding; +import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.settings.source.DbSettingConfigSource; import edu.harvard.iq.dataverse.util.BundleUtil; import java.util.Arrays; import javax.mail.internet.InternetAddress; +import org.apache.commons.lang3.StringUtils; + public class BrandingUtil { + static DataverseServiceBean dataverseService; + public static String getInstallationBrandName(String rootDataverseName) { - return rootDataverseName; + String installationName = new DbSettingConfigSource().getValue(DbSettingConfigSource.PREFIX + ".InstitutionName"); + return StringUtils.isEmpty(installationName) ? dataverseService.getRootDataverseName() : installationName; } public static String getSupportTeamName(InternetAddress systemAddress, String rootDataverseName) { @@ -35,4 +42,7 @@ public static String getContactHeader(InternetAddress systemAddress, String root return BundleUtil.getStringFromBundle("contact.header", Arrays.asList(getSupportTeamName(systemAddress, rootDataverseName))); } + public static void injectDataverseService(DataverseServiceBean dataverseService) { + BrandingUtil.dataverseService = dataverseService; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtilHelper.java b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtilHelper.java new file mode 100644 index 00000000000..07256f7bfff --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtilHelper.java @@ -0,0 +1,28 @@ +package edu.harvard.iq.dataverse.branding; + +import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; + +import javax.annotation.PostConstruct; +import javax.ejb.EJB; +import javax.ejb.Singleton; +import javax.ejb.Startup; + +/** + * This is a small helper bean for the MPCONFIG DbSettingConfigSource. + * As it is a singleton and built at application start (=deployment), it will inject the (stateless) + * settings service into the MPCONFIG POJO once it's ready. + * + * MPCONFIG requires it's sources to be POJOs. No direct dependency injection possible. + */ +@Singleton +@Startup +public class BrandingUtilHelper { + @EJB + DataverseServiceBean dataverseService; + + @PostConstruct + public void injectService() { + BrandingUtil.injectDataverseService(dataverseService); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/source/DbSettingConfigSource.java b/src/main/java/edu/harvard/iq/dataverse/settings/source/DbSettingConfigSource.java index 597fdb314d2..b54c720db4d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/source/DbSettingConfigSource.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/source/DbSettingConfigSource.java @@ -23,7 +23,7 @@ public class DbSettingConfigSource implements ConfigSource { private static final ConcurrentHashMap properties = new ConcurrentHashMap<>(); private static Instant lastUpdate; private static SettingsServiceBean settingsSvc; - static final String PREFIX = "dataverse.settings.fromdb"; + public static final String PREFIX = "dataverse.settings.fromdb"; /** * Let the SettingsServiceBean be injected by DbSettingConfigHelper with PostConstruct diff --git a/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java index a57f73bd7a7..9cd97324b52 100644 --- a/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java @@ -1,21 +1,53 @@ package edu.harvard.iq.dataverse.branding; +import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.settings.Setting; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.settings.source.DbSettingConfigSource; import edu.harvard.iq.dataverse.util.BundleUtil; import java.io.UnsupportedEncodingException; import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + import javax.mail.internet.AddressException; import javax.mail.internet.InternetAddress; import static org.junit.Assert.assertEquals; -import org.junit.Test; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.MethodOrderer.OrderAnnotation; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +@ExtendWith(MockitoExtension.class) +@TestMethodOrder(OrderAnnotation.class) public class BrandingUtilTest { + DbSettingConfigSource dbSource = new DbSettingConfigSource(); + @Mock + SettingsServiceBean settingsSvc; + @Mock + DataverseServiceBean dataverseService; + @Test + @Order(1) public void testGetInstallationBrandName() { + +// Set settings = new HashSet<>(Arrays.asList(new Setting("InstitutionName", "LibraScholar"), new Setting("FooBarI18N", "de", "hallo"))); + Mockito.when(dataverseService.getRootDataverseName()).thenReturn("LibraScholar"); + BrandingUtil.injectDataverseService(dataverseService); +// DbSettingConfigSource.injectSettingsService(settingsSvc); System.out.println("testGetInstallationBrandName"); assertEquals("LibraScholar", BrandingUtil.getInstallationBrandName("LibraScholar")); - assertEquals(null, BrandingUtil.getInstallationBrandName(null));// misconfiguration to set to null - assertEquals("", BrandingUtil.getInstallationBrandName(""));// misconfiguration to set to empty string + Set settings = new HashSet<>(Arrays.asList(new Setting("InstitutionName", "NotLibraScholar"), new Setting("FooBarI18N", "de", "hallo"))); + Mockito.when(settingsSvc.listAll()).thenReturn(settings); + DbSettingConfigSource.injectSettingsService(settingsSvc); + + assertEquals("NotLibraScholar", BrandingUtil.getInstallationBrandName(null));// misconfiguration to set to null + //assertEquals("", BrandingUtil.getInstallationBrandName(""));// misconfiguration to set to empty string } @Test From 4ed6e8d9910c2c7c81a4ba11cd95f4b2b70babf7 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 4 Mar 2021 15:24:11 -0500 Subject: [PATCH 414/576] #7373 add additional upgrade steps --- doc/release-notes/7373-solr-upgrade.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/release-notes/7373-solr-upgrade.md b/doc/release-notes/7373-solr-upgrade.md index c6c084961bf..06e7bc62e5a 100644 --- a/doc/release-notes/7373-solr-upgrade.md +++ b/doc/release-notes/7373-solr-upgrade.md @@ -9,3 +9,14 @@ appear until indexing is complete. See http://guides.dataverse.org/installation/prerequisites.html#installing-solr + +[for the additional upgrade steps section] + +Run the script updateSchemaMDB.sh to generate updated solr schema files and preserve any other custom fields in your Solr configuration. +For example: (modify the path names as needed) +cd /usr/local/solr-8.8.1/server/solr/collection1/conf +wget https://github.com/IQSS/dataverse/releases/download/v5.4/updateSchemaMDB.sh +chmod +x updateSchemaMDB.sh +./updateSchemaMDB.sh -t . + +See http://guides.dataverse.org/en/5.4/admin/metadatacustomization.html?highlight=updateschemamdb for more information. From fd2391622e212286682b76cff38db6a599d5a513 Mon Sep 17 00:00:00 2001 From: gdurand Date: Thu, 4 Mar 2021 17:04:29 -0500 Subject: [PATCH 415/576] add back variable indexing --- .../iq/dataverse/search/IndexServiceBean.java | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 5b2d63c43eb..f61f879eee7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -23,6 +23,7 @@ import edu.harvard.iq.dataverse.dataaccess.DataAccessRequest; import edu.harvard.iq.dataverse.dataaccess.StorageIO; import edu.harvard.iq.dataverse.datavariable.DataVariable; +import edu.harvard.iq.dataverse.datavariable.VariableMetadata; import edu.harvard.iq.dataverse.datavariable.VariableMetadataUtil; import edu.harvard.iq.dataverse.datavariable.VariableServiceBean; import edu.harvard.iq.dataverse.harvest.client.HarvestingClient; @@ -45,7 +46,9 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.Future; +import java.util.function.Function; import java.util.logging.Logger; +import java.util.stream.Collectors; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import javax.ejb.AsyncResult; @@ -698,7 +701,7 @@ private String addOrUpdateDataset(IndexableDataset indexableDataset) throws Sol return addOrUpdateDataset(indexableDataset, null); } - private String addOrUpdateDataset(IndexableDataset indexableDataset, Set datafilesInDraftVersion) throws SolrServerException, IOException { + private String addOrUpdateDataset(IndexableDataset indexableDataset, Set datafilesInDraftVersion) throws SolrServerException, IOException { IndexableDataset.DatasetState state = indexableDataset.getDatasetState(); Dataset dataset = indexableDataset.getDatasetVersion().getDataset(); logger.fine("adding or updating Solr document for dataset id " + dataset.getId()); @@ -934,8 +937,7 @@ private String addOrUpdateDataset(IndexableDataset indexableDataset, Set d */ if ((fileMetadata.getDataFile().isRestricted() == releasedFileMetadata.getDataFile().isRestricted())) { if (fileMetadata.contentEquals(releasedFileMetadata) - /* SEK 3/12/2020 remove variable metadata indexing*/ - // && variableMetadataUtil.compareVariableMetadata(releasedFileMetadata,fileMetadata) + && variableMetadataUtil.compareVariableMetadata(releasedFileMetadata,fileMetadata) ) { indexThisMetadata = false; logger.fine("This file metadata hasn't changed since the released version; skipping indexing."); @@ -1155,6 +1157,14 @@ private String addOrUpdateDataset(IndexableDataset indexableDataset, Set d // names and labels: if (fileMetadata.getDataFile().isTabularData()) { List variables = fileMetadata.getDataFile().getDataTable().getDataVariables(); + + Map variableMap = null; + List variablesByMetadata = variableService.findVarMetByFileMetaId(fileMetadata.getId()); + + variableMap = + variablesByMetadata.stream().collect(Collectors.toMap(VariableMetadata::getId, Function.identity())); + + for (DataVariable var : variables) { // Hard-coded search fields, for now: // TODO: eventually: review, decide how datavariables should @@ -1169,21 +1179,14 @@ private String addOrUpdateDataset(IndexableDataset indexableDataset, Set d if (var.getName() != null && !var.getName().equals("")) { datafileSolrInputDocument.addField(SearchFields.VARIABLE_NAME, var.getName()); } - -/* SEK 3/12/2020 remove variable metadata indexing - List vmList = variableService.findByDataVarIdAndFileMetaId(var.getId(), fileMetadata.getId()); - VariableMetadata vm = null; - if (vmList != null && vmList.size() >0) { - vm = vmList.get(0); - } - - if (vmList.size() == 0 ) { + + VariableMetadata vm = variableMap.get(var.getId()); + if (vm == null) { //Variable Label if (var.getLabel() != null && !var.getLabel().equals("")) { datafileSolrInputDocument.addField(SearchFields.VARIABLE_LABEL, var.getLabel()); } - - } else if (vm != null) { + } else { if (vm.getLabel() != null && !vm.getLabel().equals("") ) { datafileSolrInputDocument.addField(SearchFields.VARIABLE_LABEL, vm.getLabel()); } @@ -1204,7 +1207,6 @@ private String addOrUpdateDataset(IndexableDataset indexableDataset, Set d } } -*/ } // TABULAR DATA TAGS: From b278aa74486f0a531b261b6fd2ab0e419e84a7ac Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Mar 2021 10:12:49 -0500 Subject: [PATCH 416/576] remove rootDataverseName param and associated code --- .../dataverse/DOIDataCiteRegisterService.java | 4 ++- .../harvard/iq/dataverse/DataCitation.java | 3 +- .../edu/harvard/iq/dataverse/DatasetPage.java | 9 ++---- .../harvard/iq/dataverse/DatasetVersion.java | 19 ++---------- .../iq/dataverse/EditDatafilesPage.java | 3 +- .../harvard/iq/dataverse/MailServiceBean.java | 10 +++---- .../iq/dataverse/SendFeedbackDialog.java | 10 +++---- .../harvard/iq/dataverse/SettingsWrapper.java | 4 +-- .../harvard/iq/dataverse/api/FeedbackApi.java | 5 ++-- .../iq/dataverse/branding/BrandingUtil.java | 14 ++++++--- .../harvard/iq/dataverse/util/MailUtil.java | 2 +- .../iq/dataverse/util/json/JsonPrinter.java | 16 ++-------- .../dataverse/branding/BrandingUtilTest.java | 29 ++++++++++++------- 13 files changed, 54 insertions(+), 74 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java index ba503a18d22..815733f1b7b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java +++ b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java @@ -6,6 +6,8 @@ package edu.harvard.iq.dataverse; import edu.harvard.iq.dataverse.AbstractGlobalIdServiceBean.GlobalIdMetadataTemplate; +import edu.harvard.iq.dataverse.branding.BrandingUtil; + import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; @@ -205,7 +207,7 @@ public static String getMetadataFromDvObject(String identifier, Map getDatasetAuthors() { * @return the publisher of the version */ public String getPublisher() { - assert (null != workingVersion); - return workingVersion.getRootDataverseNameforCitation(); + return dataverseService.getRootDataverseName(); } public void downloadRsyncScript() { diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java index 067cba23d04..d28ce5175d4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java @@ -1391,21 +1391,6 @@ public String getDistributorName() { } return null; } - - // TODO: Consider renaming this method since it's also used for getting the "provider" for Schema.org JSON-LD. - public String getRootDataverseNameforCitation(){ - //Get root dataverse name for Citation - Dataverse root = this.getDataset().getOwner(); - while (root.getOwner() != null) { - root = root.getOwner(); - } - String rootDataverseName = root.getName(); - if (!StringUtil.isEmpty(rootDataverseName)) { - return rootDataverseName; - } else { - return ""; - } - } public List getDatasetDistributors() { //todo get distributors from DatasetfieldValues @@ -1888,11 +1873,11 @@ public String getJsonLd() { job.add("includedInDataCatalog", Json.createObjectBuilder() .add("@type", "DataCatalog") - .add("name", this.getRootDataverseNameforCitation()) + .add("name", BrandingUtil.getRootDataverseCollectionName()) .add("url", SystemConfig.getDataverseSiteUrlStatic()) ); - String installationBrandName = BrandingUtil.getInstallationBrandName(getRootDataverseNameforCitation()); + String installationBrandName = BrandingUtil.getInstallationBrandName(); /** * Both "publisher" and "provider" are included but they have the same * values. Some services seem to prefer one over the other. diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 89b21d84856..63b1a3113ec 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -941,8 +941,7 @@ public String save() { if (dataset.isLockedFor(DatasetLock.Reason.EditInProgress) || lockTest.isLockedFor(DatasetLock.Reason.EditInProgress)) { logger.log(Level.INFO, "Couldn''t save dataset: {0}", "It is locked." + ""); - String rootDataverseName = dataverseService.findRootDataverse().getName(); - JH.addMessage(FacesMessage.SEVERITY_FATAL, getBundleString("dataset.locked.editInProgress.message"),BundleUtil.getStringFromBundle("dataset.locked.editInProgress.message.details", Arrays.asList(BrandingUtil.getSupportTeamName(null, rootDataverseName)))); + JH.addMessage(FacesMessage.SEVERITY_FATAL, getBundleString("dataset.locked.editInProgress.message"),BundleUtil.getStringFromBundle("dataset.locked.editInProgress.message.details", Arrays.asList(BrandingUtil.getSupportTeamName(null)))); return null; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java index 13a92c9cd27..4078df48948 100644 --- a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java @@ -121,12 +121,11 @@ public boolean sendSystemEmail(String to, String subject, String messageText, bo boolean sent = false; - String rootDataverseName = dataverseService.findRootDataverse().getName(); InternetAddress systemAddress = getSystemAddress(); String body = messageText - + (isHtmlContent ? BundleUtil.getStringFromBundle("notification.email.closing.html", Arrays.asList(BrandingUtil.getSupportTeamEmailAddress(systemAddress), BrandingUtil.getSupportTeamName(systemAddress, rootDataverseName))) - : BundleUtil.getStringFromBundle("notification.email.closing", Arrays.asList(BrandingUtil.getSupportTeamEmailAddress(systemAddress), BrandingUtil.getSupportTeamName(systemAddress, rootDataverseName)))); + + (isHtmlContent ? BundleUtil.getStringFromBundle("notification.email.closing.html", Arrays.asList(BrandingUtil.getSupportTeamEmailAddress(systemAddress), BrandingUtil.getSupportTeamName(systemAddress))) + : BundleUtil.getStringFromBundle("notification.email.closing", Arrays.asList(BrandingUtil.getSupportTeamEmailAddress(systemAddress), BrandingUtil.getSupportTeamName(systemAddress)))); logger.fine("Sending email to " + to + ". Subject: <<<" + subject + ">>>. Body: " + body); try { @@ -497,13 +496,12 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio messageText += MessageFormat.format(pattern, paramArrayReturnedDataset); return messageText; case CREATEACC: - String rootDataverseName = dataverseService.findRootDataverse().getName(); InternetAddress systemAddress = getSystemAddress(); String accountCreatedMessage = BundleUtil.getStringFromBundle("notification.email.welcome", Arrays.asList( - BrandingUtil.getInstallationBrandName(rootDataverseName), + BrandingUtil.getInstallationBrandName(), systemConfig.getGuidesBaseUrl(), systemConfig.getGuidesVersion(), - BrandingUtil.getSupportTeamName(systemAddress, rootDataverseName), + BrandingUtil.getSupportTeamName(systemAddress), BrandingUtil.getSupportTeamEmailAddress(systemAddress) )); String optionalConfirmEmailAddon = confirmEmailService.optionalConfirmEmailAddonMsg(userNotification.getUser()); diff --git a/src/main/java/edu/harvard/iq/dataverse/SendFeedbackDialog.java b/src/main/java/edu/harvard/iq/dataverse/SendFeedbackDialog.java index dd27842573f..363972b48c3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/SendFeedbackDialog.java +++ b/src/main/java/edu/harvard/iq/dataverse/SendFeedbackDialog.java @@ -130,7 +130,7 @@ public void setUserSum(Long userSum) { public String getMessageTo() { if (recipient == null) { - return BrandingUtil.getSupportTeamName(systemAddress, dataverseService.findRootDataverse().getName()); + return BrandingUtil.getSupportTeamName(systemAddress); } else if (recipient.isInstanceofDataverse()) { return ((Dataverse) recipient).getDisplayName() + " " + BundleUtil.getStringFromBundle("contact.contact"); } else { @@ -140,7 +140,7 @@ public String getMessageTo() { public String getFormHeader() { if (recipient == null) { - return BrandingUtil.getContactHeader(systemAddress, dataverseService.findRootDataverse().getName()); + return BrandingUtil.getContactHeader(systemAddress); } else if (recipient.isInstanceofDataverse()) { return BundleUtil.getStringFromBundle("contact.dataverse.header"); } else { @@ -198,10 +198,8 @@ public void validateUserEmail(FacesContext context, UIComponent component, Objec } public String sendMessage() { - // FIXME: move dataverseService.findRootDataverse() to init - String rootDataverseName = dataverseService.findRootDataverse().getName(); - String installationBrandName = BrandingUtil.getInstallationBrandName(rootDataverseName); - String supportTeamName = BrandingUtil.getSupportTeamName(systemAddress, rootDataverseName); + String installationBrandName = BrandingUtil.getInstallationBrandName(); + String supportTeamName = BrandingUtil.getSupportTeamName(systemAddress); List feedbacks = FeedbackUtil.gatherFeedback(recipient, dataverseSession, messageSubject, userMessage, systemAddress, userEmail, systemConfig.getDataverseSiteUrl(), installationBrandName, supportTeamName); if (feedbacks.isEmpty()) { logger.warning("No feedback has been sent!"); diff --git a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java index f27c7c712dc..d7512dd5cf0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java +++ b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java @@ -173,13 +173,13 @@ public boolean isDataFilePIDSequentialDependent(){ public String getSupportTeamName() { String systemEmail = getValueForKey(SettingsServiceBean.Key.SystemEmail); InternetAddress systemAddress = MailUtil.parseSystemAddress(systemEmail); - return BrandingUtil.getSupportTeamName(systemAddress, dataverseService.findRootDataverse().getName()); + return BrandingUtil.getSupportTeamName(systemAddress); } public String getSupportTeamEmail() { String systemEmail = getValueForKey(SettingsServiceBean.Key.SystemEmail); InternetAddress systemAddress = MailUtil.parseSystemAddress(systemEmail); - return BrandingUtil.getSupportTeamEmailAddress(systemAddress) != null ? BrandingUtil.getSupportTeamEmailAddress(systemAddress) : BrandingUtil.getSupportTeamName(systemAddress, dataverseService.findRootDataverse().getName()); + return BrandingUtil.getSupportTeamEmailAddress(systemAddress) != null ? BrandingUtil.getSupportTeamEmailAddress(systemAddress) : BrandingUtil.getSupportTeamName(systemAddress); } public Integer getUploadMethodsCount() { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/FeedbackApi.java b/src/main/java/edu/harvard/iq/dataverse/api/FeedbackApi.java index 7c81fe1a892..d9a94ee340b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/FeedbackApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/FeedbackApi.java @@ -38,9 +38,8 @@ public Response submitFeedback(JsonObject jsonObject) throws AddressException { String userEmail = jsonObject.getString("fromEmail"); String messageSubject = jsonObject.getString("subject"); String baseUrl = systemConfig.getDataverseSiteUrl(); - String rootDataverseName = dataverseSvc.findRootDataverse().getName(); - String installationBrandName = BrandingUtil.getInstallationBrandName(rootDataverseName); - String supportTeamName = BrandingUtil.getSupportTeamName(systemAddress, rootDataverseName); + String installationBrandName = BrandingUtil.getInstallationBrandName(); + String supportTeamName = BrandingUtil.getSupportTeamName(systemAddress); JsonArrayBuilder jab = Json.createArrayBuilder(); List feedbacks = FeedbackUtil.gatherFeedback(recipient, dataverseSession, messageSubject, userMessage, systemAddress, userEmail, baseUrl, installationBrandName, supportTeamName); feedbacks.forEach((feedback) -> { diff --git a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java index abae2f53737..e083ac53349 100644 --- a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java @@ -12,18 +12,24 @@ public class BrandingUtil { static DataverseServiceBean dataverseService; - public static String getInstallationBrandName(String rootDataverseName) { + public static String getInstallationBrandName() { String installationName = new DbSettingConfigSource().getValue(DbSettingConfigSource.PREFIX + ".InstitutionName"); return StringUtils.isEmpty(installationName) ? dataverseService.getRootDataverseName() : installationName; } + + //Convenience to access root name without injecting dataverseService (e.g. in DatasetVersion) + public static String getRootDataverseCollectionName() { + return dataverseService.getRootDataverseName(); + } - public static String getSupportTeamName(InternetAddress systemAddress, String rootDataverseName) { + public static String getSupportTeamName(InternetAddress systemAddress) { if (systemAddress != null) { String personalName = systemAddress.getPersonal(); if (personalName != null) { return personalName; } } + String rootDataverseName=dataverseService.getRootDataverseName(); if (rootDataverseName != null && !rootDataverseName.isEmpty()) { return rootDataverseName + " " + BundleUtil.getStringFromBundle("contact.support"); } @@ -38,8 +44,8 @@ public static String getSupportTeamEmailAddress(InternetAddress systemAddress) { return systemAddress.getAddress(); } - public static String getContactHeader(InternetAddress systemAddress, String rootDataverseName) { - return BundleUtil.getStringFromBundle("contact.header", Arrays.asList(getSupportTeamName(systemAddress, rootDataverseName))); + public static String getContactHeader(InternetAddress systemAddress) { + return BundleUtil.getStringFromBundle("contact.header", Arrays.asList(getSupportTeamName(systemAddress))); } public static void injectDataverseService(DataverseServiceBean dataverseService) { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java index 37667d16b55..300a82d200f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java @@ -30,7 +30,7 @@ public static InternetAddress parseSystemAddress(String systemEmail) { } public static String getSubjectTextBasedOnNotification(UserNotification userNotification, String rootDataverseName, Object objectOfNotification) { - List rootDvNameAsList = Arrays.asList(BrandingUtil.getInstallationBrandName(rootDataverseName)); + List rootDvNameAsList = Arrays.asList(BrandingUtil.getInstallationBrandName()); switch (userNotification.getType()) { case ASSIGNROLE: return BundleUtil.getStringFromBundle("notification.email.assign.role.subject", rootDvNameAsList); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index c37efc3178f..06e0610832c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -35,6 +35,7 @@ import edu.harvard.iq.dataverse.authorization.providers.AuthenticationProviderRow; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; +import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; @@ -319,7 +320,7 @@ public static JsonObjectBuilder json(Dataset ds) { .add("persistentUrl", ds.getPersistentURL()) .add("protocol", ds.getProtocol()) .add("authority", ds.getAuthority()) - .add("publisher", getRootDataverseNameforCitation(ds)) + .add("publisher", BrandingUtil.getRootDataverseCollectionName()) .add("publicationDate", ds.getPublicationDateFormattedYYYYMMDD()) .add("storageIdentifier", ds.getStorageIdentifier()); } @@ -389,19 +390,6 @@ public static JsonObjectBuilder jsonDataFileList(List dataFiles){ return bld; } - private static String getRootDataverseNameforCitation(Dataset dataset) { - Dataverse root = dataset.getOwner(); - while (root.getOwner() != null) { - root = root.getOwner(); - } - String rootDataverseName = root.getName(); - if (!StringUtil.isEmpty(rootDataverseName)) { - return rootDataverseName; - } else { - return ""; - } - } - private static String getLicenseInfo(DatasetVersion dsv) { if (dsv.getTermsOfUseAndAccess().getLicense() != null && dsv.getTermsOfUseAndAccess().getLicense().equals(TermsOfUseAndAccess.License.CC0)) { return "CC0 Waiver"; diff --git a/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java index 9cd97324b52..ff656810405 100644 --- a/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java @@ -41,24 +41,29 @@ public void testGetInstallationBrandName() { BrandingUtil.injectDataverseService(dataverseService); // DbSettingConfigSource.injectSettingsService(settingsSvc); System.out.println("testGetInstallationBrandName"); - assertEquals("LibraScholar", BrandingUtil.getInstallationBrandName("LibraScholar")); + assertEquals("LibraScholar", BrandingUtil.getInstallationBrandName()); + Set settings = new HashSet<>(Arrays.asList(new Setting("InstitutionName", "NotLibraScholar"), new Setting("FooBarI18N", "de", "hallo"))); Mockito.when(settingsSvc.listAll()).thenReturn(settings); DbSettingConfigSource.injectSettingsService(settingsSvc); - - assertEquals("NotLibraScholar", BrandingUtil.getInstallationBrandName(null));// misconfiguration to set to null - //assertEquals("", BrandingUtil.getInstallationBrandName(""));// misconfiguration to set to empty string + assertEquals("NotLibraScholar", BrandingUtil.getInstallationBrandName());// misconfiguration to set to null } @Test public void testGetSupportTeamName() throws AddressException, UnsupportedEncodingException { System.out.println("testGetSupportTeamName"); - assertEquals("Support", BrandingUtil.getSupportTeamName(null, null)); - assertEquals("Support", BrandingUtil.getSupportTeamName(null, "")); - assertEquals("LibraScholar Support", BrandingUtil.getSupportTeamName(null, "LibraScholar")); - assertEquals("LibraScholar Support", BrandingUtil.getSupportTeamName(new InternetAddress("support@librascholar.edu"), "LibraScholar")); - assertEquals("LibraScholar Support Team", BrandingUtil.getSupportTeamName(new InternetAddress("support@librascholar.edu", "LibraScholar Support Team"), "LibraScholar")); - assertEquals("", BrandingUtil.getSupportTeamName(new InternetAddress("support@librascholar.edu", ""), "LibraScholar")); // misconfiguration to set to empty string + Mockito.when(dataverseService.getRootDataverseName()).thenReturn(null); + BrandingUtil.injectDataverseService(dataverseService); + assertEquals("Support", BrandingUtil.getSupportTeamName(null)); + Mockito.when(dataverseService.getRootDataverseName()).thenReturn(""); + BrandingUtil.injectDataverseService(dataverseService); + assertEquals("Support", BrandingUtil.getSupportTeamName(null)); + Mockito.when(dataverseService.getRootDataverseName()).thenReturn("LibraScholar"); + BrandingUtil.injectDataverseService(dataverseService); + assertEquals("LibraScholar Support", BrandingUtil.getSupportTeamName(null)); + assertEquals("LibraScholar Support", BrandingUtil.getSupportTeamName(new InternetAddress("support@librascholar.edu"))); + assertEquals("LibraScholar Support Team", BrandingUtil.getSupportTeamName(new InternetAddress("support@librascholar.edu", "LibraScholar Support Team"))); + assertEquals("", BrandingUtil.getSupportTeamName(new InternetAddress("support@librascholar.edu", ""))); // misconfiguration to set to empty string } @Test @@ -135,7 +140,9 @@ public void testEmailSubject() { @Test public void testGetContactHeader() { System.out.println("testGetContactHeader"); - assertEquals("Contact Support", BrandingUtil.getContactHeader(null, null)); + Mockito.when(dataverseService.getRootDataverseName()).thenReturn(null); + BrandingUtil.injectDataverseService(dataverseService); + assertEquals("Contact Support", BrandingUtil.getContactHeader(null)); } } From 76085bbfc45b2d8e4733a8e4566a40836f945d6f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Mar 2021 10:30:37 -0500 Subject: [PATCH 417/576] fix tests --- .../edu/harvard/iq/dataverse/branding/BrandingUtilTest.java | 6 +++++- .../edu/harvard/iq/dataverse/export/DDIExporterTest.java | 4 +--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java index ff656810405..4528d645ce6 100644 --- a/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java @@ -43,10 +43,14 @@ public void testGetInstallationBrandName() { System.out.println("testGetInstallationBrandName"); assertEquals("LibraScholar", BrandingUtil.getInstallationBrandName()); - Set settings = new HashSet<>(Arrays.asList(new Setting("InstitutionName", "NotLibraScholar"), new Setting("FooBarI18N", "de", "hallo"))); + Set settings = new HashSet<>(Arrays.asList(new Setting("InstitutionName", "NotLibraScholar"))); Mockito.when(settingsSvc.listAll()).thenReturn(settings); DbSettingConfigSource.injectSettingsService(settingsSvc); assertEquals("NotLibraScholar", BrandingUtil.getInstallationBrandName());// misconfiguration to set to null + + //Reset for other tests + settings.clear(); + settings.add(new Setting("InstitutionName", "LibraScholar")); } @Test diff --git a/src/test/java/edu/harvard/iq/dataverse/export/DDIExporterTest.java b/src/test/java/edu/harvard/iq/dataverse/export/DDIExporterTest.java index e70e375ace9..6090d8f4a1e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/DDIExporterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/DDIExporterTest.java @@ -145,12 +145,10 @@ public void testCitation() throws Exception { version.setVersionState(DatasetVersion.VersionState.DRAFT); Dataset dataset = new Dataset(); version.setDataset(dataset); - Dataverse dataverse = new Dataverse(); - dataset.setOwner(dataverse); String citation = version.getCitation(); System.out.println("citation: " + citation); int currentYear = Year.now().getValue(); - assertEquals("Finch, Fiona, " + currentYear + ", \"Darwin's Finches\", DRAFT VERSION", citation); + assertEquals("Finch, Fiona, " + currentYear + ", \"Darwin's Finches\", LibraScholar, DRAFT VERSION", citation); } @Test From f28511e7cc3ec989a4942a8e7339b81242cc103a Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 5 Mar 2021 11:49:43 -0500 Subject: [PATCH 418/576] changed the path where the solr files live, for the installer build. #7373 --- scripts/installer/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/installer/Makefile b/scripts/installer/Makefile index 4ef35a01471..180e2cb03d5 100644 --- a/scripts/installer/Makefile +++ b/scripts/installer/Makefile @@ -56,13 +56,13 @@ ${JHOVE_SCHEMA}: ../../conf/jhove/jhoveConfig.xsd ${INSTALLER_ZIP_DIR} @echo copying jhove schema file /bin/cp ../../conf/jhove/jhoveConfig.xsd ${INSTALLER_ZIP_DIR} -${SOLR_SCHEMA}: ../../conf/solr/7.7.2/schema.xml ../../conf/solr/7.7.2/schema_dv_mdb_fields.xml ../../conf/solr/7.7.2/schema_dv_mdb_copies.xml ../../conf/solr/7.7.2/updateSchemaMDB.sh ${INSTALLER_ZIP_DIR} +${SOLR_SCHEMA}: ../../conf/solr/8.8.1/schema.xml ../../conf/solr/8.8.1/schema_dv_mdb_fields.xml ../../conf/solr/8.8.1/schema_dv_mdb_copies.xml ../../conf/solr/8.8.1/updateSchemaMDB.sh ${INSTALLER_ZIP_DIR} @echo copying Solr schema file - /bin/cp ../../conf/solr/7.7.2/schema*.xml ../../conf/solr/7.7.2/updateSchemaMDB.sh ${INSTALLER_ZIP_DIR} + /bin/cp ../../conf/solr/8.8.1/schema*.xml ../../conf/solr/8.8.1/updateSchemaMDB.sh ${INSTALLER_ZIP_DIR} -${SOLR_CONFIG}: ../../conf/solr/7.7.2/solrconfig.xml ${INSTALLER_ZIP_DIR} +${SOLR_CONFIG}: ../../conf/solr/8.8.1/solrconfig.xml ${INSTALLER_ZIP_DIR} @echo copying Solr config file - /bin/cp ../../conf/solr/7.7.2/solrconfig.xml ${INSTALLER_ZIP_DIR} + /bin/cp ../../conf/solr/8.8.1/solrconfig.xml ${INSTALLER_ZIP_DIR} ${PYTHON_FILES}: README_python.txt install.py installConfig.py installAppServer.py installUtils.py requirements.txt default.config interactive.config ${INSTALLER_ZIP_DIR} @echo copying Python installer files From 92b0d2fe97ebea4e16d9a6ca345359712a18f150 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Mar 2021 11:55:36 -0500 Subject: [PATCH 419/576] owner still needed to create citation --- .../java/edu/harvard/iq/dataverse/export/DDIExporterTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/export/DDIExporterTest.java b/src/test/java/edu/harvard/iq/dataverse/export/DDIExporterTest.java index 6090d8f4a1e..3a5ef60a0e4 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/DDIExporterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/DDIExporterTest.java @@ -2,7 +2,6 @@ import edu.harvard.iq.dataverse.ControlledVocabularyValue; import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetFieldServiceBean; import edu.harvard.iq.dataverse.DatasetFieldType; import edu.harvard.iq.dataverse.DatasetFieldType.FieldType; import edu.harvard.iq.dataverse.mocks.MockDatasetFieldSvc; @@ -17,9 +16,7 @@ import java.nio.file.Paths; import java.time.Year; import java.util.Arrays; -import java.util.HashMap; import java.util.HashSet; -import java.util.Map; import java.util.Set; import javax.json.Json; import javax.json.JsonObject; @@ -145,6 +142,7 @@ public void testCitation() throws Exception { version.setVersionState(DatasetVersion.VersionState.DRAFT); Dataset dataset = new Dataset(); version.setDataset(dataset); + dataset.setOwner(new Dataverse()); String citation = version.getCitation(); System.out.println("citation: " + citation); int currentYear = Year.now().getValue(); From d892319d79d2b7558102c4b9d30bd38846556623 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Mar 2021 11:55:50 -0500 Subject: [PATCH 420/576] remove ordering (not needed) --- .../edu/harvard/iq/dataverse/branding/BrandingUtilTest.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java index 4528d645ce6..e75f905af1e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java @@ -13,17 +13,13 @@ import javax.mail.internet.AddressException; import javax.mail.internet.InternetAddress; import static org.junit.Assert.assertEquals; -import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestMethodOrder; -import org.junit.jupiter.api.MethodOrderer.OrderAnnotation; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) -@TestMethodOrder(OrderAnnotation.class) public class BrandingUtilTest { DbSettingConfigSource dbSource = new DbSettingConfigSource(); @@ -33,7 +29,6 @@ public class BrandingUtilTest { DataverseServiceBean dataverseService; @Test - @Order(1) public void testGetInstallationBrandName() { // Set settings = new HashSet<>(Arrays.asList(new Setting("InstitutionName", "LibraScholar"), new Setting("FooBarI18N", "de", "hallo"))); From e6d5ff8913bdcaf07e17b90e980bc030b369fb95 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Mar 2021 11:56:09 -0500 Subject: [PATCH 421/576] remove rootDataverseName from MailUtil method --- .../harvard/iq/dataverse/MailServiceBean.java | 3 +- .../confirmemail/ConfirmEmailServiceBean.java | 22 +++++-------- .../harvard/iq/dataverse/util/MailUtil.java | 2 +- .../iq/dataverse/util/MailUtilTest.java | 32 +++++++++---------- 4 files changed, 25 insertions(+), 34 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java index 4078df48948..5effffb5d43 100644 --- a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java @@ -244,8 +244,7 @@ public Boolean sendNotificationEmail(UserNotification notification, String comme Object objectOfNotification = getObjectOfNotification(notification); if (objectOfNotification != null){ String messageText = getMessageTextBasedOnNotification(notification, objectOfNotification, comment, requestor); - String rootDataverseName = dataverseService.findRootDataverse().getName(); - String subjectText = MailUtil.getSubjectTextBasedOnNotification(notification, rootDataverseName, objectOfNotification); + String subjectText = MailUtil.getSubjectTextBasedOnNotification(notification, objectOfNotification); if (!(messageText.isEmpty() || subjectText.isEmpty())){ retval = sendSystemEmail(emailAddress, subjectText, messageText, isHtmlContent); } else { diff --git a/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailServiceBean.java index bc3b70326f1..913c666fbe5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailServiceBean.java @@ -121,20 +121,14 @@ private void sendLinkOnEmailChange(AuthenticatedUser aUser, String confirmationU try { String toAddress = aUser.getEmail(); - try { - Dataverse rootDataverse = dataverseService.findRootDataverse(); - if (rootDataverse != null) { - String rootDataverseName = rootDataverse.getName(); - // FIXME: consider refactoring this into MailServiceBean.sendNotificationEmail. CONFIRMEMAIL may be the only type where we don't want an in-app notification. - UserNotification userNotification = new UserNotification(); - userNotification.setType(UserNotification.Type.CONFIRMEMAIL); - String subject = MailUtil.getSubjectTextBasedOnNotification(userNotification, rootDataverseName, null); - logger.fine("sending email to " + toAddress + " with this subject: " + subject); - mailService.sendSystemEmail(toAddress, subject, messageBody); - } - } catch (Exception e) { - logger.info("The root dataverse is not present. Don't send a notification to dataverseAdmin."); - } + + // FIXME: consider refactoring this into MailServiceBean.sendNotificationEmail. + // CONFIRMEMAIL may be the only type where we don't want an in-app notification. + UserNotification userNotification = new UserNotification(); + userNotification.setType(UserNotification.Type.CONFIRMEMAIL); + String subject = MailUtil.getSubjectTextBasedOnNotification(userNotification, null); + logger.fine("sending email to " + toAddress + " with this subject: " + subject); + mailService.sendSystemEmail(toAddress, subject, messageBody); } catch (Exception ex) { /** * @todo get more specific about the exception that's thrown when diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java index 300a82d200f..38ba3e195ba 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java @@ -29,7 +29,7 @@ public static InternetAddress parseSystemAddress(String systemEmail) { return null; } - public static String getSubjectTextBasedOnNotification(UserNotification userNotification, String rootDataverseName, Object objectOfNotification) { + public static String getSubjectTextBasedOnNotification(UserNotification userNotification, Object objectOfNotification) { List rootDvNameAsList = Arrays.asList(BrandingUtil.getInstallationBrandName()); switch (userNotification.getType()) { case ASSIGNROLE: diff --git a/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java index af0c414d356..23a69356fbc 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java @@ -7,12 +7,10 @@ public class MailUtilTest { - private String rootDataverseName; UserNotification userNotification = new UserNotification(); @Before public void setUp() { - rootDataverseName = "LibraScholar"; userNotification = new UserNotification(); } @@ -33,91 +31,91 @@ public void testParseSystemAddress() { @Test public void testSubjectCreateAccount() { userNotification.setType(UserNotification.Type.CREATEACC); - assertEquals("LibraScholar: Your account has been created", MailUtil.getSubjectTextBasedOnNotification(userNotification, rootDataverseName, null)); + assertEquals("LibraScholar: Your account has been created", MailUtil.getSubjectTextBasedOnNotification(userNotification, null)); } @Test public void testSubjectAssignRole() { userNotification.setType(UserNotification.Type.ASSIGNROLE); - assertEquals("LibraScholar: You have been assigned a role", MailUtil.getSubjectTextBasedOnNotification(userNotification, rootDataverseName, null)); + assertEquals("LibraScholar: You have been assigned a role", MailUtil.getSubjectTextBasedOnNotification(userNotification, null)); } @Test public void testSubjectCreateDataverse() { userNotification.setType(UserNotification.Type.CREATEDV); - assertEquals("LibraScholar: Your dataverse has been created", MailUtil.getSubjectTextBasedOnNotification(userNotification, rootDataverseName, null)); + assertEquals("LibraScholar: Your dataverse has been created", MailUtil.getSubjectTextBasedOnNotification(userNotification, null)); } @Test public void testSubjectRevokeRole() { userNotification.setType(UserNotification.Type.REVOKEROLE); - assertEquals("LibraScholar: Your role has been revoked", MailUtil.getSubjectTextBasedOnNotification(userNotification, rootDataverseName, null)); + assertEquals("LibraScholar: Your role has been revoked", MailUtil.getSubjectTextBasedOnNotification(userNotification, null)); } @Test public void testSubjectRequestFileAccess() { userNotification.setType(UserNotification.Type.REQUESTFILEACCESS); - assertEquals("LibraScholar: Access has been requested for a restricted file", MailUtil.getSubjectTextBasedOnNotification(userNotification, rootDataverseName, null)); + assertEquals("LibraScholar: Access has been requested for a restricted file", MailUtil.getSubjectTextBasedOnNotification(userNotification, null)); } @Test public void testSubjectGrantFileAccess() { userNotification.setType(UserNotification.Type.GRANTFILEACCESS); - assertEquals("LibraScholar: You have been granted access to a restricted file", MailUtil.getSubjectTextBasedOnNotification(userNotification, rootDataverseName, null)); + assertEquals("LibraScholar: You have been granted access to a restricted file", MailUtil.getSubjectTextBasedOnNotification(userNotification, null)); } @Test public void testSubjectRejectFileAccess() { userNotification.setType(UserNotification.Type.REJECTFILEACCESS); - assertEquals("LibraScholar: Your request for access to a restricted file has been rejected", MailUtil.getSubjectTextBasedOnNotification(userNotification, rootDataverseName, null)); + assertEquals("LibraScholar: Your request for access to a restricted file has been rejected", MailUtil.getSubjectTextBasedOnNotification(userNotification, null)); } @Test public void testSubjectCreateDataset() { userNotification.setType(UserNotification.Type.CREATEDS); - assertEquals("LibraScholar: Your dataset has been created", MailUtil.getSubjectTextBasedOnNotification(userNotification, rootDataverseName, null)); + assertEquals("LibraScholar: Your dataset has been created", MailUtil.getSubjectTextBasedOnNotification(userNotification, null)); } @Test public void testSubjectSubmittedDS() { userNotification.setType(UserNotification.Type.SUBMITTEDDS); - assertEquals("LibraScholar: Your dataset has been submitted for review", MailUtil.getSubjectTextBasedOnNotification(userNotification, rootDataverseName, null)); + assertEquals("LibraScholar: Your dataset has been submitted for review", MailUtil.getSubjectTextBasedOnNotification(userNotification, null)); } @Test public void testSubjectPublishedDS() { userNotification.setType(UserNotification.Type.PUBLISHEDDS); - assertEquals("LibraScholar: Your dataset has been published", MailUtil.getSubjectTextBasedOnNotification(userNotification, rootDataverseName, null)); + assertEquals("LibraScholar: Your dataset has been published", MailUtil.getSubjectTextBasedOnNotification(userNotification, null)); } @Test public void testSubjectReturnedDS() { userNotification.setType(UserNotification.Type.RETURNEDDS); - assertEquals("LibraScholar: Your dataset has been returned", MailUtil.getSubjectTextBasedOnNotification(userNotification, rootDataverseName, null)); + assertEquals("LibraScholar: Your dataset has been returned", MailUtil.getSubjectTextBasedOnNotification(userNotification, null)); } @Test public void testSubjectChecksumFail() { userNotification.setType(UserNotification.Type.CHECKSUMFAIL); - assertEquals("LibraScholar: Your upload failed checksum validation", MailUtil.getSubjectTextBasedOnNotification(userNotification, rootDataverseName, null)); + assertEquals("LibraScholar: Your upload failed checksum validation", MailUtil.getSubjectTextBasedOnNotification(userNotification, null)); } @Test public void testSubjectFileSystemImport() { userNotification.setType(UserNotification.Type.FILESYSTEMIMPORT); //TODO SEK add a dataset version to get the Dataset Title which is actually used in the subject now - assertEquals("Dataset LibraScholar has been successfully uploaded and verified", MailUtil.getSubjectTextBasedOnNotification(userNotification, rootDataverseName , null)); + assertEquals("Dataset LibraScholar has been successfully uploaded and verified", MailUtil.getSubjectTextBasedOnNotification(userNotification, null)); } @Test public void testSubjectChecksumImport() { userNotification.setType(UserNotification.Type.CHECKSUMIMPORT); - assertEquals("LibraScholar: Your file checksum job has completed", MailUtil.getSubjectTextBasedOnNotification(userNotification, rootDataverseName, null)); + assertEquals("LibraScholar: Your file checksum job has completed", MailUtil.getSubjectTextBasedOnNotification(userNotification, null)); } @Test public void testSubjectConfirmEmail() { userNotification.setType(UserNotification.Type.CONFIRMEMAIL); - assertEquals("LibraScholar: Verify your email address", MailUtil.getSubjectTextBasedOnNotification(userNotification, rootDataverseName, null)); + assertEquals("LibraScholar: Verify your email address", MailUtil.getSubjectTextBasedOnNotification(userNotification, null)); } } From 965c190addbbf5621a76528c4a0ec8ad81ee0b06 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Mar 2021 12:08:16 -0500 Subject: [PATCH 422/576] get root name directly --- .../iq/dataverse/AbstractGlobalIdServiceBean.java | 4 ++-- .../api/datadeposit/ServiceDocumentManagerImpl.java | 11 +++++------ .../harvest/server/web/servlet/OAIServlet.java | 2 +- .../edu/harvard/iq/dataverse/util/SystemConfig.java | 2 +- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/AbstractGlobalIdServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/AbstractGlobalIdServiceBean.java index 6305b871cc0..f6cbd01ece0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/AbstractGlobalIdServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/AbstractGlobalIdServiceBean.java @@ -62,7 +62,7 @@ protected Map addBasicMetadata(DvObject dvObjectIn, Map met metadataTemplate.setContacts(dataset.getLatestVersion().getDatasetContacts()); metadataTemplate.setProducers(dataset.getLatestVersion().getDatasetProducers()); metadataTemplate.setTitle(dvObject.getCurrentName()); - String producerString = dataverseService.findRootDataverse().getName(); + String producerString = dataverseService.getRootDataverseName(); if (producerString.isEmpty() || producerString.equals(DatasetField.NA_VALUE) ) { producerString = UNAVAILABLE; } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ServiceDocumentManagerImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ServiceDocumentManagerImpl.java index 7c80fe810d2..049b20f605b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ServiceDocumentManagerImpl.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ServiceDocumentManagerImpl.java @@ -10,6 +10,8 @@ import java.util.logging.Logger; import javax.ejb.EJB; import javax.inject.Inject; + +import org.apache.commons.lang3.StringUtils; import org.swordapp.server.AuthCredentials; import org.swordapp.server.ServiceDocument; import org.swordapp.server.ServiceDocumentManager; @@ -43,12 +45,9 @@ public ServiceDocument getServiceDocument(String sdUri, AuthCredentials authCred String warning = urlManager.processUrl(sdUri); ServiceDocument service = new ServiceDocument(); SwordWorkspace swordWorkspace = new SwordWorkspace(); - Dataverse rootDataverse = dataverseService.findRootDataverse(); - if (rootDataverse != null) { - String name = rootDataverse.getName(); - if (name != null) { - swordWorkspace.setTitle(name); - } + String name = dataverseService.getRootDataverseName(); + if (!StringUtils.isEmpty(name)) { + swordWorkspace.setTitle(name); } if (warning != null) { swordWorkspace.getWrappedWorkspace().setAttributeValue("warning", warning); diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java index c68cd98de4f..02c620d7655 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java @@ -177,7 +177,7 @@ private RepositoryConfiguration createRepositoryConfiguration() { // some of the settings below - such as the max list numbers - // need to be configurable! - String dataverseName = dataverseService.findRootDataverse().getName(); + String dataverseName = dataverseService.getRootDataverseName(); String repositoryName = StringUtils.isEmpty(dataverseName) || "Root".equals(dataverseName) ? "Test Dataverse OAI Archive" : dataverseName + " Dataverse OAI Archive"; InternetAddress internetAddress = MailUtil.parseSystemAddress(settingsService.getValueForKey(SettingsServiceBean.Key.SystemEmail)); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index a1ed4877f39..f5d129405b7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -650,7 +650,7 @@ public String getDefaultAuthProvider() { } public String getNameOfInstallation() { - return dataverseService.findRootDataverse().getName(); + return dataverseService.getRootDataverseName(); } public AbstractOAuth2AuthenticationProvider.DevOAuthAccountType getDevOAuthAccountType() { From a2d4d9f75382321a9b5b4816fec301bb39b29e68 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Mar 2021 12:10:52 -0500 Subject: [PATCH 423/576] add notes --- .../java/edu/harvard/iq/dataverse/DataverseServiceBean.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index 02bd7c841ab..7a66722121e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -196,6 +196,9 @@ public Dataverse findRootDataverse() { return em.createNamedQuery("Dataverse.findRoot", Dataverse.class).getSingleResult(); } + + //Similarly - if the above throws that exception, do we need to catch it here? + //ToDo - consider caching? public String getRootDataverseName() { Dataverse root = findRootDataverse(); if(root==null) { From 8093d2f63b58a9804ab6d4fbbb8430cd7b76c478 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Fri, 5 Mar 2021 12:18:05 -0500 Subject: [PATCH 424/576] Fixed truncate and ajax update issues [ref #6685, #7565] --- src/main/webapp/dataset.xhtml | 25 +++++++++++++------ src/main/webapp/dataverse_template.xhtml | 9 +++---- .../resources/js/dv_rebind_bootstrap_ui.js | 3 ++- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 949c97ceb4d..4d43a25ad59 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1653,19 +1653,29 @@ + summaryDescTruncation();     diff --git a/src/main/webapp/dataverse_template.xhtml b/src/main/webapp/dataverse_template.xhtml index 61487c0d1c0..8594cd0b579 100644 --- a/src/main/webapp/dataverse_template.xhtml +++ b/src/main/webapp/dataverse_template.xhtml @@ -79,14 +79,13 @@ + - - - - + + @@ -105,7 +104,7 @@ } ; }); - + // Rebind bootstrap UI components bind_bsui_components(); }); diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index a5a77009782..25a9d05e0fd 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -163,7 +163,8 @@ function sharrre(){ */ function contentTruncate(truncSelector, truncMoreBtn, truncMoreTip){ // SELECTOR ID FROM PARAMETERS - $('#' + truncSelector + ' td div').each(function () { + $('#' + truncSelector + ' td > div:first-child').each(function () { + // add responsive img class to limit width to that of container $(this).find('img').attr('class', 'img-responsive'); From 1f4d497e565675b06bdc0d77da5e0a5759c43b3c Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Mar 2021 12:20:43 -0500 Subject: [PATCH 425/576] avoid helper class --- .../iq/dataverse/branding/BrandingUtil.java | 8 ++++++ .../branding/BrandingUtilHelper.java | 28 ------------------- 2 files changed, 8 insertions(+), 28 deletions(-) delete mode 100644 src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtilHelper.java diff --git a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java index e083ac53349..407508a6898 100644 --- a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java @@ -4,12 +4,20 @@ import edu.harvard.iq.dataverse.settings.source.DbSettingConfigSource; import edu.harvard.iq.dataverse.util.BundleUtil; import java.util.Arrays; + +import javax.annotation.PostConstruct; +import javax.ejb.EJB; +import javax.ejb.Singleton; +import javax.ejb.Startup; import javax.mail.internet.InternetAddress; import org.apache.commons.lang3.StringUtils; +@Singleton +@Startup public class BrandingUtil { + @EJB static DataverseServiceBean dataverseService; public static String getInstallationBrandName() { diff --git a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtilHelper.java b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtilHelper.java deleted file mode 100644 index 07256f7bfff..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtilHelper.java +++ /dev/null @@ -1,28 +0,0 @@ -package edu.harvard.iq.dataverse.branding; - -import edu.harvard.iq.dataverse.DataverseServiceBean; -import edu.harvard.iq.dataverse.settings.SettingsServiceBean; - -import javax.annotation.PostConstruct; -import javax.ejb.EJB; -import javax.ejb.Singleton; -import javax.ejb.Startup; - -/** - * This is a small helper bean for the MPCONFIG DbSettingConfigSource. - * As it is a singleton and built at application start (=deployment), it will inject the (stateless) - * settings service into the MPCONFIG POJO once it's ready. - * - * MPCONFIG requires it's sources to be POJOs. No direct dependency injection possible. - */ -@Singleton -@Startup -public class BrandingUtilHelper { - @EJB - DataverseServiceBean dataverseService; - - @PostConstruct - public void injectService() { - BrandingUtil.injectDataverseService(dataverseService); - } -} From 7517267f8728e8470cb7b35d9fccf457c8f30114 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Mar 2021 12:50:56 -0500 Subject: [PATCH 426/576] remove #7387 tests --- .../iq/dataverse/branding/BrandingUtil.java | 5 ++-- .../dataverse/branding/BrandingUtilTest.java | 24 +------------------ 2 files changed, 3 insertions(+), 26 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java index 407508a6898..dd8ee64a607 100644 --- a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java @@ -5,7 +5,6 @@ import edu.harvard.iq.dataverse.util.BundleUtil; import java.util.Arrays; -import javax.annotation.PostConstruct; import javax.ejb.EJB; import javax.ejb.Singleton; import javax.ejb.Startup; @@ -21,8 +20,8 @@ public class BrandingUtil { static DataverseServiceBean dataverseService; public static String getInstallationBrandName() { - String installationName = new DbSettingConfigSource().getValue(DbSettingConfigSource.PREFIX + ".InstitutionName"); - return StringUtils.isEmpty(installationName) ? dataverseService.getRootDataverseName() : installationName; + //ToDo #7387 which will make this call return something different than getRootDataverseCollectionName() + return dataverseService.getRootDataverseName(); } //Convenience to access root name without injecting dataverseService (e.g. in DatasetVersion) diff --git a/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java index e75f905af1e..62bd4eed5c5 100644 --- a/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java @@ -1,15 +1,9 @@ package edu.harvard.iq.dataverse.branding; import edu.harvard.iq.dataverse.DataverseServiceBean; -import edu.harvard.iq.dataverse.settings.Setting; -import edu.harvard.iq.dataverse.settings.SettingsServiceBean; -import edu.harvard.iq.dataverse.settings.source.DbSettingConfigSource; import edu.harvard.iq.dataverse.util.BundleUtil; import java.io.UnsupportedEncodingException; import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; - import javax.mail.internet.AddressException; import javax.mail.internet.InternetAddress; import static org.junit.Assert.assertEquals; @@ -22,30 +16,14 @@ @ExtendWith(MockitoExtension.class) public class BrandingUtilTest { - DbSettingConfigSource dbSource = new DbSettingConfigSource(); - @Mock - SettingsServiceBean settingsSvc; @Mock DataverseServiceBean dataverseService; @Test public void testGetInstallationBrandName() { -// Set settings = new HashSet<>(Arrays.asList(new Setting("InstitutionName", "LibraScholar"), new Setting("FooBarI18N", "de", "hallo"))); - Mockito.when(dataverseService.getRootDataverseName()).thenReturn("LibraScholar"); - BrandingUtil.injectDataverseService(dataverseService); -// DbSettingConfigSource.injectSettingsService(settingsSvc); - System.out.println("testGetInstallationBrandName"); + System.out.println("testGetInstallationBrandName"); assertEquals("LibraScholar", BrandingUtil.getInstallationBrandName()); - - Set settings = new HashSet<>(Arrays.asList(new Setting("InstitutionName", "NotLibraScholar"))); - Mockito.when(settingsSvc.listAll()).thenReturn(settings); - DbSettingConfigSource.injectSettingsService(settingsSvc); - assertEquals("NotLibraScholar", BrandingUtil.getInstallationBrandName());// misconfiguration to set to null - - //Reset for other tests - settings.clear(); - settings.add(new Setting("InstitutionName", "LibraScholar")); } @Test From 1b4f360ad4d08eb34e350cf2ec63b11f6ac4e9d7 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Mar 2021 12:53:16 -0500 Subject: [PATCH 427/576] remove @startup --- .../java/edu/harvard/iq/dataverse/branding/BrandingUtil.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java index dd8ee64a607..67b68fc3b50 100644 --- a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java @@ -1,19 +1,14 @@ package edu.harvard.iq.dataverse.branding; import edu.harvard.iq.dataverse.DataverseServiceBean; -import edu.harvard.iq.dataverse.settings.source.DbSettingConfigSource; import edu.harvard.iq.dataverse.util.BundleUtil; import java.util.Arrays; import javax.ejb.EJB; import javax.ejb.Singleton; -import javax.ejb.Startup; import javax.mail.internet.InternetAddress; -import org.apache.commons.lang3.StringUtils; - @Singleton -@Startup public class BrandingUtil { @EJB From b273af063fcff99c5e4ccd3bf89ae73dc50da6f4 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Mar 2021 13:04:27 -0500 Subject: [PATCH 428/576] inject settingsService and rename for consistency --- .../harvard/iq/dataverse/export/ExportService.java | 3 --- .../harvard/iq/dataverse/util/json/JsonPrinter.java | 11 ++++++++--- .../iq/dataverse/util/json/JsonPrinterTest.java | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java b/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java index 9eaf0dbced0..b315b05c766 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java @@ -64,9 +64,6 @@ public static synchronized ExportService getInstance() { } public static synchronized ExportService getInstance(SettingsServiceBean settingsService) { - ExportService.settingsService = settingsService; - // We pass settingsService into the JsonPrinter so it can check the :ExcludeEmailFromExport setting in calls to JsonPrinter.jsonAsDatasetDto(). - JsonPrinter.setSettingsService(settingsService); if (service == null) { service = new ExportService(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 06e0610832c..e86d1acff7a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -57,6 +57,9 @@ import java.util.stream.Collector; import java.util.stream.Collectors; import static java.util.stream.Collectors.toList; + +import javax.ejb.EJB; +import javax.ejb.Singleton; import javax.json.JsonArray; import javax.json.JsonObject; @@ -65,14 +68,16 @@ * * @author michael */ +@Singleton public class JsonPrinter { private static final Logger logger = Logger.getLogger(JsonPrinter.class.getCanonicalName()); - static SettingsServiceBean settingsService = null; + @EJB + static SettingsServiceBean settingsService; - // Passed to DatasetFieldWalker so it can check the :ExcludeEmailFromExport setting - public static void setSettingsService(SettingsServiceBean ssb) { + // Used in Tests + public static void injectSettingsService(SettingsServiceBean ssb) { settingsService = ssb; } diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java index f8f0fdc7554..32f7b86a9c9 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java @@ -201,7 +201,7 @@ public void testDatasetContactOutOfBoxNoPrivacy() { fields.add(datasetContactField); SettingsServiceBean nullServiceBean = null; - JsonPrinter.setSettingsService(nullServiceBean); + JsonPrinter.injectSettingsService(nullServiceBean); JsonObject jsonObject = JsonPrinter.json(block, fields).build(); assertNotNull(jsonObject); @@ -242,7 +242,7 @@ public void testDatasetContactWithPrivacy() { datasetContactField.setDatasetFieldCompoundValues(vals); fields.add(datasetContactField); - JsonPrinter.setSettingsService(new MockSettingsSvc()); + JsonPrinter.injectSettingsService(new MockSettingsSvc()); JsonObject jsonObject = JsonPrinter.json(block, fields).build(); assertNotNull(jsonObject); From 6fa216bb061bd0d33a993e51afef5cb47e375b5a Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 5 Mar 2021 13:09:41 -0500 Subject: [PATCH 429/576] #7633 add alias and delete built in --- .../iq/dataverse/api/AbstractApiBean.java | 32 ++++++++++++++++++ .../edu/harvard/iq/dataverse/api/Admin.java | 13 ++++++++ .../edu/harvard/iq/dataverse/api/Roles.java | 33 ++++++++++--------- .../authorization/DataverseRole.java | 24 +++++++------- src/main/java/propertyFiles/Bundle.properties | 4 +++ 5 files changed, 80 insertions(+), 26 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index db0958800f8..b71625a36bd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -26,6 +26,7 @@ import edu.harvard.iq.dataverse.UserServiceBean; import edu.harvard.iq.dataverse.actionlogging.ActionLogServiceBean; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; +import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.authorization.RoleAssignee; import edu.harvard.iq.dataverse.authorization.groups.GroupServiceBean; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; @@ -86,6 +87,7 @@ public abstract class AbstractApiBean { private static final Logger logger = Logger.getLogger(AbstractApiBean.class.getName()); private static final String DATAVERSE_KEY_HEADER_NAME = "X-Dataverse-key"; private static final String PERSISTENT_ID_KEY=":persistentId"; + private static final String ALIAS_KEY=":alias"; public static final String STATUS_ERROR = "ERROR"; public static final String STATUS_OK = "OK"; public static final String STATUS_WF_IN_PROGRESS = "WORKFLOW_IN_PROGRESS"; @@ -483,6 +485,36 @@ protected DataFile findDataFileOrDie(String id) throws WrappedResponse { } } + + + protected DataverseRole findRoleOrDie(String id) throws WrappedResponse { + DataverseRole role; + if (id.equals(ALIAS_KEY)) { + String alias = getRequestParameter(ALIAS_KEY.substring(1)); + role = em.createNamedQuery("DataverseRole.findDataverseRoleByAlias", DataverseRole.class) + .setParameter("alias", alias) + .getSingleResult(); + } else { + + role = rolesSvc.find(Long.parseLong(id)); + + try { + role = rolesSvc.find(Long.parseLong(id)); + if (role == null) { + throw new WrappedResponse(notFound(BundleUtil.getStringFromBundle("find.datafile.error.datafile.not.found.id", Collections.singletonList(id)))); + } + + } catch (NumberFormatException nfe) { + throw new WrappedResponse( + badRequest(BundleUtil.getStringFromBundle("find.datafile.error.datafile.not.found.bad.id", Collections.singletonList(id)))); + } + } + + + throw new WrappedResponse(notFound("role with id " + id + " not found")); + + } + protected DatasetLinkingDataverse findDatasetLinkingDataverseOrDie(String datasetId, String linkingDataverseId) throws WrappedResponse { DatasetLinkingDataverse dsld; Dataverse linkingDataverse = findDataverseOrDie(linkingDataverseId); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index b52665a7747..de2f2266761 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -76,6 +76,7 @@ import java.util.List; import edu.harvard.iq.dataverse.authorization.AuthTestDataServiceBean; import edu.harvard.iq.dataverse.authorization.AuthenticationProvidersRegistrationServiceBean; +import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.authorization.RoleAssignee; import edu.harvard.iq.dataverse.authorization.UserRecordIdentifier; import edu.harvard.iq.dataverse.authorization.groups.impl.explicit.ExplicitGroupServiceBean; @@ -87,6 +88,7 @@ import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.impl.MergeInAccountCommand; import edu.harvard.iq.dataverse.engine.command.impl.ChangeUserIdentifierCommand; +import edu.harvard.iq.dataverse.engine.command.impl.DeleteRoleCommand; import edu.harvard.iq.dataverse.engine.command.impl.RegisterDvObjectCommand; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; @@ -890,6 +892,17 @@ public Response listBuiltinRoles() { return error(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); } } + + @DELETE + @Path("roles/{id}") + public Response deleteRole(@PathParam("id") String id) { + + return response(req -> { + DataverseRole doomed = findRoleOrDie(id); + execCommand(new DeleteRoleCommand(req, doomed)); + return ok("role " + doomed.getName() + " deleted."); + }); + } @Path("superuser/{identifier}") @POST diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Roles.java b/src/main/java/edu/harvard/iq/dataverse/api/Roles.java index b3f75e00c5a..72add184a24 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Roles.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Roles.java @@ -1,8 +1,10 @@ package edu.harvard.iq.dataverse.api; +import static edu.harvard.iq.dataverse.api.AbstractApiBean.error; import edu.harvard.iq.dataverse.api.dto.RoleDTO; import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; import javax.ws.rs.GET; import javax.ws.rs.POST; @@ -11,6 +13,9 @@ import static edu.harvard.iq.dataverse.util.json.JsonPrinter.*; import edu.harvard.iq.dataverse.engine.command.impl.CreateRoleCommand; import edu.harvard.iq.dataverse.engine.command.impl.DeleteRoleCommand; +import edu.harvard.iq.dataverse.util.BundleUtil; +import java.util.Arrays; +import java.util.List; import javax.ejb.Stateless; import javax.ws.rs.DELETE; import javax.ws.rs.QueryParam; @@ -26,7 +31,7 @@ public class Roles extends AbstractApiBean { @GET @Path("{id}") - public Response viewRole( @PathParam("id") Long id) { + public Response viewRole( @PathParam("id") String id) { return response( ()-> { final User user = findUserOrDie(); final DataverseRole role = findRoleOrDie(id); @@ -35,14 +40,19 @@ public Response viewRole( @PathParam("id") Long id) { }); } - @DELETE - @Path("{id}") - public Response deleteRole( @PathParam("id") Long id ) { - return response( req -> { - execCommand( new DeleteRoleCommand(req, findRoleOrDie(id)) ); - return ok("role " + id + " deleted."); + @DELETE + @Path("{id}") + public Response deleteRole(@PathParam("id") String id) { + return response(req -> { + DataverseRole role = findRoleOrDie(id); + List args = Arrays.asList(role.getName()); + if (role.getOwner() == null) { + throw new WrappedResponse(forbidden(BundleUtil.getStringFromBundle("find.dataverse.role.error.role.builtin.not.allowed", args))); + } + execCommand(new DeleteRoleCommand(req, role)); + return ok("role " + role.getName() + " deleted."); }); - } + } @POST public Response createNewRole( RoleDTO roleDto, @@ -52,11 +62,4 @@ public Response createNewRole( RoleDTO roleDto, req,findDataverseOrDie(dvoIdtf)))))); } - private DataverseRole findRoleOrDie( long id ) throws WrappedResponse { - DataverseRole role = rolesSvc.find(id); - if ( role != null ) { - return role; - } - throw new WrappedResponse(notFound( "role with id " + id + " not found")); - } } diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/DataverseRole.java b/src/main/java/edu/harvard/iq/dataverse/authorization/DataverseRole.java index 8ca72e7e9bc..12ddf817221 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/DataverseRole.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/DataverseRole.java @@ -33,18 +33,20 @@ * @author michael */ @NamedQueries({ - @NamedQuery(name = "DataverseRole.findByOwnerId", - query= "SELECT r FROM DataverseRole r WHERE r.owner.id=:ownerId ORDER BY r.name"), - @NamedQuery(name = "DataverseRole.findBuiltinRoles", - query= "SELECT r FROM DataverseRole r WHERE r.owner is null ORDER BY r.name"), + @NamedQuery(name = "DataverseRole.findByOwnerId", + query = "SELECT r FROM DataverseRole r WHERE r.owner.id=:ownerId ORDER BY r.name"), + @NamedQuery(name = "DataverseRole.findBuiltinRoles", + query = "SELECT r FROM DataverseRole r WHERE r.owner is null ORDER BY r.name"), @NamedQuery(name = "DataverseRole.findBuiltinRoleByAlias", - query= "SELECT r FROM DataverseRole r WHERE r.alias=:alias AND r.owner is null"), - @NamedQuery(name = "DataverseRole.findCustomRoleByAliasAndOwner", - query= "SELECT r FROM DataverseRole r WHERE r.alias=:alias and (r.owner is null or r.owner.id=:ownerId)"), - @NamedQuery(name = "DataverseRole.listAll", - query= "SELECT r FROM DataverseRole r"), - @NamedQuery(name = "DataverseRole.deleteById", - query= "DELETE FROM DataverseRole r WHERE r.id=:id") + query = "SELECT r FROM DataverseRole r WHERE r.alias=:alias AND r.owner is null"), + @NamedQuery(name = "DataverseRole.findDataverseRoleByAlias", + query = "SELECT r FROM DataverseRole r WHERE r.alias=:alias"), + @NamedQuery(name = "DataverseRole.findCustomRoleByAliasAndOwner", + query = "SELECT r FROM DataverseRole r WHERE r.alias=:alias and (r.owner is null or r.owner.id=:ownerId)"), + @NamedQuery(name = "DataverseRole.listAll", + query = "SELECT r FROM DataverseRole r"), + @NamedQuery(name = "DataverseRole.deleteById", + query = "DELETE FROM DataverseRole r WHERE r.id=:id") }) @Entity @Table(indexes = {@Index(columnList="owner_id") diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 8352e66c92d..eddbb905d3c 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1922,6 +1922,10 @@ find.dataverselinking.error.not.found.bad.ids=Bad dataverse ID number: {0} or da find.datafile.error.datafile.not.found.id=File with ID {0} not found. find.datafile.error.datafile.not.found.bad.id=Bad file ID number: {0}. find.datafile.error.dataset.not.found.persistentId=Datafile with Persistent ID {0} not found. +find.dataverse.role.error.role.not.found.id=Dataverse Role with ID {0} not found. +find.dataverse.role.error.role.not.found.bad.id=Bad Dataverse Role ID number: {0} +find.dataverse.role.error.role.not.found.alias=Dataverse Role with alias {0} not found. +find.dataverse.role.error.role.builtin.not.allowed=May not delete Built In Role {0}. file.addreplace.error.dataset_id_not_found=There was no dataset found for ID: file.addreplace.error.no_edit_dataset_permission=You do not have permission to edit this dataset. file.addreplace.error.filename_undetermined=The file name cannot be determined. From 883141c0aaa8beec91a19ed895dc6709ef68fafd Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 5 Mar 2021 13:21:10 -0500 Subject: [PATCH 430/576] #7373 update solr installation file --- .../source/_static/installation/files/etc/init.d/solr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr index d351c544a65..16d364fc9cf 100755 --- a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr +++ b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr @@ -5,7 +5,7 @@ # chkconfig: 35 92 08 # description: Starts and stops Apache Solr -SOLR_DIR="/usr/local/solr/solr-7.7.2" +SOLR_DIR="/usr/local/solr/solr-8.8.1" SOLR_COMMAND="bin/solr" SOLR_ARGS="-m 1g -j jetty.host=127.0.0.1" SOLR_USER=solr From 1dd75bf48770545ee63b225104eb9180ed94e7e4 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Mar 2021 13:31:29 -0500 Subject: [PATCH 431/576] use BrandingUtil in OREmap --- src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java | 3 ++- src/main/java/propertyFiles/Bundle.properties | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index 5520de3954e..289c1c8cf52 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -9,6 +9,7 @@ import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.TermsOfUseAndAccess; +import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.export.OAI_OREExporter; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.SystemConfig; @@ -247,7 +248,7 @@ public JsonObject getOREMap() throws Exception { JsonObject oremap = Json.createObjectBuilder() .add(JsonLDTerm.dcTerms("modified").getLabel(), LocalDate.now().toString()) .add(JsonLDTerm.dcTerms("creator").getLabel(), - BundleUtil.getStringFromBundle("institution.name")) + BrandingUtil.getInstallationBrandName()) .add("@type", JsonLDTerm.ore("ResourceMap").getLabel()) // Define an id for the map itself (separate from the @id of the dataset being // described diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 4d95322caf2..7ae45d2a000 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -164,7 +164,6 @@ contact.context.support.intro={0},\n\nThe following message was sent from {1}.\n contact.context.support.ending=\n\n---\n\nMessage sent from Support contact form. # dataverseuser.xhtml -institution.name=A Dataverse Instance account.info=Account Information account.edit=Edit Account account.apiToken=API Token From 511cffada01aa9bdf6bea344fd7a48ea1c14b8e4 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Mar 2021 17:32:45 -0500 Subject: [PATCH 432/576] try property inject just committing to record the code I tried to @Inject - did this and tried just a static private String, but both result in null pointer exceptions. The Provider suggests it will dynamically get new values, but the javadocs does say something about checking at the start to see if the property exists (which it doesn't for DbSettingsConfigSource?) --- .../iq/dataverse/branding/BrandingUtil.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java index 67b68fc3b50..e9fc331fccd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java @@ -1,13 +1,21 @@ package edu.harvard.iq.dataverse.branding; import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.settings.source.DbSettingConfigSource; import edu.harvard.iq.dataverse.util.BundleUtil; import java.util.Arrays; +import java.util.NoSuchElementException; import javax.ejb.EJB; import javax.ejb.Singleton; +import javax.inject.Inject; +import javax.inject.Provider; import javax.mail.internet.InternetAddress; +import org.apache.commons.lang3.StringUtils; +import org.eclipse.microprofile.config.ConfigProvider; +import org.eclipse.microprofile.config.inject.ConfigProperty; + @Singleton public class BrandingUtil { @@ -15,8 +23,15 @@ public class BrandingUtil { static DataverseServiceBean dataverseService; public static String getInstallationBrandName() { - //ToDo #7387 which will make this call return something different than getRootDataverseCollectionName() - return dataverseService.getRootDataverseName(); + String installationName=null; //new DbSettingConfigSource().getValue(DbSettingConfigSource.PREFIX + ".installation.name"); + try { + installationName = ConfigProvider.getConfig().getValue(DbSettingConfigSource.PREFIX + ".installation.name", String.class); + } catch (NoSuchElementException nse) { + installationName=null; + } + //System.out.println("fromdbsource: " + new DbSettingConfigSource().getValue(DbSettingConfigSource.PREFIX + ".installation.name")); + System.out.println("IN: " + installationName); + return StringUtils.isEmpty(installationName) ? dataverseService.getRootDataverseName() : installationName; } //Convenience to access root name without injecting dataverseService (e.g. in DatasetVersion) From 0df20d6113fd73da43cdf74056d86e72b9d6010e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Mar 2021 17:36:34 -0500 Subject: [PATCH 433/576] Bug fixes clear old properties on update invert logic to make caching work --- .../dataverse/settings/source/DbSettingConfigSource.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/source/DbSettingConfigSource.java b/src/main/java/edu/harvard/iq/dataverse/settings/source/DbSettingConfigSource.java index b54c720db4d..3bf672213e6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/source/DbSettingConfigSource.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/source/DbSettingConfigSource.java @@ -39,9 +39,10 @@ public static void injectSettingsService(SettingsServiceBean injected) { */ public static void updateProperties() { // skip if the service has not been injected yet - if (settingsSvc == null) + if (settingsSvc == null) { return; - + } + properties.clear(); Set dbSettings = settingsSvc.listAll(); dbSettings.forEach(s -> properties.put(PREFIX+"."+s.getName()+ (s.getLang() == null ? "" : "."+s.getLang()), s.getContent())); lastUpdate = Instant.now(); @@ -50,7 +51,7 @@ public static void updateProperties() { @Override public Map getProperties() { // if the cache is at least XX number of seconds old, update before serving data. - if (lastUpdate == null || Instant.now().minus(Duration.ofSeconds(60)).isBefore(lastUpdate)) { + if (lastUpdate == null || Instant.now().minus(Duration.ofSeconds(60)).isAfter(lastUpdate)) { updateProperties(); } return properties; From 6e89b9bbac36cc8ac25ef17da59073265dde85fe Mon Sep 17 00:00:00 2001 From: qqmyers Date: Sat, 6 Mar 2021 11:07:44 -0500 Subject: [PATCH 434/576] add installation setting and make including it as distr optional in DDI export stdyDcr - only use installation name if there is no distributor defined in metadata --- .../iq/dataverse/branding/BrandingUtil.java | 12 ++---- .../dataverse/export/ddi/DdiExportUtil.java | 16 +++++++- .../META-INF/microprofile-aliases.properties | 4 +- .../dataverse/branding/BrandingUtilTest.java | 38 ++++++++++++++++++- 4 files changed, 56 insertions(+), 14 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java index e9fc331fccd..1a5f13679dc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java @@ -1,36 +1,30 @@ package edu.harvard.iq.dataverse.branding; import edu.harvard.iq.dataverse.DataverseServiceBean; -import edu.harvard.iq.dataverse.settings.source.DbSettingConfigSource; import edu.harvard.iq.dataverse.util.BundleUtil; import java.util.Arrays; import java.util.NoSuchElementException; import javax.ejb.EJB; import javax.ejb.Singleton; -import javax.inject.Inject; -import javax.inject.Provider; import javax.mail.internet.InternetAddress; import org.apache.commons.lang3.StringUtils; import org.eclipse.microprofile.config.ConfigProvider; -import org.eclipse.microprofile.config.inject.ConfigProperty; @Singleton public class BrandingUtil { @EJB static DataverseServiceBean dataverseService; - + public static String getInstallationBrandName() { - String installationName=null; //new DbSettingConfigSource().getValue(DbSettingConfigSource.PREFIX + ".installation.name"); + String installationName=null; try { - installationName = ConfigProvider.getConfig().getValue(DbSettingConfigSource.PREFIX + ".installation.name", String.class); + installationName = ConfigProvider.getConfig().getValue("dataverse.branding.installation.name", String.class); } catch (NoSuchElementException nse) { installationName=null; } - //System.out.println("fromdbsource: " + new DbSettingConfigSource().getValue(DbSettingConfigSource.PREFIX + ".installation.name")); - System.out.println("IN: " + installationName); return StringUtils.isEmpty(installationName) ? dataverseService.getRootDataverseName() : installationName; } diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index dd797a55539..9aebb695d13 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -55,6 +55,8 @@ import org.xml.sax.SAXException; import org.xml.sax.SAXParseException; import org.w3c.dom.Document; +import org.apache.commons.lang3.StringUtils; +import org.eclipse.microprofile.config.ConfigProvider; import org.w3c.dom.DOMException; // For write operation @@ -195,7 +197,16 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) writeProducersElement(xmlw, version); xmlw.writeStartElement("distStmt"); - if (datasetDto.getPublisher() != null && !datasetDto.getPublisher().equals("")) { + //The default is to add Dataverse Repository as a distributor. The excludeinstallationifset setting turns that off if there is a distributor defined in the metadata + boolean distributorSet=false; + MetadataBlockDTO citationDTO= version.getMetadataBlocks().get("citation"); + if(citationDTO!=null) { + if(citationDTO.getField(DatasetFieldConstant.distributorName)!=null) { + distributorSet=true; + } + } + if (!StringUtils.isEmpty(datasetDto.getPublisher()) && !(ConfigProvider.getConfig() + .getValue("dataverse.export.distributor.excludeinstallationifset", Boolean.class) && distributorSet)) { xmlw.writeStartElement("distrbtr"); writeAttribute(xmlw, "source", "archive"); xmlw.writeCharacters(datasetDto.getPublisher()); @@ -308,7 +319,8 @@ private static void writeDocDescElement (XMLStreamWriter xmlw, DatasetDTO datase xmlw.writeEndElement(); // IDNo xmlw.writeEndElement(); // titlStmt xmlw.writeStartElement("distStmt"); - if (datasetDto.getPublisher() != null && !datasetDto.getPublisher().equals("")) { + //The doc is always published by the Dataverse Repository + if (!StringUtils.isEmpty(datasetDto.getPublisher())) { xmlw.writeStartElement("distrbtr"); writeAttribute(xmlw, "source", "archive"); xmlw.writeCharacters(datasetDto.getPublisher()); diff --git a/src/main/resources/META-INF/microprofile-aliases.properties b/src/main/resources/META-INF/microprofile-aliases.properties index 4efaf02b083..3dc688824ed 100644 --- a/src/main/resources/META-INF/microprofile-aliases.properties +++ b/src/main/resources/META-INF/microprofile-aliases.properties @@ -1,4 +1,6 @@ # NOTE # This file is a placeholder for future aliases of deprecated config settings. # Format: -# dataverse.new.config.option=dataverse.old.deprecated.option \ No newline at end of file +# dataverse.new.config.option=dataverse.old.deprecated.option +dataverse.branding.installation.name=dataverse.settings.fromdb.InstallationName +dataverse.export.distributor.excludeinstallationifset=dataverse.settings.fromdb.ExportInstallationAsDistributorOnlyWhenNotSet \ No newline at end of file diff --git a/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java index 62bd4eed5c5..e9f94b3d2a8 100644 --- a/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java @@ -1,29 +1,63 @@ package edu.harvard.iq.dataverse.branding; import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.settings.Setting; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.settings.source.DbSettingConfigSource; import edu.harvard.iq.dataverse.util.BundleUtil; import java.io.UnsupportedEncodingException; import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + import javax.mail.internet.AddressException; import javax.mail.internet.InternetAddress; import static org.junit.Assert.assertEquals; + +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.MethodOrderer.OrderAnnotation; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) +@TestMethodOrder(OrderAnnotation.class) public class BrandingUtilTest { @Mock DataverseServiceBean dataverseService; + DbSettingConfigSource dbSource = new DbSettingConfigSource(); + @Mock + SettingsServiceBean settingsSvc; @Test + @Order(1) public void testGetInstallationBrandName() { + System.out.println("testGetInstallationBrandName"); - System.out.println("testGetInstallationBrandName"); - assertEquals("LibraScholar", BrandingUtil.getInstallationBrandName()); + //Initialize the DBSettingConfigSource with empty properties + Set settings = new HashSet<>(); + Mockito.when(settingsSvc.listAll()).thenReturn(settings); + DbSettingConfigSource.injectSettingsService(settingsSvc); + + //And configure the mock DataverseService to pretend the root collection name is as shown + Mockito.when(dataverseService.getRootDataverseName()).thenReturn("LibraScholar"); + BrandingUtil.injectDataverseService(dataverseService); + + assertEquals("LibraScholar", BrandingUtil.getInstallationBrandName()); //Defaults to root collection name + + settings.add(new Setting("InstallationName", "NotLibraScholar")); + ///Mockito.when(settingsSvc.listAll()).thenReturn(settings); + DbSettingConfigSource.injectSettingsService(settingsSvc); + assertEquals("NotLibraScholar", BrandingUtil.getInstallationBrandName()); //uses setting + + // Reset for other tests + settings.clear(); + //Cause the config source to update (otherwise it will cache the old settings for ~60 seconds + DbSettingConfigSource.injectSettingsService(settingsSvc); } @Test From 72028e673cc15302d7d137402081236392df1ab9 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Sat, 6 Mar 2021 11:30:45 -0500 Subject: [PATCH 435/576] simplify logic with getOptionalValue() --- .../iq/dataverse/branding/BrandingUtil.java | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java index 1a5f13679dc..62ef336288e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java @@ -19,16 +19,12 @@ public class BrandingUtil { static DataverseServiceBean dataverseService; public static String getInstallationBrandName() { - String installationName=null; - try { - installationName = ConfigProvider.getConfig().getValue("dataverse.branding.installation.name", String.class); - } catch (NoSuchElementException nse) { - installationName=null; - } - return StringUtils.isEmpty(installationName) ? dataverseService.getRootDataverseName() : installationName; + return ConfigProvider.getConfig().getOptionalValue("dataverse.branding.installation.name", String.class) + .orElse(dataverseService.getRootDataverseName()); } - - //Convenience to access root name without injecting dataverseService (e.g. in DatasetVersion) + + // Convenience to access root name without injecting dataverseService (e.g. in + // DatasetVersion) public static String getRootDataverseCollectionName() { return dataverseService.getRootDataverseName(); } From 7942737dc1c53c282062291541eb2005dbf978ee Mon Sep 17 00:00:00 2001 From: qqmyers Date: Sat, 6 Mar 2021 11:31:27 -0500 Subject: [PATCH 436/576] don't send settingsService through ExportService --- .../edu/harvard/iq/dataverse/DatasetPage.java | 6 +++--- .../iq/dataverse/DatasetServiceBean.java | 2 +- .../edu/harvard/iq/dataverse/FilePage.java | 4 ++-- .../edu/harvard/iq/dataverse/api/Datasets.java | 2 +- .../edu/harvard/iq/dataverse/api/Files.java | 2 +- .../CuratePublishedDatasetVersionCommand.java | 2 +- .../impl/DeaccessionDatasetVersionCommand.java | 2 +- .../FinalizeDatasetPublicationCommand.java | 6 +++--- .../command/impl/RedetectFileTypeCommand.java | 2 +- .../iq/dataverse/export/ExportService.java | 15 +-------------- .../dataverse/export/HtmlCodeBookExporter.java | 2 +- .../iq/dataverse/export/OAI_OREExporter.java | 2 +- .../iq/dataverse/export/ddi/DdiExportUtil.java | 3 +-- .../harvest/server/OAIRecordServiceBean.java | 4 ++-- .../harvest/server/web/servlet/OAIServlet.java | 4 ++-- .../iq/dataverse/util/bagit/OREMap.java | 18 +++++++++++++++--- 16 files changed, 37 insertions(+), 39 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 533c8cc0840..a8189714c6c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -4198,13 +4198,13 @@ public String getTabularDataFileURL(Long fileid) { public List< String[]> getExporters(){ List retList = new ArrayList<>(); String myHostURL = getDataverseSiteUrl(); - for (String [] provider : ExportService.getInstance(settingsService).getExportersLabels() ){ + for (String [] provider : ExportService.getInstance().getExportersLabels() ){ String formatName = provider[1]; String formatDisplayName = provider[0]; Exporter exporter = null; try { - exporter = ExportService.getInstance(settingsService).getExporter(formatName); + exporter = ExportService.getInstance().getExporter(formatName); } catch (ExportException ex) { exporter = null; } @@ -5293,7 +5293,7 @@ public boolean isThisLatestReleasedVersion() { public String getJsonLd() { if (isThisLatestReleasedVersion()) { - ExportService instance = ExportService.getInstance(settingsService); + ExportService instance = ExportService.getInstance(); String jsonLd = instance.getExportAsString(dataset, SchemaDotOrgExporter.NAME); if (jsonLd != null) { logger.fine("Returning cached schema.org JSON-LD."); diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index c1efe119fd2..f77fcc78013 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -963,7 +963,7 @@ public long findStorageSize(Dataset dataset, boolean countCachedExtras, GetDatas // (i.e., the metadata exports): StorageIO datasetSIO = DataAccess.getStorageIO(dataset); - for (String[] exportProvider : ExportService.getInstance(settingsService).getExportersLabels()) { + for (String[] exportProvider : ExportService.getInstance().getExportersLabels()) { String exportLabel = "export_" + exportProvider[1] + ".cached"; try { total += datasetSIO.getAuxObjectSize(exportLabel); diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index da315424220..575b6e01fa6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -304,13 +304,13 @@ public void setVersion(String version) { public List< String[]> getExporters(){ List retList = new ArrayList<>(); String myHostURL = systemConfig.getDataverseSiteUrl(); - for (String [] provider : ExportService.getInstance(settingsService).getExportersLabels() ){ + for (String [] provider : ExportService.getInstance().getExportersLabels() ){ String formatName = provider[1]; String formatDisplayName = provider[0]; Exporter exporter = null; try { - exporter = ExportService.getInstance(settingsService).getExporter(formatName); + exporter = ExportService.getInstance().getExporter(formatName); } catch (ExportException ex) { exporter = null; } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 4ea4144fbf9..31b4ea98aee 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -265,7 +265,7 @@ public Response exportDataset(@QueryParam("persistentId") String persistentId, @ return error(Response.Status.NOT_FOUND, "A dataset with the persistentId " + persistentId + " could not be found."); } - ExportService instance = ExportService.getInstance(settingsSvc); + ExportService instance = ExportService.getInstance(); InputStream is = instance.getExport(dataset, exporter); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index defc2c4d9ab..334c89552a3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -601,7 +601,7 @@ public Response redetectDatafile(@PathParam("id") String id, @QueryParam("dryRun private void exportDatasetMetadata(SettingsServiceBean settingsServiceBean, Dataset theDataset) { try { - ExportService instance = ExportService.getInstance(settingsServiceBean); + ExportService instance = ExportService.getInstance(); instance.exportAllFormats(theDataset); } catch (ExportException ex) { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java index 373454bf694..3dcbe98a161 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java @@ -184,7 +184,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { //And the exported metadata files try { - ExportService instance = ExportService.getInstance(ctxt.settings()); + ExportService instance = ExportService.getInstance(); instance.exportAllFormats(getDataset()); } catch (ExportException ex) { // Just like with indexing, a failure to export is not a fatal condition. diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeaccessionDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeaccessionDatasetVersionCommand.java index 6aa5e0e250a..391e798e285 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeaccessionDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeaccessionDatasetVersionCommand.java @@ -75,7 +75,7 @@ public DatasetVersion execute(CommandContext ctxt) throws CommandException { boolean doNormalSolrDocCleanUp = true; - ExportService instance = ExportService.getInstance(ctxt.settings()); + ExportService instance = ExportService.getInstance(); if (managed.getDataset().getReleasedVersion() != null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java index c2f186f1e8c..ef45598f356 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java @@ -246,7 +246,7 @@ public boolean onSuccess(CommandContext ctxt, Object r) { } } - exportMetadata(dataset, ctxt.settings()); + exportMetadata(dataset); ctxt.datasets().updateLastExportTimeStamp(dataset.getId()); @@ -257,10 +257,10 @@ public boolean onSuccess(CommandContext ctxt, Object r) { * Attempting to run metadata export, for all the formats for which we have * metadata Exporters. */ - private void exportMetadata(Dataset dataset, SettingsServiceBean settingsServiceBean) { + private void exportMetadata(Dataset dataset) { try { - ExportService instance = ExportService.getInstance(settingsServiceBean); + ExportService instance = ExportService.getInstance(); instance.exportAllFormats(dataset); } catch (Exception ex) { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RedetectFileTypeCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RedetectFileTypeCommand.java index ef20ec76e12..8eeca0cb4cd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RedetectFileTypeCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RedetectFileTypeCommand.java @@ -89,7 +89,7 @@ public DataFile execute(CommandContext ctxt) throws CommandException { logger.info("Exception while reindexing files during file type redetection: " + ex.getLocalizedMessage()); } try { - ExportService instance = ExportService.getInstance(ctxt.settings()); + ExportService instance = ExportService.getInstance(); instance.exportAllFormats(dataset); } catch (ExportException ex) { // Just like with indexing, a failure to export is not a fatal condition. diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java b/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java index b315b05c766..52de3cf1d78 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java @@ -45,25 +45,12 @@ public class ExportService { private static ExportService service; private ServiceLoader loader; - static SettingsServiceBean settingsService; private ExportService() { loader = ServiceLoader.load(Exporter.class); } - /** - * @deprecated Use `getInstance(SettingsServiceBean settingsService)` - * instead. For privacy reasons, we need to pass in settingsService so that - * we can make a decision whether not not to exclude email addresses. No new - * code should call this method and it would be nice to remove calls from - * existing code. - */ - @Deprecated public static synchronized ExportService getInstance() { - return getInstance(null); - } - - public static synchronized ExportService getInstance(SettingsServiceBean settingsService) { if (service == null) { service = new ExportService(); } @@ -72,7 +59,7 @@ public static synchronized ExportService getInstance(SettingsServiceBean setting public List< String[]> getExportersLabels() { List retList = new ArrayList<>(); - Iterator exporters = ExportService.getInstance(null).loader.iterator(); + Iterator exporters = ExportService.getInstance().loader.iterator(); while (exporters.hasNext()) { Exporter e = exporters.next(); String[] temp = new String[2]; diff --git a/src/main/java/edu/harvard/iq/dataverse/export/HtmlCodeBookExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/HtmlCodeBookExporter.java index f77821a59e2..367ac4bbc5b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/HtmlCodeBookExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/HtmlCodeBookExporter.java @@ -35,7 +35,7 @@ public void exportDataset(DatasetVersion version, JsonObject json, OutputStream try { InputStream ddiInputStream; try { - ddiInputStream = ExportService.getInstance(ExportService.settingsService).getExport(version.getDataset(), "ddi"); + ddiInputStream = ExportService.getInstance().getExport(version.getDataset(), "ddi"); } catch(ExportException | IOException e) { throw new ExportException ("Cannot open export_ddi cached file"); } diff --git a/src/main/java/edu/harvard/iq/dataverse/export/OAI_OREExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/OAI_OREExporter.java index 6cfcb590681..87adc1b4c5b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/OAI_OREExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/OAI_OREExporter.java @@ -25,7 +25,7 @@ public class OAI_OREExporter implements Exporter { public void exportDataset(DatasetVersion version, JsonObject json, OutputStream outputStream) throws ExportException { try { - new OREMap(version, ExportService.settingsService.isTrueForKey(SettingsServiceBean.Key.ExcludeEmailFromExport, false)).writeOREMap(outputStream); + new OREMap(version).writeOREMap(outputStream); } catch (Exception e) { logger.severe(e.getMessage()); e.printStackTrace(); diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 9aebb695d13..9fba7cbd384 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -205,8 +205,7 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) distributorSet=true; } } - if (!StringUtils.isEmpty(datasetDto.getPublisher()) && !(ConfigProvider.getConfig() - .getValue("dataverse.export.distributor.excludeinstallationifset", Boolean.class) && distributorSet)) { + if (!StringUtils.isEmpty(datasetDto.getPublisher()) && !(ConfigProvider.getConfig().getOptionalValue("dataverse.export.distributor.excludeinstallationifset", Boolean.class).orElse(false) && distributorSet)) { xmlw.writeStartElement("distrbtr"); writeAttribute(xmlw, "source", "archive"); xmlw.writeCharacters(datasetDto.getPublisher()); diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java index 973c712b5c8..02e7675a776 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java @@ -229,7 +229,7 @@ public void markOaiRecordsAsRemoved(Collection records, Date updateTi public void exportAllFormats(Dataset dataset) { try { - ExportService exportServiceInstance = ExportService.getInstance(settingsService); + ExportService exportServiceInstance = ExportService.getInstance(); logger.log(Level.FINE, "Attempting to run export on dataset {0}", dataset.getGlobalId()); exportServiceInstance.exportAllFormats(dataset); datasetService.updateLastExportTimeStamp(dataset.getId()); @@ -240,7 +240,7 @@ public void exportAllFormats(Dataset dataset) { @TransactionAttribute(REQUIRES_NEW) public void exportAllFormatsInNewTransaction(Dataset dataset) throws ExportException { try { - ExportService exportServiceInstance = ExportService.getInstance(settingsService); + ExportService exportServiceInstance = ExportService.getInstance(); exportServiceInstance.exportAllFormats(dataset); datasetService.updateLastExportTimeStamp(dataset.getId()); } catch (Exception e) { diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java index 02c620d7655..5d4c02a87e2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java @@ -132,11 +132,11 @@ private Context createContext() { } private void addSupportedMetadataFormats(Context context) { - for (String[] provider : ExportService.getInstance(settingsService).getExportersLabels()) { + for (String[] provider : ExportService.getInstance().getExportersLabels()) { String formatName = provider[1]; Exporter exporter; try { - exporter = ExportService.getInstance(settingsService).getExporter(formatName); + exporter = ExportService.getInstance().getExporter(formatName); } catch (ExportException ex) { exporter = null; } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index 289c1c8cf52..7989a901d6c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -11,6 +11,7 @@ import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.export.OAI_OREExporter; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.json.JsonLDNamespace; @@ -25,6 +26,8 @@ import java.util.TreeMap; import java.util.Map.Entry; +import javax.ejb.Stateless; +import javax.inject.Inject; import javax.json.Json; import javax.json.JsonArray; import javax.json.JsonArrayBuilder; @@ -32,16 +35,25 @@ import javax.json.JsonObjectBuilder; import javax.json.JsonValue; +@Stateless public class OREMap { + @Inject static SettingsServiceBean settingsService; + public static final String NAME = "OREMap"; private Map localContext = new TreeMap(); private DatasetVersion version; private boolean excludeEmail = false; - public OREMap(DatasetVersion version, boolean excludeEmail) { + public OREMap(DatasetVersion version) { + this(version, settingsService.isTrueForKey(SettingsServiceBean.Key.ExcludeEmailFromExport, false)); + + } + + //Used when the ExcludeEmailFromExport needs to be overriden, i.e. for archiving + public OREMap(DatasetVersion dv, boolean exclude) { this.version = version; - this.excludeEmail = excludeEmail; + this.excludeEmail = exclude; } public void writeOREMap(OutputStream outputStream) throws Exception { @@ -167,7 +179,7 @@ public JsonObject getOREMap() throws Exception { } aggBuilder.add(JsonLDTerm.schemaOrg("includedInDataCatalog").getLabel(), - dataset.getDataverseContext().getDisplayName()); + BrandingUtil.getRootDataverseCollectionName()); // The aggregation aggregates aggregatedresources (Datafiles) which each have // their own entry and metadata From edc8b64701a85202f82b22aa7e3439b4263504b8 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Sat, 6 Mar 2021 11:31:27 -0500 Subject: [PATCH 437/576] don't send settingsService through ExportService --- .../edu/harvard/iq/dataverse/DatasetPage.java | 6 +++--- .../iq/dataverse/DatasetServiceBean.java | 2 +- .../edu/harvard/iq/dataverse/FilePage.java | 4 ++-- .../edu/harvard/iq/dataverse/api/Datasets.java | 2 +- .../edu/harvard/iq/dataverse/api/Files.java | 2 +- .../CuratePublishedDatasetVersionCommand.java | 2 +- .../impl/DeaccessionDatasetVersionCommand.java | 2 +- .../FinalizeDatasetPublicationCommand.java | 6 +++--- .../command/impl/RedetectFileTypeCommand.java | 2 +- .../iq/dataverse/export/ExportService.java | 15 +-------------- .../dataverse/export/HtmlCodeBookExporter.java | 2 +- .../iq/dataverse/export/OAI_OREExporter.java | 2 +- .../harvest/server/OAIRecordServiceBean.java | 4 ++-- .../harvest/server/web/servlet/OAIServlet.java | 4 ++-- .../iq/dataverse/util/bagit/OREMap.java | 18 +++++++++++++++--- 15 files changed, 36 insertions(+), 37 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 533c8cc0840..a8189714c6c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -4198,13 +4198,13 @@ public String getTabularDataFileURL(Long fileid) { public List< String[]> getExporters(){ List retList = new ArrayList<>(); String myHostURL = getDataverseSiteUrl(); - for (String [] provider : ExportService.getInstance(settingsService).getExportersLabels() ){ + for (String [] provider : ExportService.getInstance().getExportersLabels() ){ String formatName = provider[1]; String formatDisplayName = provider[0]; Exporter exporter = null; try { - exporter = ExportService.getInstance(settingsService).getExporter(formatName); + exporter = ExportService.getInstance().getExporter(formatName); } catch (ExportException ex) { exporter = null; } @@ -5293,7 +5293,7 @@ public boolean isThisLatestReleasedVersion() { public String getJsonLd() { if (isThisLatestReleasedVersion()) { - ExportService instance = ExportService.getInstance(settingsService); + ExportService instance = ExportService.getInstance(); String jsonLd = instance.getExportAsString(dataset, SchemaDotOrgExporter.NAME); if (jsonLd != null) { logger.fine("Returning cached schema.org JSON-LD."); diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index c1efe119fd2..f77fcc78013 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -963,7 +963,7 @@ public long findStorageSize(Dataset dataset, boolean countCachedExtras, GetDatas // (i.e., the metadata exports): StorageIO datasetSIO = DataAccess.getStorageIO(dataset); - for (String[] exportProvider : ExportService.getInstance(settingsService).getExportersLabels()) { + for (String[] exportProvider : ExportService.getInstance().getExportersLabels()) { String exportLabel = "export_" + exportProvider[1] + ".cached"; try { total += datasetSIO.getAuxObjectSize(exportLabel); diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index da315424220..575b6e01fa6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -304,13 +304,13 @@ public void setVersion(String version) { public List< String[]> getExporters(){ List retList = new ArrayList<>(); String myHostURL = systemConfig.getDataverseSiteUrl(); - for (String [] provider : ExportService.getInstance(settingsService).getExportersLabels() ){ + for (String [] provider : ExportService.getInstance().getExportersLabels() ){ String formatName = provider[1]; String formatDisplayName = provider[0]; Exporter exporter = null; try { - exporter = ExportService.getInstance(settingsService).getExporter(formatName); + exporter = ExportService.getInstance().getExporter(formatName); } catch (ExportException ex) { exporter = null; } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 4ea4144fbf9..31b4ea98aee 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -265,7 +265,7 @@ public Response exportDataset(@QueryParam("persistentId") String persistentId, @ return error(Response.Status.NOT_FOUND, "A dataset with the persistentId " + persistentId + " could not be found."); } - ExportService instance = ExportService.getInstance(settingsSvc); + ExportService instance = ExportService.getInstance(); InputStream is = instance.getExport(dataset, exporter); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index defc2c4d9ab..334c89552a3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -601,7 +601,7 @@ public Response redetectDatafile(@PathParam("id") String id, @QueryParam("dryRun private void exportDatasetMetadata(SettingsServiceBean settingsServiceBean, Dataset theDataset) { try { - ExportService instance = ExportService.getInstance(settingsServiceBean); + ExportService instance = ExportService.getInstance(); instance.exportAllFormats(theDataset); } catch (ExportException ex) { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java index 373454bf694..3dcbe98a161 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java @@ -184,7 +184,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { //And the exported metadata files try { - ExportService instance = ExportService.getInstance(ctxt.settings()); + ExportService instance = ExportService.getInstance(); instance.exportAllFormats(getDataset()); } catch (ExportException ex) { // Just like with indexing, a failure to export is not a fatal condition. diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeaccessionDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeaccessionDatasetVersionCommand.java index 6aa5e0e250a..391e798e285 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeaccessionDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeaccessionDatasetVersionCommand.java @@ -75,7 +75,7 @@ public DatasetVersion execute(CommandContext ctxt) throws CommandException { boolean doNormalSolrDocCleanUp = true; - ExportService instance = ExportService.getInstance(ctxt.settings()); + ExportService instance = ExportService.getInstance(); if (managed.getDataset().getReleasedVersion() != null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java index c2f186f1e8c..ef45598f356 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java @@ -246,7 +246,7 @@ public boolean onSuccess(CommandContext ctxt, Object r) { } } - exportMetadata(dataset, ctxt.settings()); + exportMetadata(dataset); ctxt.datasets().updateLastExportTimeStamp(dataset.getId()); @@ -257,10 +257,10 @@ public boolean onSuccess(CommandContext ctxt, Object r) { * Attempting to run metadata export, for all the formats for which we have * metadata Exporters. */ - private void exportMetadata(Dataset dataset, SettingsServiceBean settingsServiceBean) { + private void exportMetadata(Dataset dataset) { try { - ExportService instance = ExportService.getInstance(settingsServiceBean); + ExportService instance = ExportService.getInstance(); instance.exportAllFormats(dataset); } catch (Exception ex) { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RedetectFileTypeCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RedetectFileTypeCommand.java index ef20ec76e12..8eeca0cb4cd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RedetectFileTypeCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RedetectFileTypeCommand.java @@ -89,7 +89,7 @@ public DataFile execute(CommandContext ctxt) throws CommandException { logger.info("Exception while reindexing files during file type redetection: " + ex.getLocalizedMessage()); } try { - ExportService instance = ExportService.getInstance(ctxt.settings()); + ExportService instance = ExportService.getInstance(); instance.exportAllFormats(dataset); } catch (ExportException ex) { // Just like with indexing, a failure to export is not a fatal condition. diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java b/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java index b315b05c766..52de3cf1d78 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java @@ -45,25 +45,12 @@ public class ExportService { private static ExportService service; private ServiceLoader loader; - static SettingsServiceBean settingsService; private ExportService() { loader = ServiceLoader.load(Exporter.class); } - /** - * @deprecated Use `getInstance(SettingsServiceBean settingsService)` - * instead. For privacy reasons, we need to pass in settingsService so that - * we can make a decision whether not not to exclude email addresses. No new - * code should call this method and it would be nice to remove calls from - * existing code. - */ - @Deprecated public static synchronized ExportService getInstance() { - return getInstance(null); - } - - public static synchronized ExportService getInstance(SettingsServiceBean settingsService) { if (service == null) { service = new ExportService(); } @@ -72,7 +59,7 @@ public static synchronized ExportService getInstance(SettingsServiceBean setting public List< String[]> getExportersLabels() { List retList = new ArrayList<>(); - Iterator exporters = ExportService.getInstance(null).loader.iterator(); + Iterator exporters = ExportService.getInstance().loader.iterator(); while (exporters.hasNext()) { Exporter e = exporters.next(); String[] temp = new String[2]; diff --git a/src/main/java/edu/harvard/iq/dataverse/export/HtmlCodeBookExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/HtmlCodeBookExporter.java index f77821a59e2..367ac4bbc5b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/HtmlCodeBookExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/HtmlCodeBookExporter.java @@ -35,7 +35,7 @@ public void exportDataset(DatasetVersion version, JsonObject json, OutputStream try { InputStream ddiInputStream; try { - ddiInputStream = ExportService.getInstance(ExportService.settingsService).getExport(version.getDataset(), "ddi"); + ddiInputStream = ExportService.getInstance().getExport(version.getDataset(), "ddi"); } catch(ExportException | IOException e) { throw new ExportException ("Cannot open export_ddi cached file"); } diff --git a/src/main/java/edu/harvard/iq/dataverse/export/OAI_OREExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/OAI_OREExporter.java index 6cfcb590681..87adc1b4c5b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/OAI_OREExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/OAI_OREExporter.java @@ -25,7 +25,7 @@ public class OAI_OREExporter implements Exporter { public void exportDataset(DatasetVersion version, JsonObject json, OutputStream outputStream) throws ExportException { try { - new OREMap(version, ExportService.settingsService.isTrueForKey(SettingsServiceBean.Key.ExcludeEmailFromExport, false)).writeOREMap(outputStream); + new OREMap(version).writeOREMap(outputStream); } catch (Exception e) { logger.severe(e.getMessage()); e.printStackTrace(); diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java index 973c712b5c8..02e7675a776 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java @@ -229,7 +229,7 @@ public void markOaiRecordsAsRemoved(Collection records, Date updateTi public void exportAllFormats(Dataset dataset) { try { - ExportService exportServiceInstance = ExportService.getInstance(settingsService); + ExportService exportServiceInstance = ExportService.getInstance(); logger.log(Level.FINE, "Attempting to run export on dataset {0}", dataset.getGlobalId()); exportServiceInstance.exportAllFormats(dataset); datasetService.updateLastExportTimeStamp(dataset.getId()); @@ -240,7 +240,7 @@ public void exportAllFormats(Dataset dataset) { @TransactionAttribute(REQUIRES_NEW) public void exportAllFormatsInNewTransaction(Dataset dataset) throws ExportException { try { - ExportService exportServiceInstance = ExportService.getInstance(settingsService); + ExportService exportServiceInstance = ExportService.getInstance(); exportServiceInstance.exportAllFormats(dataset); datasetService.updateLastExportTimeStamp(dataset.getId()); } catch (Exception e) { diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java index 02c620d7655..5d4c02a87e2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java @@ -132,11 +132,11 @@ private Context createContext() { } private void addSupportedMetadataFormats(Context context) { - for (String[] provider : ExportService.getInstance(settingsService).getExportersLabels()) { + for (String[] provider : ExportService.getInstance().getExportersLabels()) { String formatName = provider[1]; Exporter exporter; try { - exporter = ExportService.getInstance(settingsService).getExporter(formatName); + exporter = ExportService.getInstance().getExporter(formatName); } catch (ExportException ex) { exporter = null; } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index 289c1c8cf52..7989a901d6c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -11,6 +11,7 @@ import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.export.OAI_OREExporter; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.json.JsonLDNamespace; @@ -25,6 +26,8 @@ import java.util.TreeMap; import java.util.Map.Entry; +import javax.ejb.Stateless; +import javax.inject.Inject; import javax.json.Json; import javax.json.JsonArray; import javax.json.JsonArrayBuilder; @@ -32,16 +35,25 @@ import javax.json.JsonObjectBuilder; import javax.json.JsonValue; +@Stateless public class OREMap { + @Inject static SettingsServiceBean settingsService; + public static final String NAME = "OREMap"; private Map localContext = new TreeMap(); private DatasetVersion version; private boolean excludeEmail = false; - public OREMap(DatasetVersion version, boolean excludeEmail) { + public OREMap(DatasetVersion version) { + this(version, settingsService.isTrueForKey(SettingsServiceBean.Key.ExcludeEmailFromExport, false)); + + } + + //Used when the ExcludeEmailFromExport needs to be overriden, i.e. for archiving + public OREMap(DatasetVersion dv, boolean exclude) { this.version = version; - this.excludeEmail = excludeEmail; + this.excludeEmail = exclude; } public void writeOREMap(OutputStream outputStream) throws Exception { @@ -167,7 +179,7 @@ public JsonObject getOREMap() throws Exception { } aggBuilder.add(JsonLDTerm.schemaOrg("includedInDataCatalog").getLabel(), - dataset.getDataverseContext().getDisplayName()); + BrandingUtil.getRootDataverseCollectionName()); // The aggregation aggregates aggregatedresources (Datafiles) which each have // their own entry and metadata From 97c9e4bd3a07aa810952adb4f635a843da988f8e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Sat, 6 Mar 2021 11:43:02 -0500 Subject: [PATCH 438/576] revert - want parent dv not root --- src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index 7989a901d6c..cbb0666f331 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -179,7 +179,7 @@ public JsonObject getOREMap() throws Exception { } aggBuilder.add(JsonLDTerm.schemaOrg("includedInDataCatalog").getLabel(), - BrandingUtil.getRootDataverseCollectionName()); + dataset.getDataverseContext().getDisplayName()); // The aggregation aggregates aggregatedresources (Datafiles) which each have // their own entry and metadata From cdc99d509d1c3cdd3571720b36b304c1415cb3e1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Sat, 6 Mar 2021 11:46:19 -0500 Subject: [PATCH 439/576] null is not an option --- .../java/edu/harvard/iq/dataverse/DataverseServiceBean.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index 7a66722121e..46444423b43 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -201,10 +201,6 @@ public Dataverse findRootDataverse() { //ToDo - consider caching? public String getRootDataverseName() { Dataverse root = findRootDataverse(); - if(root==null) { - logger.warning("Failed to find root dataverse!"); - return ""; - } String rootDataverseName=root.getName(); return StringUtil.isEmpty(rootDataverseName) ? "" : rootDataverseName; } From fb8d5a159281497e57520f3e387994eff177ead3 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Sat, 6 Mar 2021 11:46:19 -0500 Subject: [PATCH 440/576] null is not an option --- .../java/edu/harvard/iq/dataverse/DataverseServiceBean.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index 7a66722121e..46444423b43 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -201,10 +201,6 @@ public Dataverse findRootDataverse() { //ToDo - consider caching? public String getRootDataverseName() { Dataverse root = findRootDataverse(); - if(root==null) { - logger.warning("Failed to find root dataverse!"); - return ""; - } String rootDataverseName=root.getName(); return StringUtil.isEmpty(rootDataverseName) ? "" : rootDataverseName; } From ba3e943bb97c364be41a12935723942a14086b9d Mon Sep 17 00:00:00 2001 From: qqmyers Date: Sat, 6 Mar 2021 11:49:18 -0500 Subject: [PATCH 441/576] removing test per review Note - a similar test will reappear when BrandingUtil supports a property that can replace the rootDataverseName for getInstallationBrandName() --- .../harvard/iq/dataverse/branding/BrandingUtilTest.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java index 62bd4eed5c5..7d175c37e41 100644 --- a/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java @@ -19,13 +19,6 @@ public class BrandingUtilTest { @Mock DataverseServiceBean dataverseService; - @Test - public void testGetInstallationBrandName() { - - System.out.println("testGetInstallationBrandName"); - assertEquals("LibraScholar", BrandingUtil.getInstallationBrandName()); - } - @Test public void testGetSupportTeamName() throws AddressException, UnsupportedEncodingException { System.out.println("testGetSupportTeamName"); From d52634d879dc2d3b68807f760c5d7926db02aa88 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Sat, 6 Mar 2021 13:09:02 -0500 Subject: [PATCH 442/576] remove imports --- .../java/edu/harvard/iq/dataverse/branding/BrandingUtil.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java index 62ef336288e..6b6bc77cca8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java @@ -3,13 +3,10 @@ import edu.harvard.iq.dataverse.DataverseServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import java.util.Arrays; -import java.util.NoSuchElementException; - import javax.ejb.EJB; import javax.ejb.Singleton; import javax.mail.internet.InternetAddress; -import org.apache.commons.lang3.StringUtils; import org.eclipse.microprofile.config.ConfigProvider; @Singleton From a06c56c9d314a9b1aed6ce0683f5e39dffabdfa8 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Sat, 6 Mar 2021 13:23:08 -0500 Subject: [PATCH 443/576] settings doc --- .../source/installation/config.rst | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index bc038489c42..e07fabba6e6 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -594,7 +594,7 @@ You may also want to look at samples at https://github.com/shlake/LibraDataHomep A simpler option to brand and customize your installation is to utilize the Dataverse collection theme, which each Dataverse collection has, that allows you to change colors, add a logo, tagline or website link to the Dataverse collection header section of the page. Those options are outlined in the :doc:`/user/dataverse-management` section of the User Guide. Custom Homepage -++++++++++++++++ ++++++++++++++++ The Dataverse Software allows you to use a custom homepage or welcome page in place of the default root Dataverse collection page. This allows for complete control over the look and feel of your installation's homepage. @@ -613,7 +613,7 @@ For more background on what this curl command above is doing, see the "Database ``curl -X DELETE http://localhost:8080/api/admin/settings/:HomePageCustomizationFile`` Custom Navbar Logo -+++++++++++++++++++ +++++++++++++++++++ The Dataverse Software allows you to replace the default Dataverse Project icon and name branding in the navbar with your own custom logo. Note that this logo is separate from the *root dataverse theme* logo. @@ -667,7 +667,7 @@ Internationalization The Dataverse Software is being translated into multiple languages by the Dataverse Project Community! Please see below for how to help with this effort! Adding Multiple Languages to the Dropdown in the Header -++++++++++++++++++++++++++++++++++++++++++++++++++++++++ ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ The presence of the :ref:`:Languages` database setting adds a dropdown in the header for multiple languages. For example to add English and French to the dropdown: @@ -1465,7 +1465,7 @@ Please note that the authority cannot have a slash ("/") in it. .. _:Shoulder: :Shoulder -++++++++++++ ++++++++++ Out of the box, the DOI shoulder is set to "FK2/" but this is for testing only! When you apply for your DOI namespace, you may have requested a shoulder. The following is only an example and a trailing slash is optional. @@ -1509,9 +1509,9 @@ The chart below shows examples from each possible combination of parameters from | | randomString | sequentialNumber | | | | | +=================+===============+==================+ -| **DEPENDENT** | TJCLKP/MLGWJO | 100001/1 | +| **DEPENDENT** | TJCLKP/MLGWJO | 100001/1 | +-----------------+---------------+------------------+ -| **INDEPENDENT** | MLGWJO | 100002 | +| **INDEPENDENT** | MLGWJO | 100002 | +-----------------+---------------+------------------+ As seen above, in cases where ``:IdentifierGenerationStyle`` is set to *sequentialNumber* and ``:DataFilePIDFormat`` is set to *DEPENDENT*, each file within a dataset will be assigned a number *within* that dataset starting with "1". @@ -1536,7 +1536,7 @@ Note: File-level PID registration was added in Dataverse Software 4.9; it could .. _:IndependentHandleService: :IndependentHandleService -+++++++++++++++++++++++++++ ++++++++++++++++++++++++++ Specific for Handle PIDs. Set this setting to true if you want to use a Handle service which is setup to work 'independently' (No communication with the Global Handle Registry). By default this setting is absent and the Dataverse Software assumes it to be false. @@ -2035,7 +2035,7 @@ Set the name of the cloud environment you've integrated with your Dataverse inst .. _:PublicInstall: :PublicInstall -+++++++++++++++++++++ +++++++++++++++ Setting an installation to public will remove the ability to restrict data files or datasets. This functionality of the Dataverse Software will be disabled from your installation. @@ -2224,3 +2224,13 @@ This is the local file system path to be used with the LocalSubmitToArchiveComma +++++++++++++++++++ These are the bucket and project names to be used with the GoogleCloudSubmitToArchiveCommand class. Further information is in the :ref:`Google Cloud Configuration` section above. + +:InstallationName ++++++++++++++++++ + +By default, the name of the root Dataverse collection is used as the 'brandname' of the repository, i.e. in emails and metadata exports. If set, :InstallationName overrides this default, allowing the root collection name and brandname to be set independently. + +:ExportInstallationAsDistributorOnlyWhenNotSet +++++++++++++++++++++++++++++++++++++++++++++++ + +In the DDI metadata exports, the default behavior is to always add the repository (using its brandname - the root collection name or the value of :ref:`:InstallationName <:InstallationName>`) to the stdyDscr/distStmt/distrbtr element. If this setting is true, this will only be done when a Distributor is not already defined in the Dataset metadata. \ No newline at end of file From 36f0677859aa57a3bff818a3059e9d9da7f6321f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Sat, 6 Mar 2021 14:24:21 -0500 Subject: [PATCH 444/576] doc for the new publish parameter and /timestamps call --- doc/sphinx-guides/source/api/native-api.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index c02733629f4..741cc5bf3fb 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1020,6 +1020,8 @@ When publishing a dataset it's good to be aware of the Dataverse Software's vers If this is the first version of the dataset, its version number will be set to ``1.0``. Otherwise, the new dataset version number is determined by the most recent version number and the ``type`` parameter. Passing ``type=minor`` increases the minor version number (2.3 is updated to 2.4). Passing ``type=major`` increases the major version number (2.3 is updated to 3.0). (Superusers can pass ``type=updatecurrent`` to update metadata without changing the version number.) +This call also supports an optional boolean query parameter: ``assureIsIndexed``. If true, the call will fail, with a 409 ("CONFLICT") response, if the dataset is awaiting re-indexing. If indexing occurs during publishing, it could cause the publish request to fail, after a 202 response has been received. Using this parameter allows the caller to wait for indexing to occur and avoid this possibility. It is most useful in situations where edits are made immediately before publication. + .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. .. code-block:: bash @@ -1039,7 +1041,7 @@ The fully expanded example above (without environment variables) looks like this The quotes around the URL are required because there is more than one query parameter separated by an ampersand (``&``), which has special meaning to Unix shells such as Bash. Putting the ``&`` in quotes ensures that "type" is interpreted as one of the query parameters. -You should expect JSON output and a 200 ("OK") response in most cases. If you receive a 202 ("ACCEPTED") response, this is normal for installations that have workflows configured. Workflows are described in the :doc:`/developers/workflows` section of the Developer Guide. +You should expect JSON output and a 200 ("OK") response in most cases. If you receive a 202 ("ACCEPTED") response, this is normal for installations that have workflows configured. Workflows are described in the :doc:`/developers/workflows` section of the Developer Guide. A 409 ("CONFLICT") response is also possible if you set ``assureIsIndexed`=true. (In this case, one could then repeat the call until a 200/202 response is sent.) .. note:: POST should be used to publish a dataset. GET is supported for backward compatibility but is deprecated and may be removed: https://github.com/IQSS/dataverse/issues/2431 @@ -1742,6 +1744,15 @@ Configure a Dataset to Use a Specific File Store ``/api/datasets/$dataset-id/storageDriver`` can be used to check, configure or reset the designated file store (storage driver) for a dataset. Please see the :doc:`/admin/dataverses-datasets` section of the guide for more information on this API. +View the Timestamps on a Dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``/api/datasets/$dataset-id/timestamps`` can be used to view timestamps associated with various events in the dataset's lifecycle. For published datasets, this API call provides the ``createTime``, ``publicationTime``, ``lastMetadataExportTime`` and ``lastMajorVersionReleaseTime``, as well as two booleans - ``hasStaleIndex`` and ``hasStalePermissionIndex`` - which, if true, indicate the Dataverse displays for the dataset are up-to-date. The reponses is ``application/json`` with the timestamps included in the returned ``data`` object. + +When called by a user who can view the draft version of the dataset, additional timestamps are reported: ``lastUpdateTime``, ``lastIndexTime``, ``lastPermissionUpdateTime``, and ``globalIdCreateTime``. + +One use case where this API call could be useful is in allowing an external application to poll and wait for changes being made by the Dataverse software or other external tool to complete prior to continuing its own processing. + Files ----- From c550a949c2ed9b441e244b1d7abee3f9f32ffa5e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Sat, 6 Mar 2021 14:28:44 -0500 Subject: [PATCH 445/576] typos --- doc/sphinx-guides/source/api/native-api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 741cc5bf3fb..d50ef05d87e 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1020,7 +1020,7 @@ When publishing a dataset it's good to be aware of the Dataverse Software's vers If this is the first version of the dataset, its version number will be set to ``1.0``. Otherwise, the new dataset version number is determined by the most recent version number and the ``type`` parameter. Passing ``type=minor`` increases the minor version number (2.3 is updated to 2.4). Passing ``type=major`` increases the major version number (2.3 is updated to 3.0). (Superusers can pass ``type=updatecurrent`` to update metadata without changing the version number.) -This call also supports an optional boolean query parameter: ``assureIsIndexed``. If true, the call will fail, with a 409 ("CONFLICT") response, if the dataset is awaiting re-indexing. If indexing occurs during publishing, it could cause the publish request to fail, after a 202 response has been received. Using this parameter allows the caller to wait for indexing to occur and avoid this possibility. It is most useful in situations where edits are made immediately before publication. +This call also supports an optional boolean query parameter: ``assureIsIndexed``. If true, the call will fail with a 409 ("CONFLICT") response if the dataset is awaiting re-indexing. If indexing occurs during publishing it could cause the publish request to fail, after a 202 response has been received. Using this parameter allows the caller to wait for indexing to occur and avoid this possibility. It is most useful in situations where edits are made immediately before publication. .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. @@ -1747,7 +1747,7 @@ Configure a Dataset to Use a Specific File Store View the Timestamps on a Dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``/api/datasets/$dataset-id/timestamps`` can be used to view timestamps associated with various events in the dataset's lifecycle. For published datasets, this API call provides the ``createTime``, ``publicationTime``, ``lastMetadataExportTime`` and ``lastMajorVersionReleaseTime``, as well as two booleans - ``hasStaleIndex`` and ``hasStalePermissionIndex`` - which, if true, indicate the Dataverse displays for the dataset are up-to-date. The reponses is ``application/json`` with the timestamps included in the returned ``data`` object. +``/api/datasets/$dataset-id/timestamps`` can be used to view timestamps associated with various events in the dataset's lifecycle. For published datasets, this API call provides the ``createTime``, ``publicationTime``, ``lastMetadataExportTime`` and ``lastMajorVersionReleaseTime``, as well as two booleans - ``hasStaleIndex`` and ``hasStalePermissionIndex`` - which, if false, indicate the Dataverse displays for the dataset are up-to-date. The response is ``application/json`` with the timestamps included in the returned ``data`` object. When called by a user who can view the draft version of the dataset, additional timestamps are reported: ``lastUpdateTime``, ``lastIndexTime``, ``lastPermissionUpdateTime``, and ``globalIdCreateTime``. From 518e9eec415f5fc3a0b8e829f50987158d354b2d Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Sun, 7 Mar 2021 12:18:36 -0500 Subject: [PATCH 446/576] add checksumtype support --- .../iq/dataverse/datasetutility/AddReplaceFileHelper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index 2ca74bb1e39..0174a94b185 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -508,9 +508,10 @@ public boolean runReplaceFromUI_Phase1(Long oldFileId, //Update params to match existing file (except checksum, which should match the new file) if(fileToReplace != null) { String checksum = optionalFileParams.getCheckSum(); + ChecksumType checkSumType = optionalFileParams.getCheckSumType(); try { optionalFileParams = new OptionalFileParams(fileToReplace); - optionalFileParams.setCheckSum(checksum); + optionalFileParams.setCheckSum(checksum, checkSumType); } catch (DataFileTagException e) { // Shouldn't happen since fileToReplace should have valid tags e.printStackTrace(); From 810e917d058c4021b96fc2520349969e5798fe7b Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Mon, 8 Mar 2021 11:06:37 -0500 Subject: [PATCH 447/576] Add read less btn to dataset desc truncate, plus code clean up [ref #6685] --- src/main/java/propertyFiles/Bundle.properties | 2 ++ src/main/webapp/dataset.xhtml | 10 ++++---- src/main/webapp/resources/css/structure.css | 3 ++- .../resources/js/dv_rebind_bootstrap_ui.js | 24 +++++++++++++------ 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index b386e8084f4..8855793f7be 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -61,6 +61,8 @@ delete=Delete copyClipboard=Copy to Clipboard truncateMoreBtn=Read full {0} [+] truncateMoreTip=Click to read the full {0}. +truncateLessBtn=Collapse {0} [+] +truncateLessTip=Click to collapse the {0}. yes=Yes no=No previous=Previous diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 4d43a25ad59..fbe9a6c656b 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1663,15 +1663,17 @@ }); function summaryDescTruncation() { // truncate summary description, contentTruncate function in dv_rebind_bootstrap_ui.js - // add summary desc row id to pass as function selector + // add summary desc row id to pass as selector param var descId = 'dsDescription'; - // add metadata dynamic label text to button and tooltip text from bundle + // add metadata dynamic label text, tooltip text from bundle as param var truncMoreBtn = '#{of:format1(bundle.truncateMoreBtn, DatasetPage.datasetVersionUI.description.datasetFieldType.localeTitle)}'; var truncMoreTip = '#{of:format1(bundle.truncateMoreTip, DatasetPage.datasetVersionUI.description.datasetFieldType.localeTitle)}'; + var truncLessBtn = '#{of:format1(bundle.truncateLessBtn, DatasetPage.datasetVersionUI.description.datasetFieldType.localeTitle)}'; + var truncLessTip = '#{of:format1(bundle.truncateLessTip, DatasetPage.datasetVersionUI.description.datasetFieldType.localeTitle)}'; if (!truncactionHappened) { - // pass text params to function - contentTruncate(descId, truncMoreBtn, truncMoreTip); + // pass bundle text variabls as params to function + contentTruncate(descId, truncMoreBtn, truncMoreTip, truncLessBtn, truncLessTip); truncactionHappened = true; } diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index cd6785679ae..f1cc793df5f 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -653,8 +653,9 @@ div.edit-field div.ui-message {margin:6px 0;} white-space: pre-wrap !important; } /* TRUNCATION */ -div.more-fade-block {text-align:center; padding-top:250px; width:100%; position:absolute; bottom:0; background:linear-gradient(180deg,hsla(0,0%,100%,0),#fff 80%);} +div.more-block {text-align:center; padding-top:250px; width:100%; position:absolute; bottom:0; background:linear-gradient(180deg,hsla(0,0%,100%,0),#fff 80%);} button.desc-more-link {margin:0; padding:0;} +div.less-block {text-align:center;}; /* Primefaces vs Bootstrap selectOneMenu fix */ div.ui-selectonemenu.form-control {padding:0px;} diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index 25a9d05e0fd..fb4e3a5b971 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -161,7 +161,7 @@ function sharrre(){ /* * Truncate dataset description content */ -function contentTruncate(truncSelector, truncMoreBtn, truncMoreTip){ +function contentTruncate(truncSelector, truncMoreBtn, truncMoreTip, truncLessBtn, truncLessTip){ // SELECTOR ID FROM PARAMETERS $('#' + truncSelector + ' td > div:first-child').each(function () { @@ -175,17 +175,27 @@ function contentTruncate(truncSelector, truncMoreBtn, truncMoreTip){ // ADD A MAX-HEIGHT TO CONTAINER $(this).css({'max-height':'250px','overflow-y':'hidden','position':'relative'}); - // BTN LABEL TEXT, DYNAMIC ARIA ATTR'S, FROM BUNDLE VIA PARAMETERS + // BTN LABEL TEXT, ARIA ATTR'S, FROM BUNDLE VIA PARAMETERS var readMoreBtn = ''; - var moreFade = '
    ' + readMoreBtn + '
    '; + var moreBlock = '
    ' + readMoreBtn + '
    '; + var readLessBtn = ''; + var lessBlock = '
    ' + readLessBtn + '
    '; - // add Read More + button background fade - $(this).append(moreFade); + // add "Read full desc [+]" btn, background fade + $(this).append(moreBlock); - // ... add full description to summary block on Read More link click ++ add responsive img class + // show full description in summary block on "Read full desc [+]" btn click $(document).on('click', 'button.desc-more-link', function() { $(this).tooltip('hide').parent('div').parent('div').css({'max-height':'none','overflow-y':'visible','position':'relative'}); - $(this).parent('div.more-fade-block').remove(); + $(this).parent('div.more-block').replaceWith(lessBlock); + $('.less-block button').tooltip(); + }); + + // trucnate description in summary block on "Collapse desc [-]" btn click + $(document).on('click', 'button.desc-less-link', function() { + $(this).tooltip('hide').parent('div').parent('div').css({'max-height':'250px','overflow-y':'hidden','position':'relative'}); + $(this).parent('div.less-block').replaceWith(moreBlock); + $('.more-block button').tooltip(); }); } }); From fe6b13a535e49c110ea56bcb6be1df9a6d44c9b4 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Mon, 8 Mar 2021 11:25:10 -0500 Subject: [PATCH 448/576] Added animation and scroll target to desc collapse [ref #6685] --- src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index fb4e3a5b971..9f7e377dea6 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -195,6 +195,7 @@ function contentTruncate(truncSelector, truncMoreBtn, truncMoreTip, truncLessBtn $(document).on('click', 'button.desc-less-link', function() { $(this).tooltip('hide').parent('div').parent('div').css({'max-height':'250px','overflow-y':'hidden','position':'relative'}); $(this).parent('div.less-block').replaceWith(moreBlock); + $('html, body').animate({scrollTop: $('#' + truncSelector).offset().top - 60}, 500); $('.more-block button').tooltip(); }); } From 42534fc0d1a5a83f150264a9752975d0efd2ff67 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 8 Mar 2021 11:36:42 -0500 Subject: [PATCH 449/576] update per review --- doc/sphinx-guides/source/installation/config.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index e07fabba6e6..9995f0172d3 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2228,9 +2228,9 @@ These are the bucket and project names to be used with the GoogleCloudSubmitToAr :InstallationName +++++++++++++++++ -By default, the name of the root Dataverse collection is used as the 'brandname' of the repository, i.e. in emails and metadata exports. If set, :InstallationName overrides this default, allowing the root collection name and brandname to be set independently. +By default, the name of the root Dataverse collection is used as the 'brandname' of the repository, i.e. in emails and metadata exports. If set, :InstallationName overrides this default, allowing the root collection name and brandname to be set independently. (Note that, since metadata export files are cached, they will have to be reexported (see :doc:`/admin/metadataexport`) before they incorporate a change in this setting.) :ExportInstallationAsDistributorOnlyWhenNotSet ++++++++++++++++++++++++++++++++++++++++++++++ -In the DDI metadata exports, the default behavior is to always add the repository (using its brandname - the root collection name or the value of :ref:`:InstallationName <:InstallationName>`) to the stdyDscr/distStmt/distrbtr element. If this setting is true, this will only be done when a Distributor is not already defined in the Dataset metadata. \ No newline at end of file +In the DDI metadata exports, the default behavior is to always add the repository (using its brandname - the root collection name or the value of :ref:`:InstallationName <:InstallationName>`) to the stdyDscr/distStmt/distrbtr element. If this setting is true, this will only be done when a Distributor is not already defined in the Dataset metadata. (Note that, since metadata export files are cached, they will have to be reexported (see :doc:`/admin/metadataexport`) before they incorporate a change in this setting.) \ No newline at end of file From 7eb7f34bb0ee230eab2f1cabab7b75f64d6ce823 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Mon, 8 Mar 2021 15:26:20 -0500 Subject: [PATCH 450/576] Removed unused code from truncation [ref #6685] --- src/main/webapp/dataset.xhtml | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index fbe9a6c656b..0198e303b06 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1654,12 +1654,9 @@ - summaryDescTruncation(); From bf05205c1eaab2d517c9175e8499d6932e851eab Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 8 Mar 2021 16:35:11 -0500 Subject: [PATCH 451/576] #7633 Add integration tests --- scripts/api/data/role-test-addRole.json | 10 +++ .../iq/dataverse/api/AbstractApiBean.java | 13 ++-- .../edu/harvard/iq/dataverse/api/RolesIT.java | 69 +++++++++++++++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 36 ++++++++++ 4 files changed, 123 insertions(+), 5 deletions(-) create mode 100644 scripts/api/data/role-test-addRole.json create mode 100644 src/test/java/edu/harvard/iq/dataverse/api/RolesIT.java diff --git a/scripts/api/data/role-test-addRole.json b/scripts/api/data/role-test-addRole.json new file mode 100644 index 00000000000..7923eed916b --- /dev/null +++ b/scripts/api/data/role-test-addRole.json @@ -0,0 +1,10 @@ +{ + "alias":"testRole", + "name":"Test Role", + "description":"Test Role for adding/deleting.", + "permissions":[ + "ViewUnpublishedDataset", + "ViewUnpublishedDataverse", + "DownloadFile" + ] +} diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index b71625a36bd..7313dde5a35 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -494,14 +494,20 @@ protected DataverseRole findRoleOrDie(String id) throws WrappedResponse { role = em.createNamedQuery("DataverseRole.findDataverseRoleByAlias", DataverseRole.class) .setParameter("alias", alias) .getSingleResult(); - } else { + if (role == null) { + throw new WrappedResponse(notFound(BundleUtil.getStringFromBundle("find.datafile.error.datafile.not.found.alias", Collections.singletonList(alias)))); + } else { + return role; + } - role = rolesSvc.find(Long.parseLong(id)); + } else { try { role = rolesSvc.find(Long.parseLong(id)); if (role == null) { throw new WrappedResponse(notFound(BundleUtil.getStringFromBundle("find.datafile.error.datafile.not.found.id", Collections.singletonList(id)))); + } else { + return role; } } catch (NumberFormatException nfe) { @@ -510,9 +516,6 @@ protected DataverseRole findRoleOrDie(String id) throws WrappedResponse { } } - - throw new WrappedResponse(notFound("role with id " + id + " not found")); - } protected DatasetLinkingDataverse findDatasetLinkingDataverseOrDie(String datasetId, String linkingDataverseId) throws WrappedResponse { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/RolesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/RolesIT.java new file mode 100644 index 00000000000..85df1932a35 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/api/RolesIT.java @@ -0,0 +1,69 @@ + +package edu.harvard.iq.dataverse.api; + +import com.jayway.restassured.RestAssured; +import com.jayway.restassured.path.json.JsonPath; +import com.jayway.restassured.response.Response; +import edu.harvard.iq.dataverse.authorization.DataverseRole; +import edu.harvard.iq.dataverse.authorization.groups.impl.builtin.AuthenticatedUsers; +import java.util.logging.Logger; +import static javax.ws.rs.core.Response.Status.OK; +import static junit.framework.Assert.assertEquals; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * + * @author skraffmi + */ +public class RolesIT { + + private static final Logger logger = Logger.getLogger(AdminIT.class.getCanonicalName()); + + @BeforeClass + public static void setUp() { + RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); + } + + @Test + public void testCreateDeleteRoles() { + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + String pathToJsonFile = "scripts/api/data/role-test-addRole.json"; + Response addBuiltinRoleResponse = UtilIT.addBuiltInRole(pathToJsonFile); + addBuiltinRoleResponse.prettyPrint(); + String body = addBuiltinRoleResponse.getBody().asString(); + String status = JsonPath.from(body).getString("status"); + assertEquals("OK", status); + + //Try to delete from non-admin api - should fail. + + Response deleteBuiltinRoleResponseError = UtilIT.deleteDataverseRole("testRole", apiToken); + deleteBuiltinRoleResponseError.prettyPrint(); + body = deleteBuiltinRoleResponseError.getBody().asString(); + status = JsonPath.from(body).getString("status"); + assertEquals("ERROR", status); + + + Response deleteBuiltinRoleResponseSucceed = UtilIT.deleteBuiltInRole("testRole"); + deleteBuiltinRoleResponseSucceed.prettyPrint(); + body = deleteBuiltinRoleResponseSucceed.getBody().asString(); + status = JsonPath.from(body).getString("status"); + assertEquals("OK", status); + +/* + Response deleteUserResponse = UtilIT.deleteUser(username); + deleteUserResponse.prettyPrint(); + assertEquals(200, deleteUserResponse.getStatusCode()); +*/ + } + +} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index f3ff8f8fae4..90bad4b81cd 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2495,6 +2495,42 @@ static Response addBannerMessage(String pathToJsonFile) { return addBannerMessageResponse; } + static Response addBuiltInRole(String pathToJsonFile) { + String jsonIn = getDatasetJson(pathToJsonFile); + + Response addBannerMessageResponse = given() + .body(jsonIn) + .contentType("application/json") + .post("/api/admin/roles"); + return addBannerMessageResponse; + } + + static Response deleteBuiltInRole(String roleAlias) { + + Response addBannerMessageResponse = given() + .delete("/api/admin/roles/:alias?alias=" +roleAlias); + return addBannerMessageResponse; + } + + static Response addDataverseRole(String pathToJsonFile, String dvAlias, String apiToken) { + String jsonIn = getDatasetJson(pathToJsonFile); + + Response addBannerMessageResponse = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .body(jsonIn) + .contentType("application/json") + .post("/api/roles"); + return addBannerMessageResponse; + } + + static Response deleteDataverseRole( String roleAlias, String apiToken) { + + Response addBannerMessageResponse = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .delete("/api/roles/:alias?alias="+roleAlias); + return addBannerMessageResponse; + } + static Response getBannerMessages() { Response getBannerMessagesResponse = given() From ec27d6e97027b92d588b3674adbdbb54a1b5fc0d Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 9 Mar 2021 11:52:55 -0500 Subject: [PATCH 452/576] #7633 add integration tests --- .../iq/dataverse/api/AbstractApiBean.java | 25 +++++++------ .../edu/harvard/iq/dataverse/api/RolesIT.java | 35 ++++++++++++++++--- .../edu/harvard/iq/dataverse/api/UtilIT.java | 26 +++++++++++++- 3 files changed, 67 insertions(+), 19 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 7313dde5a35..3465cc55039 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -70,6 +70,7 @@ import javax.json.JsonValue; import javax.json.JsonValue.ValueType; import javax.persistence.EntityManager; +import javax.persistence.NoResultException; import javax.persistence.PersistenceContext; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.core.Context; @@ -484,20 +485,19 @@ protected DataFile findDataFileOrDie(String id) throws WrappedResponse { } } } - - - + protected DataverseRole findRoleOrDie(String id) throws WrappedResponse { DataverseRole role; if (id.equals(ALIAS_KEY)) { String alias = getRequestParameter(ALIAS_KEY.substring(1)); - role = em.createNamedQuery("DataverseRole.findDataverseRoleByAlias", DataverseRole.class) - .setParameter("alias", alias) - .getSingleResult(); - if (role == null) { - throw new WrappedResponse(notFound(BundleUtil.getStringFromBundle("find.datafile.error.datafile.not.found.alias", Collections.singletonList(alias)))); - } else { - return role; + try { + return em.createNamedQuery("DataverseRole.findDataverseRoleByAlias", DataverseRole.class) + .setParameter("alias", alias) + .getSingleResult(); + + //Should not be a multiple result exception due to table constraint + } catch (NoResultException nre) { + throw new WrappedResponse(notFound(BundleUtil.getStringFromBundle("find.dataverse.role.error.role.not.found.alias", Collections.singletonList(alias)))); } } else { @@ -505,17 +505,16 @@ protected DataverseRole findRoleOrDie(String id) throws WrappedResponse { try { role = rolesSvc.find(Long.parseLong(id)); if (role == null) { - throw new WrappedResponse(notFound(BundleUtil.getStringFromBundle("find.datafile.error.datafile.not.found.id", Collections.singletonList(id)))); + throw new WrappedResponse(notFound(BundleUtil.getStringFromBundle("find.dataverse.role.error.role.not.found.id", Collections.singletonList(id)))); } else { return role; } } catch (NumberFormatException nfe) { throw new WrappedResponse( - badRequest(BundleUtil.getStringFromBundle("find.datafile.error.datafile.not.found.bad.id", Collections.singletonList(id)))); + badRequest(BundleUtil.getStringFromBundle("find.dataverse.role.error.role.not.found.bad.id", Collections.singletonList(id)))); } } - } protected DatasetLinkingDataverse findDatasetLinkingDataverseOrDie(String datasetId, String linkingDataverseId) throws WrappedResponse { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/RolesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/RolesIT.java index 85df1932a35..d12641ce0c4 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/RolesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/RolesIT.java @@ -30,8 +30,10 @@ public void testCreateDeleteRoles() { Response createUser = UtilIT.createRandomUser(); createUser.prettyPrint(); + String username = UtilIT.getUsernameFromResponse(createUser); String apiToken = UtilIT.getApiTokenFromResponse(createUser); + UtilIT.makeSuperUser(username); Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); createDataverseResponse.prettyPrint(); @@ -52,6 +54,8 @@ public void testCreateDeleteRoles() { status = JsonPath.from(body).getString("status"); assertEquals("ERROR", status); + // deleteTitleViaNative.then().assertThat().body("message", equalTo("Error parsing dataset update: Empty value for field: Title ")); + Response deleteBuiltinRoleResponseSucceed = UtilIT.deleteBuiltInRole("testRole"); deleteBuiltinRoleResponseSucceed.prettyPrint(); @@ -59,11 +63,32 @@ public void testCreateDeleteRoles() { status = JsonPath.from(body).getString("status"); assertEquals("OK", status); -/* - Response deleteUserResponse = UtilIT.deleteUser(username); - deleteUserResponse.prettyPrint(); - assertEquals(200, deleteUserResponse.getStatusCode()); -*/ + //add as dataverse role + Response addDataverseRoleResponse = UtilIT.addDataverseRole(pathToJsonFile, dataverseAlias, apiToken); + addDataverseRoleResponse.prettyPrint(); + body = addBuiltinRoleResponse.getBody().asString(); + status = JsonPath.from(body).getString("status"); + assertEquals("OK", status); + + Response viewDataverseRoleResponse = UtilIT.viewDataverseRole("testRole", apiToken); + viewDataverseRoleResponse.prettyPrint(); + body = viewDataverseRoleResponse.getBody().asString(); + String idString = JsonPath.from(body).getString("data.id"); + + System.out.print("idString: " + idString); + + Response deleteDataverseRoleResponseBadAlias = UtilIT.deleteDataverseRole("badAlias", apiToken); + deleteDataverseRoleResponseBadAlias.prettyPrint(); + body = deleteDataverseRoleResponseBadAlias.getBody().asString(); + status = JsonPath.from(body).getString("status"); + assertEquals("ERROR", status); + + Response deleteDataverseRoleResponseSucceed = UtilIT.deleteDataverseRole("testRole", apiToken); + deleteDataverseRoleResponseSucceed.prettyPrint(); + body = deleteDataverseRoleResponseSucceed.getBody().asString(); + status = JsonPath.from(body).getString("status"); + assertEquals("OK", status); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 90bad4b81cd..c1887fafbe6 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2519,7 +2519,7 @@ static Response addDataverseRole(String pathToJsonFile, String dvAlias, String a .header(API_TOKEN_HTTP_HEADER, apiToken) .body(jsonIn) .contentType("application/json") - .post("/api/roles"); + .post("/api/roles?dvo="+dvAlias); return addBannerMessageResponse; } @@ -2531,6 +2531,30 @@ static Response deleteDataverseRole( String roleAlias, String apiToken) { return addBannerMessageResponse; } + static Response deleteDataverseRoleById( String id, String apiToken) { + + Response addBannerMessageResponse = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .delete("/api/roles/"+id); + return addBannerMessageResponse; + } + + static Response viewDataverseRole( String roleAlias, String apiToken) { + + Response addBannerMessageResponse = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/roles/:alias?alias="+roleAlias); + return addBannerMessageResponse; + } + + static Response viewDataverseRoleById( String id, String apiToken) { + + Response addBannerMessageResponse = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/roles/"+id); + return addBannerMessageResponse; + } + static Response getBannerMessages() { Response getBannerMessagesResponse = given() From 8c5a52b82144216920be7feb436b686fc3e7bc30 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 9 Mar 2021 15:10:41 -0500 Subject: [PATCH 453/576] #7633 add test for delete by id --- .../edu/harvard/iq/dataverse/api/RolesIT.java | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/RolesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/RolesIT.java index d12641ce0c4..d1e2ffb2426 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/RolesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/RolesIT.java @@ -4,11 +4,9 @@ import com.jayway.restassured.RestAssured; import com.jayway.restassured.path.json.JsonPath; import com.jayway.restassured.response.Response; -import edu.harvard.iq.dataverse.authorization.DataverseRole; -import edu.harvard.iq.dataverse.authorization.groups.impl.builtin.AuthenticatedUsers; import java.util.logging.Logger; -import static javax.ws.rs.core.Response.Status.OK; import static junit.framework.Assert.assertEquals; +import static org.hamcrest.CoreMatchers.equalTo; import org.junit.BeforeClass; import org.junit.Test; @@ -54,7 +52,7 @@ public void testCreateDeleteRoles() { status = JsonPath.from(body).getString("status"); assertEquals("ERROR", status); - // deleteTitleViaNative.then().assertThat().body("message", equalTo("Error parsing dataset update: Empty value for field: Title ")); + deleteBuiltinRoleResponseError.then().assertThat().body("message", equalTo("May not delete Built In Role Test Role.")); Response deleteBuiltinRoleResponseSucceed = UtilIT.deleteBuiltInRole("testRole"); @@ -81,9 +79,18 @@ public void testCreateDeleteRoles() { deleteDataverseRoleResponseBadAlias.prettyPrint(); body = deleteDataverseRoleResponseBadAlias.getBody().asString(); status = JsonPath.from(body).getString("status"); + assertEquals("ERROR", status); + deleteDataverseRoleResponseBadAlias.then().assertThat().body("message", equalTo("Dataverse Role with alias badAlias not found.")); + + Long idBad = Long.parseLong(idString) + 10; + Response deleteDataverseRoleResponseBadId = UtilIT.deleteDataverseRoleById(idBad.toString(), apiToken); + deleteDataverseRoleResponseBadId.prettyPrint(); + body = deleteDataverseRoleResponseBadId.getBody().asString(); + status = JsonPath.from(body).getString("status"); assertEquals("ERROR", status); + deleteDataverseRoleResponseBadId.then().assertThat().body("message", equalTo("Dataverse Role with ID " + idBad.toString() + " not found.")); - Response deleteDataverseRoleResponseSucceed = UtilIT.deleteDataverseRole("testRole", apiToken); + Response deleteDataverseRoleResponseSucceed = UtilIT.deleteDataverseRoleById(idString, apiToken); deleteDataverseRoleResponseSucceed.prettyPrint(); body = deleteDataverseRoleResponseSucceed.getBody().asString(); status = JsonPath.from(body).getString("status"); From 768ed3c7b5d879f6dc8792bf4d56b887d6a0a1eb Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 9 Mar 2021 16:06:29 -0500 Subject: [PATCH 454/576] flagging release note --- doc/release-notes/820-non-ascii-chars-in-search.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release-notes/820-non-ascii-chars-in-search.md b/doc/release-notes/820-non-ascii-chars-in-search.md index c449a68cb11..9e21dd83694 100644 --- a/doc/release-notes/820-non-ascii-chars-in-search.md +++ b/doc/release-notes/820-non-ascii-chars-in-search.md @@ -1,3 +1,5 @@ +(review these notes if this gets into the same release as #7645 as the steps are included there - we expect to include this in the same release) + ### Search with non-ascii characters Many languages include characters that have close analogs in ascii, e.g. (á, à, â, ç, é, è, ê, ë, í, ó, ö, ú, ù, û, ü…). This release changes the default Solr configuration to allow search to match words based on these associations, e.g. a search for Mercè would match the word Merce in a Dataset, and vice versa. This should generally be helpful, but can result in false positives.,e.g. "canon" will be found searching for "cañon". From c216aff17c86c92e258975c0bc5ad6a4e3db4a72 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 9 Mar 2021 21:22:13 -0500 Subject: [PATCH 455/576] Update doc/sphinx-guides/source/developers/s3-direct-upload-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/developers/s3-direct-upload-api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst index 4e41335b41a..fe63b2f7d41 100644 --- a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst +++ b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst @@ -81,7 +81,7 @@ If the client is unable to complete the multipart upload, it should call the abo Adding the Uploaded file to the Dataset --------------------------------------- -Once the file exists in the s3 bucket, a final API call is needed to add it to the Dataset. This call is the same call used to upload a file to Dataverse but, rather than sending the file bytes, additional metadata is added the "jsonData" parameter. +Once the file exists in the s3 bucket, a final API call is needed to add it to the Dataset. This call is the same call used to upload a file to Dataverse but, rather than sending the file bytes, additional metadata is added using the "jsonData" parameter. jsonData normally includes information such as a file description, tags, provenance, whether the file is restricted, etc. For direct uploads, the jsonData object must also include values for: * "storageIdentifier" - String, as specified in prior calls @@ -107,4 +107,4 @@ Note that this API call can be used independently of the others, e.g. supporting With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifer must be prefaced with the Dataverse store identifier used, as with the internally generated examples above. - \ No newline at end of file + From c702dfc071e164725f7628a7e065be1351156b94 Mon Sep 17 00:00:00 2001 From: Peter Kiraly Date: Wed, 10 Mar 2021 15:31:13 +0100 Subject: [PATCH 456/576] #7666: use the production citation.tsv in test, remove the copy stored in test resources. --- .../edu/harvard/iq/dataverse/api/AdminIT.java | 3 +- src/test/resources/tsv/citation.tsv | 324 ------------------ 2 files changed, 2 insertions(+), 325 deletions(-) delete mode 100644 src/test/resources/tsv/citation.tsv diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java index 84da33cd3ee..f83e9d9c839 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java @@ -689,11 +689,12 @@ public void testLoadMetadataBlock_NoErrorPath() { byte[] updatedContent = null; try { - updatedContent = Files.readAllBytes(Paths.get("src/test/resources/tsv/citation.tsv")); + updatedContent = Files.readAllBytes(Paths.get("scripts/api/data/metadatablocks/citation.tsv")); } catch (IOException e) { logger.warning(e.getMessage()); assertEquals(0,1); } + Response response = UtilIT.loadMetadataBlock(apiToken, updatedContent); assertEquals(200, response.getStatusCode()); response.then().assertThat().statusCode(OK.getStatusCode()); diff --git a/src/test/resources/tsv/citation.tsv b/src/test/resources/tsv/citation.tsv deleted file mode 100644 index d63390e9866..00000000000 --- a/src/test/resources/tsv/citation.tsv +++ /dev/null @@ -1,324 +0,0 @@ -#metadataBlock name dataverseAlias displayName blockURI - citation Citation Metadata https://dataverse.org/schema/citation/ -#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id termURI - title Title Full title by which the Dataset is known. Enter title... text 0 TRUE FALSE FALSE FALSE TRUE TRUE citation http://purl.org/dc/terms/title - subtitle Subtitle A secondary title used to amplify or state certain limitations on the main title. text 1 FALSE FALSE FALSE FALSE FALSE FALSE citation - alternativeTitle Alternative Title A title by which the work is commonly referred, or an abbreviation of the title. text 2 FALSE FALSE FALSE FALSE FALSE FALSE citation http://purl.org/dc/terms/alternative - alternativeURL Alternative URL A URL where the dataset can be viewed, such as a personal or project website. Enter full URL, starting with http:// url 3 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE citation https://schema.org/distribution - otherId Other ID Another unique identifier that identifies this Dataset (e.g., producer's or another repository's number). none 4 : FALSE FALSE TRUE FALSE FALSE FALSE citation - otherIdAgency Agency Name of agency which generated this identifier. text 5 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE otherId citation - otherIdValue Identifier Other identifier that corresponds to this Dataset. text 6 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE otherId citation - author Author The person(s), corporate body(ies), or agency(ies) responsible for creating the work. none 7 FALSE FALSE TRUE FALSE TRUE TRUE citation http://purl.org/dc/terms/creator - authorName Name The author's Family Name, Given Name or the name of the organization responsible for this Dataset. FamilyName, GivenName or Organization text 8 #VALUE TRUE FALSE FALSE TRUE TRUE TRUE author citation - authorAffiliation Affiliation The organization with which the author is affiliated. text 9 (#VALUE) TRUE FALSE FALSE TRUE TRUE FALSE author citation - authorIdentifierScheme Identifier Scheme Name of the identifier scheme (ORCID, ISNI). text 10 - #VALUE: FALSE TRUE FALSE FALSE TRUE FALSE author citation http://purl.org/spar/datacite/AgentIdentifierScheme - authorIdentifier Identifier Uniquely identifies an individual author or organization, according to various schemes. text 11 #VALUE FALSE FALSE FALSE FALSE TRUE FALSE author citation http://purl.org/spar/datacite/AgentIdentifier - datasetContact Contact The contact(s) for this Dataset. none 12 FALSE FALSE TRUE FALSE TRUE TRUE citation - datasetContactName Name The contact's Family Name, Given Name or the name of the organization. FamilyName, GivenName or Organization text 13 #VALUE FALSE FALSE FALSE FALSE TRUE FALSE datasetContact citation - datasetContactAffiliation Affiliation The organization with which the contact is affiliated. text 14 (#VALUE) FALSE FALSE FALSE FALSE TRUE FALSE datasetContact citation - datasetContactEmail E-mail The e-mail address(es) of the contact(s) for the Dataset. This will not be displayed. email 15 #EMAIL FALSE FALSE FALSE FALSE TRUE TRUE datasetContact citation - dsDescription Description A summary describing the purpose, nature, and scope of the Dataset. none 16 FALSE FALSE TRUE FALSE TRUE TRUE citation - dsDescriptionValue Text A summary describing the purpose, nature, and scope of the Dataset. textbox 17 #VALUE TRUE FALSE FALSE FALSE TRUE TRUE dsDescription citation - dsDescriptionDate Date In cases where a Dataset contains more than one description (for example, one might be supplied by the data producer and another prepared by the data repository where the data are deposited), the date attribute is used to distinguish between the two descriptions. The date attribute follows the ISO convention of YYYY-MM-DD. YYYY-MM-DD date 18 (#VALUE) FALSE FALSE FALSE FALSE TRUE FALSE dsDescription citation - subject Subject Domain-specific Subject Categories that are topically relevant to the Dataset. text 19 TRUE TRUE TRUE TRUE TRUE TRUE citation http://purl.org/dc/terms/subject - keyword Keyword Key terms that describe important aspects of the Dataset. none 20 FALSE FALSE TRUE FALSE TRUE FALSE citation - keywordValue Term Key terms that describe important aspects of the Dataset. Can be used for building keyword indexes and for classification and retrieval purposes. A controlled vocabulary can be employed. The vocab attribute is provided for specification of the controlled vocabulary in use, such as LCSH, MeSH, or others. The vocabURI attribute specifies the location for the full controlled vocabulary. text 21 #VALUE TRUE FALSE FALSE TRUE TRUE FALSE keyword citation - keywordVocabulary Vocabulary For the specification of the keyword controlled vocabulary in use, such as LCSH, MeSH, or others. text 22 (#VALUE) FALSE FALSE FALSE FALSE TRUE FALSE keyword citation - keywordVocabularyURI Vocabulary URL Keyword vocabulary URL points to the web presence that describes the keyword vocabulary, if appropriate. Enter an absolute URL where the keyword vocabulary web site is found, such as http://www.my.org. Enter full URL, starting with http:// url 23 #VALUE FALSE FALSE FALSE FALSE TRUE FALSE keyword citation - topicClassification Topic Classification The classification field indicates the broad important topic(s) and subjects that the data cover. Library of Congress subject terms may be used here. none 24 FALSE FALSE TRUE FALSE FALSE FALSE citation - topicClassValue Term Topic or Subject term that is relevant to this Dataset. text 25 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE topicClassification citation - topicClassVocab Vocabulary Provided for specification of the controlled vocabulary in use, e.g., LCSH, MeSH, etc. text 26 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE topicClassification citation - topicClassVocabURI Vocabulary URL Specifies the URL location for the full controlled vocabulary. Enter full URL, starting with http:// url 27 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE topicClassification citation - publication Related Publication Publications that use the data from this Dataset. The full list of Related Publications will be displayed on the metadata tab. none 28 FALSE FALSE TRUE FALSE TRUE FALSE citation http://purl.org/dc/terms/isReferencedBy - publicationCitation Citation The full bibliographic citation for this related publication. textbox 29 #VALUE TRUE FALSE FALSE FALSE TRUE FALSE publication citation http://purl.org/dc/terms/bibliographicCitation - publicationIDType ID Type The type of digital identifier used for this publication (e.g., Digital Object Identifier (DOI)). text 30 #VALUE: TRUE TRUE FALSE FALSE TRUE FALSE publication citation http://purl.org/spar/datacite/ResourceIdentifierScheme - publicationIDNumber ID Number The identifier for the selected ID type. text 31 #VALUE TRUE FALSE FALSE FALSE TRUE FALSE publication citation http://purl.org/spar/datacite/ResourceIdentifier - publicationURL URL Link to the publication web page (e.g., journal article page, archive record page, or other). Enter full URL, starting with http:// url 32 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE publication citation https://schema.org/distribution - notesText Notes Additional important information about the Dataset. textbox 33 FALSE FALSE FALSE FALSE TRUE FALSE citation - language Language Language of the Dataset text 34 TRUE TRUE TRUE TRUE FALSE FALSE citation http://purl.org/dc/terms/language - producer Producer Person or organization with the financial or administrative responsibility over this Dataset none 35 FALSE FALSE TRUE FALSE FALSE FALSE citation - producerName Name Producer name FamilyName, GivenName or Organization text 36 #VALUE TRUE FALSE FALSE TRUE FALSE TRUE producer citation - producerAffiliation Affiliation The organization with which the producer is affiliated. text 37 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE producer citation - producerAbbreviation Abbreviation The abbreviation by which the producer is commonly known. (ex. IQSS, ICPSR) text 38 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE producer citation - producerURL URL Producer URL points to the producer's web presence, if appropriate. Enter an absolute URL where the producer's web site is found, such as http://www.my.org. Enter full URL, starting with http:// url 39 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE producer citation - producerLogoURL Logo URL URL for the producer's logo, which points to this producer's web-accessible logo image. Enter an absolute URL where the producer's logo image is found, such as http://www.my.org/images/logo.gif. Enter full URL for image, starting with http:// url 40
    FALSE FALSE FALSE FALSE FALSE FALSE producer citation - productionDate Production Date Date when the data collection or other materials were produced (not distributed, published or archived). YYYY-MM-DD date 41 TRUE FALSE FALSE TRUE FALSE FALSE citation - productionPlace Production Place The location where the data collection and any other related materials were produced. text 42 FALSE FALSE FALSE FALSE FALSE FALSE citation - contributor Contributor The organization or person responsible for either collecting, managing, or otherwise contributing in some form to the development of the resource. none 43 : FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/contributor - contributorType Type The type of contributor of the resource. text 44 #VALUE TRUE TRUE FALSE TRUE FALSE FALSE contributor citation - contributorName Name The Family Name, Given Name or organization name of the contributor. FamilyName, GivenName or Organization text 45 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE contributor citation - grantNumber Grant Information Grant Information none 46 : FALSE FALSE TRUE FALSE FALSE FALSE citation https://schema.org/sponsor - grantNumberAgency Grant Agency Grant Number Agency text 47 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE grantNumber citation - grantNumberValue Grant Number The grant or contract number of the project that sponsored the effort. text 48 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE grantNumber citation - distributor Distributor The organization designated by the author or producer to generate copies of the particular work including any necessary editions or revisions. none 49 FALSE FALSE TRUE FALSE FALSE FALSE citation - distributorName Name Distributor name FamilyName, GivenName or Organization text 50 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE distributor citation - distributorAffiliation Affiliation The organization with which the distributor contact is affiliated. text 51 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE distributor citation - distributorAbbreviation Abbreviation The abbreviation by which this distributor is commonly known (e.g., IQSS, ICPSR). text 52 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE distributor citation - distributorURL URL Distributor URL points to the distributor's web presence, if appropriate. Enter an absolute URL where the distributor's web site is found, such as http://www.my.org. Enter full URL, starting with http:// url 53 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE distributor citation - distributorLogoURL Logo URL URL of the distributor's logo, which points to this distributor's web-accessible logo image. Enter an absolute URL where the distributor's logo image is found, such as http://www.my.org/images/logo.gif. Enter full URL for image, starting with http:// url 54
    FALSE FALSE FALSE FALSE FALSE FALSE distributor citation - distributionDate Distribution Date Date that the work was made available for distribution/presentation. YYYY-MM-DD date 55 TRUE FALSE FALSE TRUE FALSE FALSE citation - depositor Depositor The person (Family Name, Given Name) or the name of the organization that deposited this Dataset to the repository. text 56 FALSE FALSE FALSE FALSE FALSE FALSE citation - dateOfDeposit Deposit Date Date that the Dataset was deposited into the repository. YYYY-MM-DD date 57 FALSE FALSE FALSE TRUE FALSE FALSE citation http://purl.org/dc/terms/dateSubmitted - timePeriodCovered Time Period Covered Time period to which the data refer. This item reflects the time period covered by the data, not the dates of coding or making documents machine-readable or the dates the data were collected. Also known as span. none 58 ; FALSE FALSE TRUE FALSE FALSE FALSE citation https://schema.org/temporalCoverage - timePeriodCoveredStart Start Start date which reflects the time period covered by the data, not the dates of coding or making documents machine-readable or the dates the data were collected. YYYY-MM-DD date 59 #NAME: #VALUE TRUE FALSE FALSE TRUE FALSE FALSE timePeriodCovered citation - timePeriodCoveredEnd End End date which reflects the time period covered by the data, not the dates of coding or making documents machine-readable or the dates the data were collected. YYYY-MM-DD date 60 #NAME: #VALUE TRUE FALSE FALSE TRUE FALSE FALSE timePeriodCovered citation - dateOfCollection Date of Collection Contains the date(s) when the data were collected. none 61 ; FALSE FALSE TRUE FALSE FALSE FALSE citation - dateOfCollectionStart Start Date when the data collection started. YYYY-MM-DD date 62 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE dateOfCollection citation - dateOfCollectionEnd End Date when the data collection ended. YYYY-MM-DD date 63 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE dateOfCollection citation - kindOfData Kind of Data Type of data included in the file: survey data, census/enumeration data, aggregate data, clinical data, event/transaction data, program source code, machine-readable text, administrative records data, experimental data, psychological test, textual data, coded textual, coded documents, time budget diaries, observation data/ratings, process-produced data, or other. text 64 TRUE FALSE TRUE TRUE FALSE FALSE citation http://rdf-vocabulary.ddialliance.org/discovery#kindOfData - series Series Information about the Dataset series. none 65 : FALSE FALSE FALSE FALSE FALSE FALSE citation - seriesName Name Name of the dataset series to which the Dataset belongs. text 66 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE series citation - seriesInformation Information History of the series and summary of those features that apply to the series as a whole. textbox 67 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE series citation - software Software Information about the software used to generate the Dataset. none 68 , FALSE FALSE TRUE FALSE FALSE FALSE citation https://www.w3.org/TR/prov-o/#wasGeneratedBy - softwareName Name Name of software used to generate the Dataset. text 69 #VALUE FALSE TRUE FALSE FALSE FALSE FALSE software citation - softwareVersion Version Version of the software used to generate the Dataset. text 70 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE software citation - relatedMaterial Related Material Any material related to this Dataset. textbox 71 FALSE FALSE TRUE FALSE FALSE FALSE citation - relatedDatasets Related Datasets Any Datasets that are related to this Dataset, such as previous research on this subject. textbox 72 FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/relation - otherReferences Other References Any references that would serve as background or supporting material to this Dataset. text 73 FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/references - dataSources Data Sources List of books, articles, serials, or machine-readable data files that served as the sources of the data collection. textbox 74 FALSE FALSE TRUE FALSE FALSE FALSE citation https://www.w3.org/TR/prov-o/#wasDerivedFrom - originOfSources Origin of Sources For historical materials, information about the origin of the sources and the rules followed in establishing the sources should be specified. textbox 75 FALSE FALSE FALSE FALSE FALSE FALSE citation - characteristicOfSources Characteristic of Sources Noted Assessment of characteristics and source material. textbox 76 FALSE FALSE FALSE FALSE FALSE FALSE citation - accessToSources Documentation and Access to Sources Level of documentation of the original sources. textbox 77 FALSE FALSE FALSE FALSE FALSE FALSE citation -#controlledVocabulary DatasetField Value identifier displayOrder - subject Agricultural Sciences D01 0 - subject Arts and Humanities D0 1 - subject Astronomy and Astrophysics D1 2 - subject Business and Management D2 3 - subject Chemistry D3 4 - subject Computer and Information Science D7 5 - subject Earth and Environmental Sciences D4 6 - subject Engineering D5 7 - subject Law D8 8 - subject Mathematical Sciences D9 9 - subject Medicine, Health and Life Sciences D6 10 - subject Physics D10 11 - subject Social Sciences D11 12 - subject Other D12 13 - publicationIDType ark 0 - publicationIDType arXiv 1 - publicationIDType bibcode 2 - publicationIDType doi 3 - publicationIDType ean13 4 - publicationIDType eissn 5 - publicationIDType handle 6 - publicationIDType isbn 7 - publicationIDType issn 8 - publicationIDType istc 9 - publicationIDType lissn 10 - publicationIDType lsid 11 - publicationIDType pmid 12 - publicationIDType purl 13 - publicationIDType upc 14 - publicationIDType url 15 - publicationIDType urn 16 - contributorType Data Collector 0 - contributorType Data Curator 1 - contributorType Data Manager 2 - contributorType Editor 3 - contributorType Funder 4 - contributorType Hosting Institution 5 - contributorType Project Leader 6 - contributorType Project Manager 7 - contributorType Project Member 8 - contributorType Related Person 9 - contributorType Researcher 10 - contributorType Research Group 11 - contributorType Rights Holder 12 - contributorType Sponsor 13 - contributorType Supervisor 14 - contributorType Work Package Leader 15 - contributorType Other 16 - authorIdentifierScheme ORCID 0 - authorIdentifierScheme ISNI 1 - authorIdentifierScheme LCNA 2 - authorIdentifierScheme VIAF 3 - authorIdentifierScheme GND 4 - authorIdentifierScheme DAI 5 - authorIdentifierScheme ResearcherID 6 - authorIdentifierScheme ScopusID 7 - language Abkhaz 0 - language Afar 1 - language Afrikaans 2 - language Akan 3 - language Albanian 4 - language Amharic 5 - language Arabic 6 - language Aragonese 7 - language Armenian 8 - language Assamese 9 - language Avaric 10 - language Avestan 11 - language Aymara 12 - language Azerbaijani 13 - language Bambara 14 - language Bashkir 15 - language Basque 16 - language Belarusian 17 - language Bengali, Bangla 18 - language Bihari 19 - language Bislama 20 - language Bosnian 21 - language Breton 22 - language Bulgarian 23 - language Burmese 24 - language Catalan,Valencian 25 - language Chamorro 26 - language Chechen 27 - language Chichewa, Chewa, Nyanja 28 - language Chinese 29 - language Chuvash 30 - language Cornish 31 - language Corsican 32 - language Cree 33 - language Croatian 34 - language Czech 35 - language Danish 36 - language Divehi, Dhivehi, Maldivian 37 - language Dutch 38 - language Dzongkha 39 - language English 40 - language Esperanto 41 - language Estonian 42 - language Ewe 43 - language Faroese 44 - language Fijian 45 - language Finnish 46 - language French 47 - language Fula, Fulah, Pulaar, Pular 48 - language Galician 49 - language Georgian 50 - language German 51 - language Greek (modern) 52 - language Guaraní 53 - language Gujarati 54 - language Haitian, Haitian Creole 55 - language Hausa 56 - language Hebrew (modern) 57 - language Herero 58 - language Hindi 59 - language Hiri Motu 60 - language Hungarian 61 - language Interlingua 62 - language Indonesian 63 - language Interlingue 64 - language Irish 65 - language Igbo 66 - language Inupiaq 67 - language Ido 68 - language Icelandic 69 - language Italian 70 - language Inuktitut 71 - language Japanese 72 - language Javanese 73 - language Kalaallisut, Greenlandic 74 - language Kannada 75 - language Kanuri 76 - language Kashmiri 77 - language Kazakh 78 - language Khmer 79 - language Kikuyu, Gikuyu 80 - language Kinyarwanda 81 - language Kyrgyz 82 - language Komi 83 - language Kongo 84 - language Korean 85 - language Kurdish 86 - language Kwanyama, Kuanyama 87 - language Latin 88 - language Luxembourgish, Letzeburgesch 89 - language Ganda 90 - language Limburgish, Limburgan, Limburger 91 - language Lingala 92 - language Lao 93 - language Lithuanian 94 - language Luba-Katanga 95 - language Latvian 96 - language Manx 97 - language Macedonian 98 - language Malagasy 99 - language Malay 100 - language Malayalam 101 - language Maltese 102 - language Māori 103 - language Marathi (Marāṭhī) 104 - language Marshallese 105 - language Mixtepec Mixtec 106 - language Mongolian 107 - language Nauru 108 - language Navajo, Navaho 109 - language Northern Ndebele 110 - language Nepali 111 - language Ndonga 112 - language Norwegian Bokmål 113 - language Norwegian Nynorsk 114 - language Norwegian 115 - language Nuosu 116 - language Southern Ndebele 117 - language Occitan 118 - language Ojibwe, Ojibwa 119 - language Old Church Slavonic,Church Slavonic,Old Bulgarian 120 - language Oromo 121 - language Oriya 122 - language Ossetian, Ossetic 123 - language Panjabi, Punjabi 124 - language Pāli 125 - language Persian (Farsi) 126 - language Polish 127 - language Pashto, Pushto 128 - language Portuguese 129 - language Quechua 130 - language Romansh 131 - language Kirundi 132 - language Romanian 133 - language Russian 134 - language Sanskrit (Saṁskṛta) 135 - language Sardinian 136 - language Sindhi 137 - language Northern Sami 138 - language Samoan 139 - language Sango 140 - language Serbian 141 - language Scottish Gaelic, Gaelic 142 - language Shona 143 - language Sinhala, Sinhalese 144 - language Slovak 145 - language Slovene 146 - language Somali 147 - language Southern Sotho 148 - language Spanish, Castilian 149 - language Sundanese 150 - language Swahili 151 - language Swati 152 - language Swedish 153 - language Tamil 154 - language Telugu 155 - language Tajik 156 - language Thai 157 - language Tigrinya 158 - language Tibetan Standard, Tibetan, Central 159 - language Turkmen 160 - language Tagalog 161 - language Tswana 162 - language Tonga (Tonga Islands) 163 - language Turkish 164 - language Tsonga 165 - language Tatar 166 - language Twi 167 - language Tahitian 168 - language Uyghur, Uighur 169 - language Ukrainian 170 - language Urdu 171 - language Uzbek 172 - language Venda 173 - language Vietnamese 174 - language Volapük 175 - language Walloon 176 - language Welsh 177 - language Wolof 178 - language Western Frisian 179 - language Xhosa 180 - language Yiddish 181 - language Yoruba 182 - language Zhuang, Chuang 183 - language Zulu 184 - language Not applicable 185 From 17a1b01d3a9351e7b3535004b461ccbbe44ecb2b Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 10 Mar 2021 10:17:35 -0500 Subject: [PATCH 457/576] #7633 update doc --- doc/sphinx-guides/source/api/native-api.rst | 32 +++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index d50ef05d87e..d879c2a82fc 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -259,6 +259,8 @@ Where ``roles.json`` looks like this:: ] } +.. note:: Only a Dataverse installation account with superuser permissions is allowed to create roles in a Dataverse Collection. + .. _list-role-assignments-on-a-dataverse-api: List Role Assignments in a Dataverse Collection @@ -2418,9 +2420,35 @@ Roles Create a New Role in a Dataverse Collection ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Creates a new role in Dataverse collection object whose Id is ``dataverseIdtf`` (that's an id/alias):: +Creates a new role under Dataverse collection ``id``. Needs a json file with the role description: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=root + + curl -H X-Dataverse-key:$API_TOKEN -X POST $SERVER_URL/api/dataverses/$ID/roles --upload-file roles.json + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST https://demo.dataverse.org/api/dataverses/root/roles --upload-file roles.json + +Where ``roles.json`` looks like this:: + + { + "alias": "sys1", + "name": “Restricted System Role”, + "description": “A person who may only add datasets.”, + "permissions": [ + "AddDataset" + ] + } + +.. note:: Only a Dataverse installation account with superuser permissions is allowed to create roles in a Dataverse Collection. - POST http://$SERVER/api/roles?dvo=$dataverseIdtf&key=$apiKey Show Role ~~~~~~~~~ From 4e4f429eacb35abaa599eb63f03571facc62c926 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 10 Mar 2021 10:53:24 -0500 Subject: [PATCH 458/576] #7633 add delete role doc --- doc/sphinx-guides/source/api/native-api.rst | 33 +++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index d879c2a82fc..6101ce7d312 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2460,9 +2460,38 @@ Shows the role with ``id``:: Delete Role ~~~~~~~~~~~ -Deletes the role with ``id``:: +A curl example using an ``ID`` + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=24 + + curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE $SERVER_URL/api/roles/$ID + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE https://demo.dataverse.org/api/roles/24 + +A curl example using a Role alias ``ALIAS`` + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ALIAS=roleAlias + + curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/files/:alias?alias=$ALIAS" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/ROLES/:alias?alias=roleAlias" - DELETE http://$SERVER/api/roles/$id Explicit Groups --------------- From 376586bb855a9f1e73eaf35224e5ec5704ba6e69 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 10 Mar 2021 10:57:42 -0500 Subject: [PATCH 459/576] 7633 fix doc typo --- doc/sphinx-guides/source/api/native-api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 6101ce7d312..56cd8d2cad6 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2484,13 +2484,13 @@ A curl example using a Role alias ``ALIAS`` export SERVER_URL=https://demo.dataverse.org export ALIAS=roleAlias - curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/files/:alias?alias=$ALIAS" + curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/roles/:alias?alias=$ALIAS" The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/ROLES/:alias?alias=roleAlias" + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE https://demo.dataverse.org/api/roles/:alias?alias=roleAlias Explicit Groups From 807bd80b3d4cdc7a06c488f5f10424e44ecb5b90 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 10 Mar 2021 11:00:57 -0500 Subject: [PATCH 460/576] #7633 add doc for deleting built in "Global" role --- doc/sphinx-guides/source/api/native-api.rst | 35 +++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 56cd8d2cad6..7bab69bdd53 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2931,6 +2931,41 @@ Create Global Role Creates a global role in the Dataverse installation. The data POSTed are assumed to be a role JSON. :: POST http://$SERVER/api/admin/roles + +Delete Global Role +~~~~~~~~~~~~~~~~~~ + +A curl example using an ``ID`` + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=24 + + curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE $SERVER_URL/api/admin/roles/$ID + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE https://demo.dataverse.org/api/admin/roles/24 + +A curl example using a Role alias ``ALIAS`` + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ALIAS=roleAlias + + curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/admin/roles/:alias?alias=$ALIAS" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE https://demo.dataverse.org/api/admin/roles/:alias?alias=roleAlias List Users ~~~~~~~~~~ From a3bc5ee71e1f9761c392eb7cd34274ef209dcc07 Mon Sep 17 00:00:00 2001 From: Peter Kiraly Date: Wed, 10 Mar 2021 17:40:50 +0100 Subject: [PATCH 461/576] #7672: changing outdated reference from readme.txt to readme.md in testing guide and fixing an unclosed quote issue in native API guide. --- doc/sphinx-guides/source/api/native-api.rst | 2 +- doc/sphinx-guides/source/developers/testing.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index d50ef05d87e..94cb6c720c6 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1041,7 +1041,7 @@ The fully expanded example above (without environment variables) looks like this The quotes around the URL are required because there is more than one query parameter separated by an ampersand (``&``), which has special meaning to Unix shells such as Bash. Putting the ``&`` in quotes ensures that "type" is interpreted as one of the query parameters. -You should expect JSON output and a 200 ("OK") response in most cases. If you receive a 202 ("ACCEPTED") response, this is normal for installations that have workflows configured. Workflows are described in the :doc:`/developers/workflows` section of the Developer Guide. A 409 ("CONFLICT") response is also possible if you set ``assureIsIndexed`=true. (In this case, one could then repeat the call until a 200/202 response is sent.) +You should expect JSON output and a 200 ("OK") response in most cases. If you receive a 202 ("ACCEPTED") response, this is normal for installations that have workflows configured. Workflows are described in the :doc:`/developers/workflows` section of the Developer Guide. A 409 ("CONFLICT") response is also possible if you set ``assureIsIndexed=true``. (In this case, one could then repeat the call until a 200/202 response is sent.) .. note:: POST should be used to publish a dataset. GET is supported for backward compatibility but is deprecated and may be removed: https://github.com/IQSS/dataverse/issues/2431 diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst index 7f59d31410f..79850357edf 100755 --- a/doc/sphinx-guides/source/developers/testing.rst +++ b/doc/sphinx-guides/source/developers/testing.rst @@ -166,7 +166,7 @@ Near the beginning and at the end of the ec2-create-instance.sh output you will Running the full API test suite using Docker ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To run the full suite of integration tests on your laptop, we recommend using the "all in one" Docker configuration described in ``conf/docker-aio/readme.txt`` in the root of the repo. +To run the full suite of integration tests on your laptop, we recommend using the "all in one" Docker configuration described in ``conf/docker-aio/readme.md`` in the root of the repo. Alternatively, you can run tests against the app server running on your laptop by following the "getting set up" steps below. @@ -446,7 +446,7 @@ How to Run the Phoenix Tests List of Tests Run Against the Phoenix Server ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -We haven't thought much about a good way to publicly list the "IT" classes that are executed against the phoenix server. (Currently your best bet is to look at the ``Executing Maven`` line at the top of the "Full Log" of "Console Output" of ``phoenix.dataverse.org-apitest-develop`` Jenkins job mentioned above.) We endeavor to keep the list of tests in the "all-in-one" Docker environment described above in sync with the list of tests configured in Jenkins. That is to say, refer to :download:`run-test-suite.sh <../../../../conf/docker-aio/run-test-suite.sh>` mentioned in ``conf/docker-aio/readme.txt`` for the current list of IT tests that are expected to pass. Here's a dump of that file: +We haven't thought much about a good way to publicly list the "IT" classes that are executed against the phoenix server. (Currently your best bet is to look at the ``Executing Maven`` line at the top of the "Full Log" of "Console Output" of ``phoenix.dataverse.org-apitest-develop`` Jenkins job mentioned above.) We endeavor to keep the list of tests in the "all-in-one" Docker environment described above in sync with the list of tests configured in Jenkins. That is to say, refer to :download:`run-test-suite.sh <../../../../conf/docker-aio/run-test-suite.sh>` mentioned in ``conf/docker-aio/readme.md`` for the current list of IT tests that are expected to pass. Here's a dump of that file: .. literalinclude:: ../../../../conf/docker-aio/run-test-suite.sh From cca7a3c0c67e33fbb0d90f24cff3b6ce36a7493b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 11 Mar 2021 13:35:58 -0500 Subject: [PATCH 462/576] use helper --- .../iq/dataverse/util/bagit/OREMap.java | 22 ++++++++-------- .../iq/dataverse/util/bagit/OREMapHelper.java | 25 +++++++++++++++++++ 2 files changed, 37 insertions(+), 10 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMapHelper.java diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index 7989a901d6c..38303eb1f41 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -12,7 +12,6 @@ import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.export.OAI_OREExporter; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; -import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.json.JsonLDNamespace; import edu.harvard.iq.dataverse.util.json.JsonLDTerm; @@ -22,12 +21,9 @@ import java.time.LocalDate; import java.util.List; import java.util.Map; -import java.util.ResourceBundle; import java.util.TreeMap; import java.util.Map.Entry; -import javax.ejb.Stateless; -import javax.inject.Inject; import javax.json.Json; import javax.json.JsonArray; import javax.json.JsonArrayBuilder; @@ -35,24 +31,22 @@ import javax.json.JsonObjectBuilder; import javax.json.JsonValue; -@Stateless public class OREMap { - @Inject static SettingsServiceBean settingsService; + static SettingsServiceBean settingsService; public static final String NAME = "OREMap"; private Map localContext = new TreeMap(); private DatasetVersion version; - private boolean excludeEmail = false; + private Boolean excludeEmail = null; public OREMap(DatasetVersion version) { - this(version, settingsService.isTrueForKey(SettingsServiceBean.Key.ExcludeEmailFromExport, false)); - + this.version = version; } //Used when the ExcludeEmailFromExport needs to be overriden, i.e. for archiving public OREMap(DatasetVersion dv, boolean exclude) { - this.version = version; + this.version = dv; this.excludeEmail = exclude; } @@ -63,6 +57,11 @@ public void writeOREMap(OutputStream outputStream) throws Exception { public JsonObject getOREMap() throws Exception { + //Set this flag if it wasn't provided + if(excludeEmail==null) { + excludeEmail = settingsService.isTrueForKey(SettingsServiceBean.Key.ExcludeEmailFromExport, false); + } + // Add namespaces we'll definitely use to Context // Additional namespaces are added as needed below localContext.putIfAbsent(JsonLDNamespace.ore.getPrefix(), JsonLDNamespace.ore.getUrl()); @@ -392,4 +391,7 @@ private JsonLDTerm getTermFor(String type, String subType) { return null; } + public static void injectSettingsService(SettingsServiceBean settingsSvc) { + settingsService = settingsSvc; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMapHelper.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMapHelper.java new file mode 100644 index 00000000000..bced52a7752 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMapHelper.java @@ -0,0 +1,25 @@ +package edu.harvard.iq.dataverse.util.bagit; + +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; + +import javax.annotation.PostConstruct; +import javax.ejb.EJB; +import javax.ejb.Singleton; +import javax.ejb.Startup; + +/** + * This is a small helper bean + * As it is a singleton and built at application start (=deployment), it will inject the (stateless) + * settings service into the OREMap once it's ready. + */ +@Singleton +@Startup +public class OREMapHelper { + @EJB + SettingsServiceBean settingsSvc; + + @PostConstruct + public void injectService() { + OREMap.injectSettingsService(settingsSvc); + } +} From d91d4ac1f58bb1f4b28047052c44c4a82d6ba0de Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 11 Mar 2021 13:36:20 -0500 Subject: [PATCH 463/576] use helper --- .../iq/dataverse/branding/BrandingUtil.java | 12 +++++---- .../branding/BrandingUtilHelper.java | 26 +++++++++++++++++++ 2 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtilHelper.java diff --git a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java index 6b6bc77cca8..07cedb7a3ce 100644 --- a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java @@ -3,21 +3,23 @@ import edu.harvard.iq.dataverse.DataverseServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import java.util.Arrays; -import javax.ejb.EJB; -import javax.ejb.Singleton; + import javax.mail.internet.InternetAddress; import org.eclipse.microprofile.config.ConfigProvider; -@Singleton public class BrandingUtil { - @EJB static DataverseServiceBean dataverseService; + static String brandName = null; + public static String getInstallationBrandName() { - return ConfigProvider.getConfig().getOptionalValue("dataverse.branding.installation.name", String.class) + if(brandName==null) { + brandName = ConfigProvider.getConfig().getOptionalValue("dataverse.branding.installation.name", String.class) .orElse(dataverseService.getRootDataverseName()); + } + return brandName; } // Convenience to access root name without injecting dataverseService (e.g. in diff --git a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtilHelper.java b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtilHelper.java new file mode 100644 index 00000000000..0fb16088ff5 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtilHelper.java @@ -0,0 +1,26 @@ +package edu.harvard.iq.dataverse.branding; + +import javax.annotation.PostConstruct; +import javax.ejb.EJB; +import javax.ejb.Singleton; +import javax.ejb.Startup; + +import edu.harvard.iq.dataverse.DataverseServiceBean; + + /** + * This is a small helper bean + * As it is a singleton and built at application start (=deployment), it will inject the (stateless) + * dataverse service into the BrandingUtil once it's ready. + */ + @Singleton + @Startup + public class BrandingUtilHelper { + + @EJB + DataverseServiceBean dataverseSvc; + + @PostConstruct + public void injectService() { + BrandingUtil.injectDataverseService(dataverseSvc); + } + } From 2e999c3068a9762b444ada78032092be0271fc85 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 11 Mar 2021 16:10:15 -0500 Subject: [PATCH 464/576] Adds the ISO 639-3 language letter codes to the list of accepted substitute values in the citation metadata block, where an exact match was available (140 languages total; #7638) --- scripts/api/data/metadatablocks/citation.tsv | 280 +++++++++---------- 1 file changed, 140 insertions(+), 140 deletions(-) diff --git a/scripts/api/data/metadatablocks/citation.tsv b/scripts/api/data/metadatablocks/citation.tsv index d63390e9866..375a8c67cec 100644 --- a/scripts/api/data/metadatablocks/citation.tsv +++ b/scripts/api/data/metadatablocks/citation.tsv @@ -137,188 +137,188 @@ authorIdentifierScheme ResearcherID 6 authorIdentifierScheme ScopusID 7 language Abkhaz 0 - language Afar 1 - language Afrikaans 2 - language Akan 3 - language Albanian 4 - language Amharic 5 - language Arabic 6 - language Aragonese 7 - language Armenian 8 - language Assamese 9 - language Avaric 10 - language Avestan 11 - language Aymara 12 - language Azerbaijani 13 - language Bambara 14 - language Bashkir 15 - language Basque 16 - language Belarusian 17 + language Afar 1 aar + language Afrikaans 2 afr + language Akan 3 aka + language Albanian 4 sqi + language Amharic 5 amh + language Arabic 6 ara + language Aragonese 7 arg + language Armenian 8 hye + language Assamese 9 asm + language Avaric 10 ava + language Avestan 11 ave + language Aymara 12 aym + language Azerbaijani 13 aze + language Bambara 14 bam + language Bashkir 15 bak + language Basque 16 eus + language Belarusian 17 bel language Bengali, Bangla 18 language Bihari 19 - language Bislama 20 - language Bosnian 21 - language Breton 22 - language Bulgarian 23 - language Burmese 24 + language Bislama 20 bis + language Bosnian 21 bos + language Breton 22 bre + language Bulgarian 23 bul + language Burmese 24 mya language Catalan,Valencian 25 - language Chamorro 26 - language Chechen 27 + language Chamorro 26 cha + language Chechen 27 che language Chichewa, Chewa, Nyanja 28 - language Chinese 29 - language Chuvash 30 - language Cornish 31 - language Corsican 32 - language Cree 33 - language Croatian 34 - language Czech 35 - language Danish 36 + language Chinese 29 zho + language Chuvash 30 chv + language Cornish 31 cor + language Corsican 32 cos + language Cree 33 cre + language Croatian 34 hrv + language Czech 35 ces + language Danish 36 dan language Divehi, Dhivehi, Maldivian 37 - language Dutch 38 - language Dzongkha 39 - language English 40 - language Esperanto 41 - language Estonian 42 - language Ewe 43 - language Faroese 44 - language Fijian 45 - language Finnish 46 - language French 47 + language Dutch 38 nld + language Dzongkha 39 dzo + language English 40 eng + language Esperanto 41 epo + language Estonian 42 est + language Ewe 43 ewe + language Faroese 44 fao + language Fijian 45 fij + language Finnish 46 fin + language French 47 fra language Fula, Fulah, Pulaar, Pular 48 - language Galician 49 - language Georgian 50 - language German 51 + language Galician 49 glg + language Georgian 50 kat + language German 51 deu language Greek (modern) 52 language Guaraní 53 - language Gujarati 54 + language Gujarati 54 guj language Haitian, Haitian Creole 55 - language Hausa 56 + language Hausa 56 hau language Hebrew (modern) 57 - language Herero 58 - language Hindi 59 - language Hiri Motu 60 - language Hungarian 61 + language Herero 58 her + language Hindi 59 hin + language Hiri Motu 60 hmo + language Hungarian 61 hun language Interlingua 62 - language Indonesian 63 - language Interlingue 64 - language Irish 65 - language Igbo 66 - language Inupiaq 67 - language Ido 68 - language Icelandic 69 - language Italian 70 - language Inuktitut 71 - language Japanese 72 - language Javanese 73 + language Indonesian 63 ind + language Interlingue 64 ile + language Irish 65 gle + language Igbo 66 ibo + language Inupiaq 67 ipk + language Ido 68 ido + language Icelandic 69 isl + language Italian 70 ita + language Inuktitut 71 iku + language Japanese 72 jpn + language Javanese 73 jav language Kalaallisut, Greenlandic 74 - language Kannada 75 - language Kanuri 76 - language Kashmiri 77 - language Kazakh 78 - language Khmer 79 + language Kannada 75 kan + language Kanuri 76 kau + language Kashmiri 77 kas + language Kazakh 78 kaz + language Khmer 79 khm language Kikuyu, Gikuyu 80 - language Kinyarwanda 81 + language Kinyarwanda 81 kin language Kyrgyz 82 - language Komi 83 - language Kongo 84 - language Korean 85 - language Kurdish 86 + language Komi 83 kom + language Kongo 84 kon + language Korean 85 kor + language Kurdish 86 kur language Kwanyama, Kuanyama 87 - language Latin 88 + language Latin 88 lat language Luxembourgish, Letzeburgesch 89 - language Ganda 90 + language Ganda 90 lug language Limburgish, Limburgan, Limburger 91 - language Lingala 92 - language Lao 93 - language Lithuanian 94 - language Luba-Katanga 95 - language Latvian 96 - language Manx 97 - language Macedonian 98 - language Malagasy 99 + language Lingala 92 lin + language Lao 93 lao + language Lithuanian 94 lit + language Luba-Katanga 95 lub + language Latvian 96 lav + language Manx 97 glv + language Macedonian 98 mkd + language Malagasy 99 mlg language Malay 100 - language Malayalam 101 - language Maltese 102 + language Malayalam 101 mal + language Maltese 102 mlt language Māori 103 language Marathi (Marāṭhī) 104 - language Marshallese 105 - language Mixtepec Mixtec 106 - language Mongolian 107 - language Nauru 108 + language Marshallese 105 mah + language Mixtepec Mixtec 106 mix + language Mongolian 107 mon + language Nauru 108 nau language Navajo, Navaho 109 language Northern Ndebele 110 language Nepali 111 - language Ndonga 112 - language Norwegian Bokmål 113 - language Norwegian Nynorsk 114 - language Norwegian 115 + language Ndonga 112 ndo + language Norwegian Bokmål 113 nob + language Norwegian Nynorsk 114 nno + language Norwegian 115 nor language Nuosu 116 language Southern Ndebele 117 language Occitan 118 language Ojibwe, Ojibwa 119 language Old Church Slavonic,Church Slavonic,Old Bulgarian 120 - language Oromo 121 + language Oromo 121 orm language Oriya 122 language Ossetian, Ossetic 123 language Panjabi, Punjabi 124 language Pāli 125 language Persian (Farsi) 126 - language Polish 127 + language Polish 127 pol language Pashto, Pushto 128 - language Portuguese 129 - language Quechua 130 - language Romansh 131 + language Portuguese 129 por + language Quechua 130 que + language Romansh 131 roh language Kirundi 132 - language Romanian 133 - language Russian 134 + language Romanian 133 ron + language Russian 134 rus language Sanskrit (Saṁskṛta) 135 - language Sardinian 136 - language Sindhi 137 - language Northern Sami 138 - language Samoan 139 - language Sango 140 - language Serbian 141 + language Sardinian 136 srd + language Sindhi 137 snd + language Northern Sami 138 sme + language Samoan 139 smo + language Sango 140 sag + language Serbian 141 srp language Scottish Gaelic, Gaelic 142 - language Shona 143 + language Shona 143 sna language Sinhala, Sinhalese 144 - language Slovak 145 + language Slovak 145 slk language Slovene 146 - language Somali 147 - language Southern Sotho 148 + language Somali 147 som + language Southern Sotho 148 sot language Spanish, Castilian 149 - language Sundanese 150 + language Sundanese 150 sun language Swahili 151 - language Swati 152 - language Swedish 153 - language Tamil 154 - language Telugu 155 - language Tajik 156 - language Thai 157 - language Tigrinya 158 + language Swati 152 ssw + language Swedish 153 swe + language Tamil 154 tam + language Telugu 155 tel + language Tajik 156 tgk + language Thai 157 tha + language Tigrinya 158 tir language Tibetan Standard, Tibetan, Central 159 - language Turkmen 160 - language Tagalog 161 - language Tswana 162 - language Tonga (Tonga Islands) 163 - language Turkish 164 - language Tsonga 165 - language Tatar 166 - language Twi 167 - language Tahitian 168 + language Turkmen 160 tuk + language Tagalog 161 tgl + language Tswana 162 tsn + language Tonga (Tonga Islands) 163 ton + language Turkish 164 tur + language Tsonga 165 tso + language Tatar 166 tat + language Twi 167 twi + language Tahitian 168 tah language Uyghur, Uighur 169 - language Ukrainian 170 - language Urdu 171 - language Uzbek 172 - language Venda 173 - language Vietnamese 174 - language Volapük 175 - language Walloon 176 - language Welsh 177 - language Wolof 178 - language Western Frisian 179 - language Xhosa 180 - language Yiddish 181 - language Yoruba 182 + language Ukrainian 170 ukr + language Urdu 171 urd + language Uzbek 172 uzb + language Venda 173 ven + language Vietnamese 174 vie + language Volapük 175 vol + language Walloon 176 wln + language Welsh 177 cym + language Wolof 178 wol + language Western Frisian 179 fry + language Xhosa 180 xho + language Yiddish 181 yid + language Yoruba 182 yor language Zhuang, Chuang 183 - language Zulu 184 + language Zulu 184 zul language Not applicable 185 From ddc4f3acc7302804bad2cec526cc433fb6c2c41c Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 11 Mar 2021 16:28:40 -0500 Subject: [PATCH 465/576] rebrand "disable users" as "deactivate users" #2419 #4475 --- conf/docker-aio/run-test-suite.sh | 2 +- .../2419-4475-7575-disable-users.md | 4 +- .../source/admin/user-administration.rst | 6 +- doc/sphinx-guides/source/api/native-api.rst | 30 +++--- .../iq/dataverse/DataverseSession.java | 8 +- .../java/edu/harvard/iq/dataverse/Shib.java | 2 +- .../iq/dataverse/api/AbstractApiBean.java | 2 +- .../edu/harvard/iq/dataverse/api/Admin.java | 22 ++--- .../dataverse/api/datadeposit/SwordAuth.java | 2 +- .../AuthenticationServiceBean.java | 6 +- .../oauth2/OAuth2LoginBackingBean.java | 2 +- .../users/AuthenticatedUser.java | 22 ++--- .../dataverse/authorization/users/User.java | 2 +- .../confirmemail/ConfirmEmailServiceBean.java | 4 +- ...ddRoleAssigneesToExplicitGroupCommand.java | 4 +- .../command/impl/AssignRoleCommand.java | 4 +- ...ommand.java => DeactivateUserCommand.java} | 28 +++--- .../command/impl/MergeInAccountCommand.java | 4 +- .../PasswordResetServiceBean.java | 4 +- .../iq/dataverse/util/json/JsonPrinter.java | 4 +- src/main/java/propertyFiles/Bundle.properties | 8 +- .../V5.3.0.5__2419-deactivate-users.sql | 4 + .../V5.3.0.5__4475-disable-users.sql | 4 - ...bleUsersIT.java => DeactivateUsersIT.java} | 92 +++++++++---------- .../iq/dataverse/api/DeleteUsersIT.java | 8 +- .../edu/harvard/iq/dataverse/api/UtilIT.java | 16 ++-- 26 files changed, 147 insertions(+), 147 deletions(-) rename src/main/java/edu/harvard/iq/dataverse/engine/command/impl/{DisableUserCommand.java => DeactivateUserCommand.java} (52%) create mode 100644 src/main/resources/db/migration/V5.3.0.5__2419-deactivate-users.sql delete mode 100644 src/main/resources/db/migration/V5.3.0.5__4475-disable-users.sql rename src/test/java/edu/harvard/iq/dataverse/api/{DisableUsersIT.java => DeactivateUsersIT.java} (76%) diff --git a/conf/docker-aio/run-test-suite.sh b/conf/docker-aio/run-test-suite.sh index da9e56e0176..47a4c3b9576 100755 --- a/conf/docker-aio/run-test-suite.sh +++ b/conf/docker-aio/run-test-suite.sh @@ -8,4 +8,4 @@ fi # Please note the "dataverse.test.baseurl" is set to run for "all-in-one" Docker environment. # TODO: Rather than hard-coding the list of "IT" classes here, add a profile to pom.xml. -source maven/maven.sh && mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DisableUsersIT -Ddataverse.test.baseurl=$dvurl +source maven/maven.sh && mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT -Ddataverse.test.baseurl=$dvurl diff --git a/doc/release-notes/2419-4475-7575-disable-users.md b/doc/release-notes/2419-4475-7575-disable-users.md index 612200905dc..2678ebc6ec9 100644 --- a/doc/release-notes/2419-4475-7575-disable-users.md +++ b/doc/release-notes/2419-4475-7575-disable-users.md @@ -1,5 +1,5 @@ ## Release Highlights -### Disable Users API, Get User Traces API, Revoke Roles API +### Deactivate Users API, Get User Traces API, Revoke Roles API -A new API has been added to disable users to prevent them from logging in or otherwise being active in the system. Disabling a user is an alternative to deleting a user, especially when the latter is not possible due to the amount of interaction the user has had with the system. In order to learn more about a user before deleting, disabling, or merging, a new "get user traces" API is available that will show objects created, roles, group memberships, and more. Finally, the "remove all roles" button available in the superuser dashboard is now also available via API. +A new API has been added to deactivate users to prevent them from logging in or otherwise being active in the system. Deactivating a user is an alternative to deleting a user, especially when the latter is not possible due to the amount of interaction the user has had with the system. In order to learn more about a user before deleting, deactivating, or merging, a new "get user traces" API is available that will show objects created, roles, group memberships, and more. Finally, the "remove all roles" button available in the superuser dashboard is now also available via API. diff --git a/doc/sphinx-guides/source/admin/user-administration.rst b/doc/sphinx-guides/source/admin/user-administration.rst index 2999c6e0856..867f06bde8e 100644 --- a/doc/sphinx-guides/source/admin/user-administration.rst +++ b/doc/sphinx-guides/source/admin/user-administration.rst @@ -49,10 +49,10 @@ Delete a User See :ref:`delete-a-user` -Disable a User --------------- +Deactivate a User +----------------- -See :ref:`disable-a-user` +See :ref:`deactivate-a-user` Confirm Email ------------- diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index c99b903ff38..ca11e77a78a 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3044,7 +3044,7 @@ Example: ``curl -H "X-Dataverse-key: $API_TOKEN" -X POST http://demo.dataverse.o This action moves account data from jsmith2 into the account jsmith and deletes the account of jsmith2. -Note: User accounts can only be merged if they are either both enabled or both disabled. See :ref:`disable a user`. +Note: User accounts can only be merged if they are either both non-deactivated or both deactivated. See :ref:`deactivate a user`. .. _change-identifier-label: @@ -3079,14 +3079,14 @@ Deletes an ``AuthenticatedUser`` whose ``id`` is passed. :: DELETE http://$SERVER/api/admin/authenticatedUsers/id/$id -Note: If the user has performed certain actions such as creating or contributing to a Dataset or downloading a file they cannot be deleted. To see where in the database these actions are stored you can use the :ref:`show-user-traces-api` API. If a user cannot be deleted for this reason, you can choose to :ref:`disable a user`. +Note: If the user has performed certain actions such as creating or contributing to a Dataset or downloading a file they cannot be deleted. To see where in the database these actions are stored you can use the :ref:`show-user-traces-api` API. If a user cannot be deleted for this reason, you can choose to :ref:`deactivate a user`. -.. _disable-a-user: +.. _deactivate-a-user: -Disable a User -~~~~~~~~~~~~~~ +Deactivate a User +~~~~~~~~~~~~~~~~~ -Disables a user. A superuser API token is not required but the command will operate using the first superuser it finds. +Deactivates a user. A superuser API token is not required but the command will operate using the first superuser it finds. .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below. @@ -3096,13 +3096,13 @@ Disables a user. A superuser API token is not required but the command will oper export SERVER_URL=http://localhost:8080 export USERNAME=jdoe - curl -H "X-Dataverse-key:$API_TOKEN" -X POST $SERVER_URL/api/admin/authenticatedUsers/$USERNAME/disable + curl -H "X-Dataverse-key:$API_TOKEN" -X POST $SERVER_URL/api/admin/authenticatedUsers/$USERNAME/deactivate The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST http://localhost:8080/api/admin/authenticatedUsers/jdoe/disable + curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST http://localhost:8080/api/admin/authenticatedUsers/jdoe/deactivate The database ID of the user can be passed instead of the username. @@ -3112,23 +3112,23 @@ The database ID of the user can be passed instead of the username. export SERVER_URL=http://localhost:8080 export USERID=42 - curl -H "X-Dataverse-key:$API_TOKEN" -X POST $SERVER_URL/api/admin/authenticatedUsers/id/$USERID/disable + curl -H "X-Dataverse-key:$API_TOKEN" -X POST $SERVER_URL/api/admin/authenticatedUsers/id/$USERID/deactivate -Note: A primary purpose of most Dataverse installations is to serve an archive. In the archival space, there are best practices around the tracking of data access and the tracking of modifications to data and metadata. In support of these key workflows, a simple mechanism to delete users that have performed edit or access actions in the system is not provided. Providing a Disable User endpoint for users who have taken certain actions in the system alongside a Delete User endpoint to remove users that haven't taken certain actions in the system is by design. +Note: A primary purpose of most Dataverse installations is to serve an archive. In the archival space, there are best practices around the tracking of data access and the tracking of modifications to data and metadata. In support of these key workflows, a simple mechanism to delete users that have performed edit or access actions in the system is not provided. Providing a Deactivate User endpoint for users who have taken certain actions in the system alongside a Delete User endpoint to remove users that haven't taken certain actions in the system is by design. -This is an irreversible action. There is no option to undisable a user. +This is an irreversible action. There is no option to undeactivate a user. -Disabling a user with this endpoint will: +Deactivating a user with this endpoint will: -- Disable the user's ability to log in to the Dataverse installation. A message will be shown, stating that the account has been disabled. The user will not able to create a new account with the same email address, ORCID, Shibboleth, or other login type. -- Disable the user's ability to use the API +- Deactivate the user's ability to log in to the Dataverse installation. A message will be shown, stating that the account has been deactivated. The user will not able to create a new account with the same email address, ORCID, Shibboleth, or other login type. +- Deactivate the user's ability to use the API - Remove the user's access from all Dataverse collections, datasets and files - Prevent a user from being assigned any roles - Cancel any pending file access requests generated by the user - Remove the user from all groups - No longer have notifications generated or sent by the Dataverse installation -Disabling a user with this endpoint will keep: +Deactivating a user with this endpoint will keep: - The user's contributions to datasets, including dataset creation, file uploads, and publishing. - The user's access history to datafiles in the Dataverse installation, including guestbook records. diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java b/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java index 5093cd99b3f..315b7c89da7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java @@ -97,14 +97,14 @@ public User getUser() { * Sets the user and configures the session timeout. */ public void setUser(User aUser) { - // We check for disabled status here in "setUser" to ensure a common user + // We check for deactivated status here in "setUser" to ensure a common user // experience across Builtin, Shib, OAuth, and OIDC users. // If we want a different user experience for Builtin users, we can // modify getUpdateAuthenticatedUser in AuthenticationServiceBean // (and probably other places). - if (aUser instanceof AuthenticatedUser && aUser.isDisabled()) { - logger.info("Login attempt by disabled user " + aUser.getIdentifier() + "."); - JsfHelper.addErrorMessage(BundleUtil.getStringFromBundle("disabled.error")); + if (aUser instanceof AuthenticatedUser && aUser.isDeactivated()) { + logger.info("Login attempt by deactivated user " + aUser.getIdentifier() + "."); + JsfHelper.addErrorMessage(BundleUtil.getStringFromBundle("deactivated.error")); return; } FacesContext context = FacesContext.getCurrentInstance(); diff --git a/src/main/java/edu/harvard/iq/dataverse/Shib.java b/src/main/java/edu/harvard/iq/dataverse/Shib.java index cd3b4a30f52..6dd48cc782a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Shib.java +++ b/src/main/java/edu/harvard/iq/dataverse/Shib.java @@ -358,7 +358,7 @@ public String confirmAndConvertAccount() { private void logInUserAndSetShibAttributes(AuthenticatedUser au) { au.setShibIdentityProvider(shibIdp); - // setUser checks for disabled users. + // setUser checks for deactivated users. session.setUser(au); logger.fine("Groups for user " + au.getId() + " (" + au.getIdentifier() + "): " + getGroups(au)); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index e3e450a8ade..d79879be9d9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -382,7 +382,7 @@ protected AuthenticatedUser findAuthenticatedUserOrDie() throws WrappedResponse private AuthenticatedUser findAuthenticatedUserOrDie( String key, String wfid ) throws WrappedResponse { if (key != null) { - // No check for disabled user because it's done in authSvc.lookupUser. + // No check for deactivated user because it's done in authSvc.lookupUser. AuthenticatedUser authUser = authSvc.lookupUser(key); if (authUser != null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 6e8ed301a68..96add147743 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -87,7 +87,7 @@ import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.impl.MergeInAccountCommand; import edu.harvard.iq.dataverse.engine.command.impl.ChangeUserIdentifierCommand; -import edu.harvard.iq.dataverse.engine.command.impl.DisableUserCommand; +import edu.harvard.iq.dataverse.engine.command.impl.DeactivateUserCommand; import edu.harvard.iq.dataverse.engine.command.impl.RegisterDvObjectCommand; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; @@ -384,33 +384,33 @@ private Response deleteAuthenticatedUser(AuthenticatedUser au) { } @POST - @Path("authenticatedUsers/{identifier}/disable") - public Response disableAuthenticatedUser(@PathParam("identifier") String identifier) { + @Path("authenticatedUsers/{identifier}/deactivate") + public Response deactivateAuthenticatedUser(@PathParam("identifier") String identifier) { AuthenticatedUser user = authSvc.getAuthenticatedUser(identifier); if (user != null) { - return disableAuthenticatedUser(user); + return deactivateAuthenticatedUser(user); } return error(Response.Status.BAD_REQUEST, "User " + identifier + " not found."); } @POST - @Path("authenticatedUsers/id/{id}/disable") - public Response disableAuthenticatedUserById(@PathParam("id") Long id) { + @Path("authenticatedUsers/id/{id}/deactivate") + public Response deactivateAuthenticatedUserById(@PathParam("id") Long id) { AuthenticatedUser user = authSvc.findByID(id); if (user != null) { - return disableAuthenticatedUser(user); + return deactivateAuthenticatedUser(user); } return error(Response.Status.BAD_REQUEST, "User " + id + " not found."); } - private Response disableAuthenticatedUser(AuthenticatedUser userToDisable) { + private Response deactivateAuthenticatedUser(AuthenticatedUser userToDisable) { AuthenticatedUser superuser = authSvc.getAdminUser(); if (superuser == null) { - return error(Response.Status.INTERNAL_SERVER_ERROR, "Cannot find superuser to execute DisableUserCommand."); + return error(Response.Status.INTERNAL_SERVER_ERROR, "Cannot find superuser to execute DeactivateUserCommand."); } try { - execCommand(new DisableUserCommand(createDataverseRequest(superuser), userToDisable)); - return ok("User " + userToDisable.getIdentifier() + " disabled."); + execCommand(new DeactivateUserCommand(createDataverseRequest(superuser), userToDisable)); + return ok("User " + userToDisable.getIdentifier() + " deactivated."); } catch (WrappedResponse ex) { return ex.getResponse(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SwordAuth.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SwordAuth.java index 339832989d6..13fc37bdc40 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SwordAuth.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SwordAuth.java @@ -36,7 +36,7 @@ public AuthenticatedUser auth(AuthCredentials authCredentials) throws SwordAuthE throw new SwordAuthException(msg); } - // Checking if the user is disabled is done inside findUserByApiToken. + // Checking if the user is deactivated is done inside findUserByApiToken. AuthenticatedUser authenticatedUserFromToken = findUserByApiToken(username); if (authenticatedUserFromToken == null) { String msg = "User not found based on API token."; diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java index f3613e499f7..8a3d51bf1a7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java @@ -309,7 +309,7 @@ public AuthenticatedUser getUpdateAuthenticatedUser( String authenticationProvid // yay! see if we already have this user. AuthenticatedUser user = lookupUser(authenticationProviderId, resp.getUserId()); - if (user != null && !user.isDisabled()) { + if (user != null && !user.isDeactivated()) { user = userService.updateLastLogin(user); } @@ -454,10 +454,10 @@ public AuthenticatedUser lookupUser( String apiToken ) { } AuthenticatedUser user = tkn.getAuthenticatedUser(); - if (!user.isDisabled()) { + if (!user.isDeactivated()) { return user; } else { - logger.info("attempted access with token from disabled user: " + apiToken); + logger.info("attempted access with token from deactivated user: " + apiToken); return null; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java index 8daa6b2c17c..225352dec43 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java @@ -106,7 +106,7 @@ public void exchangeCodeForToken() throws IOException { } else { // login the user and redirect to HOME of intended page (if any). - // setUser checks for disabled users. + // setUser checks for deactivated users. session.setUser(dvUser); final OAuth2TokenData tokenData = oauthUser.getTokenData(); if (tokenData != null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java index 03069536287..abe0fef597f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java @@ -57,7 +57,7 @@ query="select au from AuthenticatedUser au WHERE (" + "LOWER(au.userIdentifier) like LOWER(:query) OR " + "lower(concat(au.firstName,' ',au.lastName)) like lower(:query)) " - + "AND au.disabled != true"), + + "AND au.deactivated != true"), @NamedQuery( name="AuthenticatedUser.findAdminUser", query="select au from AuthenticatedUser au WHERE " + "au.superuser = true " @@ -117,10 +117,10 @@ public class AuthenticatedUser implements User, Serializable { private boolean superuser; @Column(nullable=true) - private boolean disabled; + private boolean deactivated; @Column(nullable=true) - private Timestamp disabledTime; + private Timestamp deactivatedTime; /** * @todo Consider storing a hash of *all* potentially interesting Shibboleth @@ -315,20 +315,20 @@ public void setSuperuser(boolean superuser) { } @Override - public boolean isDisabled() { - return disabled; + public boolean isDeactivated() { + return deactivated; } - public void setDisabled(boolean disabled) { - this.disabled = disabled; + public void setDeactivated(boolean deactivated) { + this.deactivated = deactivated; } - public Timestamp getDisabledTime() { - return disabledTime; + public Timestamp getDeactivatedTime() { + return deactivatedTime; } - public void setDisabledTime(Timestamp disabledTime) { - this.disabledTime = disabledTime; + public void setDeactivatedTime(Timestamp deactivatedTime) { + this.deactivatedTime = deactivatedTime; } @OneToOne(mappedBy = "authenticatedUser") diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/users/User.java b/src/main/java/edu/harvard/iq/dataverse/authorization/users/User.java index baaf0e5f627..4655c9c9f0d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/users/User.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/users/User.java @@ -14,7 +14,7 @@ public interface User extends RoleAssignee, Serializable { public boolean isSuperuser(); - default boolean isDisabled() { + default boolean isDeactivated() { return false; } diff --git a/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailServiceBean.java index 574cfa11cde..244dd476909 100644 --- a/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailServiceBean.java @@ -169,8 +169,8 @@ public ConfirmEmailExecResponse processToken(String tokenQueried) { long nowInMilliseconds = new Date().getTime(); Timestamp emailConfirmed = new Timestamp(nowInMilliseconds); AuthenticatedUser authenticatedUser = confirmEmailData.getAuthenticatedUser(); - if (authenticatedUser.isDisabled()) { - logger.fine("User is disabled."); + if (authenticatedUser.isDeactivated()) { + logger.fine("User is deactivated."); return null; } authenticatedUser.setEmailConfirmed(emailConfirmed); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AddRoleAssigneesToExplicitGroupCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AddRoleAssigneesToExplicitGroupCommand.java index d4e47a5d449..8ba1d181609 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AddRoleAssigneesToExplicitGroupCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AddRoleAssigneesToExplicitGroupCommand.java @@ -52,8 +52,8 @@ public ExplicitGroup execute(CommandContext ctxt) throws CommandException { } else { if (ra instanceof AuthenticatedUser) { AuthenticatedUser user = (AuthenticatedUser) ra; - if (user.isDisabled()) { - throw new IllegalCommandException("User " + user.getUserIdentifier() + " is disabled and cannot be added to a group.", this); + if (user.isDeactivated()) { + throw new IllegalCommandException("User " + user.getUserIdentifier() + " is deactivated and cannot be added to a group.", this); } } try { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java index 274d485c5d2..276f52a5802 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java @@ -53,8 +53,8 @@ public AssignRoleCommand(RoleAssignee anAssignee, DataverseRole aRole, DvObject public RoleAssignment execute(CommandContext ctxt) throws CommandException { if (grantee instanceof AuthenticatedUser) { AuthenticatedUser user = (AuthenticatedUser) grantee; - if (user.isDisabled()) { - throw new IllegalCommandException("User " + user.getUserIdentifier() + " is disabled and cannot be given a role.", this); + if (user.isDeactivated()) { + throw new IllegalCommandException("User " + user.getUserIdentifier() + " is deactivated and cannot be given a role.", this); } } // TODO make sure the role is defined on the dataverse. diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DisableUserCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeactivateUserCommand.java similarity index 52% rename from src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DisableUserCommand.java rename to src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeactivateUserCommand.java index 58ee7295e22..1dab8120767 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DisableUserCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeactivateUserCommand.java @@ -13,32 +13,32 @@ // Superuser-only enforced below. @RequiredPermissions({}) -public class DisableUserCommand extends AbstractCommand { +public class DeactivateUserCommand extends AbstractCommand { private DataverseRequest request; - private AuthenticatedUser userToDisable; + private AuthenticatedUser userToDeactivate; - public DisableUserCommand(DataverseRequest request, AuthenticatedUser userToDisable) { + public DeactivateUserCommand(DataverseRequest request, AuthenticatedUser userToDeactivate) { super(request, (DvObject) null); this.request = request; - this.userToDisable = userToDisable; + this.userToDeactivate = userToDeactivate; } @Override public AuthenticatedUser execute(CommandContext ctxt) throws CommandException { if (!(getUser() instanceof AuthenticatedUser) || !getUser().isSuperuser()) { - throw new PermissionException("Disable user command can only be called by superusers.", this, null, null); + throw new PermissionException("Deactivate user command can only be called by superusers.", this, null, null); } - if (userToDisable == null) { - throw new CommandException("Cannot disable user. User not found.", this); + if (userToDeactivate == null) { + throw new CommandException("Cannot deactivate user. User not found.", this); } - ctxt.engine().submit(new RevokeAllRolesCommand(userToDisable, request)); - ctxt.authentication().removeAuthentictedUserItems(userToDisable); - ctxt.notifications().findByUser(userToDisable.getId()).forEach(ctxt.notifications()::delete); - userToDisable.setDisabled(true); - userToDisable.setDisabledTime(new Timestamp(new Date().getTime())); - AuthenticatedUser disabledUser = ctxt.authentication().save(userToDisable); - return disabledUser; + ctxt.engine().submit(new RevokeAllRolesCommand(userToDeactivate, request)); + ctxt.authentication().removeAuthentictedUserItems(userToDeactivate); + ctxt.notifications().findByUser(userToDeactivate.getId()).forEach(ctxt.notifications()::delete); + userToDeactivate.setDeactivated(true); + userToDeactivate.setDeactivatedTime(new Timestamp(new Date().getTime())); + AuthenticatedUser deactivatedUser = ctxt.authentication().save(userToDeactivate); + return deactivatedUser; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java index c0e3b3ea89a..c0dce9799e1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java @@ -59,8 +59,8 @@ public MergeInAccountCommand(DataverseRequest createDataverseRequest, Authentica @Override protected void executeImpl(CommandContext ctxt) throws CommandException { - if (consumedAU.isDisabled() && !ongoingAU.isDisabled() || !consumedAU.isDisabled() && ongoingAU.isDisabled()) { - throw new IllegalCommandException("User accounts can only be merged if they are either both enabled or both disabled.", this); + if (consumedAU.isDeactivated() && !ongoingAU.isDeactivated() || !consumedAU.isDeactivated() && ongoingAU.isDeactivated()) { + throw new IllegalCommandException("User accounts can only be merged if they are either both non-deactivated or both deactivated.", this); } List baseRAList = ctxt.roleAssignees().getAssignmentsFor(ongoingAU.getIdentifier()); diff --git a/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetServiceBean.java index 9b28be6527c..4e92d861a2f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetServiceBean.java @@ -65,8 +65,8 @@ public PasswordResetInitResponse requestReset(String emailAddress) throws Passwo logger.info("Cannot find a user based on " + emailAddress + ". Cannot reset password."); return new PasswordResetInitResponse(false); } - if (authUser.isDisabled()) { - logger.info("Cannot reset password for " + emailAddress + " because account is disabled."); + if (authUser.isDeactivated()) { + logger.info("Cannot reset password for " + emailAddress + " because account is deactivated."); return new PasswordResetInitResponse(false); } BuiltinUser user = dataverseUserService.findByUserName(authUser.getUserIdentifier()); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 668f3cbf771..172a63db983 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -107,8 +107,8 @@ public static JsonObjectBuilder json(AuthenticatedUser authenticatedUser) { .add("lastName", authenticatedUser.getLastName()) .add("email", authenticatedUser.getEmail()) .add("superuser", authenticatedUser.isSuperuser()) - .add("disabled", authenticatedUser.isDisabled()) - .add("disabledTime", authenticatedUser.getDisabledTime()) + .add("deactivated", authenticatedUser.isDeactivated()) + .add("deactivatedTime", authenticatedUser.getDeactivatedTime()) .add("affiliation", authenticatedUser.getAffiliation()) .add("position", authenticatedUser.getPosition()) .add("persistentUserId", authenticatedUser.getAuthenticatedUserLookup().getPersistentUserId()) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index a76a7111479..8b5e2c0b49e 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -364,7 +364,7 @@ shib.dataverseUsername=Dataverse Username shib.currentDataversePassword=Current Dataverse Password shib.accountInformation=Account Information shib.offerToCreateNewAccount=This information is provided by your institution and will be used to create your Dataverse account. -shib.passwordRejected=Validation Error - Your account can only be converted if you provide the correct password for your existing account. If your existing account has been disabled by an administrator, you cannot convert your account. +shib.passwordRejected=Validation Error - Your account can only be converted if you provide the correct password for your existing account. If your existing account has been deactivated by an administrator, you cannot convert your account. # oauth2/firstLogin.xhtml oauth2.btn.convertAccount=Convert Existing Account @@ -399,7 +399,7 @@ oauth2.newAccount.emailInvalid=Invalid email address. oauth2.convertAccount.explanation=Please enter your {0} account username or email and password to convert your account to the {1} log in option. Learn more about converting your account. oauth2.convertAccount.username=Existing username oauth2.convertAccount.password=Password -oauth2.convertAccount.authenticationFailed=Your account can only be converted if you provide the correct username and password for your existing account. If your existing account has been disabled by an administrator, you cannot convert your account. +oauth2.convertAccount.authenticationFailed=Your account can only be converted if you provide the correct username and password for your existing account. If your existing account has been deactivated by an administrator, you cannot convert your account. oauth2.convertAccount.buttonTitle=Convert Account oauth2.convertAccount.success=Your Dataverse account is now associated with your {0} account. @@ -407,8 +407,8 @@ oauth2.convertAccount.success=Your Dataverse account is now associated with your oauth2.callback.page.title=OAuth Callback oauth2.callback.message=Authentication Error - Dataverse could not authenticate your login with the provider that you selected. Please make sure you authorize your account to connect with Dataverse. For more details about the information being requested, see the User Guide. -# disabled user accounts -disabled.error=Sorry, your account has been disabled. +# deactivated user accounts +deactivated.error=Sorry, your account has been deactivated. # tab on dataverseuser.xhtml apitoken.title=API Token diff --git a/src/main/resources/db/migration/V5.3.0.5__2419-deactivate-users.sql b/src/main/resources/db/migration/V5.3.0.5__2419-deactivate-users.sql new file mode 100644 index 00000000000..c87fe019a52 --- /dev/null +++ b/src/main/resources/db/migration/V5.3.0.5__2419-deactivate-users.sql @@ -0,0 +1,4 @@ +-- Users can be deactivated. +ALTER TABLE authenticateduser ADD COLUMN IF NOT EXISTS deactivated BOOLEAN; +-- A timestamp of when the user was deactivated. +ALTER TABLE authenticateduser ADD COLUMN IF NOT EXISTS deactivatedtime timestamp without time zone; diff --git a/src/main/resources/db/migration/V5.3.0.5__4475-disable-users.sql b/src/main/resources/db/migration/V5.3.0.5__4475-disable-users.sql deleted file mode 100644 index 48823a72247..00000000000 --- a/src/main/resources/db/migration/V5.3.0.5__4475-disable-users.sql +++ /dev/null @@ -1,4 +0,0 @@ --- Users can be disabled. -ALTER TABLE authenticateduser ADD COLUMN IF NOT EXISTS disabled BOOLEAN; --- A timestamp of when the user was disabled. -ALTER TABLE authenticateduser ADD COLUMN IF NOT EXISTS disabledtime timestamp without time zone; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DisableUsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java similarity index 76% rename from src/test/java/edu/harvard/iq/dataverse/api/DisableUsersIT.java rename to src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java index d9c23b100e0..ed979ff785c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DisableUsersIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java @@ -16,7 +16,7 @@ import org.junit.BeforeClass; import org.junit.Test; -public class DisableUsersIT { +public class DeactivateUsersIT { @BeforeClass public static void setUp() { @@ -24,7 +24,7 @@ public static void setUp() { } @Test - public void testDisableUser() { + public void testDeactivateUser() { Response createSuperuser = UtilIT.createRandomUser(); createSuperuser.then().assertThat().statusCode(OK.getStatusCode()); @@ -51,9 +51,9 @@ public void testDisableUser() { String username = UtilIT.getUsernameFromResponse(createUser); String apiToken = UtilIT.getApiTokenFromResponse(createUser); - Response grantRoleBeforeDisable = UtilIT.grantRoleOnDataverse(dataverseAlias, DataverseRole.ADMIN.toString(), "@" + username, superuserApiToken); - grantRoleBeforeDisable.prettyPrint(); - grantRoleBeforeDisable.then().assertThat() + Response grantRoleBeforeDeactivate = UtilIT.grantRoleOnDataverse(dataverseAlias, DataverseRole.ADMIN.toString(), "@" + username, superuserApiToken); + grantRoleBeforeDeactivate.prettyPrint(); + grantRoleBeforeDeactivate.then().assertThat() .statusCode(OK.getStatusCode()) .body("data.assignee", equalTo("@" + username)) .body("data._roleAlias", equalTo("admin")); @@ -75,31 +75,31 @@ public void testDisableUser() { addToGroup.then().assertThat() .statusCode(OK.getStatusCode()); - Response userTracesBeforeDisable = UtilIT.getUserTraces(username, superuserApiToken); - userTracesBeforeDisable.prettyPrint(); - userTracesBeforeDisable.then().assertThat() + Response userTracesBeforeDeactivate = UtilIT.getUserTraces(username, superuserApiToken); + userTracesBeforeDeactivate.prettyPrint(); + userTracesBeforeDeactivate.then().assertThat() .statusCode(OK.getStatusCode()) .body("data.traces.roleAssignments.items[0].definitionPointName", equalTo(dataverseAlias)) .body("data.traces.roleAssignments.items[0].definitionPointId", equalTo(dataverseId)) .body("data.traces.explicitGroups.items[0].name", equalTo("Group for " + dataverseAlias)); - Response disableUser = UtilIT.disableUser(username); - disableUser.prettyPrint(); - disableUser.then().assertThat().statusCode(OK.getStatusCode()); + Response deactivateUser = UtilIT.deactivateUser(username); + deactivateUser.prettyPrint(); + deactivateUser.then().assertThat().statusCode(OK.getStatusCode()); Response getUser = UtilIT.getAuthenticatedUser(username, superuserApiToken); getUser.prettyPrint(); getUser.then().assertThat() .statusCode(OK.getStatusCode()) - .body("data.disabled", equalTo(true)); + .body("data.deactivated", equalTo(true)); - Response getUserDisabled = UtilIT.getAuthenticatedUserByToken(apiToken); - getUserDisabled.prettyPrint(); - getUserDisabled.then().assertThat().statusCode(BAD_REQUEST.getStatusCode()); + Response getUserDeactivated = UtilIT.getAuthenticatedUserByToken(apiToken); + getUserDeactivated.prettyPrint(); + getUserDeactivated.then().assertThat().statusCode(BAD_REQUEST.getStatusCode()); - Response userTracesAfterDisable = UtilIT.getUserTraces(username, superuserApiToken); - userTracesAfterDisable.prettyPrint(); - userTracesAfterDisable.then().assertThat() + Response userTracesAfterDeactivate = UtilIT.getUserTraces(username, superuserApiToken); + userTracesAfterDeactivate.prettyPrint(); + userTracesAfterDeactivate.then().assertThat() .statusCode(OK.getStatusCode()) /** * Here we are showing the the following were deleted: @@ -110,28 +110,28 @@ public void testDisableUser() { */ .body("data.traces", equalTo(Collections.EMPTY_MAP)); - Response grantRoleAfterDisable = UtilIT.grantRoleOnDataverse(dataverseAlias, DataverseRole.ADMIN.toString(), "@" + username, superuserApiToken); - grantRoleAfterDisable.prettyPrint(); - grantRoleAfterDisable.then().assertThat() + Response grantRoleAfterDeactivate = UtilIT.grantRoleOnDataverse(dataverseAlias, DataverseRole.ADMIN.toString(), "@" + username, superuserApiToken); + grantRoleAfterDeactivate.prettyPrint(); + grantRoleAfterDeactivate.then().assertThat() .statusCode(FORBIDDEN.getStatusCode()) - .body("message", equalTo("User " + username + " is disabled and cannot be given a role.")); + .body("message", equalTo("User " + username + " is deactivated and cannot be given a role.")); Response addToGroupAfter = UtilIT.addToGroup(dataverseAlias, aliasInOwner, roleAssigneesToAdd, superuserApiToken); addToGroupAfter.prettyPrint(); addToGroupAfter.then().assertThat() .statusCode(FORBIDDEN.getStatusCode()) - .body("message", equalTo("User " + username + " is disabled and cannot be added to a group.")); + .body("message", equalTo("User " + username + " is deactivated and cannot be added to a group.")); Response grantRoleOnDataset = UtilIT.grantRoleOnDataset(datasetPersistentId, DataverseRole.ADMIN.toString(), "@" + username, superuserApiToken); grantRoleOnDataset.prettyPrint(); grantRoleOnDataset.then().assertThat() .statusCode(FORBIDDEN.getStatusCode()) - .body("message", equalTo("User " + username + " is disabled and cannot be given a role.")); + .body("message", equalTo("User " + username + " is deactivated and cannot be given a role.")); } @Test - public void testDisableUserById() { + public void testDeactivateUserById() { Response createSuperuser = UtilIT.createRandomUser(); createSuperuser.then().assertThat().statusCode(OK.getStatusCode()); @@ -148,13 +148,13 @@ public void testDisableUserById() { Long userId = JsonPath.from(createUser.body().asString()).getLong("data.user.id"); String apiToken = UtilIT.getApiTokenFromResponse(createUser); - Response disableUser = UtilIT.disableUser(userId); - disableUser.prettyPrint(); - disableUser.then().assertThat().statusCode(OK.getStatusCode()); + Response deactivateUser = UtilIT.deactivateUser(userId); + deactivateUser.prettyPrint(); + deactivateUser.then().assertThat().statusCode(OK.getStatusCode()); } @Test - public void testMergeDisabledIntoEnabledUser() { + public void testMergeDeactivatedIntoNonDeactivatedUser() { Response createSuperuser = UtilIT.createRandomUser(); String superuserUsername = UtilIT.getUsernameFromResponse(createSuperuser); @@ -171,18 +171,18 @@ public void testMergeDisabledIntoEnabledUser() { createUserToMerge.prettyPrint(); String usernameToMerge = UtilIT.getUsernameFromResponse(createUserToMerge); - Response disableUser = UtilIT.disableUser(usernameToMerge); - disableUser.prettyPrint(); - disableUser.then().assertThat().statusCode(OK.getStatusCode()); + Response deactivateUser = UtilIT.deactivateUser(usernameToMerge); + deactivateUser.prettyPrint(); + deactivateUser.then().assertThat().statusCode(OK.getStatusCode()); - // User accounts can only be merged if they are either both enabled or both disabled. + // User accounts can only be merged if they are either both non-deactivated or both deactivated. Response mergeAccounts = UtilIT.mergeAccounts(usernameMergeTarget, usernameToMerge, superuserApiToken); mergeAccounts.prettyPrint(); mergeAccounts.then().assertThat().statusCode(BAD_REQUEST.getStatusCode()); } @Test - public void testMergeEnabledIntoDisabledUser() { + public void testMergeNonDeactivatedIntoDeactivatedUser() { Response createSuperuser = UtilIT.createRandomUser(); String superuserUsername = UtilIT.getUsernameFromResponse(createSuperuser); @@ -199,18 +199,18 @@ public void testMergeEnabledIntoDisabledUser() { createUserToMerge.prettyPrint(); String usernameToMerge = UtilIT.getUsernameFromResponse(createUserToMerge); - Response disableUser = UtilIT.disableUser(usernameMergeTarget); - disableUser.prettyPrint(); - disableUser.then().assertThat().statusCode(OK.getStatusCode()); + Response deactivateUser = UtilIT.deactivateUser(usernameMergeTarget); + deactivateUser.prettyPrint(); + deactivateUser.then().assertThat().statusCode(OK.getStatusCode()); - // User accounts can only be merged if they are either both enabled or both disabled. + // User accounts can only be merged if they are either both non-deactivated or both deactivated. Response mergeAccounts = UtilIT.mergeAccounts(usernameMergeTarget, usernameToMerge, superuserApiToken); mergeAccounts.prettyPrint(); mergeAccounts.then().assertThat().statusCode(BAD_REQUEST.getStatusCode()); } @Test - public void testMergeDisabledIntoDisabledUser() { + public void testMergeDeactivatedIntoDeactivatedUser() { Response createSuperuser = UtilIT.createRandomUser(); String superuserUsername = UtilIT.getUsernameFromResponse(createSuperuser); @@ -227,15 +227,15 @@ public void testMergeDisabledIntoDisabledUser() { createUserToMerge.prettyPrint(); String usernameToMerge = UtilIT.getUsernameFromResponse(createUserToMerge); - Response disableUserMergeTarget = UtilIT.disableUser(usernameMergeTarget); - disableUserMergeTarget.prettyPrint(); - disableUserMergeTarget.then().assertThat().statusCode(OK.getStatusCode()); + Response deactivatedUserMergeTarget = UtilIT.deactivateUser(usernameMergeTarget); + deactivatedUserMergeTarget.prettyPrint(); + deactivatedUserMergeTarget.then().assertThat().statusCode(OK.getStatusCode()); - Response disableUserToMerge = UtilIT.disableUser(usernameToMerge); - disableUserToMerge.prettyPrint(); - disableUserToMerge.then().assertThat().statusCode(OK.getStatusCode()); + Response deactivatedUserToMerge = UtilIT.deactivateUser(usernameToMerge); + deactivatedUserToMerge.prettyPrint(); + deactivatedUserToMerge.then().assertThat().statusCode(OK.getStatusCode()); - // User accounts can only be merged if they are either both enabled or both disabled. + // User accounts can only be merged if they are either both non-deactivated or both deactivated. Response mergeAccounts = UtilIT.mergeAccounts(usernameMergeTarget, usernameToMerge, superuserApiToken); mergeAccounts.prettyPrint(); mergeAccounts.then().assertThat().statusCode(OK.getStatusCode()); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DeleteUsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DeleteUsersIT.java index cbfe761234d..a7f5153d69f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DeleteUsersIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DeleteUsersIT.java @@ -53,8 +53,8 @@ * * - guestbookresponse: Definitely a concern but it's possible to null out the * user id. You can't delete a user but you can merge instead. There is talk of - * disable which would probably null out the id. In all cases the name and email - * address in the rows are left alone. + * deactivate which would probably null out the id. In all cases the name and + * email address in the rows are left alone. * * - oauth2tokendata: Not a concern. Rows are deleted. * @@ -668,10 +668,10 @@ public void testCuratorSendsCommentsToAuthor() throws InterruptedException { + " because the user has contributed to dataset version(s).")); // What should we do with curator2 instead of deleting? The only option is to merge - // curator2 into some other account. Once implemented, we'll disable curator2's account + // curator2 into some other account. Once implemented, we'll deactivate curator2's account // so that curator2 continues to be displayed as a contributor. // - // TODO: disable curator2 here + // TODO: deactivate curator2 here // Response removeRolesFromAuthor2 = UtilIT.deleteUserRoles(author2Username, superuserApiToken); removeRolesFromAuthor2.prettyPrint(); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index cc4b3e2029e..63911d09e80 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -977,16 +977,16 @@ static public Response grantRoleOnDataverse(String definitionPoint, String role, .post("api/dataverses/" + definitionPoint + "/assignments?key=" + apiToken); } - public static Response disableUser(String username) { - Response disableUserResponse = given() - .post("/api/admin/authenticatedUsers/" + username + "/disable"); - return disableUserResponse; + public static Response deactivateUser(String username) { + Response deactivateUserResponse = given() + .post("/api/admin/authenticatedUsers/" + username + "/deactivate"); + return deactivateUserResponse; } - public static Response disableUser(Long userId) { - Response disableUserResponse = given() - .post("/api/admin/authenticatedUsers/id/" + userId + "/disable"); - return disableUserResponse; + public static Response deactivateUser(Long userId) { + Response deactivateUserResponse = given() + .post("/api/admin/authenticatedUsers/id/" + userId + "/deactivate"); + return deactivateUserResponse; } public static Response deleteUser(String username) { From a0b0e8d7f36cf5cef047e1c83e682864b5c81f17 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 11 Mar 2021 16:31:22 -0500 Subject: [PATCH 466/576] a release note for the updated citation metadata block. (#7638) --- doc/release-notes/7638-citation-metadatablock-update.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/release-notes/7638-citation-metadatablock-update.md diff --git a/doc/release-notes/7638-citation-metadatablock-update.md b/doc/release-notes/7638-citation-metadatablock-update.md new file mode 100644 index 00000000000..698dea623ce --- /dev/null +++ b/doc/release-notes/7638-citation-metadatablock-update.md @@ -0,0 +1,7 @@ +### Citation metadatablock update + +Due to a minor update in the citation metadata block (extra ISO-639-3 language codes added) a block upgrade is required: + +`wget https://github.com/IQSS/dataverse/releases/download/v5.4/citation.tsv` +`curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @citation.tsv -H "Content-type: text/tab-separated-values"` + From 63bb986cd610fac5191b136237b8f453390d0e6c Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 11 Mar 2021 17:00:11 -0500 Subject: [PATCH 467/576] a quick fix that makes the harvesting clients page more useable w/ remote servers offering long lists of OAI sets. (#7638) --- .../edu/harvard/iq/dataverse/HarvestingClientsPage.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/HarvestingClientsPage.java b/src/main/java/edu/harvard/iq/dataverse/HarvestingClientsPage.java index 3cc581ee93c..a88c4833f54 100644 --- a/src/main/java/edu/harvard/iq/dataverse/HarvestingClientsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/HarvestingClientsPage.java @@ -25,6 +25,7 @@ import java.util.Arrays; import java.util.List; import java.util.Locale; +import java.util.Collections; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Pattern; @@ -915,6 +916,12 @@ public void setOaiSetsSelectItems(List oaiSetsSelectItems) { private void createOaiSetsSelectItems(List setNames) { setOaiSetsSelectItems(new ArrayList<>()); if (setNames != null) { + + // Let's sort the list - otherwise, if the list is long enough, + // using this pulldown menu may be very difficult: + + Collections.sort(setNames, String.CASE_INSENSITIVE_ORDER); + for (String set: setNames) { if (!StringUtils.isEmpty(set)) { getOaiSetsSelectItems().add(new SelectItem(set, set)); From f92dab79441f142913255c36cb4807d8f33ab7ae Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 12 Mar 2021 08:46:38 -0500 Subject: [PATCH 468/576] change the dataset-level publisher field to use brand name --- src/main/java/edu/harvard/iq/dataverse/DataCitation.java | 2 +- .../java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataCitation.java b/src/main/java/edu/harvard/iq/dataverse/DataCitation.java index 14f0a940238..2e8df48f4f1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataCitation.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataCitation.java @@ -746,7 +746,7 @@ private void getAuthorsAndProducersFrom(DatasetVersion dsv) { private String getPublisherFrom(DatasetVersion dsv) { if (!dsv.getDataset().isHarvested()) { - return BrandingUtil.getRootDataverseCollectionName(); + return BrandingUtil.getInstallationBrandName(); } else { return dsv.getDistributorName(); // remove += [distributor] SEK 8-18-2016 diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index e86d1acff7a..533bd59307b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -325,7 +325,7 @@ public static JsonObjectBuilder json(Dataset ds) { .add("persistentUrl", ds.getPersistentURL()) .add("protocol", ds.getProtocol()) .add("authority", ds.getAuthority()) - .add("publisher", BrandingUtil.getRootDataverseCollectionName()) + .add("publisher", BrandingUtil.getInstallationBrandName()) .add("publicationDate", ds.getPublicationDateFormattedYYYYMMDD()) .add("storageIdentifier", ds.getStorageIdentifier()); } From 6697e16c00d6fffdaa82956ef92789a3b2c5e47f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 12 Mar 2021 08:47:44 -0500 Subject: [PATCH 469/576] remove use of mp var seeing the following when using the alias/db sources: Caused by: java.lang.IllegalStateException: Recursive update at java.base/java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1983) at fish.payara.nucleus.microprofile.config.spi.PayaraConfig.getValue(PayaraConfig.java:157) at fish.payara.nucleus.microprofile.config.spi.PayaraConfig.getValueInternal(PayaraConfig.java:127) at fish.payara.nucleus.microprofile.config.spi.PayaraConfig.getOptionalValue(PayaraConfig.java:123) at edu.harvard.iq.dataverse.settings.source.AliasConfigSource.getValue(AliasConfigSource.java:87) at fish.payara.nucleus.microprofile.config.spi.PayaraConfig.getSourceValue(PayaraConfig.java:183) at fish.payara.nucleus.microprofile.config.spi.PayaraConfig.getValueConverted(PayaraConfig.java:166) at fish.payara.nucleus.microprofile.config.spi.PayaraConfig.lambda$getValue$1(PayaraConfig.java:161) at java.base/java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1908) at fish.payara.nucleus.microprofile.config.spi.PayaraConfig.getValue(PayaraConfig.java:157) at fish.payara.nucleus.microprofile.config.spi.PayaraConfig.getValueInternal(PayaraConfig.java:127) at fish.payara.nucleus.microprofile.config.spi.PayaraConfig.getOptionalValue(PayaraConfig.java:123) at edu.harvard.iq.dataverse.branding.BrandingUtil.getInstallationBrandName(BrandingUtil.java:19) at edu.harvard.iq.dataverse.util.MailUtil.getSubjectTextBasedOnNotification(MailUtil.java:33) at edu.harvard.iq.dataverse.MailServiceBean.sendNotificationEmail(MailServiceBean.java:247) --- .../iq/dataverse/branding/BrandingUtil.java | 19 +++++++++---------- .../branding/BrandingUtilHelper.java | 4 +++- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java index 07cedb7a3ce..479680a4230 100644 --- a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java @@ -1,24 +1,22 @@ package edu.harvard.iq.dataverse.branding; import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import java.util.Arrays; +import java.util.logging.Logger; import javax.mail.internet.InternetAddress; -import org.eclipse.microprofile.config.ConfigProvider; - public class BrandingUtil { + private static final Logger logger = Logger.getLogger(BrandingUtil.class.getCanonicalName()); + static DataverseServiceBean dataverseService; + static SettingsServiceBean settingsService; - static String brandName = null; - public static String getInstallationBrandName() { - if(brandName==null) { - brandName = ConfigProvider.getConfig().getOptionalValue("dataverse.branding.installation.name", String.class) - .orElse(dataverseService.getRootDataverseName()); - } + String brandName = settingsService.getValueForKey(SettingsServiceBean.Key.InstallationName, dataverseService.getRootDataverseName()); return brandName; } @@ -54,7 +52,8 @@ public static String getContactHeader(InternetAddress systemAddress) { return BundleUtil.getStringFromBundle("contact.header", Arrays.asList(getSupportTeamName(systemAddress))); } - public static void injectDataverseService(DataverseServiceBean dataverseService) { - BrandingUtil.dataverseService = dataverseService; + public static void injectServices(DataverseServiceBean dataverseSvc, SettingsServiceBean settingsSvc) { + dataverseService = dataverseSvc; + settingsService = settingsSvc; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtilHelper.java b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtilHelper.java index 0fb16088ff5..d0184f82c78 100644 --- a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtilHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtilHelper.java @@ -6,6 +6,7 @@ import javax.ejb.Startup; import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; /** * This is a small helper bean @@ -18,9 +19,10 @@ public class BrandingUtilHelper { @EJB DataverseServiceBean dataverseSvc; + @EJB SettingsServiceBean settingsSvc; @PostConstruct public void injectService() { - BrandingUtil.injectDataverseService(dataverseSvc); + BrandingUtil.injectServices(dataverseSvc, settingsSvc); } } From 3238a5638cc221238a8470021e8e9acf3d6e416f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 12 Mar 2021 09:28:49 -0500 Subject: [PATCH 470/576] DbSetting bug fix --- .../iq/dataverse/settings/source/DbSettingConfigSource.java | 2 +- .../iq/dataverse/settings/source/DbSettingConfigSourceTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/source/DbSettingConfigSource.java b/src/main/java/edu/harvard/iq/dataverse/settings/source/DbSettingConfigSource.java index 3bf672213e6..838cd415819 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/source/DbSettingConfigSource.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/source/DbSettingConfigSource.java @@ -44,7 +44,7 @@ public static void updateProperties() { } properties.clear(); Set dbSettings = settingsSvc.listAll(); - dbSettings.forEach(s -> properties.put(PREFIX+"."+s.getName()+ (s.getLang() == null ? "" : "."+s.getLang()), s.getContent())); + dbSettings.forEach(s -> properties.put(PREFIX+"."+s.getName().substring(1) + (s.getLang() == null ? "" : "."+s.getLang()), s.getContent())); lastUpdate = Instant.now(); } diff --git a/src/test/java/edu/harvard/iq/dataverse/settings/source/DbSettingConfigSourceTest.java b/src/test/java/edu/harvard/iq/dataverse/settings/source/DbSettingConfigSourceTest.java index c886ef0ba69..9ceca24aadf 100644 --- a/src/test/java/edu/harvard/iq/dataverse/settings/source/DbSettingConfigSourceTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/settings/source/DbSettingConfigSourceTest.java @@ -36,7 +36,7 @@ void testEmptyIfNoSettingsService() { @Test @Order(2) void testDataRetrieval() { - Set settings = new HashSet<>(Arrays.asList(new Setting("FooBar", "hello"), new Setting("FooBarI18N", "de", "hallo"))); + Set settings = new HashSet<>(Arrays.asList(new Setting(":FooBar", "hello"), new Setting(":FooBarI18N", "de", "hallo"))); Mockito.when(settingsSvc.listAll()).thenReturn(settings); DbSettingConfigSource.injectSettingsService(settingsSvc); From df8973f8a42ce166108d6f53a5a577c08153948c Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 12 Mar 2021 09:29:07 -0500 Subject: [PATCH 471/576] add new settings --- .../iq/dataverse/settings/SettingsServiceBean.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 1f53281b2fa..02637bfa8df 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -422,7 +422,18 @@ Whether Harvesting (OAI) service is enabled /** * Sort Date Facets Chronologically instead or presenting them in order of # of hits as other facets are. Default is true */ - ChronologicalDateFacets + ChronologicalDateFacets, + + /** + * Used where BrandingUtil.getInstallationBrandName is called, overides the default use of the root Dataverse collection name + */ + InstallationName, + /** + * In metadata exports that set a 'distributor' this flag determines whether the + * Installation Brand Name is always included (default/false) or is not included + * when the Distributor field (citation metadatablock) is set (true) + */ + ExportInstallationAsDistributorOnlyWhenNotSet ; @Override From 4ca9618f70af1b25fe830fd64bb5d9acb73ec361 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 12 Mar 2021 09:29:21 -0500 Subject: [PATCH 472/576] test updates to match --- .../iq/dataverse/branding/BrandingUtil.java | 7 ++- .../dataverse/branding/BrandingUtilTest.java | 45 +++++++------------ .../iq/dataverse/util/MailUtilTest.java | 27 ++++++++++- 3 files changed, 47 insertions(+), 32 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java index 479680a4230..16ee7bb9702 100644 --- a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java @@ -16,7 +16,12 @@ public class BrandingUtil { static SettingsServiceBean settingsService; public static String getInstallationBrandName() { - String brandName = settingsService.getValueForKey(SettingsServiceBean.Key.InstallationName, dataverseService.getRootDataverseName()); + + String brandName = settingsService.getValueForKey(SettingsServiceBean.Key.InstallationName); + //Separate if statement simplifies test setup, otherwise could use the getValueForKey method with a default param + if(brandName==null) { + brandName = dataverseService.getRootDataverseName(); + } return brandName; } diff --git a/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java index e9f94b3d2a8..32f6a487994 100644 --- a/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/branding/BrandingUtilTest.java @@ -1,15 +1,10 @@ package edu.harvard.iq.dataverse.branding; import edu.harvard.iq.dataverse.DataverseServiceBean; -import edu.harvard.iq.dataverse.settings.Setting; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; -import edu.harvard.iq.dataverse.settings.source.DbSettingConfigSource; import edu.harvard.iq.dataverse.util.BundleUtil; import java.io.UnsupportedEncodingException; import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; - import javax.mail.internet.AddressException; import javax.mail.internet.InternetAddress; import static org.junit.Assert.assertEquals; @@ -28,8 +23,7 @@ public class BrandingUtilTest { @Mock - DataverseServiceBean dataverseService; - DbSettingConfigSource dbSource = new DbSettingConfigSource(); + DataverseServiceBean dataverseSvc; @Mock SettingsServiceBean settingsSvc; @@ -38,39 +32,30 @@ public class BrandingUtilTest { public void testGetInstallationBrandName() { System.out.println("testGetInstallationBrandName"); - //Initialize the DBSettingConfigSource with empty properties - Set settings = new HashSet<>(); - Mockito.when(settingsSvc.listAll()).thenReturn(settings); - DbSettingConfigSource.injectSettingsService(settingsSvc); + Mockito.when(settingsSvc.getValueForKey(SettingsServiceBean.Key.InstallationName)).thenReturn(null); //And configure the mock DataverseService to pretend the root collection name is as shown - Mockito.when(dataverseService.getRootDataverseName()).thenReturn("LibraScholar"); - BrandingUtil.injectDataverseService(dataverseService); + Mockito.when(dataverseSvc.getRootDataverseName()).thenReturn("LibraScholar"); + BrandingUtil.injectServices(dataverseSvc, settingsSvc); assertEquals("LibraScholar", BrandingUtil.getInstallationBrandName()); //Defaults to root collection name - - settings.add(new Setting("InstallationName", "NotLibraScholar")); - ///Mockito.when(settingsSvc.listAll()).thenReturn(settings); - DbSettingConfigSource.injectSettingsService(settingsSvc); + + Mockito.when(settingsSvc.getValueForKey(SettingsServiceBean.Key.InstallationName)).thenReturn("NotLibraScholar"); + assertEquals("NotLibraScholar", BrandingUtil.getInstallationBrandName()); //uses setting - - // Reset for other tests - settings.clear(); - //Cause the config source to update (otherwise it will cache the old settings for ~60 seconds - DbSettingConfigSource.injectSettingsService(settingsSvc); } @Test public void testGetSupportTeamName() throws AddressException, UnsupportedEncodingException { System.out.println("testGetSupportTeamName"); - Mockito.when(dataverseService.getRootDataverseName()).thenReturn(null); - BrandingUtil.injectDataverseService(dataverseService); + Mockito.when(dataverseSvc.getRootDataverseName()).thenReturn(null); + BrandingUtil.injectServices(dataverseSvc, settingsSvc); assertEquals("Support", BrandingUtil.getSupportTeamName(null)); - Mockito.when(dataverseService.getRootDataverseName()).thenReturn(""); - BrandingUtil.injectDataverseService(dataverseService); + Mockito.when(dataverseSvc.getRootDataverseName()).thenReturn(""); + BrandingUtil.injectServices(dataverseSvc, settingsSvc); assertEquals("Support", BrandingUtil.getSupportTeamName(null)); - Mockito.when(dataverseService.getRootDataverseName()).thenReturn("LibraScholar"); - BrandingUtil.injectDataverseService(dataverseService); + Mockito.when(dataverseSvc.getRootDataverseName()).thenReturn("LibraScholar"); + BrandingUtil.injectServices(dataverseSvc, settingsSvc); assertEquals("LibraScholar Support", BrandingUtil.getSupportTeamName(null)); assertEquals("LibraScholar Support", BrandingUtil.getSupportTeamName(new InternetAddress("support@librascholar.edu"))); assertEquals("LibraScholar Support Team", BrandingUtil.getSupportTeamName(new InternetAddress("support@librascholar.edu", "LibraScholar Support Team"))); @@ -151,8 +136,8 @@ public void testEmailSubject() { @Test public void testGetContactHeader() { System.out.println("testGetContactHeader"); - Mockito.when(dataverseService.getRootDataverseName()).thenReturn(null); - BrandingUtil.injectDataverseService(dataverseService); + Mockito.when(dataverseSvc.getRootDataverseName()).thenReturn(null); + BrandingUtil.injectServices(dataverseSvc, settingsSvc); assertEquals("Contact Support", BrandingUtil.getContactHeader(null)); } diff --git a/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java index 23a69356fbc..4363d1dd3c9 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java @@ -1,17 +1,36 @@ package edu.harvard.iq.dataverse.util; +import edu.harvard.iq.dataverse.DataverseServiceBean; import edu.harvard.iq.dataverse.UserNotification; +import edu.harvard.iq.dataverse.branding.BrandingUtil; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; + import static org.junit.Assert.assertEquals; -import org.junit.Test; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.MethodOrderer.OrderAnnotation; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; import org.junit.Before; +@ExtendWith(MockitoExtension.class) +@TestMethodOrder(OrderAnnotation.class) public class MailUtilTest { UserNotification userNotification = new UserNotification(); + @Mock + DataverseServiceBean dataverseSvc; + @Mock + SettingsServiceBean settingsSvc; + @Before public void setUp() { userNotification = new UserNotification(); + } @Test @@ -29,7 +48,13 @@ public void testParseSystemAddress() { } @Test + @Order(1) public void testSubjectCreateAccount() { + Mockito.when(settingsSvc.getValueForKey(SettingsServiceBean.Key.InstallationName)).thenReturn(null); + //And configure the mock DataverseService to pretend the root collection name is as shown + Mockito.when(dataverseSvc.getRootDataverseName()).thenReturn("LibraScholar"); + BrandingUtil.injectServices(dataverseSvc, settingsSvc); + userNotification.setType(UserNotification.Type.CREATEACC); assertEquals("LibraScholar: Your account has been created", MailUtil.getSubjectTextBasedOnNotification(userNotification, null)); } From 16e1e7b92cb0d187b5972b00831c6e18b2a77fef Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 12 Mar 2021 09:36:23 -0500 Subject: [PATCH 473/576] no longer used --- src/main/resources/META-INF/microprofile-aliases.properties | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/resources/META-INF/microprofile-aliases.properties b/src/main/resources/META-INF/microprofile-aliases.properties index 3dc688824ed..ebde2910d52 100644 --- a/src/main/resources/META-INF/microprofile-aliases.properties +++ b/src/main/resources/META-INF/microprofile-aliases.properties @@ -2,5 +2,4 @@ # This file is a placeholder for future aliases of deprecated config settings. # Format: # dataverse.new.config.option=dataverse.old.deprecated.option -dataverse.branding.installation.name=dataverse.settings.fromdb.InstallationName dataverse.export.distributor.excludeinstallationifset=dataverse.settings.fromdb.ExportInstallationAsDistributorOnlyWhenNotSet \ No newline at end of file From abb8be01e7a92ef4bef66780e4ae677cd67e49f9 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 12 Mar 2021 10:53:35 -0500 Subject: [PATCH 474/576] remove API token from deactivate user examples (not needed) #2419 #4475 --- doc/sphinx-guides/source/api/native-api.rst | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index ca11e77a78a..b59f67b32b7 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3092,27 +3092,25 @@ Deactivates a user. A superuser API token is not required but the command will o .. code-block:: bash - export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=http://localhost:8080 export USERNAME=jdoe - curl -H "X-Dataverse-key:$API_TOKEN" -X POST $SERVER_URL/api/admin/authenticatedUsers/$USERNAME/deactivate + curl -X POST $SERVER_URL/api/admin/authenticatedUsers/$USERNAME/deactivate The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST http://localhost:8080/api/admin/authenticatedUsers/jdoe/deactivate + curl -X POST http://localhost:8080/api/admin/authenticatedUsers/jdoe/deactivate The database ID of the user can be passed instead of the username. .. code-block:: bash - export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=http://localhost:8080 export USERID=42 - curl -H "X-Dataverse-key:$API_TOKEN" -X POST $SERVER_URL/api/admin/authenticatedUsers/id/$USERID/deactivate + curl -X POST $SERVER_URL/api/admin/authenticatedUsers/id/$USERID/deactivate Note: A primary purpose of most Dataverse installations is to serve an archive. In the archival space, there are best practices around the tracking of data access and the tracking of modifications to data and metadata. In support of these key workflows, a simple mechanism to delete users that have performed edit or access actions in the system is not provided. Providing a Deactivate User endpoint for users who have taken certain actions in the system alongside a Delete User endpoint to remove users that haven't taken certain actions in the system is by design. From 07e4fd73ee48f29c267099cf8cf1da04a03ca43b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 12 Mar 2021 10:55:57 -0500 Subject: [PATCH 475/576] fix field type name --- .../harvard/iq/dataverse/export/ddi/DdiExportUtil.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 9fba7cbd384..4a7765c0979 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -201,11 +201,15 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) boolean distributorSet=false; MetadataBlockDTO citationDTO= version.getMetadataBlocks().get("citation"); if(citationDTO!=null) { - if(citationDTO.getField(DatasetFieldConstant.distributorName)!=null) { + if(citationDTO.getField(DatasetFieldConstant.distributor)!=null) { distributorSet=true; } } - if (!StringUtils.isEmpty(datasetDto.getPublisher()) && !(ConfigProvider.getConfig().getOptionalValue("dataverse.export.distributor.excludeinstallationifset", Boolean.class).orElse(false) && distributorSet)) { + logger.info("Dsitr set?: " + distributorSet); + logger.info("Pub?: " + datasetDto.getPublisher()); + boolean excludeRepository = ConfigProvider.getConfig().getOptionalValue("dataverse.export.distributor.excludeinstallationifset", Boolean.class).orElse(false); + logger.info("Exclude: " + excludeRepository); + if (!StringUtils.isEmpty(datasetDto.getPublisher()) && !(excludeRepository && distributorSet)) { xmlw.writeStartElement("distrbtr"); writeAttribute(xmlw, "source", "archive"); xmlw.writeCharacters(datasetDto.getPublisher()); From 51ae2f14dc96bed8d22d85b4ae5334ac274dc6d5 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 12 Mar 2021 11:09:53 -0500 Subject: [PATCH 476/576] use helpers --- .../iq/dataverse/branding/BrandingUtil.java | 4 --- .../branding/BrandingUtilHelper.java | 26 +++++++++++++++++++ .../iq/dataverse/util/bagit/OREMap.java | 24 +++++++++-------- .../iq/dataverse/util/bagit/OREMapHelper.java | 25 ++++++++++++++++++ 4 files changed, 64 insertions(+), 15 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtilHelper.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMapHelper.java diff --git a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java index 67b68fc3b50..2b6f1ec62e1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java @@ -4,14 +4,10 @@ import edu.harvard.iq.dataverse.util.BundleUtil; import java.util.Arrays; -import javax.ejb.EJB; -import javax.ejb.Singleton; import javax.mail.internet.InternetAddress; -@Singleton public class BrandingUtil { - @EJB static DataverseServiceBean dataverseService; public static String getInstallationBrandName() { diff --git a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtilHelper.java b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtilHelper.java new file mode 100644 index 00000000000..0fb16088ff5 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtilHelper.java @@ -0,0 +1,26 @@ +package edu.harvard.iq.dataverse.branding; + +import javax.annotation.PostConstruct; +import javax.ejb.EJB; +import javax.ejb.Singleton; +import javax.ejb.Startup; + +import edu.harvard.iq.dataverse.DataverseServiceBean; + + /** + * This is a small helper bean + * As it is a singleton and built at application start (=deployment), it will inject the (stateless) + * dataverse service into the BrandingUtil once it's ready. + */ + @Singleton + @Startup + public class BrandingUtilHelper { + + @EJB + DataverseServiceBean dataverseSvc; + + @PostConstruct + public void injectService() { + BrandingUtil.injectDataverseService(dataverseSvc); + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index cbb0666f331..38303eb1f41 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -12,7 +12,6 @@ import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.export.OAI_OREExporter; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; -import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.json.JsonLDNamespace; import edu.harvard.iq.dataverse.util.json.JsonLDTerm; @@ -22,12 +21,9 @@ import java.time.LocalDate; import java.util.List; import java.util.Map; -import java.util.ResourceBundle; import java.util.TreeMap; import java.util.Map.Entry; -import javax.ejb.Stateless; -import javax.inject.Inject; import javax.json.Json; import javax.json.JsonArray; import javax.json.JsonArrayBuilder; @@ -35,24 +31,22 @@ import javax.json.JsonObjectBuilder; import javax.json.JsonValue; -@Stateless public class OREMap { - @Inject static SettingsServiceBean settingsService; + static SettingsServiceBean settingsService; public static final String NAME = "OREMap"; private Map localContext = new TreeMap(); private DatasetVersion version; - private boolean excludeEmail = false; + private Boolean excludeEmail = null; public OREMap(DatasetVersion version) { - this(version, settingsService.isTrueForKey(SettingsServiceBean.Key.ExcludeEmailFromExport, false)); - + this.version = version; } //Used when the ExcludeEmailFromExport needs to be overriden, i.e. for archiving public OREMap(DatasetVersion dv, boolean exclude) { - this.version = version; + this.version = dv; this.excludeEmail = exclude; } @@ -63,6 +57,11 @@ public void writeOREMap(OutputStream outputStream) throws Exception { public JsonObject getOREMap() throws Exception { + //Set this flag if it wasn't provided + if(excludeEmail==null) { + excludeEmail = settingsService.isTrueForKey(SettingsServiceBean.Key.ExcludeEmailFromExport, false); + } + // Add namespaces we'll definitely use to Context // Additional namespaces are added as needed below localContext.putIfAbsent(JsonLDNamespace.ore.getPrefix(), JsonLDNamespace.ore.getUrl()); @@ -179,7 +178,7 @@ public JsonObject getOREMap() throws Exception { } aggBuilder.add(JsonLDTerm.schemaOrg("includedInDataCatalog").getLabel(), - dataset.getDataverseContext().getDisplayName()); + BrandingUtil.getRootDataverseCollectionName()); // The aggregation aggregates aggregatedresources (Datafiles) which each have // their own entry and metadata @@ -392,4 +391,7 @@ private JsonLDTerm getTermFor(String type, String subType) { return null; } + public static void injectSettingsService(SettingsServiceBean settingsSvc) { + settingsService = settingsSvc; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMapHelper.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMapHelper.java new file mode 100644 index 00000000000..bced52a7752 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMapHelper.java @@ -0,0 +1,25 @@ +package edu.harvard.iq.dataverse.util.bagit; + +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; + +import javax.annotation.PostConstruct; +import javax.ejb.EJB; +import javax.ejb.Singleton; +import javax.ejb.Startup; + +/** + * This is a small helper bean + * As it is a singleton and built at application start (=deployment), it will inject the (stateless) + * settings service into the OREMap once it's ready. + */ +@Singleton +@Startup +public class OREMapHelper { + @EJB + SettingsServiceBean settingsSvc; + + @PostConstruct + public void injectService() { + OREMap.injectSettingsService(settingsSvc); + } +} From 7fafd2fbecd3e2a88f34f197a878e7d92fb6e810 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 12 Mar 2021 11:24:01 -0500 Subject: [PATCH 477/576] add private to dataverseService --- .../java/edu/harvard/iq/dataverse/branding/BrandingUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java index 2b6f1ec62e1..7421f39d2ff 100644 --- a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java @@ -8,7 +8,7 @@ public class BrandingUtil { - static DataverseServiceBean dataverseService; + private static DataverseServiceBean dataverseService; public static String getInstallationBrandName() { //ToDo #7387 which will make this call return something different than getRootDataverseCollectionName() From b90621208775d1391ba0e84c4cd66b399b129b3c Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 12 Mar 2021 11:24:01 -0500 Subject: [PATCH 478/576] add private to Services --- .../java/edu/harvard/iq/dataverse/branding/BrandingUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java index 16ee7bb9702..50661ee97fc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/branding/BrandingUtil.java @@ -12,8 +12,8 @@ public class BrandingUtil { private static final Logger logger = Logger.getLogger(BrandingUtil.class.getCanonicalName()); - static DataverseServiceBean dataverseService; - static SettingsServiceBean settingsService; + private static DataverseServiceBean dataverseService; + private static SettingsServiceBean settingsService; public static String getInstallationBrandName() { From 3713104c35e0b526bba374795c6ba9da3dd5242e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 12 Mar 2021 12:05:19 -0500 Subject: [PATCH 479/576] fix error handing for revoke all roles (bubble up msg) #2419 #4475 --- src/main/java/edu/harvard/iq/dataverse/api/Users.java | 4 ++-- .../java/edu/harvard/iq/dataverse/api/DeleteUsersIT.java | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Users.java b/src/main/java/edu/harvard/iq/dataverse/api/Users.java index 6e0476711ec..ce226ea14b8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Users.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Users.java @@ -204,8 +204,8 @@ public Response removeUserRoles(@PathParam("identifier") String identifier) { } execCommand(new RevokeAllRolesCommand(userToModify, createDataverseRequest(findUserOrDie()))); return ok("Roles removed for user " + identifier + "."); - } catch (Exception ex) { - return error(Response.Status.BAD_REQUEST, "Unable to revoke all roles: " + ex.getLocalizedMessage()); + } catch (WrappedResponse wr) { + return wr.getResponse(); } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DeleteUsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DeleteUsersIT.java index a7f5153d69f..cae1d0e210a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DeleteUsersIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DeleteUsersIT.java @@ -16,6 +16,7 @@ import static javax.ws.rs.core.Response.Status.BAD_REQUEST; import static javax.ws.rs.core.Response.Status.CREATED; import static javax.ws.rs.core.Response.Status.OK; +import static javax.ws.rs.core.Response.Status.UNAUTHORIZED; import static junit.framework.Assert.assertEquals; import static org.hamcrest.CoreMatchers.equalTo; import org.junit.BeforeClass; @@ -673,6 +674,13 @@ public void testCuratorSendsCommentsToAuthor() throws InterruptedException { // // TODO: deactivate curator2 here // + // Show the error if you don't have permission. + Response failToRemoveRole = UtilIT.deleteUserRoles(author2Username, curator2ApiToken); + failToRemoveRole.prettyPrint(); + failToRemoveRole.then().assertThat() + .statusCode(UNAUTHORIZED.getStatusCode()) + .body("message", equalTo("User @" + curator2Username + " is not permitted to perform requested action.")); + Response removeRolesFromAuthor2 = UtilIT.deleteUserRoles(author2Username, superuserApiToken); removeRolesFromAuthor2.prettyPrint(); removeRolesFromAuthor2.then().assertThat() From eecd60a40e1c87ebe7e0df43745062bce967171b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 12 Mar 2021 12:47:24 -0500 Subject: [PATCH 480/576] send message with failure - per review catch --- src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java index 42feaf6d2c2..0ba6a718edf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java @@ -506,7 +506,7 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio case WORKFLOW_FAILURE: version = (DatasetVersion) targetObject; pattern = BundleUtil.getStringFromBundle("notification.email.workflow.failure"); - String[] paramArrayWorkflowFailure = {version.getDataset().getDisplayName(), getDatasetLink(version.getDataset())}; + String[] paramArrayWorkflowFailure = {version.getDataset().getDisplayName(), getDatasetLink(version.getDataset()), comment}; messageText += MessageFormat.format(pattern, paramArrayWorkflowFailure); return messageText; case CREATEACC: From a68a777f5930935dc42014f853e52d85b2744642 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 12 Mar 2021 12:57:36 -0500 Subject: [PATCH 481/576] Don't imply success is a problem :-) --- .../iq/dataverse/workflow/step/Success.java | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java b/src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java index 621afdc61f6..c9967290134 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java @@ -4,34 +4,37 @@ * Result returned when step execution succeeds. */ public class Success implements WorkflowStepResult { - + private final String reason; private final String message; - - public Success( String reason ) { + + public Success(String reason) { this(reason, null); } - + /** * Constructs a new success message. - * @param reason Technical reason (for logs etc.). - * @param message Human readable reason. + * + * @param reason Technical comment (for logs etc.). + * @param message Human readable comment. */ public Success(String reason, String message) { this.reason = reason; this.message = message; } - + /** - * Holds the technical reason for the success, useful for debugging the problem. - * @return the technical reason for the problem. + * Holds a technical comment about the success. + * + * @return the technical comment about the processing. */ public String getReason() { return reason; } /** - * Holds the user-friendly message explaining the failure. + * Holds the user-friendly message describing what was successfully done. + * * @return user-friendly message for the success. */ public String getMessage() { @@ -42,6 +45,5 @@ public String getMessage() { public String toString() { return "WorkflowStepResult.Success{" + "reason=" + reason + ", message=" + message + '}'; } - - + } From 54c5c15a9cd15ad59af8114eafe19209ca075e07 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 12 Mar 2021 14:58:50 -0500 Subject: [PATCH 482/576] Add messages to notification panel per review --- src/main/java/propertyFiles/Bundle.properties | 3 +++ src/main/webapp/dataverseuser.xhtml | 22 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index b70c5117d23..515530a7cb4 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -202,6 +202,9 @@ notification.wasSubmittedForReview={0} was submitted for review to be published notification.wasReturnedByReviewer={0} was returned by the curator of {1}. notification.wasPublished={0} was published in {1}. notification.publishFailedPidReg={0} in {1} could not be published due to a failure to register, or update the Global Identifier for the dataset or one of the files in it. Contact support if this continues to happen. +notification.workflowFailed=An external workflow run on {0} in {1} has failed. Check your email and/or view the Dataset page which may have additional details. Contact support if this continues to happen. +notification.workflowSucceeded=An external workflow run on {0} in {1} has succeeded. Check your email and/or view the Dataset page which may have additional details. + notification.ingestCompleted=Dataset {1} ingest has successfully finished. notification.ingestCompletedWithErrors=Dataset {1} ingest has finished with errors. notification.generic.objectDeleted=The dataverse, dataset, or file for this notification has been deleted. diff --git a/src/main/webapp/dataverseuser.xhtml b/src/main/webapp/dataverseuser.xhtml index 5de0154f49c..3e48e16404c 100644 --- a/src/main/webapp/dataverseuser.xhtml +++ b/src/main/webapp/dataverseuser.xhtml @@ -312,6 +312,28 @@     + + + + + #{item.theObject.getDataset().getDisplayName()} + + + #{item.theObject.getDataset().getOwner().getDisplayName()} + + + + + + + + #{item.theObject.getDataset().getDisplayName()} + + + #{item.theObject.getDataset().getOwner().getDisplayName()} + + + From 2aa6c022f98b46b050495f5a632a081e904374ed Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 12 Mar 2021 15:34:33 -0500 Subject: [PATCH 483/576] modification of GDCC pull #4 https://github.com/GlobalDataverseCommunityConsortium/dataverse/pull/4 --- .../FinalizeDatasetPublicationCommand.java | 2 +- .../command/impl/PublishDatasetCommand.java | 2 +- .../workflow/WorkflowServiceBean.java | 92 ++++++++++++++----- 3 files changed, 71 insertions(+), 25 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java index c2f186f1e8c..5764e5cdfe2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java @@ -186,7 +186,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { //Should this be in onSuccess()? ctxt.workflows().getDefaultWorkflow(TriggerType.PostPublishDataset).ifPresent(wf -> { try { - ctxt.workflows().start(wf, buildContext(ds, TriggerType.PostPublishDataset, datasetExternallyReleased)); + ctxt.workflows().start(wf, buildContext(ds, TriggerType.PostPublishDataset, datasetExternallyReleased), false); } catch (CommandException ex) { ctxt.datasets().removeDatasetLocks(ds, DatasetLock.Reason.Workflow); logger.log(Level.SEVERE, "Error invoking post-publish workflow: " + ex.getMessage(), ex); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java index 41645078a25..f1bab1901d0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java @@ -96,7 +96,7 @@ public PublishDatasetResult execute(CommandContext ctxt) throws CommandException // We start a workflow theDataset = ctxt.em().merge(theDataset); ctxt.em().flush(); - ctxt.workflows().start(prePubWf.get(), buildContext(theDataset, TriggerType.PrePublishDataset, datasetExternallyReleased)); + ctxt.workflows().start(prePubWf.get(), buildContext(theDataset, TriggerType.PrePublishDataset, datasetExternallyReleased), true); return new PublishDatasetResult(theDataset, Status.Workflow); } else{ diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java index b6aa35ede7b..539ade60828 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java @@ -99,18 +99,43 @@ public WorkflowServiceBean() { */ //ToDo - should this be @Async? or just the forward() method? @Asynchronous - public void start(Workflow wf, WorkflowContext ctxt) throws CommandException { - - // Since we are calling this asynchronously anyway - sleep here - // for a few seconds, just in case, to make sure the database update of - // the dataset initiated by the PublishDatasetCommand has finished, - // to avoid any concurrency/optimistic lock issues. - try { - Thread.sleep(1000); - } catch (Exception ex) { - logger.warning("Failed to sleep for a second."); + public void start(Workflow wf, WorkflowContext ctxt, boolean findDataset) throws CommandException { + /* + * Workflows appear to start running prior to the caller's transaction + * completing which can result in exceptions in setting the lock below. To avoid + * this, there are two work-arounds - wait briefly for that transaction to end, + * or refresh the dataset from the db - so the lock is written based on the + * current db state. The latter works for pre-publication workflows (since the + * only changes to the Dataset in the Publish command are edits to the version + * number in the draft version (which aren't valid for the draft anyway)), while + * the former is required for post-publication workflows which may need to see + * the final version number, update times and other changes made in the Finalize + * Publication command. Not waiting saves significant time when many datasets + * are processed, so is prefereable when it makes sense. + * + * This code should be reconsidered if/when the launching of pre/post + * publication workflows is moved to command onSuccess methods (and when + * onSuccess methods are guaranteed to be after the transaction completes (see + * #7568) or other changes are made that can guarantee the dataset in the + * WorkflowContext is up-to-date/usable in further transactions in the workflow. + * (e.g. if this method is not asynchronous) + * + */ + + if (!findDataset) { + /* + * Sleep here briefly to make sure the database update from the callers + * transaction completes which avoids any concurrency/optimistic lock issues. + * Note: 1 second appears long enough, but shorter delays may work + */ + try { + Thread.sleep(1000); + } catch (Exception ex) { + logger.warning("Failed to sleep for a second."); + } } - ctxt = refresh(ctxt, retrieveRequestedSettings( wf.getRequiredSettings()), getCurrentApiToken(ctxt.getRequest().getAuthenticatedUser())); + //Refresh will only em.find the dataset if findDataset is true. (otherwise the dataset is em.merged) + ctxt = refresh(ctxt, retrieveRequestedSettings( wf.getRequiredSettings()), getCurrentApiToken(ctxt.getRequest().getAuthenticatedUser()), findDataset); lockDataset(ctxt, new DatasetLock(DatasetLock.Reason.Workflow, ctxt.getRequest().getAuthenticatedUser())); forward(wf, ctxt); } @@ -466,18 +491,39 @@ private WorkflowStep createStep(WorkflowStepData wsd) { private WorkflowContext refresh( WorkflowContext ctxt ) { return refresh(ctxt, ctxt.getSettings(), ctxt.getApiToken()); } - - private WorkflowContext refresh( WorkflowContext ctxt, Map settings, ApiToken apiToken ) { - /* An earlier version of this class used em.find() to 'refresh' the Dataset in the context. - * For a PostPublication workflow, this had the consequence of hiding/removing changes to the Dataset - * made in the FinalizeDatasetPublicationCommand (i.e. the fact that the draft version is now released and - * has a version number). It is not clear to me if the em.merge below is needed or if it handles the case of - * resumed workflows. (The overall method is needed to allow the context to be updated in the start() method with the - * settings and APItoken retrieved by the WorkflowServiceBean) - JM - 9/18. - */ - WorkflowContext newCtxt =new WorkflowContext( ctxt.getRequest(), - em.merge(ctxt.getDataset()), ctxt.getNextVersionNumber(), - ctxt.getNextMinorVersionNumber(), ctxt.getType(), settings, apiToken, ctxt.getDatasetExternallyReleased(), ctxt.getInvocationId(), ctxt.getLockId()); + + private WorkflowContext refresh(WorkflowContext ctxt, Map settings, ApiToken apiToken) { + return refresh(ctxt, settings, apiToken, false); + } + + private WorkflowContext refresh(WorkflowContext ctxt, Map settings, ApiToken apiToken, + boolean findDataset) { + /* + * An earlier version of this class used em.find() to 'refresh' the Dataset in + * the context. For a PostPublication workflow, this had the consequence of + * hiding/removing changes to the Dataset made in the + * FinalizeDatasetPublicationCommand (i.e. the fact that the draft version is + * now released and has a version number). It is not clear to me if the em.merge + * below is needed or if it handles the case of resumed workflows. (The overall + * method is needed to allow the context to be updated in the start() method + * with the settings and APItoken retrieved by the WorkflowServiceBean) - JM - + * 9/18. + */ + /* + * Introduced the findDataset boolean to optionally revert above change. + * Refreshing the Dataset just before trying to set the workflow lock greatly + * reduces the number of OptimisticLockExceptions. JvM 2/21 + */ + WorkflowContext newCtxt; + if (findDataset) { + newCtxt = new WorkflowContext(ctxt.getRequest(), datasets.find(ctxt.getDataset().getId()), + ctxt.getNextVersionNumber(), ctxt.getNextMinorVersionNumber(), ctxt.getType(), settings, apiToken, + ctxt.getDatasetExternallyReleased(), ctxt.getInvocationId(), ctxt.getLockId()); + } else { + newCtxt = new WorkflowContext(ctxt.getRequest(), em.merge(ctxt.getDataset()), ctxt.getNextVersionNumber(), + ctxt.getNextMinorVersionNumber(), ctxt.getType(), settings, apiToken, + ctxt.getDatasetExternallyReleased(), ctxt.getInvocationId(), ctxt.getLockId()); + } return newCtxt; } From c2b7b0aaf38c332f21da2adcae0b1233122afbfe Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 12 Mar 2021 17:10:42 -0500 Subject: [PATCH 484/576] add "deactivated" to user dashboard, list-users API #2419 #4475 --- .../java/edu/harvard/iq/dataverse/UserServiceBean.java | 8 ++++++-- .../dataverse/authorization/users/AuthenticatedUser.java | 3 +++ src/main/java/propertyFiles/Bundle.properties | 1 + src/main/webapp/dashboard-users.xhtml | 1 + .../edu/harvard/iq/dataverse/api/DeactivateUsersIT.java | 9 +++++++++ 5 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/UserServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/UserServiceBean.java index e395e9a90ec..5707f477a87 100644 --- a/src/main/java/edu/harvard/iq/dataverse/UserServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/UserServiceBean.java @@ -140,7 +140,10 @@ private AuthenticatedUser createAuthenticatedUserForView (Object[] dbRowValues, user.setAuthProviderId(UserUtil.getStringOrNull(dbRowValues[11])); user.setAuthProviderFactoryAlias(UserUtil.getStringOrNull(dbRowValues[12])); - + + user.setDeactivated((Boolean)(dbRowValues[13])); + user.setDeactivatedTime(UserUtil.getTimestampOrNull(dbRowValues[14])); + user.setRoles(roles); return user; } @@ -417,7 +420,8 @@ private List getUserListCore(String searchTerm, qstr += " u.affiliation, u.superuser,"; qstr += " u.position,"; qstr += " u.createdtime, u.lastlogintime, u.lastapiusetime, "; - qstr += " prov.id, prov.factoryalias"; + qstr += " prov.id, prov.factoryalias, "; + qstr += " u.deactivated, u.deactivatedtime "; qstr += " FROM authenticateduser u,"; qstr += " authenticateduserlookup prov_lookup,"; qstr += " authenticationproviderrow prov"; diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java index abe0fef597f..0f4bf8b5878 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java @@ -388,6 +388,9 @@ public JsonObjectBuilder toJson() { authenicatedUserJson.add("lastLoginTime", UserUtil.getTimestampStringOrNull(this.lastLoginTime)); authenicatedUserJson.add("lastApiUseTime", UserUtil.getTimestampStringOrNull(this.lastApiUseTime)); + authenicatedUserJson.add("deactivated", this.deactivated); + authenicatedUserJson.add("deactivatedTime", UserUtil.getTimestampStringOrNull(this.deactivatedTime)); + return authenicatedUserJson; } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 8b5e2c0b49e..030d80e4d5c 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -607,6 +607,7 @@ dashboard.list_users.tbl_header.authProviderFactoryAlias=Authentication dashboard.list_users.tbl_header.createdTime=Created Time dashboard.list_users.tbl_header.lastLoginTime=Last Login Time dashboard.list_users.tbl_header.lastApiUseTime=Last API Use Time +dashboard.list_users.tbl_header.deactivated=deactivated dashboard.list_users.tbl_header.roles.removeAll=Remove All dashboard.list_users.tbl_header.roles.removeAll.header=Remove All Roles dashboard.list_users.tbl_header.roles.removeAll.confirmationText=Are you sure you want to remove all roles for user {0}? diff --git a/src/main/webapp/dashboard-users.xhtml b/src/main/webapp/dashboard-users.xhtml index a9e7461f1fb..ebb447dfe30 100644 --- a/src/main/webapp/dashboard-users.xhtml +++ b/src/main/webapp/dashboard-users.xhtml @@ -65,6 +65,7 @@ + diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java index ed979ff785c..f7f5979fcbd 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java @@ -13,6 +13,7 @@ import static javax.ws.rs.core.Response.Status.OK; import static javax.ws.rs.core.Response.Status.UNAUTHORIZED; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.startsWith; import org.junit.BeforeClass; import org.junit.Test; @@ -93,6 +94,14 @@ public void testDeactivateUser() { .statusCode(OK.getStatusCode()) .body("data.deactivated", equalTo(true)); + Response findUser = UtilIT.filterAuthenticatedUsers(superuserApiToken, username, null, 100, null); + findUser.prettyPrint(); + findUser.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.users[0].userIdentifier", equalTo(username)) + .body("data.users[0].deactivated", equalTo(true)) + .body("data.users[0].deactivatedTime", startsWith("2")); + Response getUserDeactivated = UtilIT.getAuthenticatedUserByToken(apiToken); getUserDeactivated.prettyPrint(); getUserDeactivated.then().assertThat().statusCode(BAD_REQUEST.getStatusCode()); From d51ecc47f59a26915f34c8f07163420d9996df35 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 15 Mar 2021 15:46:35 +0100 Subject: [PATCH 485/576] fix(engine): Fix class imports for HTTPException in DeletePidCommand #7684 --- .../dataverse/engine/command/impl/DeletePidCommand.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeletePidCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeletePidCommand.java index 841c84793ec..274aeb3c3fd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeletePidCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeletePidCommand.java @@ -13,12 +13,12 @@ import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; +import org.apache.commons.httpclient.HttpException; + import java.util.Arrays; import java.util.Collections; import java.util.logging.Logger; -import javax.xml.ws.http.HTTPException; - /** * No required permissions because we check for superuser status. */ @@ -51,8 +51,8 @@ protected void executeImpl(CommandContext ctxt) throws CommandException { dataset.setGlobalIdCreateTime(null); dataset.setIdentifierRegistered(false); ctxt.datasets().merge(dataset); - } catch (HTTPException hex) { - String message = BundleUtil.getStringFromBundle("pids.deletePid.failureExpected", Arrays.asList(dataset.getGlobalId().asString(), Integer.toString(hex.getStatusCode()))); + } catch (HttpException hex) { + String message = BundleUtil.getStringFromBundle("pids.deletePid.failureExpected", Arrays.asList(dataset.getGlobalId().asString(), Integer.toString(hex.getReasonCode()))); logger.info(message); throw new IllegalCommandException(message, this); } catch (Exception ex) { From 25ec376ae7110cd07222d21bfd4eb77ed7f1a3e6 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 15 Mar 2021 13:06:59 -0400 Subject: [PATCH 486/576] bug fixes, per update --- .../command/impl/CuratePublishedDatasetVersionCommand.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java index 373454bf694..a308e6637f3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java @@ -61,7 +61,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { // final DatasetVersion editVersion = getDataset().getEditVersion(); tidyUpFields(updateVersion); - // Merge the new version into out JPA context + // Merge the new version into our JPA context ctxt.em().merge(updateVersion); @@ -71,7 +71,8 @@ public Dataset execute(CommandContext ctxt) throws CommandException { updateVersion.setTermsOfUseAndAccess(newTerms); //Put old terms on version that will be deleted.... getDataset().getEditVersion().setTermsOfUseAndAccess(oldTerms); - + //Also set the fileaccessrequest boolean on the dataset to match the new terms + getDataset().setFileAccessRequest(updateVersion.getTermsOfUseAndAccess().isFileAccessRequest()); List newComments = getDataset().getEditVersion().getWorkflowComments(); if (newComments!=null && newComments.size() >0) { for(WorkflowComment wfc: newComments) { @@ -162,7 +163,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { // Update modification time on the published version and the dataset updateVersion.setLastUpdateTime(getTimestamp()); tempDataset.setModificationTime(getTimestamp()); - + ctxt.em().merge(updateVersion); Dataset savedDataset = ctxt.em().merge(tempDataset); // Flush before calling DeleteDatasetVersion which calls From 99a92cb9156e0a28277b15771c5211c6be6f3a06 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 15 Mar 2021 13:07:44 -0400 Subject: [PATCH 487/576] Revert "bug fixes, per update" This reverts commit 25ec376ae7110cd07222d21bfd4eb77ed7f1a3e6. --- .../command/impl/CuratePublishedDatasetVersionCommand.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java index a308e6637f3..373454bf694 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java @@ -61,7 +61,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { // final DatasetVersion editVersion = getDataset().getEditVersion(); tidyUpFields(updateVersion); - // Merge the new version into our JPA context + // Merge the new version into out JPA context ctxt.em().merge(updateVersion); @@ -71,8 +71,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { updateVersion.setTermsOfUseAndAccess(newTerms); //Put old terms on version that will be deleted.... getDataset().getEditVersion().setTermsOfUseAndAccess(oldTerms); - //Also set the fileaccessrequest boolean on the dataset to match the new terms - getDataset().setFileAccessRequest(updateVersion.getTermsOfUseAndAccess().isFileAccessRequest()); + List newComments = getDataset().getEditVersion().getWorkflowComments(); if (newComments!=null && newComments.size() >0) { for(WorkflowComment wfc: newComments) { @@ -163,7 +162,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { // Update modification time on the published version and the dataset updateVersion.setLastUpdateTime(getTimestamp()); tempDataset.setModificationTime(getTimestamp()); - ctxt.em().merge(updateVersion); + Dataset savedDataset = ctxt.em().merge(tempDataset); // Flush before calling DeleteDatasetVersion which calls From 25062f0882b6c89400fcd583525fce11895a5d45 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 15 Mar 2021 14:57:07 -0400 Subject: [PATCH 488/576] Revert "Revert "bug fixes, per update"" This reverts commit 99a92cb9156e0a28277b15771c5211c6be6f3a06. --- .../command/impl/CuratePublishedDatasetVersionCommand.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java index 373454bf694..a308e6637f3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java @@ -61,7 +61,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { // final DatasetVersion editVersion = getDataset().getEditVersion(); tidyUpFields(updateVersion); - // Merge the new version into out JPA context + // Merge the new version into our JPA context ctxt.em().merge(updateVersion); @@ -71,7 +71,8 @@ public Dataset execute(CommandContext ctxt) throws CommandException { updateVersion.setTermsOfUseAndAccess(newTerms); //Put old terms on version that will be deleted.... getDataset().getEditVersion().setTermsOfUseAndAccess(oldTerms); - + //Also set the fileaccessrequest boolean on the dataset to match the new terms + getDataset().setFileAccessRequest(updateVersion.getTermsOfUseAndAccess().isFileAccessRequest()); List newComments = getDataset().getEditVersion().getWorkflowComments(); if (newComments!=null && newComments.size() >0) { for(WorkflowComment wfc: newComments) { @@ -162,7 +163,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { // Update modification time on the published version and the dataset updateVersion.setLastUpdateTime(getTimestamp()); tempDataset.setModificationTime(getTimestamp()); - + ctxt.em().merge(updateVersion); Dataset savedDataset = ctxt.em().merge(tempDataset); // Flush before calling DeleteDatasetVersion which calls From 3e59e70b64194a66debbf72738fc43e1ef007c33 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 15 Mar 2021 15:39:28 -0400 Subject: [PATCH 489/576] in session, ensure user hasn't been deleted or deactivated #2419 #4475 --- .../iq/dataverse/DataverseSession.java | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java b/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java index 315b7c89da7..95c0e124faf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java @@ -4,6 +4,7 @@ import edu.harvard.iq.dataverse.PermissionServiceBean.StaticPermissionQuery; import edu.harvard.iq.dataverse.actionlogging.ActionLogRecord; import edu.harvard.iq.dataverse.actionlogging.ActionLogServiceBean; +import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.GuestUser; import edu.harvard.iq.dataverse.authorization.users.User; @@ -56,7 +57,10 @@ public class DataverseSession implements Serializable{ @EJB BannerMessageServiceBean bannerMessageService; - + + @EJB + AuthenticationServiceBean authenticationService; + private static final Logger logger = Logger.getLogger(DataverseSession.class.getCanonicalName()); private boolean statusDismissed = false; @@ -89,7 +93,19 @@ public User getUser() { if ( user == null ) { user = GuestUser.get(); } - + if (user instanceof AuthenticatedUser) { + AuthenticatedUser auFromSession = (AuthenticatedUser) user; + AuthenticatedUser auFreshLookup = authenticationService.findByID(auFromSession.getId()); + if (auFreshLookup == null) { + logger.fine("getUser found user no longer exists (was deleted). Returning GuestUser."); + user = GuestUser.get(); + } else { + if (auFreshLookup.isDeactivated()) { + logger.fine("getUser found user is deactivated. Returning GuestUser."); + user = GuestUser.get(); + } + } + } return user; } From 284cc6ef223bf27931a28ce235ddb6a569d2adeb Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 16 Mar 2021 09:34:22 -0400 Subject: [PATCH 490/576] Changes to make dataset block imports update controlled vocab. alternate values. (#7638) --- .../dataverse/ControlledVocabularyValue.java | 2 +- .../dataverse/api/DatasetFieldServiceApi.java | 27 ++++++++++++------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ControlledVocabularyValue.java b/src/main/java/edu/harvard/iq/dataverse/ControlledVocabularyValue.java index 0b868c73c42..c88fac00c27 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ControlledVocabularyValue.java +++ b/src/main/java/edu/harvard/iq/dataverse/ControlledVocabularyValue.java @@ -100,7 +100,7 @@ public void setDatasetFieldType(DatasetFieldType datasetFieldType) { this.datasetFieldType = datasetFieldType; } - @OneToMany(mappedBy = "controlledVocabularyValue", cascade = {CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST}) + @OneToMany(mappedBy = "controlledVocabularyValue", cascade = {CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST}, orphanRemoval=true) private Collection controlledVocabAlternates = new ArrayList<>(); public Collection getControlledVocabAlternates() { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java index 3cb5f900cd6..e7aa9ea661c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Iterator; import javax.ejb.EJB; import javax.ejb.EJBException; import javax.json.Json; @@ -470,15 +471,23 @@ private String parseControlledVocabulary(String[] values) { if (cvv == null) { cvv = new ControlledVocabularyValue(); cvv.setDatasetFieldType(dsv); - //Alt is only for dataload so only add to new - for (int i = 5; i < values.length; i++) { - ControlledVocabAlternate alt = new ControlledVocabAlternate(); - alt.setDatasetFieldType(dsv); - alt.setControlledVocabularyValue(cvv); - alt.setStrValue(values[i]); - cvv.getControlledVocabAlternates().add(alt); - } - } + } + + // Alternate variants for this controlled vocab. value: + + // Note that these are overwritten every time: + cvv.getControlledVocabAlternates().clear(); + // - meaning, if an alternate has been removed from the tsv file, + // it will be removed from the database! -- L.A. 5.4 + + for (int i = 5; i < values.length; i++) { + ControlledVocabAlternate alt = new ControlledVocabAlternate(); + alt.setDatasetFieldType(dsv); + alt.setControlledVocabularyValue(cvv); + alt.setStrValue(values[i]); + cvv.getControlledVocabAlternates().add(alt); + } + cvv.setStrValue(values[2]); cvv.setIdentifier(values[3]); cvv.setDisplayOrder(Integer.parseInt(values[4])); From 4f81fbde7f9ff4a6a0752619fff2c033c7265354 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 16 Mar 2021 11:37:06 -0400 Subject: [PATCH 491/576] prevent accounts from being merged into themselves #2419 #4475 Otherwise, the account is deleted! --- .../command/impl/MergeInAccountCommand.java | 4 ++++ .../iq/dataverse/api/DeactivateUsersIT.java | 21 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java index c0dce9799e1..950357a1ad2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java @@ -59,6 +59,10 @@ public MergeInAccountCommand(DataverseRequest createDataverseRequest, Authentica @Override protected void executeImpl(CommandContext ctxt) throws CommandException { + if (consumedAU.getId() == ongoingAU.getId()) { + throw new IllegalCommandException("You cannot merge an account into itself.", this); + } + if (consumedAU.isDeactivated() && !ongoingAU.isDeactivated() || !consumedAU.isDeactivated() && ongoingAU.isDeactivated()) { throw new IllegalCommandException("User accounts can only be merged if they are either both non-deactivated or both deactivated.", this); } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java index f7f5979fcbd..cc79f39345a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java @@ -250,4 +250,25 @@ public void testMergeDeactivatedIntoDeactivatedUser() { mergeAccounts.then().assertThat().statusCode(OK.getStatusCode()); } + @Test + public void testMergeUserIntoSelf() { + + Response createSuperuser = UtilIT.createRandomUser(); + String superuserUsername = UtilIT.getUsernameFromResponse(createSuperuser); + String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser); + Response toggleSuperuser = UtilIT.makeSuperUser(superuserUsername); + toggleSuperuser.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response createUserToMerge = UtilIT.createRandomUser(); + createUserToMerge.prettyPrint(); + String usernameToMerge = UtilIT.getUsernameFromResponse(createUserToMerge); + + String usernameMergeTarget = usernameToMerge; + + Response mergeAccounts = UtilIT.mergeAccounts(usernameMergeTarget, usernameToMerge, superuserApiToken); + mergeAccounts.prettyPrint(); + mergeAccounts.then().assertThat().statusCode(OK.getStatusCode()); + } + } From 010962adba56828da7733e234b22273b8f0c3da5 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 16 Mar 2021 12:46:17 -0400 Subject: [PATCH 492/576] removing an import that's not needed (#7638) --- .../edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java index e7aa9ea661c..646abf51b6c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java @@ -20,7 +20,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Iterator; import javax.ejb.EJB; import javax.ejb.EJBException; import javax.json.Json; From 9fdaf4183769921110c5ac76cc6d1df77b9f40b9 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 16 Mar 2021 14:06:19 -0400 Subject: [PATCH 493/576] fix "merge into self" test #2419 #4475 This should have been included in 4f81fbde7 --- .../java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java index cc79f39345a..a34b3076445 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java @@ -268,7 +268,7 @@ public void testMergeUserIntoSelf() { Response mergeAccounts = UtilIT.mergeAccounts(usernameMergeTarget, usernameToMerge, superuserApiToken); mergeAccounts.prettyPrint(); - mergeAccounts.then().assertThat().statusCode(OK.getStatusCode()); + mergeAccounts.then().assertThat().statusCode(BAD_REQUEST.getStatusCode()); } } From 071ac7b36e3e4bd38750c7bb5e6c1c422f812be1 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 16 Mar 2021 14:17:35 -0400 Subject: [PATCH 494/576] adding script in usual spot --- scripts/issues/7687/file_access_flag_update_bug.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 scripts/issues/7687/file_access_flag_update_bug.txt diff --git a/scripts/issues/7687/file_access_flag_update_bug.txt b/scripts/issues/7687/file_access_flag_update_bug.txt new file mode 100644 index 00000000000..a96b9dfab1b --- /dev/null +++ b/scripts/issues/7687/file_access_flag_update_bug.txt @@ -0,0 +1,11 @@ +-- this query will identify datasets where a superuser has run the Curate command and the update included a change to the fileaccessrequest flag, resulting in the file access request updates not being reflected in the published version + +select da.id, dv.id, ta.id, da.fileaccessrequest, ta.fileaccessrequest, dv.releasetime +from datasetversion dv, termsofuseandaccess ta, dataset da +where dv.dataset_id=da.id +and dv.termsofuseandaccess_id=ta.id +and ta.fileaccessrequest != da.fileaccessrequest +and dv.versionstate='RELEASED' +and dv.releasetime in (select max(releasetime) +from datasetversion +where dataset_id=da.id); \ No newline at end of file From c3ef44418dd274e7a032275c2868821e0643d6ef Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 16 Mar 2021 14:35:08 -0400 Subject: [PATCH 495/576] adding release note with link to script --- doc/release-notes/7687-curate-command.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/release-notes/7687-curate-command.md diff --git a/doc/release-notes/7687-curate-command.md b/doc/release-notes/7687-curate-command.md new file mode 100644 index 00000000000..69a1d757769 --- /dev/null +++ b/doc/release-notes/7687-curate-command.md @@ -0,0 +1,9 @@ +## Notes for Dataverse Installation Administrators + +### DB Cleanup for Superusers Releasing without Version Updates + +In datasets where a superuser has run the Curate command and the update included a change to the fileaccessrequest flag, those changes would not be reflected appropriately in the published version. This should be a rare occurrence. + +Instead of an automated solution, we recommend inspecting the affected datasets and correcting the fileaccessrequest flag as appropriate. You can identify the affected datasets this via a query, which is available in the folder here: + +https://github.com/IQSS/dataverse/raw/develop/scripts/issues/7687/ \ No newline at end of file From fbded77e18e7fc38399586e748a63a30651a531e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 16 Mar 2021 14:49:25 -0400 Subject: [PATCH 496/576] prevent deactivated accounts from being converted to OAuth #2419 #4475 --- doc/sphinx-guides/source/user/account.rst | 2 ++ .../authorization/providers/oauth2/OAuth2FirstLoginPage.java | 4 ++++ src/main/java/propertyFiles/Bundle.properties | 1 + 3 files changed, 7 insertions(+) diff --git a/doc/sphinx-guides/source/user/account.rst b/doc/sphinx-guides/source/user/account.rst index 18a44bcb85d..e4fa58d3274 100755 --- a/doc/sphinx-guides/source/user/account.rst +++ b/doc/sphinx-guides/source/user/account.rst @@ -132,6 +132,8 @@ If you already have a Dataverse installation account associated with the Usernam #. Enter your username and password for your Dataverse installation account and click "Convert Account". #. Now you have finished converting your Dataverse installation account to use ORCID for log in. +Note that you cannot go through this conversion process if your Dataverse installation account associated with the Username/Email log in option has been deactivated. + Convert your Dataverse installation account away from ORCID for log in ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2FirstLoginPage.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2FirstLoginPage.java index bd65e296d52..44f00f797a0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2FirstLoginPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2FirstLoginPage.java @@ -208,6 +208,10 @@ public String convertExistingAccount() { auReq.putCredential(creds.get(1).getKey(), getPassword()); try { AuthenticatedUser existingUser = authenticationSvc.getUpdateAuthenticatedUser(BuiltinAuthenticationProvider.PROVIDER_ID, auReq); + if (existingUser.isDeactivated()) { + JsfHelper.addErrorMessage(BundleUtil.getStringFromBundle("oauth2.convertAccount.failedDeactivated")); + return null; + } authenticationSvc.updateProvider(existingUser, newUser.getServiceId(), newUser.getIdInService()); builtinUserSvc.removeUser(existingUser.getUserIdentifier()); diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 030d80e4d5c..5c23e3dd34e 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -402,6 +402,7 @@ oauth2.convertAccount.password=Password oauth2.convertAccount.authenticationFailed=Your account can only be converted if you provide the correct username and password for your existing account. If your existing account has been deactivated by an administrator, you cannot convert your account. oauth2.convertAccount.buttonTitle=Convert Account oauth2.convertAccount.success=Your Dataverse account is now associated with your {0} account. +oauth2.convertAccount.failedDeactivated=Your existing account cannot be converted because it has been deactivated. # oauth2/callback.xhtml oauth2.callback.page.title=OAuth Callback From 9d872d083b225b792c0ecba9fc9b4d0aeb3d7880 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 16 Mar 2021 16:36:07 -0400 Subject: [PATCH 497/576] #7633 fix duplicate alias error message --- .../iq/dataverse/ManagePermissionsPage.java | 2 +- .../command/impl/CreateRoleCommand.java | 17 +++++++- src/main/java/propertyFiles/Bundle.properties | 2 + .../command/impl/CreateRoleCommandTest.java | 41 +++++++++++++++++++ 4 files changed, 59 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java index d73aaeb8dbd..79a3ca800e2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java @@ -590,7 +590,7 @@ public void updateRole(ActionEvent e) { } catch (PermissionException ex) { JH.addMessage(FacesMessage.SEVERITY_ERROR, BundleUtil.getStringFromBundle("permission.roleNotSaved"), BundleUtil.getStringFromBundle("permission.permissionsMissing", Arrays.asList(ex.getRequiredPermissions().toString()))); } catch (CommandException ex) { - JH.addMessage(FacesMessage.SEVERITY_FATAL, BundleUtil.getStringFromBundle("permission.roleNotSaved")); + JH.addMessage(FacesMessage.SEVERITY_ERROR, BundleUtil.getStringFromBundle("permission.roleNotSaved").concat(" " + ex.getMessage()) ); logger.log(Level.SEVERE, "Error saving role: " + ex.getMessage(), ex); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateRoleCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateRoleCommand.java index ff28021146d..cb9b0a3c774 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateRoleCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateRoleCommand.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.api.AbstractApiBean; import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; @@ -11,6 +12,8 @@ import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; +import edu.harvard.iq.dataverse.util.BundleUtil; +import javax.persistence.NoResultException; /** * Create a new role in a dataverse. @@ -34,9 +37,19 @@ public DataverseRole execute(CommandContext ctxt) throws CommandException { User user = getUser(); //todo: temporary for 4.0 - only superusers can create and edit roles if ((!(user instanceof AuthenticatedUser) || !user.isSuperuser())) { - throw new IllegalCommandException("Roles can only be created or edited by superusers.",this); + throw new IllegalCommandException(BundleUtil.getStringFromBundle("permission.role.must.be.created.by.superuser"),this); + } + //Test to see if the role already exists in DB + try { + DataverseRole testRole = ctxt.em().createNamedQuery("DataverseRole.findDataverseRoleByAlias", DataverseRole.class) + .setParameter("alias", created.getAlias()) + .getSingleResult(); + if (!(testRole == null)) { + throw new IllegalCommandException(BundleUtil.getStringFromBundle("permission.role.not.created.alias.already.exists"), this); + } + } catch (NoResultException nre) { + // we want no results because that meand we can create a role } - dv.addRole(created); return ctxt.roles().save(created); } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 0c50276ea0d..40b15230f42 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2239,6 +2239,8 @@ permission.permissionsMissing=Permissions {0} missing. permission.CannotAssigntDefaultPermissions=Cannot assign default permissions. permission.default.contributor.role.none.decription=A person who has no permissions on a newly created dataset. Not recommended for dataverses with human contributors. permission.default.contributor.role.none.name=None +permission.role.must.be.created.by.superuser=Roles can only be created or edited by superusers. +permission.role.not.created.alias.already.exists=Role with this alias already exists. #ManageFilePermissionsPage.java permission.roleNotAbleToBeRemoved=The role assignment was not able to be removed. diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateRoleCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateRoleCommandTest.java index ea39bb1bc77..243285e69ab 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateRoleCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateRoleCommandTest.java @@ -9,13 +9,19 @@ import edu.harvard.iq.dataverse.authorization.users.GuestUser; import edu.harvard.iq.dataverse.engine.TestCommandContext; import edu.harvard.iq.dataverse.engine.TestDataverseEngine; +import edu.harvard.iq.dataverse.engine.TestEntityManager; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; import edu.harvard.iq.dataverse.mocks.MocksFactory; +import javax.persistence.EntityManager; +import javax.persistence.TypedQuery; import static org.junit.Assert.assertTrue; import org.junit.Before; import org.junit.Test; +import org.mockito.Matchers; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** * @@ -36,6 +42,12 @@ public DataverseRole save(DataverseRole aRole) { } }; } + + @Override + public EntityManager em() { + return new LocalTestEntityManager(); + + } }); @Before @@ -94,4 +106,33 @@ public void testGuestUsersCantAddRoles() throws CommandException { engine.submit(sut); } + private class LocalTestEntityManager extends TestEntityManager { + + @Override + public T merge(T entity) { + return entity; + } + + @Override + public void persist(Object entity) { + // + } + + @Override + public void flush() { + //nothing to do here + } + + @Override + public TypedQuery createNamedQuery(String name, Class resultClass) { + //Mocking a query to return no results when + //checking for existing role in DB + TypedQuery mockedQuery = mock(TypedQuery.class); + when(mockedQuery.setParameter(Matchers.anyString(), Matchers.anyObject())).thenReturn(mockedQuery); + when(mockedQuery.getSingleResult()).thenReturn(null); + return mockedQuery; + } + + } + } From fdc9cbe448d5c10b39a550be25adaa864aeab273 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 16 Mar 2021 16:56:18 -0400 Subject: [PATCH 498/576] prevent deactivated accounts from being converted to Shib #2419 #4475 --- doc/sphinx-guides/source/user/account.rst | 2 + .../java/edu/harvard/iq/dataverse/Shib.java | 4 ++ .../edu/harvard/iq/dataverse/api/Admin.java | 4 ++ src/main/java/propertyFiles/Bundle.properties | 1 + .../edu/harvard/iq/dataverse/api/AdminIT.java | 57 +++++++++++++++++++ 5 files changed, 68 insertions(+) diff --git a/doc/sphinx-guides/source/user/account.rst b/doc/sphinx-guides/source/user/account.rst index e4fa58d3274..4c343ff85d4 100755 --- a/doc/sphinx-guides/source/user/account.rst +++ b/doc/sphinx-guides/source/user/account.rst @@ -99,6 +99,8 @@ If you already have a Dataverse installation account associated with the Usernam #. Enter your current password for your Dataverse installation account and click "Convert Account". #. Now you have finished converting your Dataverse installation account to use your institutional log in. +Note that you cannot go through this conversion process if your Dataverse installation account associated with the Username/Email log in option has been deactivated. + Convert your Dataverse installation account away from your Institutional Log In ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/src/main/java/edu/harvard/iq/dataverse/Shib.java b/src/main/java/edu/harvard/iq/dataverse/Shib.java index 6dd48cc782a..4ad50320f23 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Shib.java +++ b/src/main/java/edu/harvard/iq/dataverse/Shib.java @@ -338,6 +338,10 @@ public String confirmAndConvertAccount() { logger.fine("builtin username: " + builtinUsername); AuthenticatedUser builtInUserToConvert = authSvc.canLogInAsBuiltinUser(builtinUsername, builtinPassword); if (builtInUserToConvert != null) { + if (builtInUserToConvert.isDeactivated()) { + JsfHelper.addErrorMessage(BundleUtil.getStringFromBundle("shib.convert.fail.deactivated")); + return null; + } // TODO: Switch from authSvc.convertBuiltInToShib to authSvc.convertBuiltInUserToRemoteUser AuthenticatedUser au = authSvc.convertBuiltInToShib(builtInUserToConvert, shibAuthProvider.getId(), userIdentifier); if (au != null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 96add147743..92866dfc15a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -695,6 +695,10 @@ public Response builtin2shib(String content) { password); if (authenticatedUser != null) { knowsExistingPassword = true; + if (builtInUserToConvert.isDeactivated()) { + problems.add("builtin account has been deactivated"); + return error(Status.BAD_REQUEST, problems.build().toString()); + } AuthenticatedUser convertedUser = authSvc.convertBuiltInToShib(builtInUserToConvert, shibProviderId, newUserIdentifierInLookupTable); if (convertedUser != null) { diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 5c23e3dd34e..ae7064762eb 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2217,6 +2217,7 @@ shib.invalidEmailAddress=The SAML assertion contained an invalid email address: shib.emailAddress.error=A single valid address could not be found. shib.nullerror=The SAML assertion for "{0}" was null. Please contact support. dataverse.shib.success=Your Dataverse account is now associated with your institutional account. +shib.convert.fail.deactivated=Your existing account cannot be converted because it has been deactivated. shib.createUser.fail=Couldn't create user. shib.duplicate.email.error=Cannot login, because the e-mail address associated with it has changed since previous login and is already in use by another account. diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java index 84da33cd3ee..e7c42132b0d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java @@ -358,6 +358,63 @@ public void testConvertShibUserToBuiltin() throws Exception { } + /** + * Here we are asserting that deactivated users cannot be converted into + * shib users. + */ + @Test + public void testConvertDeactivateUserToShib() { + + Response createUserToConvert = UtilIT.createRandomUser(); + createUserToConvert.then().assertThat().statusCode(OK.getStatusCode()); + createUserToConvert.prettyPrint(); + + long idOfUserToConvert = createUserToConvert.body().jsonPath().getLong("data.authenticatedUser.id"); + String emailOfUserToConvert = createUserToConvert.body().jsonPath().getString("data.authenticatedUser.email"); + String usernameOfUserToConvert = UtilIT.getUsernameFromResponse(createUserToConvert); + + Response deactivateUser = UtilIT.deactivateUser(usernameOfUserToConvert); + deactivateUser.prettyPrint(); + deactivateUser.then().assertThat().statusCode(OK.getStatusCode()); + + String password = usernameOfUserToConvert; + String newEmailAddressToUse = "builtin2shib." + UUID.randomUUID().toString().substring(0, 8) + "@mailinator.com"; + String data = emailOfUserToConvert + ":" + password + ":" + newEmailAddressToUse; + + Response builtinToShibAnon = UtilIT.migrateBuiltinToShib(data, ""); + builtinToShibAnon.prettyPrint(); + builtinToShibAnon.then().assertThat().statusCode(FORBIDDEN.getStatusCode()); + + Response createSuperuser = UtilIT.createRandomUser(); + String superuserUsername = UtilIT.getUsernameFromResponse(createSuperuser); + String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser); + Response toggleSuperuser = UtilIT.makeSuperUser(superuserUsername); + toggleSuperuser.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response getAuthProviders = UtilIT.getAuthProviders(superuserApiToken); + getAuthProviders.prettyPrint(); + if (!getAuthProviders.body().asString().contains(BuiltinAuthenticationProvider.PROVIDER_ID)) { + System.out.println("Can't proceed with test without builtin provider."); + return; + } + + Response makeShibUser = UtilIT.migrateBuiltinToShib(data, superuserApiToken); + makeShibUser.prettyPrint(); + makeShibUser.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo("[\"builtin account has been deactivated\"]")); + + Response userIsStillBuiltin = UtilIT.getAuthenticatedUser(usernameOfUserToConvert, superuserApiToken); + userIsStillBuiltin.prettyPrint(); + userIsStillBuiltin.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.id", equalTo(Long.valueOf(idOfUserToConvert).intValue())) + .body("data.identifier", equalTo("@" + usernameOfUserToConvert)) + .body("data.authenticationProviderId", equalTo("builtin")); + + } + @Test public void testConvertOAuthUserToBuiltin() throws Exception { From e753683babb231a252a3355a5b29bc45873ff5a8 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 17 Mar 2021 09:24:20 -0400 Subject: [PATCH 499/576] #7633 add content type to add roles doc --- doc/sphinx-guides/source/api/native-api.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 910e1b1b63a..32f752ceaec 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -246,7 +246,7 @@ The fully expanded example above (without environment variables) looks like this .. code-block:: bash - curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST https://demo.dataverse.org/api/dataverses/root/roles --upload-file roles.json + curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST -H "Content-type:application/json" https://demo.dataverse.org/api/dataverses/root/roles --upload-file roles.json Where ``roles.json`` looks like this:: @@ -2428,13 +2428,13 @@ Creates a new role under Dataverse collection ``id``. Needs a json file with the export SERVER_URL=https://demo.dataverse.org export ID=root - curl -H X-Dataverse-key:$API_TOKEN -X POST $SERVER_URL/api/dataverses/$ID/roles --upload-file roles.json + curl -H X-Dataverse-key:$API_TOKEN -X POST -H "Content-type:application/json" $SERVER_URL/api/dataverses/$ID/roles --upload-file roles.json The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST https://demo.dataverse.org/api/dataverses/root/roles --upload-file roles.json + curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST -H "Content-type:application/json" https://demo.dataverse.org/api/dataverses/root/roles --upload-file roles.json Where ``roles.json`` looks like this:: From 0efee2810d555c9e241f8e2fa7728389d5483aba Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 17 Mar 2021 11:45:14 -0400 Subject: [PATCH 500/576] exclude deactivated users from role assignment autocomplete #2419 #4475 Restore the query to what it was in previous releases. Put the logic in the method. --- .../edu/harvard/iq/dataverse/RoleAssigneeServiceBean.java | 4 +++- .../iq/dataverse/authorization/users/AuthenticatedUser.java | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/RoleAssigneeServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/RoleAssigneeServiceBean.java index b31b55b2e4f..6b207ed0e75 100644 --- a/src/main/java/edu/harvard/iq/dataverse/RoleAssigneeServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/RoleAssigneeServiceBean.java @@ -376,7 +376,9 @@ public List filterRoleAssignees(String query, DvObject dvObject, L .getResultList().stream() .filter(ra -> roleAssignSelectedRoleAssignees == null || !roleAssignSelectedRoleAssignees.contains(ra)) .forEach((ra) -> { - roleAssigneeList.add(ra); + if (!ra.isDeactivated()) { + roleAssigneeList.add(ra); + } }); // now we add groups to the list, both global and explicit diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java index 0f4bf8b5878..db6164e0ac7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java @@ -56,8 +56,7 @@ @NamedQuery( name="AuthenticatedUser.filter", query="select au from AuthenticatedUser au WHERE (" + "LOWER(au.userIdentifier) like LOWER(:query) OR " - + "lower(concat(au.firstName,' ',au.lastName)) like lower(:query)) " - + "AND au.deactivated != true"), + + "lower(concat(au.firstName,' ',au.lastName)) like lower(:query))"), @NamedQuery( name="AuthenticatedUser.findAdminUser", query="select au from AuthenticatedUser au WHERE " + "au.superuser = true " From 4b2ac9644977db21cf20d5bf2a93321e6a7c3f42 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 17 Mar 2021 11:59:09 -0400 Subject: [PATCH 501/576] cleanup, add shib/oauth convert to list for admins #2419 #4475 --- doc/sphinx-guides/source/api/native-api.rst | 1 + .../authorization/AuthenticationServiceBean.java | 3 --- .../iq/dataverse/passwordreset/PasswordResetPage.java | 10 +++++----- .../passwordreset/PasswordResetServiceBean.java | 2 +- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index ff496df4f20..9f3f89406e5 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3136,6 +3136,7 @@ Deactivating a user with this endpoint will: - Cancel any pending file access requests generated by the user - Remove the user from all groups - No longer have notifications generated or sent by the Dataverse installation +- Prevent the account from being converted into an OAuth or Shibboleth account. Deactivating a user with this endpoint will keep: diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java index 8a3d51bf1a7..349a86301a6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java @@ -200,9 +200,6 @@ public boolean isOrcidEnabled() { * Before calling this method, make sure you've deleted all the stuff tied * to the user, including stuff they've created, role assignments, group * assignments, etc. See the "removeAuthentictedUserItems" (sic) method. - * - * Longer term, the intention is to have a "disableAuthenticatedUser" - * method/command. See https://github.com/IQSS/dataverse/issues/2419 */ public void deleteAuthenticatedUser(Object pk) { AuthenticatedUser user = em.find(AuthenticatedUser.class, pk); diff --git a/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetPage.java b/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetPage.java index f27621cef74..aea910c496e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetPage.java @@ -121,14 +121,14 @@ public String sendPasswordResetLink() { actionLogSvc.log( new ActionLogRecord(ActionLogRecord.ActionType.BuiltinUser, "passwordResetSent") .setInfo("Email Address: " + emailAddress) ); } else { - logger.log(Level.INFO, "Cannot find account (or it's disabled) given {0}", emailAddress); + logger.log(Level.INFO, "Cannot find account (or it's deactivated) given {0}", emailAddress); } /** * We show this "an email will be sent" message no matter what (if - * the account can be found or not, if the account has been disabled - * or not) to prevent hackers from figuring out if you have an - * account based on your email address. Yes, this is a white lie - * sometimes, in the name of security. + * the account can be found or not, if the account has been + * deactivated or not) to prevent hackers from figuring out if you + * have an account based on your email address. Yes, this is a white + * lie sometimes, in the name of security. */ FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("passwdVal.passwdReset.resetInitiated"), BundleUtil.getStringFromBundle("passwdReset.successSubmit.tip", Arrays.asList(emailAddress)))); diff --git a/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetServiceBean.java index 4e92d861a2f..c8db23985d8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetServiceBean.java @@ -53,7 +53,7 @@ public class PasswordResetServiceBean { * * @param emailAddress * @return {@link PasswordResetInitResponse} with empty PasswordResetData if - * the reset won't continue (no user, disabled user). + * the reset won't continue (no user, deactivated user). * @throws edu.harvard.iq.dataverse.passwordreset.PasswordResetException */ // inspired by Troy Hunt: Everything you ever wanted to know about building a secure password reset feature - http://www.troyhunt.com/2012/05/everything-you-ever-wanted-to-know.html From d0191c394e345f02cfb99b71a48845b0fb1b1632 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 18 Mar 2021 13:56:36 -0400 Subject: [PATCH 502/576] more efficient check for disabled or deleted users #2419 #4475 Previously, in 3e59e70 we added logic to check if the user in the session has been deleted or deactivated. The problem is that this code is called often so now we only do this check if the user tries to go to the account page or tries to execute a command. --- .../harvard/iq/dataverse/DataversePage.java | 2 +- .../iq/dataverse/DataverseSession.java | 15 ++++++++++++++- .../iq/dataverse/EjbDataverseEngine.java | 19 ++++++++++++++++++- .../providers/builtin/DataverseUserPage.java | 2 +- src/main/java/propertyFiles/Bundle.properties | 2 ++ 5 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java index c7f816ce219..2e5ec9840ed 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java @@ -695,7 +695,7 @@ public String save() { } catch (CommandException ex) { logger.log(Level.SEVERE, "Unexpected Exception calling dataverse command", ex); - String errMsg = create ? BundleUtil.getStringFromBundle("dataverse.create.failure") : BundleUtil.getStringFromBundle("dataverse.update.failure"); + String errMsg = create ? ex.getLocalizedMessage() : BundleUtil.getStringFromBundle("dataverse.update.failure"); JH.addMessage(FacesMessage.SEVERITY_FATAL, errMsg); return null; } catch (Exception e) { diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java b/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java index 95c0e124faf..c6016939c08 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java @@ -90,10 +90,23 @@ public void setDismissedMessages(List dismissedMessages) { private Boolean debug; public User getUser() { + return getUser(false); + } + + /** + * For performance reasons, we only lookup the authenticated user again (to + * check if it has been deleted or deactivated, for example) when we have + * to. + * + * @param lookupAuthenticatedUserAgain A boolean to indicate if we should go + * to the database again to lookup the user to get the latest values that + * may have been updated outside the session. + */ + public User getUser(boolean lookupAuthenticatedUserAgain) { if ( user == null ) { user = GuestUser.get(); } - if (user instanceof AuthenticatedUser) { + if (lookupAuthenticatedUserAgain && user instanceof AuthenticatedUser) { AuthenticatedUser auFromSession = (AuthenticatedUser) user; AuthenticatedUser auFreshLookup = authenticationService.findByID(auFromSession.getId()); if (auFreshLookup == null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java b/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java index da9aba43498..0937f6f6cf7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java +++ b/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java @@ -8,6 +8,7 @@ import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.groups.GroupServiceBean; import edu.harvard.iq.dataverse.authorization.groups.impl.explicit.ExplicitGroupServiceBean; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.confirmemail.ConfirmEmailServiceBean; import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleServiceBean; import edu.harvard.iq.dataverse.engine.command.Command; @@ -30,8 +31,10 @@ import edu.harvard.iq.dataverse.search.SolrIndexServiceBean; import edu.harvard.iq.dataverse.search.savedsearch.SavedSearchServiceBean; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.workflow.WorkflowServiceBean; +import java.util.Arrays; import java.util.EnumSet; import java.util.Stack; import java.util.logging.Level; @@ -215,7 +218,21 @@ public R submit(Command aCommand) throws CommandException { } DataverseRequest dvReq = aCommand.getRequest(); - + + AuthenticatedUser authenticatedUser = dvReq.getAuthenticatedUser(); + if (authenticatedUser != null) { + AuthenticatedUser auFreshLookup = authentication.findByID(authenticatedUser.getId()); + if (auFreshLookup == null) { + logger.fine("submit method found user no longer exists (was deleted)."); + throw new CommandException(BundleUtil.getStringFromBundle("command.exception.user.deleted", Arrays.asList(aCommand.getClass().getSimpleName())), aCommand); + } else { + if (auFreshLookup.isDeactivated()) { + logger.fine("submit method found user is deactivated."); + throw new CommandException(BundleUtil.getStringFromBundle("command.exception.user.deactivated", Arrays.asList(aCommand.getClass().getSimpleName())), aCommand); + } + } + } + Map affectedDvObjects = aCommand.getAffectedDvObjects(); logRec.setInfo(aCommand.describe()); for (Map.Entry> pair : requiredMap.entrySet()) { diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java index 663cdf4b4b2..65cb11b5351 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java @@ -162,7 +162,7 @@ public String init() { } } - if ( session.getUser().isAuthenticated() ) { + if (session.getUser(true).isAuthenticated()) { setCurrentUser((AuthenticatedUser) session.getUser()); userAuthProvider = authenticationService.lookupProvider(currentUser); notificationsList = userNotificationService.findByUser(currentUser.getId()); diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index ff9522b6bf4..e78c5433bd9 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2283,6 +2283,8 @@ pid.allowedCharacters=^[A-Za-z0-9._/:\\-]* #General Command Exception command.exception.only.superusers={1} can only be called by superusers. +command.exception.user.deactivated={0} failed: User account has been deactivated. +command.exception.user.deleted={0} failed: User account has been deleted. #Admin-API admin.api.auth.mustBeSuperUser=Forbidden. You must be a superuser. From 7625fc2af7bb24a4dc283c9b617a672b2e52e0e8 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 18 Mar 2021 13:57:27 -0400 Subject: [PATCH 503/576] #7634 order dvobjects by display name --- .../engine/command/impl/ListDataverseContentCommand.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseContentCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseContentCommand.java index 26e1e988fac..3c606d35881 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseContentCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseContentCommand.java @@ -7,7 +7,9 @@ import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.EnumSet; import java.util.List; import java.util.Map; @@ -32,11 +34,14 @@ public ListDataverseContentCommand(DataverseRequest aRequest, Dataverse anAffect @Override public List execute(CommandContext ctxt) throws CommandException { + List availableObjects = new ArrayList(); if (getRequest().getUser().isSuperuser()) { - return ctxt.dvObjects().findByOwnerId(dvToList.getId()); + availableObjects = ctxt.dvObjects().findByOwnerId(dvToList.getId()); } else { - return ctxt.permissions().whichChildrenHasPermissionsForOrReleased(getRequest(), dvToList, EnumSet.of(Permission.ViewUnpublishedDataverse, Permission.ViewUnpublishedDataset)); + availableObjects = ctxt.permissions().whichChildrenHasPermissionsForOrReleased(getRequest(), dvToList, EnumSet.of(Permission.ViewUnpublishedDataverse, Permission.ViewUnpublishedDataset)); } + availableObjects.sort(Comparator.comparing(DvObject::getDisplayName)); + return availableObjects; } @Override From b6b2298c77b5901901c876d12271757ace010d22 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 18 Mar 2021 14:19:49 -0400 Subject: [PATCH 504/576] correct flyway number --- .../{V5.5.0.2__7564-workflow.sql => V5.3.0.5__7564-workflow.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V5.5.0.2__7564-workflow.sql => V5.3.0.5__7564-workflow.sql} (100%) diff --git a/src/main/resources/db/migration/V5.5.0.2__7564-workflow.sql b/src/main/resources/db/migration/V5.3.0.5__7564-workflow.sql similarity index 100% rename from src/main/resources/db/migration/V5.5.0.2__7564-workflow.sql rename to src/main/resources/db/migration/V5.3.0.5__7564-workflow.sql From 24433cfd8599c71d2eb3616fc4c9d030c47f56c2 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 19 Mar 2021 14:17:18 -0400 Subject: [PATCH 505/576] #7675 add pub reminders to save success messages --- .../edu/harvard/iq/dataverse/DatasetPage.java | 46 +++++++++++++++---- src/main/java/propertyFiles/Bundle.properties | 6 ++- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 5f596d1b556..15cbbd9d7d1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -1107,6 +1107,29 @@ public String getComputeUrl() throws IOException { } + + //get a string to add to save success message + //depends on dataset state and user privleges + private String getReminderString(Dataset dataset) { + + //dataset id is null before first save -- needed for create mode + if( dataset.getId() == null || !dataset.isReleased() ){ + //messages for draft state. + if (canPublishDataset()){ + return BundleUtil.getStringFromBundle("dataset.message.publish.remind.draft"); + } else { + return BundleUtil.getStringFromBundle("dataset.message.submit.remind.draft"); + } + } else{ + //messages for new version - post-publish + if (canPublishDataset()){ + return BundleUtil.getStringFromBundle("dataset.message.publish.remind.version"); + } else { + return BundleUtil.getStringFromBundle("dataset.message.submit.remind.version"); + } + } + } + //For a single file public String getComputeUrl(FileMetadata metadata) { SwiftAccessIO swiftObject = null; @@ -1280,6 +1303,11 @@ public boolean canUpdateDataset() { public boolean canPublishDataverse() { return permissionsWrapper.canIssuePublishDataverseCommand(dataset.getOwner()); } + + public boolean canPublishDataset(){ + permissionsWrapper.doesSessionUserHaveDataSetPermission(dvRequestService.getDataverseRequest(), dataset, Permission.PublishDataset); + return permissionsWrapper.canIssuePublishDatasetCommand(dataset); + } public boolean canViewUnpublishedDataset() { return permissionsWrapper.canViewUnpublishedDataset( dvRequestService.getDataverseRequest(), dataset); @@ -3347,6 +3375,8 @@ public String save() { return ""; } + + // Use the Create or Update command to save the dataset: Command cmd; Map deleteStorageLocations = null; @@ -3450,7 +3480,7 @@ public String save() { } if (addFilesSuccess && dataset.getFiles().size() > 0) { if (nNewFiles == dataset.getFiles().size()) { - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.createSuccess")); + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.createSuccess").concat(" ").concat(getReminderString(dataset))); } else { String partialSuccessMessage = BundleUtil.getStringFromBundle("dataset.message.createSuccess.partialSuccessSavingFiles"); partialSuccessMessage = partialSuccessMessage.replace("{0}", "" + dataset.getFiles().size() + ""); @@ -3460,26 +3490,26 @@ public String save() { } else { JsfHelper.addWarningMessage(BundleUtil.getStringFromBundle("dataset.message.createSuccess.failedToSaveFiles")); } - } else { - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.createSuccess")); + } else { + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.createSuccess").concat(" ").concat(getReminderString(dataset))); } } if (editMode.equals(EditMode.METADATA)) { - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.metadataSuccess")); + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.metadataSuccess").concat(" ").concat(getReminderString(dataset))); } if (editMode.equals(EditMode.LICENSE)) { - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.termsSuccess")); + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.termsSuccess").concat(" ").concat(getReminderString(dataset))); } if (editMode.equals(EditMode.FILE)) { - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.filesSuccess")); + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.filesSuccess").concat(" ").concat(getReminderString(dataset))); } } else { // must have been a bulk file update or delete: if (bulkFileDeleteInProgress) { - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.bulkFileDeleteSuccess")); + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.bulkFileDeleteSuccess").concat(" ").concat(getReminderString(dataset))); } else { - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.bulkFileUpdateSuccess")); + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.bulkFileUpdateSuccess").concat(" ").concat(getReminderString(dataset))); } } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index abcd502d638..0fd0f443794 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1403,10 +1403,14 @@ dataset.message.createSuccess.failedToSaveFiles=Partial Success: The dataset has dataset.message.createSuccess.partialSuccessSavingFiles=Partial Success: The dataset has been created. But only {0} out of {1} files have been saved. Please try uploading the missing file(s) again. dataset.message.linkSuccess= {0} has been successfully linked to {1}. dataset.message.metadataSuccess=The metadata for this dataset has been updated. -dataset.message.termsSuccess=The terms for this dataset has been updated. +dataset.message.termsSuccess=The terms for this dataset have been updated. dataset.message.filesSuccess=The files for this dataset have been updated. dataset.message.addFiles.Failure=Failed to add files to the dataset. Please try uploading the file(s) again. dataset.message.addFiles.partialSuccess=Partial success: only {0} files out of {1} have been saved. Please try uploading the missing file(s) again. +dataset.message.publish.remind.draft=If it's ready for sharing, please publish it. +dataset.message.submit.remind.draft=If it's ready for sharing, please submit it for review. +dataset.message.publish.remind.version=If it\u2019s ready for sharing, please publish it so that others can see these changes. +dataset.message.submit.remind.version=If it\u2019s ready for sharing, please submit it for review so that others can see these changes. dataset.message.publishSuccess=This dataset has been published. dataset.message.only.authenticatedUsers=Only authenticated users may release Datasets. dataset.message.deleteSuccess=This dataset has been deleted. From fd3b3f153d4438236bb0c85429c2a6e2c6692cc1 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 19 Mar 2021 15:06:23 -0400 Subject: [PATCH 506/576] #7675 remove unused code --- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 15cbbd9d7d1..b377c57b68b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -1305,7 +1305,6 @@ public boolean canPublishDataverse() { } public boolean canPublishDataset(){ - permissionsWrapper.doesSessionUserHaveDataSetPermission(dvRequestService.getDataverseRequest(), dataset, Permission.PublishDataset); return permissionsWrapper.canIssuePublishDatasetCommand(dataset); } From 6b2a89ffcb089062fa8673c0fea0679f6f0cc5b0 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 19 Mar 2021 15:07:33 -0400 Subject: [PATCH 507/576] #7675 get standard apostrophe --- src/main/java/propertyFiles/Bundle.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 0fd0f443794..46998453ae1 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1409,8 +1409,8 @@ dataset.message.addFiles.Failure=Failed to add files to the dataset. Please try dataset.message.addFiles.partialSuccess=Partial success: only {0} files out of {1} have been saved. Please try uploading the missing file(s) again. dataset.message.publish.remind.draft=If it's ready for sharing, please publish it. dataset.message.submit.remind.draft=If it's ready for sharing, please submit it for review. -dataset.message.publish.remind.version=If it\u2019s ready for sharing, please publish it so that others can see these changes. -dataset.message.submit.remind.version=If it\u2019s ready for sharing, please submit it for review so that others can see these changes. +dataset.message.publish.remind.version=If it's ready for sharing, please publish it so that others can see these changes. +dataset.message.submit.remind.version=If it's ready for sharing, please submit it for review so that others can see these changes. dataset.message.publishSuccess=This dataset has been published. dataset.message.only.authenticatedUsers=Only authenticated users may release Datasets. dataset.message.deleteSuccess=This dataset has been deleted. From a306707d7c0ea878b20683ebbb533d60f99f6fda Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 19 Mar 2021 16:59:37 -0400 Subject: [PATCH 508/576] #7633 give feedback on required fields --- src/main/java/propertyFiles/Bundle.properties | 2 ++ src/main/webapp/roles-edit.xhtml | 22 ++++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 40b15230f42..c085f18edc6 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -967,6 +967,8 @@ dataverse.permissions.roles.add=Add New Role dataverse.permissions.roles.description=All the roles set up in your dataverse, that you can assign to users and groups. dataverse.permissions.roles.edit=Edit Role dataverse.permissions.roles.copy=Copy Role +dataverse.permissions.roles.alias.required=Please enter a unique identifier for this role. +dataverse.permissions.roles.name.required=Please enter a name for this role. # permissions-manage-files.xhtml dataverse.permissionsFiles.title=Restricted File Permissions diff --git a/src/main/webapp/roles-edit.xhtml b/src/main/webapp/roles-edit.xhtml index a59d53bcdae..4748f24df5d 100644 --- a/src/main/webapp/roles-edit.xhtml +++ b/src/main/webapp/roles-edit.xhtml @@ -8,14 +8,18 @@
    +
    - +
    @@ -24,12 +28,17 @@ #{bundle['dataverse.permissions.roles.id']} + + - + +
    +
    @@ -65,9 +74,12 @@
    + oncomplete="if (args && !args.validationFailed) addRoleCommand();" + update="roleInputTextFragment @([id$=Messages])" + > + + + From 4da006434ab0cad7274f5ce8e47037bdbfa32936 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 22 Mar 2021 11:21:32 -0400 Subject: [PATCH 509/576] handle lower case keys, consolidate common code, fix typos. --- .../internalspi/AuthorizedExternalStep.java | 26 +-------- .../internalspi/PauseWithMessageStep.java | 32 +---------- .../iq/dataverse/workflows/WorkflowUtil.java | 57 +++++++++++++++++++ 3 files changed, 62 insertions(+), 53 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/AuthorizedExternalStep.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/AuthorizedExternalStep.java index 244abdb017c..bbe200aaeb3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/AuthorizedExternalStep.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/AuthorizedExternalStep.java @@ -9,6 +9,8 @@ import edu.harvard.iq.dataverse.workflow.step.Success; import edu.harvard.iq.dataverse.workflow.step.WorkflowStep; import edu.harvard.iq.dataverse.workflow.step.WorkflowStepResult; +import edu.harvard.iq.dataverse.workflows.WorkflowUtil; + import static edu.harvard.iq.dataverse.workflow.step.WorkflowStepResult.OK; import java.io.StringReader; @@ -77,29 +79,7 @@ public WorkflowStepResult run(WorkflowContext context) { @Override public WorkflowStepResult resume(WorkflowContext context, Map internalData, String externalData) { - try (StringReader reader = new StringReader(externalData)) { - JsonObject response = Json.createReader(reader).readObject(); - String status = response.getString("Status"); - String reason = null; - String message = null; - if (response.containsKey("Reason")) { - reason = response.getString("Reason"); - } - if (response.containsKey("Message")) { - message = response.getString("Message"); - } - switch (status) { - case "Success": - logger.log(Level.FINE, "AuthExt Worfklow Step Succeeded: " + reason); - return new Success(reason, message); - case "Failure": - logger.log(Level.WARNING, "Remote system indicates workflow failed: {0}", reason); - return new Failure(reason, message); - } - } catch (Exception e) { - logger.log(Level.WARNING, "Remote system returned a bad reposonse: {0}", externalData); - } - return new Failure("Workflow failure: Response from remote server could not be parsed:" + externalData, null); + return WorkflowUtil.parseResponse(externalData); } @Override diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/PauseWithMessageStep.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/PauseWithMessageStep.java index c19de39f2ab..f7332611697 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/PauseWithMessageStep.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/PauseWithMessageStep.java @@ -3,18 +3,14 @@ import edu.harvard.iq.dataverse.workflow.WorkflowContext; import edu.harvard.iq.dataverse.workflow.step.Failure; import edu.harvard.iq.dataverse.workflow.step.Pending; -import edu.harvard.iq.dataverse.workflow.step.Success; import edu.harvard.iq.dataverse.workflow.step.WorkflowStep; import edu.harvard.iq.dataverse.workflow.step.WorkflowStepResult; +import edu.harvard.iq.dataverse.workflows.WorkflowUtil; -import java.io.StringReader; import java.util.HashMap; import java.util.Map; -import java.util.logging.Level; import java.util.logging.Logger; -import javax.json.Json; -import javax.json.JsonObject; /** * A sample step that pauses the workflow. @@ -23,8 +19,6 @@ */ public class PauseWithMessageStep implements WorkflowStep { - private static final Logger logger = Logger.getLogger(PauseWithMessageStep.class.getName()); - /** Constant used by testing to simulate a failed step. */ public static final String FAILURE_RESPONSE="fail"; @@ -43,29 +37,7 @@ public WorkflowStepResult run(WorkflowContext context) { @Override public WorkflowStepResult resume(WorkflowContext context, Map internalData, String externalData) { - try (StringReader reader = new StringReader(externalData)) { - JsonObject response = Json.createReader(reader).readObject(); - String status = response.getString("Status"); - String reason = null; - String message = null; - if (response.containsKey("Reason")) { - reason = response.getString("Reason"); - } - if (response.containsKey("Message")) { - message = response.getString("Message"); - } - switch (status) { - case "Success": - logger.log(Level.FINE, "AuthExt Worfklow Step Succeeded: " + reason); - return new Success(reason, message); - case "Failure": - logger.log(Level.WARNING, "Remote system indicates workflow failed: {0}", reason); - return new Failure(reason, message); - } - } catch (Exception e) { - logger.log(Level.WARNING, "Remote system returned a bad reposonse: {0}", externalData); - } - return new Failure("Workflow failure: Response from remote server could not be parsed:" + externalData, null); + return WorkflowUtil.parseResponse(externalData); } @Override diff --git a/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowUtil.java b/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowUtil.java index e28f8525c9a..3ce57233f5d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowUtil.java @@ -2,14 +2,29 @@ import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.api.Util; + +import java.io.StringReader; import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + import javax.json.Json; import javax.json.JsonArrayBuilder; +import javax.json.JsonObject; + import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; +import edu.harvard.iq.dataverse.workflow.internalspi.PauseWithMessageStep; +import edu.harvard.iq.dataverse.workflow.step.Failure; +import edu.harvard.iq.dataverse.workflow.step.Success; +import edu.harvard.iq.dataverse.workflow.step.WorkflowStepResult; + import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; +import static org.apache.commons.lang3.StringEscapeUtils.escapeHtml4; public class WorkflowUtil { + private static final Logger logger = Logger.getLogger(WorkflowUtil.class.getName()); + public static JsonArrayBuilder getAllWorkflowComments(DatasetVersion datasetVersion) { JsonArrayBuilder workflowCommentsAsJson = Json.createArrayBuilder(); List workflowComments = datasetVersion.getWorkflowComments(); @@ -26,4 +41,46 @@ public static JsonArrayBuilder getAllWorkflowComments(DatasetVersion datasetVers } return workflowCommentsAsJson; } + + public static WorkflowStepResult parseResponse(String externalData) { + try (StringReader reader = new StringReader(externalData)) { + JsonObject response = Json.createReader(reader).readObject(); + String status = null; + //Lower case is documented, upper case is deprecated + if(response.containsKey("status")) { + status= response.getString("status"); + }else if(response.containsKey("Status")) { + status= response.getString("Status"); + } + String reason = null; + String message = null; + if (response.containsKey("reason")) { + reason = response.getString("reason"); + }else if (response.containsKey("Reason")) { + reason = response.getString("Reason"); + } + if (response.containsKey("message")) { + message = response.getString("message"); + }else if (response.containsKey("Message")) { + message = response.getString("Message"); + } + switch (status) { + case "Success": + logger.log(Level.FINE, "AuthExt Worfklow Step Succeeded: " + reason); + return new Success(reason, message); + case "Failure": + logger.log(Level.WARNING, "Remote system indicates workflow failed: {0}", reason); + return new Failure(reason, message); + default: + logger.log(Level.WARNING, "Remote system returned a response with no \"status\" key: {0}", externalData); + } + } catch (Exception e) { + logger.log(Level.WARNING, "Remote system returned a bad response: {0}", externalData); + } + //In general, the remote workflow service creating the response is trusted, but, if it's causing an error, escape the result to avoid issues in the UI + return new Failure("Workflow failure: Response from remote server could not be parsed:" + escapeHtml4(externalData), null); + + } + + } From 1a0ad95ca608c17f5f61af2572c44d79e9eacfa8 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 22 Mar 2021 12:05:47 -0400 Subject: [PATCH 510/576] add example in docs --- .../source/developers/workflows.rst | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/doc/sphinx-guides/source/developers/workflows.rst b/doc/sphinx-guides/source/developers/workflows.rst index bc94b8ba356..25b60ad8afd 100644 --- a/doc/sphinx-guides/source/developers/workflows.rst +++ b/doc/sphinx-guides/source/developers/workflows.rst @@ -77,10 +77,10 @@ pause/message A variant of the pause step that pauses the workflow and allows the external process to send a success/failure message. The workflow is paused until a POST request is sent to ``/api/workflows/{invocation-id}``. The response in the POST body (Content-type:application/json) should be a json object (the same as for the http/extauth step) containing: -- "Status" - can be "Success" or "Failure" -- "Reason" - a message that will be logged -- "Message" - a message to send to the user that will be sent as a notification and as a banner on the relevant dataset page. -An unparsable reponse will be considered a Failure that will be logged with no user message. +- "status" - can be "success" or "failure" +- "reason" - a message that will be logged +- "message" - a message to send to the user that will be sent as a notification and as a banner on the relevant dataset page. +An unparsable reponse will be considered a Failure that will be logged with no user message. (See the http/authext step for an example POST call) .. code:: json @@ -135,9 +135,17 @@ The url, content type, and message body can use data from the workflow context, The workflow is restarted when the external system replies with a POST request to ``/api/workflows/{invocation-id}`` (Content-Type: application/json). The response has is expected to be a json object with three keys: -- "Status" - can be "Success" or "Failure" -- "Reason" - a message that will be logged -- "Message" - a message to send to the user that will be sent as a notification and as a banner on the relevant dataset page. +- "status" - can be "success" or "failure" +- "reason" - a message that will be logged +- "message" - a message to send to the user that will be sent as a notification and as a banner on the relevant dataset page. + +.. code-block:: bash + + export INVOCATION_ID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export MESSAGE={"status":"success", "reason":"Workflow completed in 10 seconds", "message":"An external workflow to virus check your data was successfully run prior to publication of your data"} + + curl -H 'Content-Type:application/json' -X POST -d $MESSAGE "$SERVER_URL/api/workflows/$INVOCATION_ID" .. code:: json From 4c03de35ac2079eb474e9344b6973309d7520cdd Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 22 Mar 2021 12:06:05 -0400 Subject: [PATCH 511/576] lower case for message types --- .../java/edu/harvard/iq/dataverse/workflows/WorkflowUtil.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowUtil.java b/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowUtil.java index 3ce57233f5d..327eecc4de4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowUtil.java @@ -65,9 +65,11 @@ public static WorkflowStepResult parseResponse(String externalData) { message = response.getString("Message"); } switch (status) { + case "success": case "Success": logger.log(Level.FINE, "AuthExt Worfklow Step Succeeded: " + reason); return new Success(reason, message); + case "failure": case "Failure": logger.log(Level.WARNING, "Remote system indicates workflow failed: {0}", reason); return new Failure(reason, message); From b4953423f19436d27c6c9b8438ef3d5091bff20d Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 22 Mar 2021 12:14:45 -0400 Subject: [PATCH 512/576] handle null message --- src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java | 6 ++++++ src/main/java/propertyFiles/Bundle.properties | 1 + 2 files changed, 7 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java index 0ba6a718edf..4b9e595374d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java @@ -500,6 +500,9 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio case WORKFLOW_SUCCESS: version = (DatasetVersion) targetObject; pattern = BundleUtil.getStringFromBundle("notification.email.workflow.success"); + if (comment == null) { + comment = BundleUtil.getStringFromBundle("notification.email.workflow.nullMessage"); + } String[] paramArrayWorkflowSuccess = {version.getDataset().getDisplayName(), getDatasetLink(version.getDataset()), comment}; messageText += MessageFormat.format(pattern, paramArrayWorkflowSuccess); return messageText; @@ -507,6 +510,9 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio version = (DatasetVersion) targetObject; pattern = BundleUtil.getStringFromBundle("notification.email.workflow.failure"); String[] paramArrayWorkflowFailure = {version.getDataset().getDisplayName(), getDatasetLink(version.getDataset()), comment}; + if (comment == null) { + comment = BundleUtil.getStringFromBundle("notification.email.workflow.nullMessage"); + } messageText += MessageFormat.format(pattern, paramArrayWorkflowFailure); return messageText; case CREATEACC: diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 515530a7cb4..4e02fb8f8ac 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -663,6 +663,7 @@ notification.email.workflow.success.subject={0}: Your dataset has been processed notification.email.workflow.success=A workflow running on {0} (view at {1}) succeeded: {2} notification.email.workflow.failure.subject={0}: Failed to process your dataset notification.email.workflow.failure=A workflow running on {0} (view at {1}) failed: {2} +notification.email.workflow.nullMessage=No additional message sent from the workflow. notification.email.create.account.subject={0}: Your account has been created notification.email.assign.role.subject={0}: You have been assigned a role notification.email.revoke.role.subject={0}: Your role has been revoked From 626d1ce9d4943250d347e824e1b2ef20bbee2332 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 22 Mar 2021 12:16:10 -0400 Subject: [PATCH 513/576] typo --- doc/sphinx-guides/source/developers/workflows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/workflows.rst b/doc/sphinx-guides/source/developers/workflows.rst index 25b60ad8afd..c982edc08bb 100644 --- a/doc/sphinx-guides/source/developers/workflows.rst +++ b/doc/sphinx-guides/source/developers/workflows.rst @@ -18,7 +18,7 @@ Steps can be internal (say, writing some data to the log) or external. External The external system reports the step result back to the Dataverse installation, by sending a HTTP ``POST`` command to ``api/workflows/{invocation-id}`` with Content-Type: text/plain. The body of the request is passed to the paused step for further processing. -Steps can define messages to send to the log and to users. If defined, the message to users is sent as a user notification (creating an email and showing in the user notification tab) and will show once for the given user if/when they view the relevant dataset page. The latter provides a means for the asynchronous workflow execution to report success or failure anologous to the way the publication and other processes report on the page. +Steps can define messages to send to the log and to users. If defined, the message to users is sent as a user notification (creating an email and showing in the user notification tab) and will show once for the given user if/when they view the relevant dataset page. The latter provides a means for the asynchronous workflow execution to report success or failure analogous to the way the publication and other processes report on the page. If a step in a workflow fails, the Dataverse installation makes an effort to roll back all the steps that preceded it. Some actions, such as writing to the log, cannot be rolled back. If such an action has a public external effect (e.g. send an EMail to a mailing list) it is advisable to put it in the post-release workflow. From 530dab34dbbbe0f3a07bdc12d35b5cae330470ec Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 22 Mar 2021 14:37:19 -0400 Subject: [PATCH 514/576] move logic up (earlier) #2419 #4475 --- src/main/java/edu/harvard/iq/dataverse/api/Admin.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 92866dfc15a..c86dcd8118f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -689,16 +689,16 @@ public Response builtin2shib(String content) { boolean knowsExistingPassword = false; BuiltinUser oldBuiltInUser = builtinUserService.findByUserName(builtInUserToConvert.getUserIdentifier()); if (oldBuiltInUser != null) { + if (builtInUserToConvert.isDeactivated()) { + problems.add("builtin account has been deactivated"); + return error(Status.BAD_REQUEST, problems.build().toString()); + } String usernameOfBuiltinAccountToConvert = oldBuiltInUser.getUserName(); response.add("old username", usernameOfBuiltinAccountToConvert); AuthenticatedUser authenticatedUser = authSvc.canLogInAsBuiltinUser(usernameOfBuiltinAccountToConvert, password); if (authenticatedUser != null) { knowsExistingPassword = true; - if (builtInUserToConvert.isDeactivated()) { - problems.add("builtin account has been deactivated"); - return error(Status.BAD_REQUEST, problems.build().toString()); - } AuthenticatedUser convertedUser = authSvc.convertBuiltInToShib(builtInUserToConvert, shibProviderId, newUserIdentifierInLookupTable); if (convertedUser != null) { From df32ec0128ea2a054ae4fb1bb01871960a9d01df Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 22 Mar 2021 14:55:19 -0400 Subject: [PATCH 515/576] change "non-deactivated" to "active" #2419 #4475 --- doc/sphinx-guides/source/api/native-api.rst | 2 +- .../engine/command/impl/MergeInAccountCommand.java | 2 +- .../edu/harvard/iq/dataverse/api/DeactivateUsersIT.java | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 9f3f89406e5..d047952a834 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3055,7 +3055,7 @@ Example: ``curl -H "X-Dataverse-key: $API_TOKEN" -X POST http://demo.dataverse.o This action moves account data from jsmith2 into the account jsmith and deletes the account of jsmith2. -Note: User accounts can only be merged if they are either both non-deactivated or both deactivated. See :ref:`deactivate a user`. +Note: User accounts can only be merged if they are either both active or both deactivated. See :ref:`deactivate a user`. .. _change-identifier-label: diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java index 950357a1ad2..1ec51764d73 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java @@ -64,7 +64,7 @@ protected void executeImpl(CommandContext ctxt) throws CommandException { } if (consumedAU.isDeactivated() && !ongoingAU.isDeactivated() || !consumedAU.isDeactivated() && ongoingAU.isDeactivated()) { - throw new IllegalCommandException("User accounts can only be merged if they are either both non-deactivated or both deactivated.", this); + throw new IllegalCommandException("User accounts can only be merged if they are either both active or both deactivated.", this); } List baseRAList = ctxt.roleAssignees().getAssignmentsFor(ongoingAU.getIdentifier()); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java index a34b3076445..25fa03dca79 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java @@ -184,7 +184,7 @@ public void testMergeDeactivatedIntoNonDeactivatedUser() { deactivateUser.prettyPrint(); deactivateUser.then().assertThat().statusCode(OK.getStatusCode()); - // User accounts can only be merged if they are either both non-deactivated or both deactivated. + // User accounts can only be merged if they are either both active or both deactivated. Response mergeAccounts = UtilIT.mergeAccounts(usernameMergeTarget, usernameToMerge, superuserApiToken); mergeAccounts.prettyPrint(); mergeAccounts.then().assertThat().statusCode(BAD_REQUEST.getStatusCode()); @@ -212,7 +212,7 @@ public void testMergeNonDeactivatedIntoDeactivatedUser() { deactivateUser.prettyPrint(); deactivateUser.then().assertThat().statusCode(OK.getStatusCode()); - // User accounts can only be merged if they are either both non-deactivated or both deactivated. + // User accounts can only be merged if they are either both active or both deactivated. Response mergeAccounts = UtilIT.mergeAccounts(usernameMergeTarget, usernameToMerge, superuserApiToken); mergeAccounts.prettyPrint(); mergeAccounts.then().assertThat().statusCode(BAD_REQUEST.getStatusCode()); @@ -244,7 +244,7 @@ public void testMergeDeactivatedIntoDeactivatedUser() { deactivatedUserToMerge.prettyPrint(); deactivatedUserToMerge.then().assertThat().statusCode(OK.getStatusCode()); - // User accounts can only be merged if they are either both non-deactivated or both deactivated. + // User accounts can only be merged if they are either both active or both deactivated. Response mergeAccounts = UtilIT.mergeAccounts(usernameMergeTarget, usernameToMerge, superuserApiToken); mergeAccounts.prettyPrint(); mergeAccounts.then().assertThat().statusCode(OK.getStatusCode()); From 746955f2ee640802d6fb1a333d0bb6073d9774a9 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 22 Mar 2021 15:42:26 -0400 Subject: [PATCH 516/576] add reason logging, better log msgs --- .../edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java | 2 ++ .../java/edu/harvard/iq/dataverse/workflows/WorkflowUtil.java | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java index 88f107bc4af..b5ecaeea7d2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java @@ -193,6 +193,7 @@ private void doResume(PendingWorkflowInvocation pending, String body) { final WorkflowContext ctxt = refresh(newCtxt,retrieveRequestedSettings( wf.getRequiredSettings()), getCurrentApiToken(newCtxt.getRequest().getAuthenticatedUser())); WorkflowStepResult res = pendingStep.resume(ctxt, pending.getLocalData(), body); if (res instanceof Failure) { + logger.warning(((Failure) res).getReason()); userNotificationService.sendNotification(ctxt.getRequest().getAuthenticatedUser(), Timestamp.from(Instant.now()), UserNotification.Type.WORKFLOW_FAILURE, ctxt.getDataset().getLatestVersion().getId(), ((Failure) res).getMessage()); //UserNotification isn't meant to be a long-term record and doesn't store the comment, so we'll also keep it as a workflow comment WorkflowComment wfc = new WorkflowComment(ctxt.getDataset().getLatestVersion(), WorkflowComment.Type.WORKFLOW_FAILURE, ((Failure) res).getMessage(), ctxt.getRequest().getAuthenticatedUser()); @@ -202,6 +203,7 @@ private void doResume(PendingWorkflowInvocation pending, String body) { pauseAndAwait(wf, ctxt, (Pending) res, pending.getPendingStepIdx()); } else { if (res instanceof Success) { + logger.info(((Success) res).getReason()); userNotificationService.sendNotification(ctxt.getRequest().getAuthenticatedUser(), Timestamp.from(Instant.now()), UserNotification.Type.WORKFLOW_SUCCESS, ctxt.getDataset().getLatestVersion().getId(), ((Success) res).getMessage()); //UserNotification isn't meant to be a long-term record and doesn't store the comment, so we'll also keep it as a workflow comment WorkflowComment wfc = new WorkflowComment(ctxt.getDataset().getLatestVersion(), WorkflowComment.Type.WORKFLOW_SUCCESS, ((Success) res).getMessage(), ctxt.getRequest().getAuthenticatedUser()); diff --git a/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowUtil.java b/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowUtil.java index 327eecc4de4..aeb8bcf6c87 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowUtil.java @@ -74,7 +74,8 @@ public static WorkflowStepResult parseResponse(String externalData) { logger.log(Level.WARNING, "Remote system indicates workflow failed: {0}", reason); return new Failure(reason, message); default: - logger.log(Level.WARNING, "Remote system returned a response with no \"status\" key: {0}", externalData); + logger.log(Level.WARNING, "Remote system returned a response with no \"status\" key or bad status value: {0}", escapeHtml4(externalData)); + return new Failure("Workflow failure: Response from remote server doesn't have valid \"status\":" + escapeHtml4(externalData), null); } } catch (Exception e) { logger.log(Level.WARNING, "Remote system returned a bad response: {0}", externalData); From 8bb9dec0fc05cc516eda3858eb497292a5a9da25 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 23 Mar 2021 09:15:08 -0400 Subject: [PATCH 517/576] #7634 order by id within type --- src/main/java/edu/harvard/iq/dataverse/DvObject.java | 2 +- .../engine/command/impl/ListDataverseContentCommand.java | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObject.java b/src/main/java/edu/harvard/iq/dataverse/DvObject.java index f1041303fdd..09a2ef85893 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DvObject.java +++ b/src/main/java/edu/harvard/iq/dataverse/DvObject.java @@ -33,7 +33,7 @@ @NamedQuery(name = "DvObject.findByProtocolIdentifierAuthority", query = "SELECT o FROM DvObject o WHERE o.identifier=:identifier and o.authority=:authority and o.protocol=:protocol"), @NamedQuery(name = "DvObject.findByOwnerId", - query = "SELECT o FROM DvObject o WHERE o.owner.id=:ownerId"), + query = "SELECT o FROM DvObject o WHERE o.owner.id=:ownerId order by o.dtype desc, o.id"), @NamedQuery(name = "DvObject.findByAuthenticatedUserId", query = "SELECT o FROM DvObject o WHERE o.creator.id=:ownerId or o.releaseUser.id=:releaseUserId") }) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseContentCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseContentCommand.java index 3c606d35881..2e0f969bf0a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseContentCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseContentCommand.java @@ -7,9 +7,7 @@ import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; -import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.EnumSet; import java.util.List; import java.util.Map; @@ -34,14 +32,11 @@ public ListDataverseContentCommand(DataverseRequest aRequest, Dataverse anAffect @Override public List execute(CommandContext ctxt) throws CommandException { - List availableObjects = new ArrayList(); if (getRequest().getUser().isSuperuser()) { - availableObjects = ctxt.dvObjects().findByOwnerId(dvToList.getId()); + return ctxt.dvObjects().findByOwnerId(dvToList.getId()); } else { - availableObjects = ctxt.permissions().whichChildrenHasPermissionsForOrReleased(getRequest(), dvToList, EnumSet.of(Permission.ViewUnpublishedDataverse, Permission.ViewUnpublishedDataset)); + return ctxt.permissions().whichChildrenHasPermissionsForOrReleased(getRequest(), dvToList, EnumSet.of(Permission.ViewUnpublishedDataverse, Permission.ViewUnpublishedDataset)); } - availableObjects.sort(Comparator.comparing(DvObject::getDisplayName)); - return availableObjects; } @Override From 5dc0f0dac512db5caecab16328ee8adacbc6b6e1 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 23 Mar 2021 09:30:09 -0400 Subject: [PATCH 518/576] #7634 update doc to reflect collection ordering. --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 94cb6c720c6..54339d5749b 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -130,7 +130,7 @@ The fully expanded example above (without environment variables) looks like this Show Contents of a Dataverse Collection ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -|CORS| Lists all the Dataverse collections and datasets directly under a Dataverse collection (direct children only, not recursive) specified by database id or alias. If you pass your API token and have access, unpublished Dataverse collections and datasets will be included in the response. +|CORS| Lists all the Dataverse collections and datasets directly under a Dataverse collection (direct children only, not recursive) specified by database id or alias. If you pass your API token and have access, unpublished Dataverse collections and datasets will be included in the response. The list will be ordered by database id within type of object. That is, all Dataverse collections will be listed first and ordered by database id, then all datasets will be listed ordered by database id. .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. From 582250ea919511f0d055edf5ea970e18d3eaa880 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 23 Mar 2021 10:20:51 -0400 Subject: [PATCH 519/576] terminology updates --- .../source/developers/s3-direct-upload-api.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst index fe63b2f7d41..bbb1d70ae06 100644 --- a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst +++ b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst @@ -3,19 +3,19 @@ Direct DataFile Upload API The direct Datafile Upload API is used internally to support direct upload of files to S3 storage and by tools such as the DVUploader. -Direct upload involves a series of three activities: +Direct upload involves a series of three activities, each involving interacting with the server for a Dataverse installation: -* Requesting initiation of a transfer from the Dataverse server +* Requesting initiation of a transfer from the server * Use of the pre-signed URL(s) returned in that call to perform an upload/multipart-upload of the file to S3 -* A call to the Dataverse server to register the file as part of the dataset and/or to cancel the transfer +* A call to the server to register the file as part of the dataset and/or to cancel the transfer This API is only enabled when a Dataset is configured with a data store supporting direct S3 upload. -Administrators should be aware that partial transfers, where a client starts uploading the file/parts of the file and does not contact the Dataverse server to complete/cancel the transfer, will result in data stored in S3 that is not referenced in Dataverse (e.g. should be considered temporary and deleted.) +Administrators should be aware that partial transfers, where a client starts uploading the file/parts of the file and does not contact the server to complete/cancel the transfer, will result in data stored in S3 that is not referenced in the Dataverse installation (e.g. should be considered temporary and deleted.) Requesting Direct Upload of a DataFile -------------------------------------- -To initiate a transfer of a file to S3, make a call to the Dataverse Server indicating the size of the file to upload. The response will include a pre-signed URL(s) that allow the client to transfer the file. Pre-signed URLs include a short-lived token authorizing the action represented by the URL. +To initiate a transfer of a file to S3, make a call to the Dataverse installation indicating the size of the file to upload. The response will include a pre-signed URL(s) that allow the client to transfer the file. Pre-signed URLs include a short-lived token authorizing the action represented by the URL. .. code-block:: bash @@ -41,7 +41,7 @@ Single URL: when the file is smaller than the size at which uploads must be brok } } -Multiple URLs: when the file must be uploaded in multiple parts. The part size is set by the Dataverse server and, for AWS-based storage, range from 5 MB to 5 GB +Multiple URLs: when the file must be uploaded in multiple parts. The part size is set by the Dataverse installation and, for AWS-based storage, range from 5 MB to 5 GB .. code-block:: bash @@ -81,7 +81,7 @@ If the client is unable to complete the multipart upload, it should call the abo Adding the Uploaded file to the Dataset --------------------------------------- -Once the file exists in the s3 bucket, a final API call is needed to add it to the Dataset. This call is the same call used to upload a file to Dataverse but, rather than sending the file bytes, additional metadata is added using the "jsonData" parameter. +Once the file exists in the s3 bucket, a final API call is needed to add it to the Dataset. This call is the same call used to upload a file to a Dataverse installation but, rather than sending the file bytes, additional metadata is added using the "jsonData" parameter. jsonData normally includes information such as a file description, tags, provenance, whether the file is restricted, etc. For direct uploads, the jsonData object must also include values for: * "storageIdentifier" - String, as specified in prior calls @@ -104,7 +104,7 @@ The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.Data curl -X POST -H 'X-Dataverse-key: $API_TOKEN' "$SERVER_URL/api/datasets/:persistentId/add?persistentId=#PERSISTENT_IDENTIFIER" -F 'jsonData=$JSON_DATA' Note that this API call can be used independently of the others, e.g. supporting use cases in which the file already exists in S3/has been uploaded via some out-of-band method. -With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifer must be prefaced with the Dataverse store identifier used, as with the internally generated examples above. +With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifer must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above. From 82e884ef8fbd13dbf4caf6de2f2cca80d26be33b Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 23 Mar 2021 10:38:15 -0400 Subject: [PATCH 520/576] #7634 fix typo --- .../engine/command/impl/ListDataverseContentCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseContentCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseContentCommand.java index 2e0f969bf0a..26e1e988fac 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseContentCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseContentCommand.java @@ -33,7 +33,7 @@ public ListDataverseContentCommand(DataverseRequest aRequest, Dataverse anAffect @Override public List execute(CommandContext ctxt) throws CommandException { if (getRequest().getUser().isSuperuser()) { - return ctxt.dvObjects().findByOwnerId(dvToList.getId()); + return ctxt.dvObjects().findByOwnerId(dvToList.getId()); } else { return ctxt.permissions().whichChildrenHasPermissionsForOrReleased(getRequest(), dvToList, EnumSet.of(Permission.ViewUnpublishedDataverse, Permission.ViewUnpublishedDataset)); } From c71db98590811d28782429c890a5332a057fe716 Mon Sep 17 00:00:00 2001 From: Leonhard Maylein Date: Tue, 23 Mar 2021 16:07:55 +0100 Subject: [PATCH 521/576] Update harvestserver.rst Add an example for OAI Set search queries (subtreePaths). --- doc/sphinx-guides/source/admin/harvestserver.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/sphinx-guides/source/admin/harvestserver.rst b/doc/sphinx-guides/source/admin/harvestserver.rst index 73b458ee84b..95e68168893 100644 --- a/doc/sphinx-guides/source/admin/harvestserver.rst +++ b/doc/sphinx-guides/source/admin/harvestserver.rst @@ -73,6 +73,14 @@ Some useful examples of search queries to define OAI sets: ``parentId:NNN`` + where NNN is the database id of the Dataverse collection object (consult the Dataverse table of the SQL database used by the application to verify the database id). + + Note that this query does **not** provide datasets that are linked into the specified Dataverse collection. + +- A query to create a set to include the datasets from a specific Dataverse collection including datasets that have been deposited into other Dataverse collections but linked into the specified Dataverse collection: + + ``subtreePaths:"/NNN"`` + where NNN is the database id of the Dataverse collection object (consult the Dataverse table of the SQL database used by the application to verify the database id). - A query to find all the dataset by a certain author: From 5584602d0a17c094861b13db60ab1ab122d300bb Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 23 Mar 2021 13:28:44 -0400 Subject: [PATCH 522/576] same change for JsonPrinter - use helper --- .../iq/dataverse/util/json/JsonPrinter.java | 1 - .../util/json/JsonPrinterHelper.java | 25 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinterHelper.java diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index e86d1acff7a..96cb9c84dab 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -76,7 +76,6 @@ public class JsonPrinter { @EJB static SettingsServiceBean settingsService; - // Used in Tests public static void injectSettingsService(SettingsServiceBean ssb) { settingsService = ssb; } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinterHelper.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinterHelper.java new file mode 100644 index 00000000000..62f3569bb8d --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinterHelper.java @@ -0,0 +1,25 @@ +package edu.harvard.iq.dataverse.util.json; + +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; + +import javax.annotation.PostConstruct; +import javax.ejb.EJB; +import javax.ejb.Singleton; +import javax.ejb.Startup; + +/** + * This is a small helper bean + * As it is a singleton and built at application start (=deployment), it will inject the (stateless) + * settings service into the OREMap once it's ready. + */ +@Singleton +@Startup +public class JsonPrinterHelper { + @EJB + SettingsServiceBean settingsSvc; + + @PostConstruct + public void injectService() { + JsonPrinter.injectSettingsService(settingsSvc); + } +} From f28fcba5a15096b6a03d7119c33e7cf039c9f479 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 23 Mar 2021 14:15:42 -0400 Subject: [PATCH 523/576] #7633 reformat required messaging --- src/main/webapp/roles-edit.xhtml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/webapp/roles-edit.xhtml b/src/main/webapp/roles-edit.xhtml index 4748f24df5d..e236ef180b3 100644 --- a/src/main/webapp/roles-edit.xhtml +++ b/src/main/webapp/roles-edit.xhtml @@ -12,31 +12,31 @@
    - + /> +
    - - + +
    From 99d47bcc2726dc0909c46edb11827b39f588842f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 23 Mar 2021 16:12:39 -0400 Subject: [PATCH 524/576] fix null when sent in Workflow Failure --- src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java index 4b9e595374d..df35ce16304 100644 --- a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java @@ -500,6 +500,7 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio case WORKFLOW_SUCCESS: version = (DatasetVersion) targetObject; pattern = BundleUtil.getStringFromBundle("notification.email.workflow.success"); + if (comment == null) { comment = BundleUtil.getStringFromBundle("notification.email.workflow.nullMessage"); } @@ -509,10 +510,10 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio case WORKFLOW_FAILURE: version = (DatasetVersion) targetObject; pattern = BundleUtil.getStringFromBundle("notification.email.workflow.failure"); - String[] paramArrayWorkflowFailure = {version.getDataset().getDisplayName(), getDatasetLink(version.getDataset()), comment}; if (comment == null) { comment = BundleUtil.getStringFromBundle("notification.email.workflow.nullMessage"); } + String[] paramArrayWorkflowFailure = {version.getDataset().getDisplayName(), getDatasetLink(version.getDataset()), comment}; messageText += MessageFormat.format(pattern, paramArrayWorkflowFailure); return messageText; case CREATEACC: From d00d0ec9493c6d820be29ea38f1429975cde0a62 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 23 Mar 2021 16:51:30 -0400 Subject: [PATCH 525/576] rename SQL script #2419 #4475 --- ...9-deactivate-users.sql => V5.3.0.6__2419-deactivate-users.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V5.3.0.5__2419-deactivate-users.sql => V5.3.0.6__2419-deactivate-users.sql} (100%) diff --git a/src/main/resources/db/migration/V5.3.0.5__2419-deactivate-users.sql b/src/main/resources/db/migration/V5.3.0.6__2419-deactivate-users.sql similarity index 100% rename from src/main/resources/db/migration/V5.3.0.5__2419-deactivate-users.sql rename to src/main/resources/db/migration/V5.3.0.6__2419-deactivate-users.sql From 81274c9f65ac839df101e850d50d9fd153052648 Mon Sep 17 00:00:00 2001 From: Gustavo Durand Date: Wed, 24 Mar 2021 12:07:12 -0400 Subject: [PATCH 526/576] Update DataversePage.java --- src/main/java/edu/harvard/iq/dataverse/DataversePage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java index 2e5ec9840ed..c7f816ce219 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java @@ -695,7 +695,7 @@ public String save() { } catch (CommandException ex) { logger.log(Level.SEVERE, "Unexpected Exception calling dataverse command", ex); - String errMsg = create ? ex.getLocalizedMessage() : BundleUtil.getStringFromBundle("dataverse.update.failure"); + String errMsg = create ? BundleUtil.getStringFromBundle("dataverse.create.failure") : BundleUtil.getStringFromBundle("dataverse.update.failure"); JH.addMessage(FacesMessage.SEVERITY_FATAL, errMsg); return null; } catch (Exception e) { From 492e0b2b226378678ddbfa3b0fdc9a728c2a0ba2 Mon Sep 17 00:00:00 2001 From: "don.sizemore" Date: Wed, 24 Mar 2021 14:59:30 -0400 Subject: [PATCH 527/576] #7722 correct datavarse typo, fix underscore syntax --- doc/sphinx-guides/source/admin/solr-search-index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/admin/solr-search-index.rst b/doc/sphinx-guides/source/admin/solr-search-index.rst index adf30b90425..5685672eceb 100644 --- a/doc/sphinx-guides/source/admin/solr-search-index.rst +++ b/doc/sphinx-guides/source/admin/solr-search-index.rst @@ -67,13 +67,13 @@ Reindexing Dataverse Collections Dataverse collections must be referenced by database object ID. If you have direct database access an SQL query such as -``select id from dataverse where alias='datavarsealias';`` +``select id from dataverse where alias='dataversealias';`` should work, or you may click the Dataverse Software's "Edit" menu and look for dataverseId= in the URLs produced by the drop-down. Then, to re-index: ``curl http://localhost:8080/api/admin/index/dataverses/135`` -which should return: _{"status":"OK","data":{"message":"starting reindex of dataverse 135"}}_ +which should return: *{"status":"OK","data":{"message":"starting reindex of dataverse 135"}}* Reindexing Datasets ++++++++++++++++++++ From 569136f5ae4967c442b527aa0cb0066961c7abba Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 25 Mar 2021 14:55:47 -0400 Subject: [PATCH 528/576] prevent user table, user-list API from blowing up #2419 #4475 The createAuthenticatedUserForView method can't handle a null for the deactivated boolean so we'll set the boolean to false for existing users. For new users, this is set to false already. --- .../resources/db/migration/V5.3.0.6__2419-deactivate-users.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/resources/db/migration/V5.3.0.6__2419-deactivate-users.sql b/src/main/resources/db/migration/V5.3.0.6__2419-deactivate-users.sql index c87fe019a52..81f01107be3 100644 --- a/src/main/resources/db/migration/V5.3.0.6__2419-deactivate-users.sql +++ b/src/main/resources/db/migration/V5.3.0.6__2419-deactivate-users.sql @@ -1,4 +1,6 @@ -- Users can be deactivated. ALTER TABLE authenticateduser ADD COLUMN IF NOT EXISTS deactivated BOOLEAN; +-- Prevent old users from having null for deactivated. +UPDATE authenticateduser SET deactivated = false; -- A timestamp of when the user was deactivated. ALTER TABLE authenticateduser ADD COLUMN IF NOT EXISTS deactivatedtime timestamp without time zone; From c326af4c5b9d2c8f421cf9eb0bb5e5267303c481 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 25 Mar 2021 16:54:37 -0400 Subject: [PATCH 529/576] deactivated users cannot become superusers #2419 #4475 --- doc/sphinx-guides/source/api/native-api.rst | 1 + .../edu/harvard/iq/dataverse/api/Admin.java | 3 +++ .../impl/GrantSuperuserStatusCommand.java | 4 ++++ src/main/webapp/dashboard-users.xhtml | 3 ++- .../iq/dataverse/api/DeactivateUsersIT.java | 18 ++++++++++++++++++ 5 files changed, 28 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 5f76bb4ba0b..85e65d26694 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3229,6 +3229,7 @@ Deactivating a user with this endpoint will: - Remove the user from all groups - No longer have notifications generated or sent by the Dataverse installation - Prevent the account from being converted into an OAuth or Shibboleth account. +- Prevent the user from becoming a superuser. Deactivating a user with this endpoint will keep: diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 7343b11ed3f..f0a9f8cf780 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -945,6 +945,9 @@ public Response toggleSuperuser(@PathParam("identifier") String identifier) { .setInfo(identifier); try { AuthenticatedUser user = authSvc.getAuthenticatedUser(identifier); + if (user.isDeactivated()) { + return error(Status.BAD_REQUEST, "You cannot make a deactivated user a superuser."); + } user.setSuperuser(!user.isSuperuser()); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GrantSuperuserStatusCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GrantSuperuserStatusCommand.java index 29f1b891c91..42af43b7247 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GrantSuperuserStatusCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GrantSuperuserStatusCommand.java @@ -40,6 +40,10 @@ protected void executeImpl(CommandContext ctxt) throws CommandException { this, null, null); } + if (targetUser.isDeactivated()) { + throw new CommandException("User " + targetUser.getIdentifier() + " has been deactivated and cannot become a superuser.", this); + } + try { targetUser.setSuperuser(true); ctxt.em().merge(targetUser); diff --git a/src/main/webapp/dashboard-users.xhtml b/src/main/webapp/dashboard-users.xhtml index ebb447dfe30..3f6087cf01c 100644 --- a/src/main/webapp/dashboard-users.xhtml +++ b/src/main/webapp/dashboard-users.xhtml @@ -85,7 +85,8 @@ - + + diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java index 25fa03dca79..374a19f453d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java @@ -271,4 +271,22 @@ public void testMergeUserIntoSelf() { mergeAccounts.then().assertThat().statusCode(BAD_REQUEST.getStatusCode()); } + @Test + public void testTurnDeactivatedUserIntoSuperuser() { + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String username = UtilIT.getUsernameFromResponse(createUser); + + Response deactivateUser = UtilIT.deactivateUser(username); + deactivateUser.prettyPrint(); + deactivateUser.then().assertThat().statusCode(OK.getStatusCode()); + + Response toggleSuperuser = UtilIT.makeSuperUser(username); + toggleSuperuser.prettyPrint(); + toggleSuperuser.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()); + + } + } From ca93b5bcd140961a65ec58d567b84432ccc132b5 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 25 Mar 2021 22:47:33 -0400 Subject: [PATCH 530/576] after battling with python and rst previewer, I have completed my minor edit --- .../source/developers/s3-direct-upload-api.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst index bbb1d70ae06..eaeba4fcfaa 100644 --- a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst +++ b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst @@ -88,9 +88,10 @@ jsonData normally includes information such as a file description, tags, provena * "fileName" - String * "mimeType" - String * "fileSize" - number of bytes -* fixity/checksum: either: -** "md5Hash" - String with MD5 hash value, or -** "checksum" - Json Object with "@type" field specifying the algorithm used and "@value" field with the value from that algorithm, both Strings +* fixity/checksum: either: + + * "md5Hash" - String with MD5 hash value, or + * "checksum" - Json Object with "@type" field specifying the algorithm used and "@value" field with the value from that algorithm, both Strings The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.DataFile.CheckSumType class and currently include MD5, SHA-1, SHA-256, and SHA-512 From 3e9b3536c245a00ebdccf40a8082050d71206ae6 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 26 Mar 2021 12:22:11 -0400 Subject: [PATCH 531/576] revert to using setting (versus microprofile) per discussion in issues --- .../dataverse/export/ddi/DdiExportUtil.java | 13 +++++++--- .../export/ddi/DdiExportUtilHelper.java | 25 +++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilHelper.java diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 4a7765c0979..4e3eab65cde 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -27,6 +27,8 @@ import static edu.harvard.iq.dataverse.export.DDIExportServiceBean.NOTE_TYPE_TAG; import static edu.harvard.iq.dataverse.export.DDIExportServiceBean.NOTE_TYPE_UNF; import edu.harvard.iq.dataverse.export.DDIExporter; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; + import static edu.harvard.iq.dataverse.util.SystemConfig.FQDN; import static edu.harvard.iq.dataverse.util.SystemConfig.SITE_URL; import edu.harvard.iq.dataverse.util.json.JsonUtil; @@ -56,7 +58,6 @@ import org.xml.sax.SAXParseException; import org.w3c.dom.Document; import org.apache.commons.lang3.StringUtils; -import org.eclipse.microprofile.config.ConfigProvider; import org.w3c.dom.DOMException; // For write operation @@ -81,8 +82,8 @@ public class DdiExportUtil { public static final String LEVEL_DV = "dv"; - @EJB - VariableServiceBean variableService; + + static SettingsServiceBean settingsService; public static final String NOTE_TYPE_CONTENTTYPE = "DATAVERSE:CONTENTTYPE"; public static final String NOTE_SUBJECT_CONTENTTYPE = "Content/MIME Type"; @@ -207,7 +208,7 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) } logger.info("Dsitr set?: " + distributorSet); logger.info("Pub?: " + datasetDto.getPublisher()); - boolean excludeRepository = ConfigProvider.getConfig().getOptionalValue("dataverse.export.distributor.excludeinstallationifset", Boolean.class).orElse(false); + boolean excludeRepository = settingsService.isTrueForKey(SettingsServiceBean.Key.ExportInstallationAsDistributorOnlyWhenNotSet, false); logger.info("Exclude: " + excludeRepository); if (!StringUtils.isEmpty(datasetDto.getPublisher()) && !(excludeRepository && distributorSet)) { xmlw.writeStartElement("distrbtr"); @@ -1799,4 +1800,8 @@ public static void datasetHtmlDDI(InputStream datafile, OutputStream outputStrea } + public static void injectSettingsService(SettingsServiceBean settingsSvc) { + settingsService=settingsSvc; + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilHelper.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilHelper.java new file mode 100644 index 00000000000..149c6791a7e --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilHelper.java @@ -0,0 +1,25 @@ +package edu.harvard.iq.dataverse.export.ddi; + +import javax.annotation.PostConstruct; +import javax.ejb.EJB; +import javax.ejb.Singleton; +import javax.ejb.Startup; + +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; + + /** + * This is a small helper bean + * As it is a singleton and built at application start (=deployment), it will inject the (stateless) + * dataverse service into the DdiExportUtil once it's ready. + */ + @Singleton + @Startup + public class DdiExportUtilHelper { + + @EJB SettingsServiceBean settingsSvc; + + @PostConstruct + public void injectService() { + DdiExportUtil.injectSettingsService(settingsSvc); + } + } \ No newline at end of file From 848f5331b45d0eb6e6d10cd5c4f427f058a13ec2 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 26 Mar 2021 12:50:24 -0400 Subject: [PATCH 532/576] update tests --- .../export/ddi/DdiExportUtilTest.java | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilTest.java index a76ce8475f2..dce34385274 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilTest.java @@ -1,30 +1,42 @@ package edu.harvard.iq.dataverse.export.ddi; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.xml.XmlPrinter; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; -import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Paths; import java.util.logging.Logger; import edu.harvard.iq.dataverse.util.xml.html.HtmlPrinter; -import org.junit.Test; -import javax.json.JsonObject; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.MethodOrderer.OrderAnnotation; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; import static org.junit.Assert.*; +@ExtendWith(MockitoExtension.class) +@TestMethodOrder(OrderAnnotation.class) public class DdiExportUtilTest { private static final Logger logger = Logger.getLogger(DdiExportUtilTest.class.getCanonicalName()); + @Mock + SettingsServiceBean settingsSvc; + @Test + @Order(1) public void testJson2DdiNoFiles() throws Exception { + Mockito.when(settingsSvc.isTrueForKey(SettingsServiceBean.Key.ExportInstallationAsDistributorOnlyWhenNotSet, false)).thenReturn(false); + DdiExportUtil.injectSettingsService(settingsSvc); File datasetVersionJson = new File("src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.json"); String datasetVersionAsJson = new String(Files.readAllBytes(Paths.get(datasetVersionJson.getAbsolutePath()))); File ddiFile = new File("src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml"); From 36f25bb179d4527aeb08d5b90554f330c8771318 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 26 Mar 2021 15:20:37 -0400 Subject: [PATCH 533/576] #2419 retest session user on save --- .../authorization/providers/builtin/DataverseUserPage.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java index 59843495b94..673839450d6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java @@ -284,6 +284,12 @@ public void validateNewPassword(FacesContext context, UIComponent toValidate, Ob public String save() { boolean passwordChanged = false; + + //First reget user to make sure they weren't deactivated or deleted + if (session.getUser().isAuthenticated() && !session.getUser(true).isAuthenticated()) { + return "dataverse.xhtml?alias=" + dataverseService.findRootDataverse().getAlias() + "&faces-redirect=true"; + } + if (editMode == EditMode.CHANGE_PASSWORD) { final AuthenticationProvider prv = getUserAuthProvider(); if (prv.isPasswordUpdateAllowed()) { From 5cef7ca8314384bf241d5ddd051d5a50806f43d7 Mon Sep 17 00:00:00 2001 From: gdurand Date: Fri, 26 Mar 2021 16:16:40 -0400 Subject: [PATCH 534/576] adding replace to kebabs plus clean uo --- .../edu/harvard/iq/dataverse/DataFile.java | 6 +-- .../iq/dataverse/DataFileServiceBean.java | 49 ----------------- .../edu/harvard/iq/dataverse/DatasetPage.java | 9 ++++ .../iq/dataverse/EditDatafilesPage.java | 43 ++++++++++----- .../edu/harvard/iq/dataverse/FilePage.java | 52 ++----------------- .../datasetutility/FileReplacePageHelper.java | 2 +- src/main/java/propertyFiles/Bundle.properties | 10 ++-- src/main/webapp/editdatafiles.xhtml | 16 +++--- .../webapp/file-edit-button-fragment.xhtml | 26 +++------- .../webapp/file-edit-popup-fragment.xhtml | 20 +++++-- src/main/webapp/file.xhtml | 43 ++++----------- src/main/webapp/filesFragment.xhtml | 13 ++++- 12 files changed, 105 insertions(+), 184 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java index cb2b6e221e8..2b371d520b3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java @@ -239,13 +239,13 @@ public DataFile(String contentType) { */ @Transient - private boolean deleted; + private Boolean deleted; - public boolean isDeleted() { + public Boolean isDeleted() { return deleted; } - public void setDeleted(boolean deleted) { + public void setDeleted(Boolean deleted) { this.deleted = deleted; } diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java index 094e20edf28..811be6ab40e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java @@ -1363,55 +1363,6 @@ public boolean isFileClassPackage (DataFile file) { public void populateFileSearchCard(SolrSearchResult solrSearchResult) { solrSearchResult.setEntity(this.findCheapAndEasy(solrSearchResult.getEntityId())); } - - - /** - * Does this file have a replacement. - * Any file should have AT MOST 1 replacement - * - * @param df - * @return - * @throws java.lang.Exception if a DataFile has more than 1 replacement - * or is unpublished and has a replacement. - */ - public boolean hasReplacement(DataFile df) throws Exception{ - - if (df.getId() == null){ - // An unsaved file cannot have a replacment - return false; - } - - - List dataFiles = em.createQuery("select o from DataFile o" + - " WHERE o.previousDataFileId = :dataFileId", DataFile.class) - .setParameter("dataFileId", df.getId()) - .getResultList(); - - if (dataFiles.isEmpty()){ - return false; - } - - if (!df.isReleased()){ - // An unpublished SHOULD NOT have a replacment - String errMsg = "DataFile with id: [" + df.getId() + "] is UNPUBLISHED with a REPLACEMENT. This should NOT happen."; - logger.severe(errMsg); - - throw new Exception(errMsg); - } - - - - else if (dataFiles.size() == 1){ - return true; - }else{ - - String errMsg = "DataFile with id: [" + df.getId() + "] has more than one replacment!"; - logger.severe(errMsg); - - throw new Exception(errMsg); - } - - } public boolean hasBeenDeleted(DataFile df){ Dataset dataset = df.getOwner(); diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 45a4531e60a..e143f4a263f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -5510,4 +5510,13 @@ public boolean isFileAccessRequest() { public void setFileAccessRequest(boolean fileAccessRequest) { this.fileAccessRequest = fileAccessRequest; } + + // wrapper method to see if the file has been deleted (or replaced) in the current version + public boolean isFileDeleted (DataFile dataFile) { + if (dataFile.isDeleted() == null) { + dataFile.setDeleted(datafileService.hasBeenDeleted(dataFile)); + } + + return dataFile.isDeleted(); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index f788fd95cae..058d7be230a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -92,7 +92,11 @@ public class EditDatafilesPage implements java.io.Serializable { public enum FileEditMode { - EDIT, UPLOAD, CREATE, SINGLE, SINGLE_REPLACE + EDIT, UPLOAD, CREATE, REPLACE, TEST + }; + + public enum Referrer { + DATASET, FILE }; @EJB @@ -138,6 +142,7 @@ public enum FileEditMode { private String selectedFileIdsString = null; private FileEditMode mode; + private Referrer referrer = Referrer.DATASET; private List selectedFileIdsList = new ArrayList<>(); private List fileMetadatas = new ArrayList<>();; @@ -207,6 +212,16 @@ public FileEditMode getMode() { public void setMode(FileEditMode mode) { this.mode = mode; } + + public Referrer getReferrer() { + return referrer; + } + + public void setReferrer(Referrer referrer) { + this.referrer = referrer; + } + + public List getFileMetadatas() { @@ -503,9 +518,9 @@ public String init() { // ------------------------------------------- // Is this a file replacement operation? // ------------------------------------------- - if (mode == FileEditMode.SINGLE_REPLACE){ + if (mode == FileEditMode.REPLACE){ /* - http://localhost:8080/editdatafiles.xhtml?mode=SINGLE_REPLACE&datasetId=26&fid=726 + http://localhost:8080/editdatafiles.xhtml?mode=REPLACE&datasetId=26&fid=726 */ DataFile fileToReplace = loadFileToReplace(); if (fileToReplace == null){ @@ -527,7 +542,7 @@ public String init() { populateFileMetadatas(); singleFile = getFileToReplace(); - }else if (mode == FileEditMode.EDIT || mode == FileEditMode.SINGLE) { + }else if (mode == FileEditMode.EDIT) { if (selectedFileIdsString != null) { String[] ids = selectedFileIdsString.split(","); @@ -541,7 +556,7 @@ public String init() { test = null; } if (test != null) { - if (mode == FileEditMode.SINGLE) { + if (FileEditMode.EDIT == mode && Referrer.FILE == referrer) { singleFile = datafileService.find(test); } selectedFileIdsList.add(test); @@ -569,7 +584,7 @@ public String init() { return permissionsWrapper.notFound(); } - if (FileEditMode.SINGLE == mode){ + if (FileEditMode.EDIT == mode && Referrer.FILE == referrer){ if (fileMetadatas.get(0).getDatasetVersion().getId() != null){ versionString = "DRAFT"; } @@ -896,7 +911,11 @@ public String saveReplacementFile() throws FileReplaceException{ if (fileReplacePageHelper.runSaveReplacementFile_Phase2()){ JsfHelper.addSuccessMessage(getBundleString("file.message.replaceSuccess")); // It worked!!! Go to page of new file!! - return returnToFileLandingPageAfterReplace(fileReplacePageHelper.getFirstNewlyAddedFile()); + if (Referrer.FILE == referrer) { + return returnToFileLandingPageAfterReplace(fileReplacePageHelper.getFirstNewlyAddedFile()); + } else { + return returnToDraftVersion(); + } }else{ // Uh oh. String errMsg = fileReplacePageHelper.getErrorMessages(); @@ -1114,7 +1133,7 @@ public String save() { workingVersion = dataset.getEditVersion(); logger.fine("working version id: "+workingVersion.getId()); - if (mode == FileEditMode.SINGLE){ + if (FileEditMode.EDIT == mode && Referrer.FILE == referrer){ JsfHelper.addSuccessMessage(getBundleString("file.message.editSuccess")); } else { @@ -1137,7 +1156,7 @@ public String save() { ingestService.startIngestJobsForDataset(dataset, (AuthenticatedUser) session.getUser()); } - if (mode == FileEditMode.SINGLE && fileMetadatas.size() > 0) { + if (FileEditMode.EDIT == mode && Referrer.FILE == referrer && fileMetadatas.size() > 0) { // If this was a "single file edit", i.e. an edit request sent from // the individual File Landing page, we want to redirect back to // the landing page. BUT ONLY if the file still exists - i.e., if @@ -1197,7 +1216,7 @@ public String cancel() { FileUtil.deleteTempFile(newFile, dataset, ingestService); } - if (mode == FileEditMode.SINGLE || mode == FileEditMode.SINGLE_REPLACE ) { + if (Referrer.FILE == referrer) { return returnToFileLandingPage(); } if (workingVersion.getId() != null) { @@ -1254,7 +1273,7 @@ private HttpClient getClient() { * @return */ public boolean isFileReplaceOperation(){ - return (mode == FileEditMode.SINGLE_REPLACE)&&(fileReplacePageHelper!= null); + return (mode == FileEditMode.REPLACE)&&(fileReplacePageHelper!= null); } public boolean allowMultipleFileUpload(){ @@ -1263,7 +1282,7 @@ public boolean allowMultipleFileUpload(){ } public boolean showFileUploadFragment(){ - return mode == FileEditMode.UPLOAD || mode == FileEditMode.CREATE || mode == FileEditMode.SINGLE_REPLACE; + return mode == FileEditMode.UPLOAD || mode == FileEditMode.CREATE || mode == FileEditMode.REPLACE; } diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index 575b6e01fa6..ac237793ed3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -786,57 +786,15 @@ private List allRelatedFiles() { return dataFiles; } - public boolean isDraftReplacementFile(){ - /* - This method tests to see if the file has been replaced in a draft version of the dataset - Since it must must work when you are on prior versions of the dataset - it must accrue all replacement files that may have been created - */ - if(null == dataset) { - dataset = fileMetadata.getDataFile().getOwner(); - } - - DataFile dataFileToTest = fileMetadata.getDataFile(); - - DatasetVersion currentVersion = dataset.getLatestVersion(); - - if (!currentVersion.isDraft()){ - return false; - } - - if (dataset.getReleasedVersion() == null){ - return false; - } - - List dataFiles = new ArrayList<>(); - - dataFiles.add(dataFileToTest); - - while (datafileService.findReplacementFile(dataFileToTest.getId()) != null ){ - dataFiles.add(datafileService.findReplacementFile(dataFileToTest.getId())); - dataFileToTest = datafileService.findReplacementFile(dataFileToTest.getId()); - } - - if(dataFiles.size() <2){ - return false; - } - - int numFiles = dataFiles.size(); - - DataFile current = dataFiles.get(numFiles - 1 ); - - DatasetVersion publishedVersion = dataset.getReleasedVersion(); - - if( datafileService.findFileMetadataByDatasetVersionIdAndDataFileId(publishedVersion.getId(), current.getId()) == null){ - return true; + // wrappermethod to see if the file has been deleted (or replaced) in the current version + public boolean isDeletedFile () { + if (file.isDeleted() == null) { + file.setDeleted(datafileService.hasBeenDeleted(file)); } - return false; + return file.isDeleted(); } - - - /** * To help with replace development * @return diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/FileReplacePageHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/FileReplacePageHelper.java index 6d9d3f1801c..6e0c16a93e0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/FileReplacePageHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/FileReplacePageHelper.java @@ -19,7 +19,7 @@ * Phase 1: File successfully uploaded and unpersisted DataFile is in memory * Phase 2: Save the files * - http://localhost:8080/editdatafiles.xhtml?mode=SINGLE_REPLACE&datasetId=26&fid=726 + http://localhost:8080/editdatafiles.xhtml?mode=REPLACE&datasetId=26&fid=726 * This is messy, trying to contain some of it--give me APIs or more time, more time:) * * @author rmp553 diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 33a3d2be286..2f211d81c36 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1531,10 +1531,8 @@ file.notFound.search=There are no files that match your search. Please change th file.noSelectedFiles.tip=There are no selected files to display. file.noUploadedFiles.tip=Files you upload will appear here. file.replace=Replace -file.replaced.warning.header=Edit File -file.replaced.warning.draft.warningMessage=You can not replace a published file that has already been replaced in a dataset draft. You can replace the file that is now in the draft version. -file.replaced.warning.previous.warningMessage=You can not edit a file that has been replaced in a previous dataset version. In order to edit it you must go to the most recently published version of the file. -file.alreadyDeleted.previous.warningMessage=This file has already been deleted in current version. It may not be edited. +file.alreadyDeleted.warning.header=Edit File +file.alreadyDeleted.previous.warningMessage=This file has already been deleted (or replaced) in the current version. It may not be edited. file.delete=Delete file.delete.duplicate.multiple=Delete Duplicate Files file.delete.duplicate.single=Delete Duplicate File @@ -1914,9 +1912,9 @@ file.metadata.datafiletag.not_tabular=You cannot add Tabular Data Tags to a non- file.metadata.filedirectory.invalidCharacters=Directory Name cannot contain invalid characters. Valid characters are a-Z, 0-9, '_', '-', '.', '\\', '/' and ' ' (white space). # File Edit Success -file.message.editSuccess=This file has been updated. +file.message.editSuccess=The file has been updated. file.message.deleteSuccess=The file has been deleted. -file.message.replaceSuccess=This file has been replaced. +file.message.replaceSuccess=The file has been replaced. # File Add/Replace operation messages file.addreplace.file_size_ok=File size is in range. diff --git a/src/main/webapp/editdatafiles.xhtml b/src/main/webapp/editdatafiles.xhtml index 01c5ccc9123..aa4eb5f228c 100644 --- a/src/main/webapp/editdatafiles.xhtml +++ b/src/main/webapp/editdatafiles.xhtml @@ -15,7 +15,7 @@ - + @@ -25,14 +25,15 @@ + - - + + @@ -53,9 +54,8 @@
    - +
    diff --git a/src/main/webapp/file-edit-button-fragment.xhtml b/src/main/webapp/file-edit-button-fragment.xhtml index 55b20c081e8..ef5df7acb3d 100644 --- a/src/main/webapp/file-edit-button-fragment.xhtml +++ b/src/main/webapp/file-edit-button-fragment.xhtml @@ -22,7 +22,6 @@ dataserVersion - the datasetversion object to associate with this view fileMetadata - for single file, the fileMetadata object of that file fileMetadataForAction - used by DatasetPage popups to identify single file (also clears for selected file) - isDraftReplacementFile - for single file, if the file is a draft and already replacing a past file configureTools - for single file, list of configureTools for the file bean - the named value of the backing bean for the below method(s), also used by isFilePg param unrestrictFileAction - name of the method on the above bean to call for unrestrict (method must take a boolean) @@ -38,7 +37,7 @@
  • @@ -71,22 +70,13 @@ - - - -
  • - - - -
    • - - -
  • - - - -
    • -     + + +
  • + + + +
    • diff --git a/src/main/webapp/file-edit-popup-fragment.xhtml b/src/main/webapp/file-edit-popup-fragment.xhtml index 7fe8fb4c083..707a5f26721 100644 --- a/src/main/webapp/file-edit-popup-fragment.xhtml +++ b/src/main/webapp/file-edit-popup-fragment.xhtml @@ -13,7 +13,17 @@ restrictFileAction - name of the method on the above bean to call for restrict (method must take a boolean) deleteFileAction - name of method on the above bean to delete files --> - + + + +

    #{bundle['file.alreadyDeleted.previous.warningMessage']}

    +
    + +
    +     @@ -71,8 +81,8 @@ - - + +
    - - + + \ No newline at end of file diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml index 34717bfae3b..06ff420a9d1 100644 --- a/src/main/webapp/file.xhtml +++ b/src/main/webapp/file.xhtml @@ -210,15 +210,15 @@
    - + #{bundle['file.editBtn']} - +
    + and !FilePage.deletedFile}"> @@ -642,31 +641,7 @@ #{bundle.close}
    - - -

    #{bundle['file.replaced.warning.draft.warningMessage']}

    -
    - -
    -     - - - -

    #{bundle['file.replaced.warning.previous.warningMessage']}

    -     - -

    #{bundle['file.alreadyDeleted.previous.warningMessage']}

    -     -
    - -
    -     +

    #{bundle['file.compute.fileAccessDenied']}

    diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index f3523b63853..c18dcf0d41a 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -530,7 +530,18 @@
    + and DatasetPage.canUpdateDataset() + and DatasetPage.isFileDeleted(fileMetadata.dataFile)}"> + + #{bundle['file.optionsBtn']} + +
    +
    Date: Fri, 26 Mar 2021 16:26:54 -0400 Subject: [PATCH 535/576] removed accidental TEST from enum --- src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 058d7be230a..3eace887e36 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -92,7 +92,7 @@ public class EditDatafilesPage implements java.io.Serializable { public enum FileEditMode { - EDIT, UPLOAD, CREATE, REPLACE, TEST + EDIT, UPLOAD, CREATE, REPLACE }; public enum Referrer { From 65e07e094db8929c2215e4cf25769ec27e823b86 Mon Sep 17 00:00:00 2001 From: gdurand Date: Fri, 26 Mar 2021 16:52:40 -0400 Subject: [PATCH 536/576] changed isDeleted to getDeleted (since I changed from a boolean to a Boolean) --- src/main/java/edu/harvard/iq/dataverse/DataFile.java | 2 +- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 4 ++-- src/main/java/edu/harvard/iq/dataverse/FilePage.java | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java index 2b371d520b3..39ac0e62695 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java @@ -241,7 +241,7 @@ public DataFile(String contentType) { @Transient private Boolean deleted; - public Boolean isDeleted() { + public Boolean getDeleted() { return deleted; } diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index e143f4a263f..949ea022987 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -5513,10 +5513,10 @@ public void setFileAccessRequest(boolean fileAccessRequest) { // wrapper method to see if the file has been deleted (or replaced) in the current version public boolean isFileDeleted (DataFile dataFile) { - if (dataFile.isDeleted() == null) { + if (dataFile.getDeleted() == null) { dataFile.setDeleted(datafileService.hasBeenDeleted(dataFile)); } - return dataFile.isDeleted(); + return dataFile.getDeleted(); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index ac237793ed3..d863500d137 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -788,11 +788,11 @@ private List allRelatedFiles() { // wrappermethod to see if the file has been deleted (or replaced) in the current version public boolean isDeletedFile () { - if (file.isDeleted() == null) { + if (file.getDeleted() == null) { file.setDeleted(datafileService.hasBeenDeleted(file)); } - return file.isDeleted(); + return file.getDeleted(); } /** From 274f150d7cd1382245856fdbdc9b64f5b9412e85 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 29 Mar 2021 09:34:53 -0400 Subject: [PATCH 537/576] #2419 update update query --- .../resources/db/migration/V5.3.0.6__2419-deactivate-users.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/db/migration/V5.3.0.6__2419-deactivate-users.sql b/src/main/resources/db/migration/V5.3.0.6__2419-deactivate-users.sql index 81f01107be3..a5e4b69e00b 100644 --- a/src/main/resources/db/migration/V5.3.0.6__2419-deactivate-users.sql +++ b/src/main/resources/db/migration/V5.3.0.6__2419-deactivate-users.sql @@ -1,6 +1,6 @@ -- Users can be deactivated. ALTER TABLE authenticateduser ADD COLUMN IF NOT EXISTS deactivated BOOLEAN; -- Prevent old users from having null for deactivated. -UPDATE authenticateduser SET deactivated = false; +UPDATE authenticateduser SET deactivated = false WHERE deactivated IS NULL; -- A timestamp of when the user was deactivated. ALTER TABLE authenticateduser ADD COLUMN IF NOT EXISTS deactivatedtime timestamp without time zone; From 7946f187254cc9c8124b647bf1583dd70c911690 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Mon, 29 Mar 2021 10:22:31 -0400 Subject: [PATCH 538/576] add period, add formatting to messages --- src/main/java/propertyFiles/Bundle.properties | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index ef698acaa71..ccca93e6c5e 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1407,7 +1407,7 @@ dataset.message.locked.downloadNotAllowedInReview=Dataset file(s) may not be dow dataset.message.locked.downloadNotAllowed=Dataset file(s) may not be downloaded due to dataset lock. dataset.message.locked.editNotAllowed=Dataset cannot be edited due to dataset lock. dataset.message.locked.publishNotAllowed=Dataset cannot be published due to dataset lock. -dataset.message.createSuccess=This dataset has been created +dataset.message.createSuccess=This dataset has been created. dataset.message.createSuccess.failedToSaveFiles=Partial Success: The dataset has been created. But the file(s) could not be saved. Please try uploading the file(s) again. dataset.message.createSuccess.partialSuccessSavingFiles=Partial Success: The dataset has been created. But only {0} out of {1} files have been saved. Please try uploading the missing file(s) again. dataset.message.linkSuccess= {0} has been successfully linked to {1}. @@ -1416,10 +1416,10 @@ dataset.message.termsSuccess=The terms for this dataset have been updated. dataset.message.filesSuccess=The files for this dataset have been updated. dataset.message.addFiles.Failure=Failed to add files to the dataset. Please try uploading the file(s) again. dataset.message.addFiles.partialSuccess=Partial success: only {0} files out of {1} have been saved. Please try uploading the missing file(s) again. -dataset.message.publish.remind.draft=If it's ready for sharing, please publish it. -dataset.message.submit.remind.draft=If it's ready for sharing, please submit it for review. -dataset.message.publish.remind.version=If it's ready for sharing, please publish it so that others can see these changes. -dataset.message.submit.remind.version=If it's ready for sharing, please submit it for review so that others can see these changes. +dataset.message.publish.remind.draft=If it's ready for sharing, please publish it. +dataset.message.submit.remind.draft=If it's ready for sharing, please submit it for review. +dataset.message.publish.remind.version=If it's ready for sharing, please publish it so that others can see these changes. +dataset.message.submit.remind.version=If it's ready for sharing, please submit it for review so that others can see these changes. dataset.message.publishSuccess=This dataset has been published. dataset.message.only.authenticatedUsers=Only authenticated users may release Datasets. dataset.message.deleteSuccess=This dataset has been deleted. From 699f99c5e6a6333e151d26f795d53d0d50f35248 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 30 Mar 2021 10:15:20 -0400 Subject: [PATCH 539/576] initial docs, adding in the first few source docs --- .../2419-4475-7575-disable-users.md | 5 -- .../{4259-java-11.md => 5.4-release-notes.md} | 70 ++++++++++++++++++- .../7084-crawlable-file-access.md | 29 -------- doc/release-notes/7373-solr-upgrade.md | 3 +- .../820-non-ascii-chars-in-search.md | 10 --- 5 files changed, 71 insertions(+), 46 deletions(-) delete mode 100644 doc/release-notes/2419-4475-7575-disable-users.md rename doc/release-notes/{4259-java-11.md => 5.4-release-notes.md} (56%) delete mode 100644 doc/release-notes/7084-crawlable-file-access.md delete mode 100644 doc/release-notes/820-non-ascii-chars-in-search.md diff --git a/doc/release-notes/2419-4475-7575-disable-users.md b/doc/release-notes/2419-4475-7575-disable-users.md deleted file mode 100644 index 2678ebc6ec9..00000000000 --- a/doc/release-notes/2419-4475-7575-disable-users.md +++ /dev/null @@ -1,5 +0,0 @@ -## Release Highlights - -### Deactivate Users API, Get User Traces API, Revoke Roles API - -A new API has been added to deactivate users to prevent them from logging in or otherwise being active in the system. Deactivating a user is an alternative to deleting a user, especially when the latter is not possible due to the amount of interaction the user has had with the system. In order to learn more about a user before deleting, deactivating, or merging, a new "get user traces" API is available that will show objects created, roles, group memberships, and more. Finally, the "remove all roles" button available in the superuser dashboard is now also available via API. diff --git a/doc/release-notes/4259-java-11.md b/doc/release-notes/5.4-release-notes.md similarity index 56% rename from doc/release-notes/4259-java-11.md rename to doc/release-notes/5.4-release-notes.md index 1e5ee19230a..3e86d4c3838 100644 --- a/doc/release-notes/4259-java-11.md +++ b/doc/release-notes/5.4-release-notes.md @@ -1,10 +1,55 @@ +# Dataverse 5.4 + +This release brings new features, enhancements, and bug fixes to Dataverse. Thank you to all of the community members who contributed code, suggestions, bug reports, and other assistance across the project. + ## Release Highlights +### Search with non-ascii characters + +Many languages include characters that have close analogs in ascii, e.g. (á, à, â, ç, é, è, ê, ë, í, ó, ö, ú, ù, û, ü…). This release changes the default Solr configuration to allow search to match words based on these associations, e.g. a search for Mercè would match the word Merce in a Dataset, and vice versa. This should generally be helpful, but can result in false positives.,e.g. "canon" will be found searching for "cañon". + +### Deactivate Users API, Get User Traces API, Revoke Roles API + +A new API has been added to deactivate users to prevent them from logging in or otherwise being active in the system. Deactivating a user is an alternative to deleting a user, especially when the latter is not possible due to the amount of interaction the user has had with the system. In order to learn more about a user before deleting, deactivating, or merging, a new "get user traces" API is available that will show objects created, roles, group memberships, and more. Finally, the "remove all roles" button available in the superuser dashboard is now also available via API. + +### New File Access API + +A new api offers *crawlable* access view of the folders and files within a datset: + +``` + /api/datasets//dirindex/ +``` + +will output a simple html listing, based on the standard Apache +directory index, with Access API download links for individual files, +and recursive calls to the API above for sub-folders. (See the +documentation entry in the guides for more information). + +Using this API, ``wget --recursive`` (or similar crawling client) can +be used to download all the files in a dataset, preserving the file +names and folder structure; without having to use the download-as-zip +API. In addition to being faster (zipping is a relatively +resource-intensive operation on the server side), this process can be +restarted if interrupted (with ``wget --continue`` or equivalent) - +unlike zipped multi-file downloads that always have to start from the +beginning. + +On a system that uses S3 with download redirects, the individual file +downloads will be handled by S3 directly, without having to be proxied +through the Dataverse application. + ### Java 11 The Dataverse software now runs and requires Java 11. This will provide performance and security enhancements, allows developers to take advantage of new and updated Java features, and moves the project to a platform with better longer term support. This upgrade requires a few extra steps in the release process, outlined below. -## Notes to Admins +## Major Use Cases + +Newly-supported use cases in this release include: + +- Deactivate Users (Issue #xxx, PR #xxx) +- Non-ascii characters + +## Notes for Dataverse Installation Administrators ### Java 11 Upgrade @@ -42,8 +87,30 @@ For installations that were previously running on Dataverse Software 4.x, the fo - `[9|]--add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED` - `[9|]--add-opens=java.logging/java.util.logging=ALL-UNNAMED` +## Notes for Tool Developers and Integrators + +### UTF-8 Characters and Spaces in File Names + +UTF-8 characters in filenames are now preserved when downloaded. + +Dataverse Installations will no longer replace spaces in file names of downloaded files with the + character. If your tool or integration has any special handling around this, you may need to make further adjustments to maintain backwards compatibility while also supporting Dataverse installations on 5.4+. + +Note that this follows a change from 5.1 that only corrected this for installations running with S3 storage. This makes the behavior consistent across installations running all types of file storage. + +## Complete List of Changes + +For the complete list of code changes in this release, see the [5.4 Milestone](https://github.com/IQSS/dataverse/milestone/94?closed=1) in Github. + +For help with upgrading, installing, or general questions please post to the [Dataverse Google Group](https://groups.google.com/forum/#!forum/dataverse-community) or email support@dataverse.org. + +## Installation + +If this is a new installation, please see our [Installation Guide](https://guides.dataverse.org/en/5.3/installation/). + ## Upgrade Instructions +0\. These instructions assume that you've already successfully upgraded from Dataverse 4.x to Dataverse 5 following the instructions in the [Dataverse 5 Release Notes](https://github.com/IQSS/dataverse/releases/tag/v5.0). After upgrading from the 4.x series to 5.0, you should progress through the other 5.x releases before attempting the upgrade to 5.4. + 1\. Undeploy the previous version. - `/bin/asadmin list-applications` @@ -109,3 +176,4 @@ Add the following JVM options to the domain>/dirindex/ -``` - -will output a simple html listing, based on the standard Apache -directory index, with Access API download links for individual files, -and recursive calls to the API above for sub-folders. (See the -documentation entry in the guides for more information). - -Using this API, ``wget --recursive`` (or similar crawling client) can -be used to download all the files in a dataset, preserving the file -names and folder structure; without having to use the download-as-zip -API. In addition to being faster (zipping is a relatively -resource-intensive operation on the server side), this process can be -restarted if interrupted (with ``wget --continue`` or equivalent) - -unlike zipped multi-file downloads that always have to start from the -beginning. - -On a system that uses S3 with download redirects, the individual file -downloads will be handled by S3 directly, without having to be proxied -through the Dataverse application. - - diff --git a/doc/release-notes/7373-solr-upgrade.md b/doc/release-notes/7373-solr-upgrade.md index 06e7bc62e5a..63ed0a61dca 100644 --- a/doc/release-notes/7373-solr-upgrade.md +++ b/doc/release-notes/7373-solr-upgrade.md @@ -7,7 +7,6 @@ Before you start the "index all", Dataverse will appear to be empty because the search results come from Solr. As indexing progresses, partial results will appear until indexing is complete. - See http://guides.dataverse.org/installation/prerequisites.html#installing-solr [for the additional upgrade steps section] @@ -20,3 +19,5 @@ chmod +x updateSchemaMDB.sh ./updateSchemaMDB.sh -t . See http://guides.dataverse.org/en/5.4/admin/metadatacustomization.html?highlight=updateschemamdb for more information. + +1. You will need to replace or modify your `schema.xml` and restart solr. Re-indexing is required to get full-functionality from this change - the standard instructions for an incremental reindex could be added here. \ No newline at end of file diff --git a/doc/release-notes/820-non-ascii-chars-in-search.md b/doc/release-notes/820-non-ascii-chars-in-search.md deleted file mode 100644 index 9e21dd83694..00000000000 --- a/doc/release-notes/820-non-ascii-chars-in-search.md +++ /dev/null @@ -1,10 +0,0 @@ -(review these notes if this gets into the same release as #7645 as the steps are included there - we expect to include this in the same release) - -### Search with non-ascii characters - -Many languages include characters that have close analogs in ascii, e.g. (á, à, â, ç, é, è, ê, ë, í, ó, ö, ú, ù, û, ü…). This release changes the default Solr configuration to allow search to match words based on these associations, e.g. a search for Mercè would match the word Merce in a Dataset, and vice versa. This should generally be helpful, but can result in false positives.,e.g. "canon" will be found searching for "cañon". - -## Upgrade Instructions - -1. You will need to replace or modify your `schema.xml` and restart solr. Re-indexing is required to get full-functionality from this change - the standard instructions for an incremental reindex could be added here. - \ No newline at end of file From 7641b2963fd24f6cc257f414c251d1c4171cd5bb Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 30 Mar 2021 10:17:31 -0400 Subject: [PATCH 540/576] fix failing test for deactivate by id #7629 Use the id from authenticateduser rather than builtinuser. Also remove superuser, no longer needed. --- .../harvard/iq/dataverse/api/DeactivateUsersIT.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java index 374a19f453d..2b7e71341ec 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java @@ -142,19 +142,11 @@ public void testDeactivateUser() { @Test public void testDeactivateUserById() { - Response createSuperuser = UtilIT.createRandomUser(); - createSuperuser.then().assertThat().statusCode(OK.getStatusCode()); - String superuserUsername = UtilIT.getUsernameFromResponse(createSuperuser); - String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser); - Response toggleSuperuser = UtilIT.makeSuperUser(superuserUsername); - toggleSuperuser.then().assertThat() - .statusCode(OK.getStatusCode()); - Response createUser = UtilIT.createRandomUser(); createUser.prettyPrint(); createUser.then().assertThat().statusCode(OK.getStatusCode()); String username = UtilIT.getUsernameFromResponse(createUser); - Long userId = JsonPath.from(createUser.body().asString()).getLong("data.user.id"); + Long userId = JsonPath.from(createUser.body().asString()).getLong("data.authenticatedUser.id"); String apiToken = UtilIT.getApiTokenFromResponse(createUser); Response deactivateUser = UtilIT.deactivateUser(userId); From f74ca49619e1b23373b05a833f44486784d2ccbe Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 30 Mar 2021 10:21:41 -0400 Subject: [PATCH 541/576] consolidating more documents, wow this is a big release --- doc/release-notes/5.4-release-notes.md | 22 ++++++++++++++ doc/release-notes/7188-utf-8-filenames.md | 9 ------ doc/release-notes/7205-orig-file-size.md | 11 ------- doc/release-notes/7337-replace-draft-file.md | 3 -- doc/release-notes/7374-postgresql-update.md | 9 ------ .../7398-saved-search-performance.md | 30 ------------------- 6 files changed, 22 insertions(+), 62 deletions(-) delete mode 100644 doc/release-notes/7188-utf-8-filenames.md delete mode 100644 doc/release-notes/7205-orig-file-size.md delete mode 100644 doc/release-notes/7337-replace-draft-file.md delete mode 100644 doc/release-notes/7374-postgresql-update.md delete mode 100644 doc/release-notes/7398-saved-search-performance.md diff --git a/doc/release-notes/5.4-release-notes.md b/doc/release-notes/5.4-release-notes.md index 3e86d4c3838..47488e95a83 100644 --- a/doc/release-notes/5.4-release-notes.md +++ b/doc/release-notes/5.4-release-notes.md @@ -48,6 +48,7 @@ Newly-supported use cases in this release include: - Deactivate Users (Issue #xxx, PR #xxx) - Non-ascii characters +- Users can now replace files in draft datasets. This functionality was previously only available on published datasets. Issue #7149/PR #7337 ## Notes for Dataverse Installation Administrators @@ -66,6 +67,22 @@ There are some things to note and keep in mind regarding the move to Java 11: - We targeted and tested Java 11, but 11+ will likely work. Java 11 was targeted because of its long term support. - If you're moving from a Dataverse installation that was previously running Glassfish 4.x (typically this would be Dataverse Software 4.x), you will need to adjust some JVM options in domain.xml as part of the upgrade process. We've provided these optional steps below. These steps are not required if your first installed Dataverse version was running Payara 5.x (typically Dataverse Software 5.x). +### PostgreSQL Version Support + +Up until this release our installation guide "strongly recommended" to install PostgreSQL v. 9.6. While that version is known to be very stable, it is nearing its end-of-life (in Nov. 2021). Dataverse Software has now been tested with versions up to 13. If you decide to upgrade PostgreSQL, the tested and recommended way of doing that is as follows: + +- Export your current database with ``pg_dumpall``; +- Install the new version of PostgreSQL; (make sure it's running on the same port, etc. so that no changes are needed in the Payara configuration) +- Re-import the database with ``psql``, as the postgres user. + +Consult the PostgreSQL upgrade documentation for more information, for example https://www.postgresql.org/docs/13/upgrading.html#UPGRADING-VIA-PGDUMPALL. + +### Retroactively Store Original File Size + +Beginning in Dataverse Software 4.10, the size of the saved original file (for an ingested tabular datafile) was stored in the database. For files added before this change, we provide an API that retrieves and permanently stores the sizes for any already existing saved originals. See [Datafile Integrity API](https://guides.dataverse.org/en/5.4/api/native-api.html#datafile-integrity) for more information. + +This was documented as a step in previous release notes, but we are noting it in these release notes to give it more visibility. + ### New JVM and DB Options #### New JVM Options @@ -177,3 +194,8 @@ Add the following JVM options to the domain> Date: Tue, 30 Mar 2021 10:24:18 -0400 Subject: [PATCH 542/576] integrating more notes --- doc/release-notes/5.4-release-notes.md | 39 ++++++++++++++++++++++- doc/release-notes/7409-remove-worldmap.md | 9 ------ doc/release-notes/7501-guides-updates.md | 5 --- 3 files changed, 38 insertions(+), 15 deletions(-) delete mode 100644 doc/release-notes/7409-remove-worldmap.md delete mode 100644 doc/release-notes/7501-guides-updates.md diff --git a/doc/release-notes/5.4-release-notes.md b/doc/release-notes/5.4-release-notes.md index 47488e95a83..5221d09edd5 100644 --- a/doc/release-notes/5.4-release-notes.md +++ b/doc/release-notes/5.4-release-notes.md @@ -42,6 +42,20 @@ through the Dataverse application. The Dataverse software now runs and requires Java 11. This will provide performance and security enhancements, allows developers to take advantage of new and updated Java features, and moves the project to a platform with better longer term support. This upgrade requires a few extra steps in the release process, outlined below. +### Saved Search Performance Improvements + +A refactoring has greatly improved Saved Search performance in the application. If your installation has multiple, potentially long-running Saved Searches in place, this greatly improves the probability that those search jobs will complete without timing out. + +### Worldmap/Geoconnect Integration Now Obsolete + +As of this release, the Geoconnect/Worldmap integration is no longer available. The Harvard University Worldmap is going through a migration process, and instead of updating this code to work with the new infrastructure, the decision was made to pursue future Geospatial exploration/analysis through other tools, following the External Tools Framework in the Dataverse Software. + +### Guides Updates + +The Dataverse Guides have been updated to follow recent changes to how different terms are used across the Dataverse Project. For more information, see Mercè's note to the community: + + + ## Major Use Cases Newly-supported use cases in this release include: @@ -83,6 +97,23 @@ Beginning in Dataverse Software 4.10, the size of the saved original file (for a This was documented as a step in previous release notes, but we are noting it in these release notes to give it more visibility. +### DB Cleanup for Saved Searches + +A previous version of dataverse changed the indexing logic so that when a user links a dataverse, its children are also indexed as linked. This means that the children do not need to be separately linked, and in this version we removed the logic that creates a saved search to create those links when a dataverse is linked. + +We recommend cleaning up the db to a) remove these saved searches and b) remove the links for the objects. We can do this via a few queries, which are available in the folder here: + +https://github.com/IQSS/dataverse/raw/develop/scripts/issues/7398/ + +There are four sets of queries available, and they should be run in this order: + +- ss_for_deletion.txt to identify the Saved Searches to be deleted +- delete_ss.txt to delete the Saved Searches identified in the previous query +- dld_for_deletion.txt to identify the linked datasets and dataverses to be deleted +- delete_dld.txt to delete the linked datasets and dataverses identified in the previous queries + +Note: removing these saved searches and links should not affect what users will see as linked due to the aforementioned indexing change. Similarly, not removing these saved searches and links should not affect anything, but is a cleanup of unnecessary rows in the database. + ### New JVM and DB Options #### New JVM Options @@ -198,4 +229,10 @@ Additional Release Steps 1\. Retroactively store original file size -Use the [Datafile Integrity API](https://guides.dataverse.org/en/5.4/api/native-api.html#datafile-integrity) to ensure that the sizes of all original files are stored in the database. \ No newline at end of file +Use the [Datafile Integrity API](https://guides.dataverse.org/en/5.4/api/native-api.html#datafile-integrity) to ensure that the sizes of all original files are stored in the database. + +2\. (Optional, but recommended) DB Cleanup + +Perform the DB Cleanup for Saved Searches and Linked Objects, summarized in the "Notes for Dataverse Installation Administrators" section above. + +3\. Take a backup of the Worldmap links, if any. \ No newline at end of file diff --git a/doc/release-notes/7409-remove-worldmap.md b/doc/release-notes/7409-remove-worldmap.md deleted file mode 100644 index 3e6ea3ed06e..00000000000 --- a/doc/release-notes/7409-remove-worldmap.md +++ /dev/null @@ -1,9 +0,0 @@ -## Release Highlights - -### Worldmap/Geoconnect Integration Now Obsolete - -As of this release, the Geoconnect/Worldmap integration is no longer available. The Harvard University Worldmap is going through a migration process, and instead of updating this code to work with the new infrastructure, the decision was made to pursue future Geospatial exploration/analysis through other tools, following the External Tools Framework in the Dataverse Software. - -## Notes to Admins - -Take a backup of the Worldmap links, if any. \ No newline at end of file diff --git a/doc/release-notes/7501-guides-updates.md b/doc/release-notes/7501-guides-updates.md deleted file mode 100644 index 29bc7557f05..00000000000 --- a/doc/release-notes/7501-guides-updates.md +++ /dev/null @@ -1,5 +0,0 @@ -## Guides Updates - -The Dataverse Guides have been updated to follow recent changes to how different terms are used across the Dataverse Project. For more information, see Mercè's note to the community: - - From 770819a7640b9d3561c78820dd975eaf03cf8d5f Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 30 Mar 2021 10:24:54 -0400 Subject: [PATCH 543/576] removing file - we already note full index in release notes for solr --- doc/release-notes/7502-more-mime-types.md | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 doc/release-notes/7502-more-mime-types.md diff --git a/doc/release-notes/7502-more-mime-types.md b/doc/release-notes/7502-more-mime-types.md deleted file mode 100644 index 0c57dc3e389..00000000000 --- a/doc/release-notes/7502-more-mime-types.md +++ /dev/null @@ -1,7 +0,0 @@ -## Upgrade Steps - -In addition to mapping friendly names to these file types, the types are further mapped to aggregate file types facets on the homepage. A full reindex is required for the facets to be refreshed. - -Kick off full reindex - -http://guides.dataverse.org/en/4.20/admin/solr-search-index.html From c268eca9c72ae560de5f0add5c6e08b6787f4ed1 Mon Sep 17 00:00:00 2001 From: Gustavo Durand Date: Tue, 30 Mar 2021 10:26:30 -0400 Subject: [PATCH 544/576] Removed a couple of extraneous lines --- .../edu/harvard/iq/dataverse/api/DeactivateUsersIT.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java index 2b7e71341ec..de2a1d422c0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DeactivateUsersIT.java @@ -141,14 +141,12 @@ public void testDeactivateUser() { @Test public void testDeactivateUserById() { - + Response createUser = UtilIT.createRandomUser(); createUser.prettyPrint(); createUser.then().assertThat().statusCode(OK.getStatusCode()); - String username = UtilIT.getUsernameFromResponse(createUser); + Long userId = JsonPath.from(createUser.body().asString()).getLong("data.authenticatedUser.id"); - String apiToken = UtilIT.getApiTokenFromResponse(createUser); - Response deactivateUser = UtilIT.deactivateUser(userId); deactivateUser.prettyPrint(); deactivateUser.then().assertThat().statusCode(OK.getStatusCode()); From 05132ecedcfa709d5c25d1b6fb23ca83d329fbfa Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 30 Mar 2021 10:28:20 -0400 Subject: [PATCH 545/576] adding info about conditionally required fields --- doc/release-notes/5.4-release-notes.md | 37 +++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/doc/release-notes/5.4-release-notes.md b/doc/release-notes/5.4-release-notes.md index 5221d09edd5..9f36eb2fa8b 100644 --- a/doc/release-notes/5.4-release-notes.md +++ b/doc/release-notes/5.4-release-notes.md @@ -56,6 +56,9 @@ The Dataverse Guides have been updated to follow recent changes to how different +### Conditionally Required Metadata Fields + + ## Major Use Cases Newly-supported use cases in this release include: @@ -63,6 +66,7 @@ Newly-supported use cases in this release include: - Deactivate Users (Issue #xxx, PR #xxx) - Non-ascii characters - Users can now replace files in draft datasets. This functionality was previously only available on published datasets. Issue #7149/PR #7337 +- Metadata designers can now set subfields of compound fields as **conditionally required**, that is, the field is required if and only if any of its "sibling" fields are entered. For example, Producer Name is now conditionally required in the citation metadata block. A user does not have to enter a Producer, but if they do, they have to enter a Producer Name. ## Notes for Dataverse Installation Administrators @@ -81,6 +85,26 @@ There are some things to note and keep in mind regarding the move to Java 11: - We targeted and tested Java 11, but 11+ will likely work. Java 11 was targeted because of its long term support. - If you're moving from a Dataverse installation that was previously running Glassfish 4.x (typically this would be Dataverse Software 4.x), you will need to adjust some JVM options in domain.xml as part of the upgrade process. We've provided these optional steps below. These steps are not required if your first installed Dataverse version was running Payara 5.x (typically Dataverse Software 5.x). +## Conditionally Required Metadata Fields + +Prior to this release, when defining metadata for compound fields (via their dataset field types), fields could be either be optional or required, i.e. if required you must always have (at least one) value for that field. For example, Author Name being required means you must have at least one Author with an nonempty Author name. + +In order to support more robust metadata (and specifically to resolve #7551), we need to allow a third case: Conditionally Required, that is, the field is required if and only if any of its "sibling" fields are entered. For example, Producer Name is now conditionally required in the citation metadata block. A user does not have to enter a Producer, but if they do, they have to enter a Producer Name. + +This change required some modifications to how "required" is defined in the metadata .tsv files (for compound fields). + +Prior to this release, the value of required for the parent compound field did not matter and so was set to false. + +Going forward: + +- For optional, the parent compound field would be required = false and all children would be required = false. +- For required, the parent compound field would be required = true and at least one child would be required = true. +- For conditionally required, the parent compound field would be required = false and at least one child would be required = true. + +This release updates the citation .tsv file that is distributed with the software for the required parent compound fields (e.g. author), as well as sets Producer Name to be conditionally required. No other distributed .tsv files were updated, as they did not have any required compound values. + +**If you have created any custom metadata .tsv files**, you will need to make the same (type of) changes there. + ### PostgreSQL Version Support Up until this release our installation guide "strongly recommended" to install PostgreSQL v. 9.6. While that version is known to be very stable, it is nearing its end-of-life (in Nov. 2021). Dataverse Software has now been tested with versions up to 13. If you decide to upgrade PostgreSQL, the tested and recommended way of doing that is as follows: @@ -235,4 +259,15 @@ Use the [Datafile Integrity API](https://guides.dataverse.org/en/5.4/api/native- Perform the DB Cleanup for Saved Searches and Linked Objects, summarized in the "Notes for Dataverse Installation Administrators" section above. -3\. Take a backup of the Worldmap links, if any. \ No newline at end of file +3\. Take a backup of the Worldmap links, if any. + +4\. Reload Citation Metadata Block: + + `wget https://github.com/IQSS/dataverse/releases/download/v5.4/citation.tsv` + `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @citation.tsv -H "Content-type: text/tab-separated-values"` + +5\. Update any custom metadata blocks (if used): + +For any subfield that has a required value of TRUE, find the corresponding parent field and change its required value to TRUE. + +Note: As there is an accompanying Flyway script that updates the values directly in the database, you do not need to reload these metadata .tsv files via API, unless you make additional changes, e.g set some compound fields to be conditionally required. \ No newline at end of file From 7acb50655ed60d156ff1376eac89a9cfafbdaea6 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 30 Mar 2021 10:41:25 -0400 Subject: [PATCH 546/576] #7675 add remind to file upload --- .../edu/harvard/iq/dataverse/DatasetPage.java | 37 ++++--------------- .../iq/dataverse/DatasetServiceBean.java | 22 +++++++++++ .../iq/dataverse/EditDatafilesPage.java | 6 ++- 3 files changed, 34 insertions(+), 31 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 45a4531e60a..ea9b9ff9d8f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -1109,29 +1109,6 @@ public String getComputeUrl() throws IOException { } - - //get a string to add to save success message - //depends on dataset state and user privleges - private String getReminderString(Dataset dataset) { - - //dataset id is null before first save -- needed for create mode - if( dataset.getId() == null || !dataset.isReleased() ){ - //messages for draft state. - if (canPublishDataset()){ - return BundleUtil.getStringFromBundle("dataset.message.publish.remind.draft"); - } else { - return BundleUtil.getStringFromBundle("dataset.message.submit.remind.draft"); - } - } else{ - //messages for new version - post-publish - if (canPublishDataset()){ - return BundleUtil.getStringFromBundle("dataset.message.publish.remind.version"); - } else { - return BundleUtil.getStringFromBundle("dataset.message.submit.remind.version"); - } - } - } - //For a single file public String getComputeUrl(FileMetadata metadata) { SwiftAccessIO swiftObject = null; @@ -3499,7 +3476,7 @@ public String save() { } if (addFilesSuccess && dataset.getFiles().size() > 0) { if (nNewFiles == dataset.getFiles().size()) { - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.createSuccess").concat(" ").concat(getReminderString(dataset))); + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.createSuccess").concat(" ").concat(datasetService.getReminderString(dataset, canPublishDataset()))); } else { String partialSuccessMessage = BundleUtil.getStringFromBundle("dataset.message.createSuccess.partialSuccessSavingFiles"); partialSuccessMessage = partialSuccessMessage.replace("{0}", "" + dataset.getFiles().size() + ""); @@ -3510,25 +3487,25 @@ public String save() { JsfHelper.addWarningMessage(BundleUtil.getStringFromBundle("dataset.message.createSuccess.failedToSaveFiles")); } } else { - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.createSuccess").concat(" ").concat(getReminderString(dataset))); + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.createSuccess").concat(" ").concat(datasetService.getReminderString(dataset, canPublishDataset()))); } } if (editMode.equals(EditMode.METADATA)) { - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.metadataSuccess").concat(" ").concat(getReminderString(dataset))); + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.metadataSuccess").concat(" ").concat(datasetService.getReminderString(dataset, canPublishDataset()))); } if (editMode.equals(EditMode.LICENSE)) { - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.termsSuccess").concat(" ").concat(getReminderString(dataset))); + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.termsSuccess").concat(" ").concat(datasetService.getReminderString(dataset, canPublishDataset()))); } if (editMode.equals(EditMode.FILE)) { - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.filesSuccess").concat(" ").concat(getReminderString(dataset))); + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.filesSuccess").concat(" ").concat(datasetService.getReminderString(dataset, canPublishDataset()))); } } else { // must have been a bulk file update or delete: if (bulkFileDeleteInProgress) { - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.bulkFileDeleteSuccess").concat(" ").concat(getReminderString(dataset))); + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.bulkFileDeleteSuccess").concat(" ").concat(datasetService.getReminderString(dataset, canPublishDataset()))); } else { - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.bulkFileUpdateSuccess").concat(" ").concat(getReminderString(dataset))); + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.bulkFileUpdateSuccess").concat(" ").concat(datasetService.getReminderString(dataset, canPublishDataset()))); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index b43f64b8c11..6174a8e0993 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -19,6 +19,7 @@ import edu.harvard.iq.dataverse.harvest.server.OAIRecordServiceBean; import edu.harvard.iq.dataverse.search.IndexServiceBean; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.workflows.WorkflowComment; import java.io.File; @@ -727,6 +728,27 @@ public void exportAllDatasets(boolean forceReExport) { } + //get a string to add to save success message + //depends on dataset state and user privleges + public String getReminderString(Dataset dataset, boolean canPublishDataset) { + + if( dataset.getId() == null || !dataset.isReleased() ){ + //messages for draft state. + if (canPublishDataset){ + return BundleUtil.getStringFromBundle("dataset.message.publish.remind.draft"); + } else { + return BundleUtil.getStringFromBundle("dataset.message.submit.remind.draft"); + } + } else{ + //messages for new version - post-publish + if (canPublishDataset){ + return BundleUtil.getStringFromBundle("dataset.message.publish.remind.version"); + } else { + return BundleUtil.getStringFromBundle("dataset.message.submit.remind.version"); + } + } + } + public void updateLastExportTimeStamp(Long datasetId) { Date now = new Date(); em.createNativeQuery("UPDATE Dataset SET lastExportTime='"+now.toString()+"' WHERE id="+datasetId).executeUpdate(); diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index f788fd95cae..f737710f711 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -1120,7 +1120,7 @@ public String save() { } else { int nFilesTotal = workingVersion.getFileMetadatas().size(); if (nNewFiles == 0 || nFilesTotal == nExpectedFilesTotal) { - JsfHelper.addSuccessMessage(getBundleString("dataset.message.filesSuccess")); + JsfHelper.addSuccessMessage(getBundleString("dataset.message.filesSuccess").concat(" ").concat(datasetService.getReminderString(dataset, canPublishDataset()))); } else if (nFilesTotal == nOldFiles) { JsfHelper.addErrorMessage(getBundleString("dataset.message.addFiles.Failure")); } else { @@ -1150,6 +1150,10 @@ public String save() { return returnToDraftVersion(); } + public boolean canPublishDataset(){ + return permissionsWrapper.canIssuePublishDatasetCommand(dataset); + } + private void populateDatasetUpdateFailureMessage(){ JH.addMessage(FacesMessage.SEVERITY_FATAL, getBundleString("dataset.message.filesFailure")); From 2f543e5397a69a803ab3830cf2c5ad838cbfc638 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 30 Mar 2021 13:30:09 -0400 Subject: [PATCH 547/576] all notes consolidated, onto the main doc --- doc/release-notes/5.4-release-notes.md | 79 +++++++++++++++++-- doc/release-notes/7373-solr-upgrade.md | 23 ------ ...panded-compound-datasetfield-validation.md | 36 --------- .../7619-restricted-summary-starts.md | 14 ---- .../7638-citation-metadatablock-update.md | 7 -- doc/release-notes/7687-curate-command.md | 9 --- 6 files changed, 72 insertions(+), 96 deletions(-) delete mode 100644 doc/release-notes/7373-solr-upgrade.md delete mode 100644 doc/release-notes/7551-expanded-compound-datasetfield-validation.md delete mode 100644 doc/release-notes/7619-restricted-summary-starts.md delete mode 100644 doc/release-notes/7638-citation-metadatablock-update.md delete mode 100644 doc/release-notes/7687-curate-command.md diff --git a/doc/release-notes/5.4-release-notes.md b/doc/release-notes/5.4-release-notes.md index 9f36eb2fa8b..b7bc43d9567 100644 --- a/doc/release-notes/5.4-release-notes.md +++ b/doc/release-notes/5.4-release-notes.md @@ -58,6 +58,23 @@ The Dataverse Guides have been updated to follow recent changes to how different ### Conditionally Required Metadata Fields +Add Text + +### Restricted Files and DDI "dataDscr" Information (Summary Statistics, Variable Names, Variable Labels) + +In previous releases, DDI "dataDscr" information (summary statistics, variable names, and variable labels, sometimes known as "variable metadata") for tabular files that were ingested successfully were available even if files were restricted. This has been changed in the following ways: + +- At the dataset level, DDI exports no longer show "dataDscr" information for restricted files. There is only one version of this export and it is the version that's suitable for public consumption with the "dataDscr" information hidden for restricted files. +- Similarly, at the dataset level, the DDI HTML Codebook no longer shows "dataDscr" information for restricted files. +- At the file level, "dataDscr" information is no longer publicly available for restricted files. In practice, it was only possible to get this publicly via API (the download/access button was hidden). +- At the file level, "dataDscr" (variable metadata) information can still be downloaded for restricted files if you have access to download the file. + +After upgrading, you should re-export to replace cached DDI exports with restricted summary stats with DDI exports fit for public consumption: + + curl http://localhost:8080/api/admin/metadata/reExportAll + +For details on this operation, see https://guides.dataverse.org/en/5.4/admin/metadataexport.html + ## Major Use Cases @@ -85,7 +102,7 @@ There are some things to note and keep in mind regarding the move to Java 11: - We targeted and tested Java 11, but 11+ will likely work. Java 11 was targeted because of its long term support. - If you're moving from a Dataverse installation that was previously running Glassfish 4.x (typically this would be Dataverse Software 4.x), you will need to adjust some JVM options in domain.xml as part of the upgrade process. We've provided these optional steps below. These steps are not required if your first installed Dataverse version was running Payara 5.x (typically Dataverse Software 5.x). -## Conditionally Required Metadata Fields +### Conditionally Required Metadata Fields Prior to this release, when defining metadata for compound fields (via their dataset field types), fields could be either be optional or required, i.e. if required you must always have (at least one) value for that field. For example, Author Name being required means you must have at least one Author with an nonempty Author name. @@ -105,6 +122,13 @@ This release updates the citation .tsv file that is distributed with the softwar **If you have created any custom metadata .tsv files**, you will need to make the same (type of) changes there. +### Citation metadatablock update + +Due to a minor update in the citation metadata block (extra ISO-639-3 language codes added) a block upgrade is required: + +`wget https://github.com/IQSS/dataverse/releases/download/v5.4/citation.tsv` +`curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @citation.tsv -H "Content-type: text/tab-separated-values"` + ### PostgreSQL Version Support Up until this release our installation guide "strongly recommended" to install PostgreSQL v. 9.6. While that version is known to be very stable, it is nearing its end-of-life (in Nov. 2021). Dataverse Software has now been tested with versions up to 13. If you decide to upgrade PostgreSQL, the tested and recommended way of doing that is as follows: @@ -115,6 +139,17 @@ Up until this release our installation guide "strongly recommended" to install P Consult the PostgreSQL upgrade documentation for more information, for example https://www.postgresql.org/docs/13/upgrading.html#UPGRADING-VIA-PGDUMPALL. +### Solr Upgrade + +With this release we upgrade to the latest available stable release in the Solr 8.x branch. We recommend a fresh installation of Solr 8.8.1 (the index will be empty) +followed by an "index all". + +Before you start the "index all", Dataverse will appear to be empty because +the search results come from Solr. As indexing progresses, partial results will +appear until indexing is complete. + +See http://guides.dataverse.org/en/5.4/installation/prerequisites.html#installing-solr + ### Retroactively Store Original File Size Beginning in Dataverse Software 4.10, the size of the saved original file (for an ingested tabular datafile) was stored in the database. For files added before this change, we provide an API that retrieves and permanently stores the sizes for any already existing saved originals. See [Datafile Integrity API](https://guides.dataverse.org/en/5.4/api/native-api.html#datafile-integrity) for more information. @@ -138,6 +173,15 @@ There are four sets of queries available, and they should be run in this order: Note: removing these saved searches and links should not affect what users will see as linked due to the aforementioned indexing change. Similarly, not removing these saved searches and links should not affect anything, but is a cleanup of unnecessary rows in the database. +### DB Cleanup for Superusers Releasing without Version Updates + +In datasets where a superuser has run the Curate command and the update included a change to the fileaccessrequest flag, those changes would not be reflected appropriately in the published version. This should be a rare occurrence. + +Instead of an automated solution, we recommend inspecting the affected datasets and correcting the fileaccessrequest flag as appropriate. You can identify the affected datasets this via a query, which is available in the folder here: + +https://github.com/IQSS/dataverse/raw/develop/scripts/issues/7687/ + + ### New JVM and DB Options #### New JVM Options @@ -249,22 +293,43 @@ Add the following JVM options to the domain>