From fa804197bbae4d6da4f32167eff4a298546cf05b Mon Sep 17 00:00:00 2001
From: Vera Clemens <clemens@zbmed.de>
Date: Fri, 27 Sep 2024 13:54:53 +0200
Subject: [PATCH 01/43] feat: index numerical and date fields in Solr with
appropriate types
---
conf/solr/schema.xml | 2 ++
.../java/edu/harvard/iq/dataverse/DatasetFieldType.java | 9 ++++-----
.../harvard/iq/dataverse/search/IndexServiceBean.java | 4 +++-
.../harvard/iq/dataverse/search/SearchServiceBean.java | 2 +-
.../java/edu/harvard/iq/dataverse/search/SolrField.java | 2 +-
5 files changed, 11 insertions(+), 8 deletions(-)
diff --git a/conf/solr/schema.xml b/conf/solr/schema.xml
index 2aed50e9998..02e699722f7 100644
--- a/conf/solr/schema.xml
+++ b/conf/solr/schema.xml
@@ -814,6 +814,8 @@
<!-- KD-tree versions of date fields -->
<fieldType name="pdate" class="solr.DatePointField" docValues="true"/>
<fieldType name="pdates" class="solr.DatePointField" docValues="true" multiValued="true"/>
+
+ <fieldType name="date_range" class="solr.DateRangeField"/>
<!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings -->
<fieldType name="binary" class="solr.BinaryField"/>
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java
index 01785359e0e..2c385268fa5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java
@@ -531,15 +531,14 @@ public String getDisplayName() {
public SolrField getSolrField() {
SolrField.SolrType solrType = SolrField.SolrType.TEXT_EN;
if (fieldType != null) {
-
- /**
- * @todo made more decisions based on fieldType: index as dates,
- * integers, and floats so we can do range queries etc.
- */
if (fieldType.equals(FieldType.DATE)) {
solrType = SolrField.SolrType.DATE;
} else if (fieldType.equals(FieldType.EMAIL)) {
solrType = SolrField.SolrType.EMAIL;
+ } else if (fieldType.equals(FieldType.INT)) {
+ solrType = SolrField.SolrType.INTEGER;
+ } else if (fieldType.equals(FieldType.FLOAT)) {
+ solrType = SolrField.SolrType.FLOAT;
}
Boolean parentAllowsMultiplesBoolean = false;
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
index a8cf9ed519b..e73b8d2f679 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
@@ -1061,6 +1061,8 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
// no-op. we want to keep email address out of Solr per
// https://github.com/IQSS/dataverse/issues/759
} else if (dsfType.getSolrField().getSolrType().equals(SolrField.SolrType.DATE)) {
+ // we index dates as full strings (YYYY, YYYY-MM or YYYY-MM-DD)
+ // for use in facets, we index only the year (YYYY)
String dateAsString = "";
if (!dsf.getValues_nondisplay().isEmpty()) {
dateAsString = dsf.getValues_nondisplay().get(0);
@@ -1080,7 +1082,7 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
logger.fine("YYYY only: " + datasetFieldFlaggedAsDate);
// solrInputDocument.addField(solrFieldSearchable,
// Integer.parseInt(datasetFieldFlaggedAsDate));
- solrInputDocument.addField(solrFieldSearchable, datasetFieldFlaggedAsDate);
+ solrInputDocument.addField(solrFieldSearchable, dateAsString);
if (dsfType.getSolrField().isFacetable()) {
// solrInputDocument.addField(solrFieldFacetable,
// Integer.parseInt(datasetFieldFlaggedAsDate));
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
index ee93c49ad34..8b1959ef7d4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
@@ -271,7 +271,7 @@ public SolrQueryResponse search(
List<DatasetFieldType> datasetFields = datasetFieldService.findAllOrderedById();
Map<String, String> solrFieldsToHightlightOnMap = new HashMap<>();
if (addHighlights) {
- solrQuery.setHighlight(true).setHighlightSnippets(1);
+ solrQuery.setHighlight(true).setHighlightSnippets(1).setHighlightRequireFieldMatch(true);
Integer fragSize = systemConfig.getSearchHighlightFragmentSize();
if (fragSize != null) {
solrQuery.setHighlightFragsize(fragSize);
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrField.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrField.java
index ca9805b6c57..7092a01beb1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SolrField.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrField.java
@@ -63,7 +63,7 @@ public enum SolrType {
* support range queries) in
* https://github.com/IQSS/dataverse/issues/370
*/
- STRING("string"), TEXT_EN("text_en"), INTEGER("int"), LONG("long"), DATE("text_en"), EMAIL("text_en");
+ STRING("string"), TEXT_EN("text_en"), INTEGER("plong"), FLOAT("pdouble"), DATE("date_range"), EMAIL("text_en");
private String type;
From 3f5919b6ac5eced7b7bfa25eb6ce1a2f3b448326 Mon Sep 17 00:00:00 2001
From: Vera Clemens <clemens@zbmed.de>
Date: Fri, 27 Sep 2024 15:43:05 +0200
Subject: [PATCH 02/43] docs: add release note snippet for #10887
---
doc/release-notes/10887-solr-field-types.md | 11 +++++++++++
1 file changed, 11 insertions(+)
create mode 100644 doc/release-notes/10887-solr-field-types.md
diff --git a/doc/release-notes/10887-solr-field-types.md b/doc/release-notes/10887-solr-field-types.md
new file mode 100644
index 00000000000..ca5b210cb21
--- /dev/null
+++ b/doc/release-notes/10887-solr-field-types.md
@@ -0,0 +1,11 @@
+This release enhances how numerical and date fields are indexed in Solr. Previously, all fields were indexed as English text (text_en), but with this update:
+
+* Integer fields are indexed as `plong`
+* Float fields are indexed as `pdouble`
+* Date fields are indexed as `date_range` (`solr.DateRangeField`)
+
+This enables range queries via the search bar or API, such as `exampleIntegerField:[25 TO 50]` or `exampleDateField:[2000-11-01 TO 2014-12-01]`.
+
+To activate this feature, Dataverse administrators must update their Solr schema.xml (manually or by rerunning `update-fields.sh`) and reindex all datasets.
+
+Additionally, search result highlighting is now more accurate, ensuring that only fields relevant to the query are highlighted in search results. If the query is specifically limited to certain fields, the highlighting is now limited to those fields as well.
\ No newline at end of file
From 42f64cb2e11ff73f8d1668ffa58219e15f19a84d Mon Sep 17 00:00:00 2001
From: Vera Clemens <clemens@zbmed.de>
Date: Thu, 17 Oct 2024 14:28:20 +0200
Subject: [PATCH 03/43] test: add test for range search queries for ints,
floats and dates
---
.../harvard/iq/dataverse/api/SearchIT.java | 193 ++++++++++++++++++
1 file changed, 193 insertions(+)
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java
index 3a2b684c421..8850b7ce7c2 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java
@@ -1269,6 +1269,199 @@ public void testGeospatialSearchInvalid() {
}
+ @Test
+ public void testRangeQueries() {
+
+ Response createUser = UtilIT.createRandomUser();
+ createUser.prettyPrint();
+ String username = UtilIT.getUsernameFromResponse(createUser);
+ String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+ Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+ createDataverseResponse.prettyPrint();
+ String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+ // Using the "astrophysics" block because it contains all field types relevant for range queries
+ // (int, float and date)
+ Response setMetadataBlocks = UtilIT.setMetadataBlocks(dataverseAlias, Json.createArrayBuilder().add("citation").add("astrophysics"), apiToken);
+ setMetadataBlocks.prettyPrint();
+ setMetadataBlocks.then().assertThat().statusCode(OK.getStatusCode());
+
+ JsonObjectBuilder datasetJson = Json.createObjectBuilder()
+ .add("datasetVersion", Json.createObjectBuilder()
+ .add("metadataBlocks", Json.createObjectBuilder()
+ .add("citation", Json.createObjectBuilder()
+ .add("fields", Json.createArrayBuilder()
+ .add(Json.createObjectBuilder()
+ .add("typeName", "title")
+ .add("value", "Test Astrophysics Dataset")
+ .add("typeClass", "primitive")
+ .add("multiple", false)
+ )
+ .add(Json.createObjectBuilder()
+ .add("value", Json.createArrayBuilder()
+ .add(Json.createObjectBuilder()
+ .add("authorName",
+ Json.createObjectBuilder()
+ .add("value", "Simpson, Homer")
+ .add("typeClass", "primitive")
+ .add("multiple", false)
+ .add("typeName", "authorName"))
+ )
+ )
+ .add("typeClass", "compound")
+ .add("multiple", true)
+ .add("typeName", "author")
+ )
+ .add(Json.createObjectBuilder()
+ .add("value", Json.createArrayBuilder()
+ .add(Json.createObjectBuilder()
+ .add("datasetContactEmail",
+ Json.createObjectBuilder()
+ .add("value", "hsimpson@mailinator.com")
+ .add("typeClass", "primitive")
+ .add("multiple", false)
+ .add("typeName", "datasetContactEmail"))
+ )
+ )
+ .add("typeClass", "compound")
+ .add("multiple", true)
+ .add("typeName", "datasetContact")
+ )
+ .add(Json.createObjectBuilder()
+ .add("value", Json.createArrayBuilder()
+ .add(Json.createObjectBuilder()
+ .add("dsDescriptionValue",
+ Json.createObjectBuilder()
+ .add("value", "This is a test dataset.")
+ .add("typeClass", "primitive")
+ .add("multiple", false)
+ .add("typeName", "dsDescriptionValue"))
+ )
+ )
+ .add("typeClass", "compound")
+ .add("multiple", true)
+ .add("typeName", "dsDescription")
+ )
+ .add(Json.createObjectBuilder()
+ .add("value", Json.createArrayBuilder()
+ .add("Other")
+ )
+ .add("typeClass", "controlledVocabulary")
+ .add("multiple", true)
+ .add("typeName", "subject")
+ )
+ )
+ )
+ .add("astrophysics", Json.createObjectBuilder()
+ .add("fields", Json.createArrayBuilder()
+ .add(Json.createObjectBuilder()
+ .add("typeName", "coverage.Temporal")
+ .add("typeClass", "compound")
+ .add("multiple", true)
+ .add("value", Json.createArrayBuilder()
+ .add(Json.createObjectBuilder()
+ .add("coverage.Temporal.StartTime",
+ Json.createObjectBuilder()
+ .add("value", "2015-01-01")
+ .add("typeClass", "primitive")
+ .add("multiple", false)
+ .add("typeName", "coverage.Temporal.StartTime")
+ )
+ )
+ )
+ )
+ .add(Json.createObjectBuilder()
+ .add("typeName", "coverage.ObjectCount")
+ .add("typeClass", "primitive")
+ .add("multiple", false)
+ .add("value", "9000")
+ )
+ .add(Json.createObjectBuilder()
+ .add("typeName", "coverage.SkyFraction")
+ .add("typeClass", "primitive")
+ .add("multiple", false)
+ .add("value", "0.002")
+ )
+ )
+ )
+ ));
+
+ Response createDatasetResponse = UtilIT.createDataset(dataverseAlias, datasetJson, apiToken);
+ createDatasetResponse.prettyPrint();
+ Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse);
+ String datasetPid = JsonPath.from(createDatasetResponse.getBody().asString()).getString("data.persistentId");
+
+ // Integer range query: Hit
+ Response search1 = UtilIT.search("id:dataset_" + datasetId + "_draft AND coverage.ObjectCount:[1000 TO 10000]", apiToken, "&show_entity_ids=true");
+ search1.prettyPrint();
+ search1.then().assertThat()
+ .statusCode(OK.getStatusCode())
+ .body("data.total_count", CoreMatchers.is(1))
+ .body("data.count_in_response", CoreMatchers.is(1))
+ .body("data.items[0].entity_id", CoreMatchers.is(datasetId));
+
+ // Integer range query: Miss
+ Response search2 = UtilIT.search("id:dataset_" + datasetId + "_draft AND coverage.ObjectCount:[* TO 1000]", apiToken);
+ search2.prettyPrint();
+ search2.then().assertThat()
+ .statusCode(OK.getStatusCode())
+ .body("data.total_count", CoreMatchers.is(0))
+ .body("data.count_in_response", CoreMatchers.is(0));
+
+ // Float range query: Hit
+ Response search3 = UtilIT.search("id:dataset_" + datasetId + "_draft AND coverage.SkyFraction:[0 TO 0.5]", apiToken, "&show_entity_ids=true");
+ search3.prettyPrint();
+ search3.then().assertThat()
+ .statusCode(OK.getStatusCode())
+ .body("data.total_count", CoreMatchers.is(1))
+ .body("data.count_in_response", CoreMatchers.is(1))
+ .body("data.items[0].entity_id", CoreMatchers.is(datasetId));
+
+ // Float range query: Miss
+ Response search4 = UtilIT.search("id:dataset_" + datasetId + "_draft AND coverage.SkyFraction:[0.5 TO 1]", apiToken);
+ search4.prettyPrint();
+ search4.then().assertThat()
+ .statusCode(OK.getStatusCode())
+ .body("data.total_count", CoreMatchers.is(0))
+ .body("data.count_in_response", CoreMatchers.is(0));
+
+ // Date range query: Hit
+ Response search5 = UtilIT.search("id:dataset_" + datasetId + "_draft AND coverage.Temporal.StartTime:2015", apiToken, "&show_entity_ids=true");
+ search5.prettyPrint();
+ search5.then().assertThat()
+ .statusCode(OK.getStatusCode())
+ .body("data.total_count", CoreMatchers.is(1))
+ .body("data.count_in_response", CoreMatchers.is(1))
+ .body("data.items[0].entity_id", CoreMatchers.is(datasetId));
+
+ // Date range query: Miss
+ Response search6 = UtilIT.search("id:dataset_" + datasetId + "_draft AND coverage.Temporal.StartTime:[2020 TO *]", apiToken);
+ search6.prettyPrint();
+ search6.then().assertThat()
+ .statusCode(OK.getStatusCode())
+ .body("data.total_count", CoreMatchers.is(0))
+ .body("data.count_in_response", CoreMatchers.is(0));
+
+ // Combining all three range queries: Hit
+ Response search7 = UtilIT.search("id:dataset_" + datasetId + "_draft AND coverage.ObjectCount:[1000 TO 10000] AND coverage.SkyFraction:[0 TO 0.5] AND coverage.Temporal.StartTime:2015", apiToken, "&show_entity_ids=true");
+ search7.prettyPrint();
+ search7.then().assertThat()
+ .statusCode(OK.getStatusCode())
+ .body("data.total_count", CoreMatchers.is(1))
+ .body("data.count_in_response", CoreMatchers.is(1))
+ .body("data.items[0].entity_id", CoreMatchers.is(datasetId));
+
+ // Combining all three range queries: Miss
+ Response search8 = UtilIT.search("id:dataset_" + datasetId + "_draft AND coverage.ObjectCount:[* TO 1000] AND coverage.SkyFraction:[0.5 TO 1] AND coverage.Temporal.StartTime:[2020 TO *]", apiToken);
+ search8.prettyPrint();
+ search8.then().assertThat()
+ .statusCode(OK.getStatusCode())
+ .body("data.total_count", CoreMatchers.is(0))
+ .body("data.count_in_response", CoreMatchers.is(0));
+
+ }
+
@AfterEach
public void tearDownDataverse() {
File treesThumb = new File("scripts/search/data/binary/trees.png.thumb48");
From ed7e38ec57e302f2a40f7491b3360878c6eb187b Mon Sep 17 00:00:00 2001
From: Vera Clemens <clemens@zbmed.de>
Date: Fri, 18 Oct 2024 15:59:18 +0200
Subject: [PATCH 04/43] feat: skip indexing of field instead of entire dataset
when encountering invalid ints, floats or dates
---
.../iq/dataverse/search/IndexServiceBean.java | 102 ++++++++++----
.../harvard/iq/dataverse/api/SearchIT.java | 128 ++++++++++++++++++
2 files changed, 207 insertions(+), 23 deletions(-)
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
index e73b8d2f679..17dc6726a5a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
@@ -27,6 +27,8 @@
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
@@ -44,6 +46,7 @@
import java.util.function.Function;
import java.util.logging.Level;
import java.util.logging.Logger;
+import java.util.regex.Pattern;
import java.util.stream.Collectors;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
@@ -1060,36 +1063,89 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
if (dsfType.getSolrField().getSolrType().equals(SolrField.SolrType.EMAIL)) {
// no-op. we want to keep email address out of Solr per
// https://github.com/IQSS/dataverse/issues/759
+ } else if (dsfType.getSolrField().getSolrType().equals(SolrField.SolrType.INTEGER)) {
+ // we need to filter invalid integer values, because otherwise the whole document will
+ // fail to be indexed
+ Pattern intPattern = Pattern.compile("^-?\\d+$");
+ List<String> indexableValues = dsf.getValuesWithoutNaValues().stream()
+ .filter(s -> intPattern.matcher(s).find())
+ .collect(Collectors.toList());
+ solrInputDocument.addField(solrFieldSearchable, indexableValues);
+ if (dsfType.getSolrField().isFacetable()) {
+ solrInputDocument.addField(solrFieldFacetable, indexableValues);
+ }
+ } else if (dsfType.getSolrField().getSolrType().equals(SolrField.SolrType.FLOAT)) {
+ // same as for integer values, we need to filter invalid float values
+ List<String> indexableValues = dsf.getValuesWithoutNaValues().stream()
+ .filter(s -> {
+ try {
+ Double.parseDouble(s);
+ return true;
+ } catch (NumberFormatException e) {
+ return false;
+ }
+ })
+ .collect(Collectors.toList());
+ solrInputDocument.addField(solrFieldSearchable, indexableValues);
+ if (dsfType.getSolrField().isFacetable()) {
+ solrInputDocument.addField(solrFieldFacetable, indexableValues);
+ }
} else if (dsfType.getSolrField().getSolrType().equals(SolrField.SolrType.DATE)) {
- // we index dates as full strings (YYYY, YYYY-MM or YYYY-MM-DD)
- // for use in facets, we index only the year (YYYY)
+ // Solr accepts dates in the ISO-8601 format, e.g. YYYY-MM-DDThh:mm:ssZ, YYYYY-MM-DD, YYYY-MM, YYYY
+ // See: https://solr.apache.org/guide/solr/latest/indexing-guide/date-formatting-math.html
+ // If dates have been entered in other formats, we need to skip or convert them
+ // TODO at the moment we are simply skipping, but converting them would offer more value for search
+ // For use in facets, we index only the year (YYYY)
String dateAsString = "";
if (!dsf.getValues_nondisplay().isEmpty()) {
- dateAsString = dsf.getValues_nondisplay().get(0);
- }
+ dateAsString = dsf.getValues_nondisplay().get(0).trim();
+ }
+
logger.fine("date as string: " + dateAsString);
+
if (dateAsString != null && !dateAsString.isEmpty()) {
- SimpleDateFormat inputDateyyyy = new SimpleDateFormat("yyyy", Locale.ENGLISH);
- try {
- /**
- * @todo when bean validation is working we
- * won't have to convert strings into dates
- */
- logger.fine("Trying to convert " + dateAsString + " to a YYYY date from dataset " + dataset.getId());
- Date dateAsDate = inputDateyyyy.parse(dateAsString);
- SimpleDateFormat yearOnly = new SimpleDateFormat("yyyy");
- String datasetFieldFlaggedAsDate = yearOnly.format(dateAsDate);
- logger.fine("YYYY only: " + datasetFieldFlaggedAsDate);
- // solrInputDocument.addField(solrFieldSearchable,
- // Integer.parseInt(datasetFieldFlaggedAsDate));
- solrInputDocument.addField(solrFieldSearchable, dateAsString);
- if (dsfType.getSolrField().isFacetable()) {
- // solrInputDocument.addField(solrFieldFacetable,
+ boolean dateValid = false;
+
+ DateTimeFormatter[] possibleFormats = {
+ DateTimeFormatter.ISO_INSTANT,
+ DateTimeFormatter.ofPattern("yyyy-MM-dd"),
+ DateTimeFormatter.ofPattern("yyyy-MM"),
+ DateTimeFormatter.ofPattern("yyyy")
+ };
+ for (DateTimeFormatter format : possibleFormats){
+ try {
+ format.parse(dateAsString);
+ dateValid = true;
+ } catch (DateTimeParseException e) {
+ // no-op, date is invalid
+ }
+ }
+
+ if (!dateValid) {
+ logger.fine("couldn't index " + dsf.getDatasetFieldType().getName() + ":" + dsf.getValues() + " because it's not a valid date format according to Solr");
+ } else {
+ SimpleDateFormat inputDateyyyy = new SimpleDateFormat("yyyy", Locale.ENGLISH);
+ try {
+ /**
+ * @todo when bean validation is working we
+ * won't have to convert strings into dates
+ */
+ logger.fine("Trying to convert " + dateAsString + " to a YYYY date from dataset " + dataset.getId());
+ Date dateAsDate = inputDateyyyy.parse(dateAsString);
+ SimpleDateFormat yearOnly = new SimpleDateFormat("yyyy");
+ String datasetFieldFlaggedAsDate = yearOnly.format(dateAsDate);
+ logger.fine("YYYY only: " + datasetFieldFlaggedAsDate);
+ // solrInputDocument.addField(solrFieldSearchable,
// Integer.parseInt(datasetFieldFlaggedAsDate));
- solrInputDocument.addField(solrFieldFacetable, datasetFieldFlaggedAsDate);
+ solrInputDocument.addField(solrFieldSearchable, dateAsString);
+ if (dsfType.getSolrField().isFacetable()) {
+ // solrInputDocument.addField(solrFieldFacetable,
+ // Integer.parseInt(datasetFieldFlaggedAsDate));
+ solrInputDocument.addField(solrFieldFacetable, datasetFieldFlaggedAsDate);
+ }
+ } catch (Exception ex) {
+ logger.info("unable to convert " + dateAsString + " into YYYY format and couldn't index it (" + dsfType.getName() + ")");
}
- } catch (Exception ex) {
- logger.info("unable to convert " + dateAsString + " into YYYY format and couldn't index it (" + dsfType.getName() + ")");
}
}
} else {
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java
index 8850b7ce7c2..6058ab17d72 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java
@@ -1462,6 +1462,134 @@ public void testRangeQueries() {
}
+ @Test
+ public void testSearchWithInvalidDateField() {
+
+ Response createUser = UtilIT.createRandomUser();
+ createUser.prettyPrint();
+ String username = UtilIT.getUsernameFromResponse(createUser);
+ String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+ Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+ createDataverseResponse.prettyPrint();
+ String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+ Response setMetadataBlocks = UtilIT.setMetadataBlocks(dataverseAlias, Json.createArrayBuilder().add("citation"), apiToken);
+ setMetadataBlocks.prettyPrint();
+ setMetadataBlocks.then().assertThat().statusCode(OK.getStatusCode());
+
+ // Adding a dataset with a date in the "timePeriodCoveredStart" field that doesn't match Solr's date format
+ // (ISO-8601 format, e.g. YYYY-MM-DDThh:mm:ssZ, YYYYY-MM-DD, YYYY-MM, YYYY)
+ // (See: https://solr.apache.org/guide/solr/latest/indexing-guide/date-formatting-math.html)
+ // So the date currently cannot be indexed
+ JsonObjectBuilder datasetJson = Json.createObjectBuilder()
+ .add("datasetVersion", Json.createObjectBuilder()
+ .add("metadataBlocks", Json.createObjectBuilder()
+ .add("citation", Json.createObjectBuilder()
+ .add("fields", Json.createArrayBuilder()
+ .add(Json.createObjectBuilder()
+ .add("typeName", "title")
+ .add("value", "Test Dataset")
+ .add("typeClass", "primitive")
+ .add("multiple", false)
+ )
+ .add(Json.createObjectBuilder()
+ .add("value", Json.createArrayBuilder()
+ .add(Json.createObjectBuilder()
+ .add("authorName",
+ Json.createObjectBuilder()
+ .add("value", "Simpson, Homer")
+ .add("typeClass", "primitive")
+ .add("multiple", false)
+ .add("typeName", "authorName"))
+ )
+ )
+ .add("typeClass", "compound")
+ .add("multiple", true)
+ .add("typeName", "author")
+ )
+ .add(Json.createObjectBuilder()
+ .add("value", Json.createArrayBuilder()
+ .add(Json.createObjectBuilder()
+ .add("datasetContactEmail",
+ Json.createObjectBuilder()
+ .add("value", "hsimpson@mailinator.com")
+ .add("typeClass", "primitive")
+ .add("multiple", false)
+ .add("typeName", "datasetContactEmail"))
+ )
+ )
+ .add("typeClass", "compound")
+ .add("multiple", true)
+ .add("typeName", "datasetContact")
+ )
+ .add(Json.createObjectBuilder()
+ .add("value", Json.createArrayBuilder()
+ .add(Json.createObjectBuilder()
+ .add("dsDescriptionValue",
+ Json.createObjectBuilder()
+ .add("value", "This is a test dataset.")
+ .add("typeClass", "primitive")
+ .add("multiple", false)
+ .add("typeName", "dsDescriptionValue"))
+ )
+ )
+ .add("typeClass", "compound")
+ .add("multiple", true)
+ .add("typeName", "dsDescription")
+ )
+ .add(Json.createObjectBuilder()
+ .add("value", Json.createArrayBuilder()
+ .add("Other")
+ )
+ .add("typeClass", "controlledVocabulary")
+ .add("multiple", true)
+ .add("typeName", "subject")
+ )
+ .add(Json.createObjectBuilder()
+ .add("typeName", "timePeriodCovered")
+ .add("typeClass", "compound")
+ .add("multiple", true)
+ .add("value", Json.createArrayBuilder()
+ .add(Json.createObjectBuilder()
+ .add("timePeriodCoveredStart",
+ Json.createObjectBuilder()
+ .add("value", "15-01-01")
+ .add("typeClass", "primitive")
+ .add("multiple", false)
+ .add("typeName", "timePeriodCoveredStart")
+ )
+ )
+ )
+ )
+ )
+ )
+ ));
+
+ Response createDatasetResponse = UtilIT.createDataset(dataverseAlias, datasetJson, apiToken);
+ createDatasetResponse.prettyPrint();
+ Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse);
+ String datasetPid = JsonPath.from(createDatasetResponse.getBody().asString()).getString("data.persistentId");
+
+ // When querying on the date field: miss (because the date field was skipped during indexing)
+ Response search1 = UtilIT.search("id:dataset_" + datasetId + "_draft AND timePeriodCoveredStart:[2000 TO 2020]", apiToken);
+ search1.prettyPrint();
+ search1.then().assertThat()
+ .statusCode(OK.getStatusCode())
+ .body("data.total_count", CoreMatchers.is(0))
+ .body("data.count_in_response", CoreMatchers.is(0));
+
+ // When querying not on the date field: the dataset can be found (only the date field was skipped during indexing, not the entire dataset)
+ Response search2 = UtilIT.search("id:dataset_" + datasetId + "_draft", apiToken, "&show_entity_ids=true");
+ search2.prettyPrint();
+ search2.then().assertThat()
+ .statusCode(OK.getStatusCode())
+ .body("data.total_count", CoreMatchers.is(1))
+ .body("data.count_in_response", CoreMatchers.is(1))
+ .body("data.items[0].entity_id", CoreMatchers.is(datasetId));
+
+ }
+
@AfterEach
public void tearDownDataverse() {
File treesThumb = new File("scripts/search/data/binary/trees.png.thumb48");
From a8f09e62667d5a6245cf35594b865db339df623e Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Thu, 31 Oct 2024 14:32:57 -0400
Subject: [PATCH 05/43] Published datasets should contain files
---
...published-datasets-should-contain-files.md | 9 +++
doc/sphinx-guides/source/api/native-api.rst | 1 +
.../edu/harvard/iq/dataverse/Dataverse.java | 11 +++-
.../harvard/iq/dataverse/api/Dataverses.java | 2 +-
.../command/impl/PublishDatasetCommand.java | 20 +++++-
.../impl/UpdateDataverseAttributeCommand.java | 33 ++++++----
.../iq/dataverse/util/json/JsonParser.java | 3 +
.../iq/dataverse/util/json/JsonPrinter.java | 3 +
src/main/java/propertyFiles/Bundle.properties | 1 +
src/main/resources/db/migration/V6.5.0.1.sql | 2 +
.../harvard/iq/dataverse/api/DatasetsIT.java | 66 +++++++++++++++++--
11 files changed, 132 insertions(+), 19 deletions(-)
create mode 100644 doc/release-notes/10981-published-datasets-should-contain-files.md
create mode 100644 src/main/resources/db/migration/V6.5.0.1.sql
diff --git a/doc/release-notes/10981-published-datasets-should-contain-files.md b/doc/release-notes/10981-published-datasets-should-contain-files.md
new file mode 100644
index 00000000000..73c76744164
--- /dev/null
+++ b/doc/release-notes/10981-published-datasets-should-contain-files.md
@@ -0,0 +1,9 @@
+## Feature: Prevent publishing Datasets without files
+A new attribute was added to Collections in order to control the publishing of Datasets without files. Once set, the publishing of a Dataset within a Collection or Collection's hierarchy, without files, will be blocked for all non Admin users.
+In order to configure a Collection to block publishing an Admin must set the attribute "requireFilesToPublishDataset" to true.
+Any Collection created under a Collection with this attribute will also be bound by this blocking. Setting this attribute on the Root Dataverse will essentially block the publishing of Datasets without files for the entire installation.
+```shell
+curl -X PUT -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/attribute/requireFilesToPublishDataset?value=true"
+```
+
+See also #10981.
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index f8b8620f121..8fab30a884b 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -1005,6 +1005,7 @@ The following attributes are supported:
* ``description`` Description
* ``affiliation`` Affiliation
* ``filePIDsEnabled`` ("true" or "false") Restricted to use by superusers and only when the :ref:`:AllowEnablingFilePIDsPerCollection <:AllowEnablingFilePIDsPerCollection>` setting is true. Enables or disables registration of file-level PIDs in datasets within the collection (overriding the instance-wide setting).
+* ``requireFilesToPublishDataset`` ("true" or "false") Dataset needs files in order to be published. Restricted to use by Administrators. If any TRUE found in the ownership tree publishing will be blocked. Publishing by an Administrator will not be blocked.
.. _collection-storage-quotas:
diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java
index 86e2e0207c1..3bbf02fd611 100644
--- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java
@@ -602,7 +602,16 @@ public List<DatasetFieldType> getCitationDatasetFieldTypes() {
public void setCitationDatasetFieldTypes(List<DatasetFieldType> citationDatasetFieldTypes) {
this.citationDatasetFieldTypes = citationDatasetFieldTypes;
}
-
+
+ @Column(nullable = true)
+ private Boolean requireFilesToPublishDataset;
+ public Boolean getRequireFilesToPublishDataset() {
+ return requireFilesToPublishDataset;
+ }
+ public void setRequireFilesToPublishDataset(boolean requireFilesToPublishDataset) {
+ this.requireFilesToPublishDataset = requireFilesToPublishDataset;
+ }
+
/**
* @Note: this setting is Nullable, with {@code null} indicating that the
* desired behavior is not explicitly configured for this specific collection.
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
index 2be6b1e51c2..f549003f70b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
@@ -631,7 +631,7 @@ public Response updateAttribute(@Context ContainerRequestContext crc, @PathParam
}
private Object formatAttributeValue(String attribute, String value) throws WrappedResponse {
- if (attribute.equals("filePIDsEnabled")) {
+ if (List.of("filePIDsEnabled","requireFilesToPublishDataset").contains(attribute)) {
return parseBooleanOrDie(value);
}
return value;
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java
index 1ac41105237..dfe2bb44b20 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java
@@ -2,6 +2,7 @@
import edu.harvard.iq.dataverse.Dataset;
import edu.harvard.iq.dataverse.DatasetLock;
+import edu.harvard.iq.dataverse.Dataverse;
import edu.harvard.iq.dataverse.authorization.Permission;
import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
import edu.harvard.iq.dataverse.engine.command.CommandContext;
@@ -9,8 +10,11 @@
import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
+import edu.harvard.iq.dataverse.util.BundleUtil;
import edu.harvard.iq.dataverse.workflow.Workflow;
import edu.harvard.iq.dataverse.workflow.WorkflowContext.TriggerType;
+
+import java.util.List;
import java.util.Optional;
import java.util.logging.Logger;
import static java.util.stream.Collectors.joining;
@@ -218,9 +222,23 @@ private void verifyCommandArguments(CommandContext ctxt) throws IllegalCommandEx
if (minorRelease && !getDataset().getLatestVersion().isMinorUpdate()) {
throw new IllegalCommandException("Cannot release as minor version. Re-try as major release.", this);
}
+
+ if (getDataset().getFiles().isEmpty() && requiresFilesToPublishDataset()) {
+ throw new IllegalCommandException(BundleUtil.getStringFromBundle("dataset.mayNotPublish.FilesRequired"), this);
+ }
}
}
-
+ private boolean requiresFilesToPublishDataset() {
+ if (!getUser().isSuperuser()) {
+ List<Dataverse> owners = getDataset().getOwner().getOwners();
+ for(Dataverse owner : owners) {
+ if (owner.getRequireFilesToPublishDataset() != null && owner.getRequireFilesToPublishDataset()) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
@Override
public boolean onSuccess(CommandContext ctxt, Object r) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseAttributeCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseAttributeCommand.java
index 57ac20fcee6..ab12d8eea26 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseAttributeCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseAttributeCommand.java
@@ -24,7 +24,7 @@ public class UpdateDataverseAttributeCommand extends AbstractCommand<Dataverse>
private static final String ATTRIBUTE_DESCRIPTION = "description";
private static final String ATTRIBUTE_AFFILIATION = "affiliation";
private static final String ATTRIBUTE_FILE_PIDS_ENABLED = "filePIDsEnabled";
-
+ private static final String ATTRIBUTE_REQUIRE_FILES_TO_PUBLISH_DATASET = "requireFilesToPublishDataset";
private final Dataverse dataverse;
private final String attributeName;
private final Object attributeValue;
@@ -45,8 +45,9 @@ public Dataverse execute(CommandContext ctxt) throws CommandException {
case ATTRIBUTE_AFFILIATION:
setStringAttribute(attributeName, attributeValue);
break;
+ case ATTRIBUTE_REQUIRE_FILES_TO_PUBLISH_DATASET:
case ATTRIBUTE_FILE_PIDS_ENABLED:
- setBooleanAttributeForFilePIDs(ctxt);
+ setBooleanAttribute(ctxt, true);
break;
default:
throw new IllegalCommandException("'" + attributeName + "' is not a supported attribute", this);
@@ -86,25 +87,33 @@ private void setStringAttribute(String attributeName, Object attributeValue) thr
}
/**
- * Helper method to handle the "filePIDsEnabled" boolean attribute.
+ * Helper method to handle boolean attributes.
*
* @param ctxt The command context.
+ * @param adminOnly True if this attribute can only be modified by an Administrator
* @throws PermissionException if the user doesn't have permission to modify this attribute.
*/
- private void setBooleanAttributeForFilePIDs(CommandContext ctxt) throws CommandException {
- if (!getRequest().getUser().isSuperuser()) {
+ private void setBooleanAttribute(CommandContext ctxt, boolean adminOnly) throws CommandException {
+ if (adminOnly && !getRequest().getUser().isSuperuser()) {
throw new PermissionException("You must be a superuser to change this setting",
this, Collections.singleton(Permission.EditDataset), dataverse);
}
- if (!ctxt.settings().isTrueForKey(SettingsServiceBean.Key.AllowEnablingFilePIDsPerCollection, false)) {
- throw new PermissionException("Changing File PID policy per collection is not enabled on this server",
- this, Collections.singleton(Permission.EditDataset), dataverse);
- }
if (!(attributeValue instanceof Boolean)) {
- throw new IllegalCommandException("'" + ATTRIBUTE_FILE_PIDS_ENABLED + "' requires a boolean value", this);
+ throw new IllegalCommandException("'" + attributeName + "' requires a boolean value", this);
+ }
+ switch (attributeName) {
+ case ATTRIBUTE_FILE_PIDS_ENABLED:
+ if (!ctxt.settings().isTrueForKey(SettingsServiceBean.Key.AllowEnablingFilePIDsPerCollection, false)) {
+ throw new PermissionException("Changing File PID policy per collection is not enabled on this server",
+ this, Collections.singleton(Permission.EditDataset), dataverse);
+ }
+ dataverse.setFilePIDsEnabled((Boolean) attributeValue);
+ case ATTRIBUTE_REQUIRE_FILES_TO_PUBLISH_DATASET:
+ dataverse.setRequireFilesToPublishDataset((Boolean) attributeValue);
+ break;
+ default:
+ throw new IllegalCommandException("Unsupported boolean attribute: " + attributeName, this);
}
-
- dataverse.setFilePIDsEnabled((Boolean) attributeValue);
}
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java
index 2f01c9bc2f2..50caf2c6732 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java
@@ -160,6 +160,9 @@ public Dataverse parseDataverse(JsonObject jobj) throws JsonParseException {
if (jobj.containsKey("filePIDsEnabled")) {
dv.setFilePIDsEnabled(jobj.getBoolean("filePIDsEnabled"));
}
+ if (jobj.containsKey("requireFilesToPublishDataset")) {
+ dv.setRequireFilesToPublishDataset(jobj.getBoolean("requireFilesToPublishDataset"));
+ }
/* We decided that subject is not user set, but gotten from the subject of the dataverse's
datasets - leavig this code in for now, in case we need to go back to it at some point
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index 1bdee48b14d..8f5f97512aa 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -292,6 +292,9 @@ public static JsonObjectBuilder json(Dataverse dv, Boolean hideEmail, Boolean re
if (dv.getFilePIDsEnabled() != null) {
bld.add("filePIDsEnabled", dv.getFilePIDsEnabled());
}
+ if (dv.getRequireFilesToPublishDataset() != null) {
+ bld.add("requireFilesToPublishDataset", dv.getRequireFilesToPublishDataset());
+ }
bld.add("isReleased", dv.isReleased());
List<DataverseFieldTypeInputLevel> inputLevels = dv.getDataverseFieldTypeInputLevels();
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 149e6a7e828..ebc09a1d731 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -1542,6 +1542,7 @@ dataset.mayNotPublish.administrator= This dataset cannot be published until {0}
dataset.mayNotPublish.both= This dataset cannot be published until {0} is published. Would you like to publish both right now?
dataset.mayNotPublish.twoGenerations= This dataset cannot be published until {0} and {1} are published.
dataset.mayNotBePublished.both.button=Yes, Publish Both
+dataset.mayNotPublish.FilesRequired=This dataset cannot be published without uploaded files.
dataset.viewVersion.unpublished=View Unpublished Version
dataset.viewVersion.published=View Published Version
dataset.link.title=Link Dataset
diff --git a/src/main/resources/db/migration/V6.5.0.1.sql b/src/main/resources/db/migration/V6.5.0.1.sql
new file mode 100644
index 00000000000..661924b54af
--- /dev/null
+++ b/src/main/resources/db/migration/V6.5.0.1.sql
@@ -0,0 +1,2 @@
+-- files are required to publish datasets
+ALTER TABLE dataverse ADD COLUMN IF NOT EXISTS requirefilestopublishdataset bool;
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 93f1024ae7a..0f24e6b73a9 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -16,16 +16,14 @@
import edu.harvard.iq.dataverse.util.SystemConfig;
import edu.harvard.iq.dataverse.util.json.JSONLDUtil;
import edu.harvard.iq.dataverse.util.json.JsonUtil;
+import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder;
import io.restassured.RestAssured;
import io.restassured.http.ContentType;
import io.restassured.parsing.Parser;
import io.restassured.path.json.JsonPath;
import io.restassured.path.xml.XmlPath;
import io.restassured.response.Response;
-import jakarta.json.Json;
-import jakarta.json.JsonArray;
-import jakarta.json.JsonObject;
-import jakarta.json.JsonObjectBuilder;
+import jakarta.json.*;
import jakarta.ws.rs.core.Response.Status;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
@@ -5168,4 +5166,64 @@ public void testGetCanDownloadAtLeastOneFile() {
Response getUserPermissionsOnDatasetInvalidIdResponse = UtilIT.getCanDownloadAtLeastOneFile("testInvalidId", DS_VERSION_LATEST, secondUserApiToken);
getUserPermissionsOnDatasetInvalidIdResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
}
+
+ @Test
+ public void testRequireFilesToPublishDatasets() {
+ // Create superuser and regular user
+ Response createUserResponse = UtilIT.createRandomUser();
+ createUserResponse.then().assertThat().statusCode(OK.getStatusCode());
+ String usernameAdmin = UtilIT.getUsernameFromResponse(createUserResponse);
+ String apiTokenAdmin = UtilIT.getApiTokenFromResponse(createUserResponse);
+ Response makeSuperUser = UtilIT.makeSuperUser(usernameAdmin);
+ assertEquals(200, makeSuperUser.getStatusCode());
+
+ createUserResponse = UtilIT.createRandomUser();
+ createUserResponse.then().assertThat().statusCode(OK.getStatusCode());
+ String apiToken = UtilIT.getApiTokenFromResponse(createUserResponse);
+
+ // Create and publish a top level Dataverse (under root) with a requireFilesToPublishDataset set to true
+ Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+ String ownerAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+ // Only admin can set this attribute
+ Response setDataverseAttributeResponse = UtilIT.setCollectionAttribute(ownerAlias, "requireFilesToPublishDataset", "true", apiToken);
+ setDataverseAttributeResponse.prettyPrint();
+ setDataverseAttributeResponse.then().assertThat().statusCode(UNAUTHORIZED.getStatusCode());
+ setDataverseAttributeResponse = UtilIT.setCollectionAttribute(ownerAlias, "requireFilesToPublishDataset", "true", apiTokenAdmin);
+ setDataverseAttributeResponse.prettyPrint();
+ setDataverseAttributeResponse.then().assertThat().statusCode(OK.getStatusCode());
+ setDataverseAttributeResponse.then().assertThat().body("data.requireFilesToPublishDataset",equalTo(true));
+ Response publishDataverseResponse = UtilIT.publishDataverseViaNativeApi(ownerAlias, apiTokenAdmin);
+ publishDataverseResponse.prettyPrint();
+ publishDataverseResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+ // Create and publish a new Dataverse under the above Dataverse with requireFilesToPublishDataset not set (default null)
+ String alias = "dv2-" + UtilIT.getRandomIdentifier();
+ createDataverseResponse = UtilIT.createSubDataverse(alias, null, apiToken, ownerAlias);
+ createDataverseResponse.prettyPrint();
+ createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+ publishDataverseResponse = UtilIT.publishDataverseViaNativeApi(alias, apiToken);
+ publishDataverseResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+ // Create a Dataset under the 2nd level Dataverse
+ Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(alias, apiToken);
+ createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+ Integer id = UtilIT.getDatasetIdFromResponse(createDatasetResponse);
+
+ // Try to publish with no files (minimum is 1 file from the top level Dataverse)
+ Response publishDatasetResponse = UtilIT.publishDatasetViaNativeApi(String.valueOf(id), "major", apiToken);
+ publishDatasetResponse.prettyPrint();
+ publishDatasetResponse.then().assertThat().statusCode(FORBIDDEN.getStatusCode());
+ publishDatasetResponse.then().assertThat().body("message", containsString(
+ BundleUtil.getStringFromBundle("dataset.mayNotPublish.FilesRequired")
+ ));
+
+ // Upload 1 file and try to publish again
+ String pathToFile = "src/main/webapp/resources/images/dataverseproject.png";
+ Response uploadResponse = UtilIT.uploadFileViaNative(String.valueOf(id), pathToFile, apiToken);
+ uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+ publishDatasetResponse = UtilIT.publishDatasetViaNativeApi(String.valueOf(id), "major", apiToken);
+ publishDatasetResponse.prettyPrint();
+ publishDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
+ }
}
From 1159134aa8b3cbb987a5e9f3d566de9914f17ff7 Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Thu, 31 Oct 2024 14:42:51 -0400
Subject: [PATCH 06/43] cosmetic fixes
---
src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 0f24e6b73a9..3bc15bfc363 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -16,14 +16,16 @@
import edu.harvard.iq.dataverse.util.SystemConfig;
import edu.harvard.iq.dataverse.util.json.JSONLDUtil;
import edu.harvard.iq.dataverse.util.json.JsonUtil;
-import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder;
import io.restassured.RestAssured;
import io.restassured.http.ContentType;
import io.restassured.parsing.Parser;
import io.restassured.path.json.JsonPath;
import io.restassured.path.xml.XmlPath;
import io.restassured.response.Response;
-import jakarta.json.*;
+import jakarta.json.Json;
+import jakarta.json.JsonArray;
+import jakarta.json.JsonObject;
+import jakarta.json.JsonObjectBuilder;
import jakarta.ws.rs.core.Response.Status;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
From c0c4c89aee8e9dd5acfb979dbc0c36f1803da57a Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Thu, 31 Oct 2024 15:56:01 -0400
Subject: [PATCH 07/43] change Admin to superuser in docs
---
.../10981-published-datasets-should-contain-files.md | 4 ++--
doc/sphinx-guides/source/api/native-api.rst | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/doc/release-notes/10981-published-datasets-should-contain-files.md b/doc/release-notes/10981-published-datasets-should-contain-files.md
index 73c76744164..74c932f853a 100644
--- a/doc/release-notes/10981-published-datasets-should-contain-files.md
+++ b/doc/release-notes/10981-published-datasets-should-contain-files.md
@@ -1,6 +1,6 @@
## Feature: Prevent publishing Datasets without files
-A new attribute was added to Collections in order to control the publishing of Datasets without files. Once set, the publishing of a Dataset within a Collection or Collection's hierarchy, without files, will be blocked for all non Admin users.
-In order to configure a Collection to block publishing an Admin must set the attribute "requireFilesToPublishDataset" to true.
+A new attribute was added to Collections in order to control the publishing of Datasets without files. Once set, the publishing of a Dataset within a Collection or Collection's hierarchy, without files, will be blocked for all non superusers.
+In order to configure a Collection to block publishing a superuser must set the attribute "requireFilesToPublishDataset" to true.
Any Collection created under a Collection with this attribute will also be bound by this blocking. Setting this attribute on the Root Dataverse will essentially block the publishing of Datasets without files for the entire installation.
```shell
curl -X PUT -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/attribute/requireFilesToPublishDataset?value=true"
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 8fab30a884b..d174d4a87cd 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -1005,7 +1005,7 @@ The following attributes are supported:
* ``description`` Description
* ``affiliation`` Affiliation
* ``filePIDsEnabled`` ("true" or "false") Restricted to use by superusers and only when the :ref:`:AllowEnablingFilePIDsPerCollection <:AllowEnablingFilePIDsPerCollection>` setting is true. Enables or disables registration of file-level PIDs in datasets within the collection (overriding the instance-wide setting).
-* ``requireFilesToPublishDataset`` ("true" or "false") Dataset needs files in order to be published. Restricted to use by Administrators. If any TRUE found in the ownership tree publishing will be blocked. Publishing by an Administrator will not be blocked.
+* ``requireFilesToPublishDataset`` ("true" or "false") Dataset needs files in order to be published. Restricted to use by superusers. If any TRUE found in the ownership tree publishing will be blocked. Publishing by a superusers will not be blocked.
.. _collection-storage-quotas:
From 85af55f8f1a6dfe0165a28d425cdcc6f51f6ea2d Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Fri, 1 Nov 2024 10:45:06 -0400
Subject: [PATCH 08/43] change how hierarchy works
---
.../10981-published-datasets-should-contain-files.md | 8 +++++---
doc/sphinx-guides/source/api/native-api.rst | 2 +-
src/main/java/edu/harvard/iq/dataverse/Dataverse.java | 9 +++++++++
.../engine/command/impl/PublishDatasetCommand.java | 9 +++++----
4 files changed, 20 insertions(+), 8 deletions(-)
diff --git a/doc/release-notes/10981-published-datasets-should-contain-files.md b/doc/release-notes/10981-published-datasets-should-contain-files.md
index 74c932f853a..964e7ff1937 100644
--- a/doc/release-notes/10981-published-datasets-should-contain-files.md
+++ b/doc/release-notes/10981-published-datasets-should-contain-files.md
@@ -1,7 +1,9 @@
## Feature: Prevent publishing Datasets without files
-A new attribute was added to Collections in order to control the publishing of Datasets without files. Once set, the publishing of a Dataset within a Collection or Collection's hierarchy, without files, will be blocked for all non superusers.
-In order to configure a Collection to block publishing a superuser must set the attribute "requireFilesToPublishDataset" to true.
-Any Collection created under a Collection with this attribute will also be bound by this blocking. Setting this attribute on the Root Dataverse will essentially block the publishing of Datasets without files for the entire installation.
+A new attribute was added to Collections in order to control the publishing of Datasets without files.
+Once set to "True", the publishing of a Dataset within a Collection, without files, will be blocked for all non superusers.
+In order to configure a Collection to block publishing a superuser must set the attribute "requireFilesToPublishDataset" to "True".
+The collection's hierarchy will be checked if the collection's "requireFilesToPublishDataset" attribute is not set explicitly to "True" or "False".
+
```shell
curl -X PUT -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/attribute/requireFilesToPublishDataset?value=true"
```
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index d174d4a87cd..8d73dd9dd24 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -1005,7 +1005,7 @@ The following attributes are supported:
* ``description`` Description
* ``affiliation`` Affiliation
* ``filePIDsEnabled`` ("true" or "false") Restricted to use by superusers and only when the :ref:`:AllowEnablingFilePIDsPerCollection <:AllowEnablingFilePIDsPerCollection>` setting is true. Enables or disables registration of file-level PIDs in datasets within the collection (overriding the instance-wide setting).
-* ``requireFilesToPublishDataset`` ("true" or "false") Dataset needs files in order to be published. Restricted to use by superusers. If any TRUE found in the ownership tree publishing will be blocked. Publishing by a superusers will not be blocked.
+* ``requireFilesToPublishDataset`` ("true" or "false") Restricted to use by superusers. Defines if Dataset needs files in order to be published. If not set the determination will be made through inheritance by checking the owners of this collection. Publishing by a superusers will not be blocked.
.. _collection-storage-quotas:
diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java
index 3bbf02fd611..5b6fbdee6ba 100644
--- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java
@@ -605,6 +605,15 @@ public void setCitationDatasetFieldTypes(List<DatasetFieldType> citationDatasetF
@Column(nullable = true)
private Boolean requireFilesToPublishDataset;
+ /**
+ * Specifies whether the existance of files in a dataset is required when publishing
+ * @return {@code Boolean.TRUE} if explicitly enabled, {@code Boolean.FALSE} if explicitly disabled.
+ * {@code null} indicates that the behavior is not explicitly defined, in which
+ * case the behavior should follow the explicit configuration of the first
+ * direct ancestor collection.
+ * @Note: If present, this configuration therefore by default applies to all
+ * the sub-collections, unless explicitly overwritten there.
+ */
public Boolean getRequireFilesToPublishDataset() {
return requireFilesToPublishDataset;
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java
index dfe2bb44b20..50800f72271 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java
@@ -230,11 +230,12 @@ private void verifyCommandArguments(CommandContext ctxt) throws IllegalCommandEx
}
private boolean requiresFilesToPublishDataset() {
if (!getUser().isSuperuser()) {
- List<Dataverse> owners = getDataset().getOwner().getOwners();
- for(Dataverse owner : owners) {
- if (owner.getRequireFilesToPublishDataset() != null && owner.getRequireFilesToPublishDataset()) {
- return true;
+ Dataverse parent = getDataset().getOwner();
+ while (parent != null) {
+ if (parent.getRequireFilesToPublishDataset() != null) {
+ return parent.getRequireFilesToPublishDataset();
}
+ parent = parent.getOwner();
}
}
return false;
From 4b1955cdf05e041291c403045e7d67b593edf936 Mon Sep 17 00:00:00 2001
From: stevenferey <steven.ferey@gmail.com>
Date: Wed, 30 Oct 2024 15:01:05 +0100
Subject: [PATCH 09/43] improvement and internationalization of harvest status
---
...-internationalization-of-harvest-status.md | 6 ++
.../harvest/client/ClientHarvestRun.java | 57 +++++++++++--------
.../harvest/client/HarvesterServiceBean.java | 15 +++--
.../harvest/client/HarvestingClient.java | 4 +-
.../client/HarvestingClientServiceBean.java | 9 ++-
src/main/java/propertyFiles/Bundle.properties | 7 +++
6 files changed, 66 insertions(+), 32 deletions(-)
create mode 100644 doc/release-notes/9294-improvement-and-internationalization-of-harvest-status.md
diff --git a/doc/release-notes/9294-improvement-and-internationalization-of-harvest-status.md b/doc/release-notes/9294-improvement-and-internationalization-of-harvest-status.md
new file mode 100644
index 00000000000..f9fc465292c
--- /dev/null
+++ b/doc/release-notes/9294-improvement-and-internationalization-of-harvest-status.md
@@ -0,0 +1,6 @@
+## Improvement and internationalization of harvest status
+
+Added a harvest status to differentiate a complete harvest with errors (Completed with failures) and without errors (Completed)
+Harvest status labels are now internationalized
+
+For more information, see issue [#9294](https://github.com/IQSS/dataverse/issues/9294)
\ No newline at end of file
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java
index ba6f5c3dec2..153a4deea0e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java
@@ -7,6 +7,8 @@
import java.io.Serializable;
import java.util.Date;
+
+import edu.harvard.iq.dataverse.util.BundleUtil;
import jakarta.persistence.Entity;
import jakarta.persistence.GeneratedValue;
import jakarta.persistence.GenerationType;
@@ -40,13 +42,7 @@ public void setId(Long id) {
this.id = id;
}
- public enum RunResultType { SUCCESS, FAILURE, INPROGRESS, INTERRUPTED };
-
- private static String RESULT_LABEL_SUCCESS = "SUCCESS";
- private static String RESULT_LABEL_FAILURE = "FAILED";
- private static String RESULT_LABEL_INPROGRESS = "IN PROGRESS";
- private static String RESULT_DELETE_IN_PROGRESS = "DELETE IN PROGRESS";
- private static String RESULT_LABEL_INTERRUPTED = "INTERRUPTED";
+ public enum RunResultType { COMPLETED, COMPLETED_WITH_FAILLURES, FAILURE, INPROGRESS, INTERRUPTED };
@ManyToOne
@JoinColumn(nullable = false)
@@ -68,36 +64,41 @@ public RunResultType getResult() {
public String getResultLabel() {
if (harvestingClient != null && harvestingClient.isDeleteInProgress()) {
- return RESULT_DELETE_IN_PROGRESS;
+ return BundleUtil.getStringFromBundle("harvestclients.result.deleteInProgress");
}
- if (isSuccess()) {
- return RESULT_LABEL_SUCCESS;
+ if (isCompleted()) {
+ return BundleUtil.getStringFromBundle("harvestclients.result.completed");
+ } else if (isCompletedWithFaillures()) {
+ return BundleUtil.getStringFromBundle("harvestclients.result.completedWithFaillures");
} else if (isFailed()) {
- return RESULT_LABEL_FAILURE;
+ return BundleUtil.getStringFromBundle("harvestclients.result.failure");
} else if (isInProgress()) {
- return RESULT_LABEL_INPROGRESS;
+ return BundleUtil.getStringFromBundle("harvestclients.result.inProgess");
} else if (isInterrupted()) {
- return RESULT_LABEL_INTERRUPTED;
+ return BundleUtil.getStringFromBundle("harvestclients.result.interrupted");
}
return null;
}
public String getDetailedResultLabel() {
if (harvestingClient != null && harvestingClient.isDeleteInProgress()) {
- return RESULT_DELETE_IN_PROGRESS;
+ return BundleUtil.getStringFromBundle("harvestclients.result.deleteInProgress");
}
- if (isSuccess() || isInterrupted()) {
+ if (isCompleted() || isCompletedWithFaillures() || isInterrupted()) {
String resultLabel = getResultLabel();
+
+ String details = BundleUtil.getStringFromBundle("harvestclients.result.details");
+ details = details.replace("{0}", String.valueOf(harvestedDatasetCount));
+ details = details.replace("{1}", String.valueOf(deletedDatasetCount));
+ details = details.replace("{2}", String.valueOf(failedDatasetCount));
- resultLabel = resultLabel.concat("; "+harvestedDatasetCount+" harvested, ");
- resultLabel = resultLabel.concat(deletedDatasetCount+" deleted, ");
- resultLabel = resultLabel.concat(failedDatasetCount+" failed.");
+ resultLabel = resultLabel.concat("; " + details);
return resultLabel;
} else if (isFailed()) {
- return RESULT_LABEL_FAILURE;
+ return BundleUtil.getStringFromBundle("harvestclients.result.failure");
} else if (isInProgress()) {
- return RESULT_LABEL_INPROGRESS;
+ return BundleUtil.getStringFromBundle("harvestclients.result.inProgess");
}
return null;
}
@@ -106,12 +107,20 @@ public void setResult(RunResultType harvestResult) {
this.harvestResult = harvestResult;
}
- public boolean isSuccess() {
- return RunResultType.SUCCESS == harvestResult;
+ public boolean isCompleted() {
+ return RunResultType.COMPLETED == harvestResult;
+ }
+
+ public void setCompleted() {
+ harvestResult = RunResultType.COMPLETED;
+ }
+
+ public boolean isCompletedWithFaillures() {
+ return RunResultType.COMPLETED_WITH_FAILLURES == harvestResult;
}
- public void setSuccess() {
- harvestResult = RunResultType.SUCCESS;
+ public void setCompletedWithFaillures() {
+ harvestResult = RunResultType.COMPLETED_WITH_FAILLURES;
}
public boolean isFailed() {
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
index e0b5c2dfbfb..8603dbbe128 100644
--- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
@@ -163,7 +163,7 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId
try {
if (harvestingClientConfig.isHarvestingNow()) {
- hdLogger.log(Level.SEVERE, "Cannot start harvest, client " + harvestingClientConfig.getName() + " is already harvesting.");
+ hdLogger.log(Level.SEVERE, String.format("Cannot start harvest, client %s is already harvesting.", harvestingClientConfig.getName()));
} else {
harvestingClientService.resetHarvestInProgress(harvestingClientId);
@@ -176,9 +176,16 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId
} else {
throw new IOException("Unsupported harvest type");
}
- harvestingClientService.setHarvestSuccess(harvestingClientId, new Date(), harvestedDatasetIds.size(), failedIdentifiers.size(), deletedIdentifiers.size());
- hdLogger.log(Level.INFO, "COMPLETED HARVEST, server=" + harvestingClientConfig.getArchiveUrl() + ", metadataPrefix=" + harvestingClientConfig.getMetadataPrefix());
- hdLogger.log(Level.INFO, "Datasets created/updated: " + harvestedDatasetIds.size() + ", datasets deleted: " + deletedIdentifiers.size() + ", datasets failed: " + failedIdentifiers.size());
+
+ if (failedIdentifiers.isEmpty()) {
+ harvestingClientService.setHarvestCompleted(harvestingClientId, new Date(), harvestedDatasetIds.size(), failedIdentifiers.size(), deletedIdentifiers.size());
+ hdLogger.log(Level.INFO, String.format("\"COMPLETED HARVEST, server=%s, metadataPrefix=%s", harvestingClientConfig.getArchiveUrl(), harvestingClientConfig.getMetadataPrefix()));
+ } else {
+ harvestingClientService.setHarvestCompletedWithFaillure(harvestingClientId, new Date(), harvestedDatasetIds.size(), failedIdentifiers.size(), deletedIdentifiers.size());
+ hdLogger.log(Level.INFO, String.format("\"COMPLETED HARVEST WITH FAILLURE, server=%s, metadataPrefix=%s", harvestingClientConfig.getArchiveUrl(), harvestingClientConfig.getMetadataPrefix()));
+ }
+
+ hdLogger.log(Level.INFO, String.format("Datasets created/updated: %s, datasets deleted: %s, datasets failed: %s", harvestedDatasetIds.size(), deletedIdentifiers.size(), failedIdentifiers.size()));
}
} catch (StopHarvestException she) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java
index 0667f5594ce..ad83a4ec342 100644
--- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java
@@ -289,7 +289,7 @@ public ClientHarvestRun getLastSuccessfulRun() {
int i = harvestHistory.size() - 1;
while (i > -1) {
- if (harvestHistory.get(i).isSuccess()) {
+ if (harvestHistory.get(i).isCompleted()) {
return harvestHistory.get(i);
}
i--;
@@ -306,7 +306,7 @@ ClientHarvestRun getLastNonEmptyRun() {
int i = harvestHistory.size() - 1;
while (i > -1) {
- if (harvestHistory.get(i).isSuccess()) {
+ if (harvestHistory.get(i).isCompleted()) {
if (harvestHistory.get(i).getHarvestedDatasetCount().longValue() > 0 ||
harvestHistory.get(i).getDeletedDatasetCount().longValue() > 0) {
return harvestHistory.get(i);
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java
index 7ec6d75a41c..4a8e3992916 100644
--- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java
@@ -164,8 +164,13 @@ public void deleteClient(Long clientId) {
}
@TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
- public void setHarvestSuccess(Long hcId, Date currentTime, int harvestedCount, int failedCount, int deletedCount) {
- recordHarvestJobStatus(hcId, currentTime, harvestedCount, failedCount, deletedCount, ClientHarvestRun.RunResultType.SUCCESS);
+ public void setHarvestCompleted(Long hcId, Date currentTime, int harvestedCount, int failedCount, int deletedCount) {
+ recordHarvestJobStatus(hcId, currentTime, harvestedCount, failedCount, deletedCount, ClientHarvestRun.RunResultType.COMPLETED);
+ }
+
+ @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
+ public void setHarvestCompletedWithFaillure(Long hcId, Date currentTime, int harvestedCount, int failedCount, int deletedCount) {
+ recordHarvestJobStatus(hcId, currentTime, harvestedCount, failedCount, deletedCount, ClientHarvestRun.RunResultType.COMPLETED_WITH_FAILLURES);
}
@TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 149e6a7e828..1cca32fedcd 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -630,6 +630,13 @@ harvestclients.viewEditDialog.archiveDescription.tip=Description of the archival
harvestclients.viewEditDialog.archiveDescription.default.generic=This Dataset is harvested from our partners. Clicking the link will take you directly to the archival source of the data.
harvestclients.viewEditDialog.btn.save=Save Changes
harvestclients.newClientDialog.title.edit=Edit Group {0}
+harvestclients.result.completed=Completed
+harvestclients.result.completedWithFaillures=Completed with failures
+harvestclients.result.failure=FAILED
+harvestclients.result.inProgess=IN PROGRESS
+harvestclients.result.deleteInProgress=DELETE IN PROGRESS
+harvestclients.result.interrupted=INTERRUPTED
+harvestclients.result.details={0} harvested, {1} deleted, {2} failed.
#harvestset.xhtml
harvestserver.title=Manage Harvesting Server
From a7f539606a8900c553c925c58c5f14e32ea884f7 Mon Sep 17 00:00:00 2001
From: Ludovic DANIEL <ludovic.daniel@smile.fr>
Date: Wed, 13 Nov 2024 16:13:27 +0100
Subject: [PATCH 10/43] minor fixes
---
.../harvest/client/ClientHarvestRun.java | 37 ++++++++++---------
.../harvest/client/HarvesterServiceBean.java | 4 +-
.../client/HarvestingClientServiceBean.java | 4 +-
src/main/java/propertyFiles/Bundle.properties | 2 +-
4 files changed, 25 insertions(+), 22 deletions(-)
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java
index 153a4deea0e..6a85219cc3c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java
@@ -6,6 +6,7 @@
package edu.harvard.iq.dataverse.harvest.client;
import java.io.Serializable;
+import java.util.Arrays;
import java.util.Date;
import edu.harvard.iq.dataverse.util.BundleUtil;
@@ -42,7 +43,7 @@ public void setId(Long id) {
this.id = id;
}
- public enum RunResultType { COMPLETED, COMPLETED_WITH_FAILLURES, FAILURE, INPROGRESS, INTERRUPTED };
+ public enum RunResultType { COMPLETED, COMPLETED_WITH_FAILURES, FAILURE, IN_PROGRESS, INTERRUPTED }
@ManyToOne
@JoinColumn(nullable = false)
@@ -66,11 +67,11 @@ public String getResultLabel() {
if (harvestingClient != null && harvestingClient.isDeleteInProgress()) {
return BundleUtil.getStringFromBundle("harvestclients.result.deleteInProgress");
}
-
+
if (isCompleted()) {
return BundleUtil.getStringFromBundle("harvestclients.result.completed");
- } else if (isCompletedWithFaillures()) {
- return BundleUtil.getStringFromBundle("harvestclients.result.completedWithFaillures");
+ } else if (isCompletedWithFailures()) {
+ return BundleUtil.getStringFromBundle("harvestclients.result.completedWithFailures");
} else if (isFailed()) {
return BundleUtil.getStringFromBundle("harvestclients.result.failure");
} else if (isInProgress()) {
@@ -85,15 +86,17 @@ public String getDetailedResultLabel() {
if (harvestingClient != null && harvestingClient.isDeleteInProgress()) {
return BundleUtil.getStringFromBundle("harvestclients.result.deleteInProgress");
}
- if (isCompleted() || isCompletedWithFaillures() || isInterrupted()) {
+ if (isCompleted() || isCompletedWithFailures() || isInterrupted()) {
String resultLabel = getResultLabel();
- String details = BundleUtil.getStringFromBundle("harvestclients.result.details");
- details = details.replace("{0}", String.valueOf(harvestedDatasetCount));
- details = details.replace("{1}", String.valueOf(deletedDatasetCount));
- details = details.replace("{2}", String.valueOf(failedDatasetCount));
-
- resultLabel = resultLabel.concat("; " + details);
+ String details = BundleUtil.getStringFromBundle("harvestclients.result.details", Arrays.asList(
+ harvestedDatasetCount.toString(),
+ deletedDatasetCount.toString(),
+ failedDatasetCount.toString()
+ ));
+ if(details != null) {
+ resultLabel = resultLabel + "; " + details;
+ }
return resultLabel;
} else if (isFailed()) {
return BundleUtil.getStringFromBundle("harvestclients.result.failure");
@@ -115,12 +118,12 @@ public void setCompleted() {
harvestResult = RunResultType.COMPLETED;
}
- public boolean isCompletedWithFaillures() {
- return RunResultType.COMPLETED_WITH_FAILLURES == harvestResult;
+ public boolean isCompletedWithFailures() {
+ return RunResultType.COMPLETED_WITH_FAILURES == harvestResult;
}
- public void setCompletedWithFaillures() {
- harvestResult = RunResultType.COMPLETED_WITH_FAILLURES;
+ public void setCompletedWithFailures() {
+ harvestResult = RunResultType.COMPLETED_WITH_FAILURES;
}
public boolean isFailed() {
@@ -132,12 +135,12 @@ public void setFailed() {
}
public boolean isInProgress() {
- return RunResultType.INPROGRESS == harvestResult ||
+ return RunResultType.IN_PROGRESS == harvestResult ||
(harvestResult == null && startTime != null && finishTime == null);
}
public void setInProgress() {
- harvestResult = RunResultType.INPROGRESS;
+ harvestResult = RunResultType.IN_PROGRESS;
}
public boolean isInterrupted() {
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
index 8603dbbe128..16580f8f9f1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
@@ -181,8 +181,8 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId
harvestingClientService.setHarvestCompleted(harvestingClientId, new Date(), harvestedDatasetIds.size(), failedIdentifiers.size(), deletedIdentifiers.size());
hdLogger.log(Level.INFO, String.format("\"COMPLETED HARVEST, server=%s, metadataPrefix=%s", harvestingClientConfig.getArchiveUrl(), harvestingClientConfig.getMetadataPrefix()));
} else {
- harvestingClientService.setHarvestCompletedWithFaillure(harvestingClientId, new Date(), harvestedDatasetIds.size(), failedIdentifiers.size(), deletedIdentifiers.size());
- hdLogger.log(Level.INFO, String.format("\"COMPLETED HARVEST WITH FAILLURE, server=%s, metadataPrefix=%s", harvestingClientConfig.getArchiveUrl(), harvestingClientConfig.getMetadataPrefix()));
+ harvestingClientService.setHarvestCompletedWithFailures(harvestingClientId, new Date(), harvestedDatasetIds.size(), failedIdentifiers.size(), deletedIdentifiers.size());
+ hdLogger.log(Level.INFO, String.format("\"COMPLETED HARVEST WITH FAILURES, server=%s, metadataPrefix=%s", harvestingClientConfig.getArchiveUrl(), harvestingClientConfig.getMetadataPrefix()));
}
hdLogger.log(Level.INFO, String.format("Datasets created/updated: %s, datasets deleted: %s, datasets failed: %s", harvestedDatasetIds.size(), deletedIdentifiers.size(), failedIdentifiers.size()));
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java
index 4a8e3992916..2f76fed1a11 100644
--- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java
@@ -169,8 +169,8 @@ public void setHarvestCompleted(Long hcId, Date currentTime, int harvestedCount,
}
@TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
- public void setHarvestCompletedWithFaillure(Long hcId, Date currentTime, int harvestedCount, int failedCount, int deletedCount) {
- recordHarvestJobStatus(hcId, currentTime, harvestedCount, failedCount, deletedCount, ClientHarvestRun.RunResultType.COMPLETED_WITH_FAILLURES);
+ public void setHarvestCompletedWithFailures(Long hcId, Date currentTime, int harvestedCount, int failedCount, int deletedCount) {
+ recordHarvestJobStatus(hcId, currentTime, harvestedCount, failedCount, deletedCount, ClientHarvestRun.RunResultType.COMPLETED_WITH_FAILURES);
}
@TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 1cca32fedcd..3d693b81daf 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -631,7 +631,7 @@ harvestclients.viewEditDialog.archiveDescription.default.generic=This Dataset is
harvestclients.viewEditDialog.btn.save=Save Changes
harvestclients.newClientDialog.title.edit=Edit Group {0}
harvestclients.result.completed=Completed
-harvestclients.result.completedWithFaillures=Completed with failures
+harvestclients.result.completedWithFailures=Completed with failures
harvestclients.result.failure=FAILED
harvestclients.result.inProgess=IN PROGRESS
harvestclients.result.deleteInProgress=DELETE IN PROGRESS
From 939d6803ca6e0aae7b6eebcc2f17b6f96e55454a Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Fri, 15 Nov 2024 16:21:04 -0500
Subject: [PATCH 11/43] remove date_range for now, revert to schema.xml from
#11024 #10887
---
conf/solr/schema.xml | 30 ++++++++++++++----------------
1 file changed, 14 insertions(+), 16 deletions(-)
diff --git a/conf/solr/schema.xml b/conf/solr/schema.xml
index 02e699722f7..e9a5364d6fd 100644
--- a/conf/solr/schema.xml
+++ b/conf/solr/schema.xml
@@ -291,12 +291,12 @@
<field name="coverage.Temporal.StopTime" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="dataCollectionSituation" type="text_en" multiValued="false" stored="true" indexed="true"/>
<field name="dataCollector" type="text_en" multiValued="false" stored="true" indexed="true"/>
- <field name="dataSources" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="datasetContact" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="datasetContactAffiliation" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="datasetContactEmail" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="datasetContactName" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="datasetLevelErrorNotes" type="text_en" multiValued="false" stored="true" indexed="true"/>
+ <field name="dataSources" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="dateOfCollection" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="dateOfCollectionEnd" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="dateOfCollectionStart" type="text_en" multiValued="true" stored="true" indexed="true"/>
@@ -327,8 +327,8 @@
<field name="journalVolume" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="journalVolumeIssue" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="keyword" type="text_en" multiValued="true" stored="true" indexed="true"/>
- <field name="keywordValue" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="keywordTermURI" type="text_en" multiValued="true" stored="true" indexed="true"/>
+ <field name="keywordValue" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="keywordVocabulary" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="keywordVocabularyURI" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="kindOfData" type="text_en" multiValued="true" stored="true" indexed="true"/>
@@ -352,9 +352,9 @@
<field name="productionPlace" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="publication" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="publicationCitation" type="text_en" multiValued="true" stored="true" indexed="true"/>
- <field name="publicationRelationType" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="publicationIDNumber" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="publicationIDType" type="text_en" multiValued="true" stored="true" indexed="true"/>
+ <field name="publicationRelationType" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="publicationURL" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="redshiftType" type="text_en" multiValued="false" stored="true" indexed="true"/>
<field name="relatedDatasets" type="text_en" multiValued="true" stored="true" indexed="true"/>
@@ -384,13 +384,13 @@
<field name="studyAssayOrganism" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="studyAssayOtherMeasurmentType" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="studyAssayOtherOrganism" type="text_en" multiValued="true" stored="true" indexed="true"/>
- <field name="studyAssayPlatform" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="studyAssayOtherPlatform" type="text_en" multiValued="true" stored="true" indexed="true"/>
- <field name="studyAssayTechnologyType" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="studyAssayOtherTechnologyType" type="text_en" multiValued="true" stored="true" indexed="true"/>
+ <field name="studyAssayPlatform" type="text_en" multiValued="true" stored="true" indexed="true"/>
+ <field name="studyAssayTechnologyType" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="studyDesignType" type="text_en" multiValued="true" stored="true" indexed="true"/>
- <field name="studyOtherDesignType" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="studyFactorType" type="text_en" multiValued="true" stored="true" indexed="true"/>
+ <field name="studyOtherDesignType" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="studyOtherFactorType" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="subject" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="subtitle" type="text_en" multiValued="false" stored="true" indexed="true"/>
@@ -402,10 +402,10 @@
<field name="timePeriodCoveredEnd" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="timePeriodCoveredStart" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="title" type="text_en" multiValued="false" stored="true" indexed="true"/>
+ <field name="topicClassification" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="topicClassValue" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="topicClassVocab" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="topicClassVocabURI" type="text_en" multiValued="true" stored="true" indexed="true"/>
- <field name="topicClassification" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="unitOfAnalysis" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="universe" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="weighting" type="text_en" multiValued="false" stored="true" indexed="true"/>
@@ -533,12 +533,12 @@
<copyField source="coverage.Temporal.StopTime" dest="_text_" maxChars="3000"/>
<copyField source="dataCollectionSituation" dest="_text_" maxChars="3000"/>
<copyField source="dataCollector" dest="_text_" maxChars="3000"/>
- <copyField source="dataSources" dest="_text_" maxChars="3000"/>
<copyField source="datasetContact" dest="_text_" maxChars="3000"/>
<copyField source="datasetContactAffiliation" dest="_text_" maxChars="3000"/>
<copyField source="datasetContactEmail" dest="_text_" maxChars="3000"/>
<copyField source="datasetContactName" dest="_text_" maxChars="3000"/>
<copyField source="datasetLevelErrorNotes" dest="_text_" maxChars="3000"/>
+ <copyField source="dataSources" dest="_text_" maxChars="3000"/>
<copyField source="dateOfCollection" dest="_text_" maxChars="3000"/>
<copyField source="dateOfCollectionEnd" dest="_text_" maxChars="3000"/>
<copyField source="dateOfCollectionStart" dest="_text_" maxChars="3000"/>
@@ -569,8 +569,8 @@
<copyField source="journalVolume" dest="_text_" maxChars="3000"/>
<copyField source="journalVolumeIssue" dest="_text_" maxChars="3000"/>
<copyField source="keyword" dest="_text_" maxChars="3000"/>
- <copyField source="keywordValue" dest="_text_" maxChars="3000"/>
<copyField source="keywordTermURI" dest="_text_" maxChars="3000"/>
+ <copyField source="keywordValue" dest="_text_" maxChars="3000"/>
<copyField source="keywordVocabulary" dest="_text_" maxChars="3000"/>
<copyField source="keywordVocabularyURI" dest="_text_" maxChars="3000"/>
<copyField source="kindOfData" dest="_text_" maxChars="3000"/>
@@ -594,9 +594,9 @@
<copyField source="productionPlace" dest="_text_" maxChars="3000"/>
<copyField source="publication" dest="_text_" maxChars="3000"/>
<copyField source="publicationCitation" dest="_text_" maxChars="3000"/>
- <copyField source="publicationRelationType" dest="_text_" maxChars="3000"/>
<copyField source="publicationIDNumber" dest="_text_" maxChars="3000"/>
<copyField source="publicationIDType" dest="_text_" maxChars="3000"/>
+ <copyField source="publicationRelationType" dest="_text_" maxChars="3000"/>
<copyField source="publicationURL" dest="_text_" maxChars="3000"/>
<copyField source="redshiftType" dest="_text_" maxChars="3000"/>
<copyField source="relatedDatasets" dest="_text_" maxChars="3000"/>
@@ -626,13 +626,13 @@
<copyField source="studyAssayOrganism" dest="_text_" maxChars="3000"/>
<copyField source="studyAssayOtherMeasurmentType" dest="_text_" maxChars="3000"/>
<copyField source="studyAssayOtherOrganism" dest="_text_" maxChars="3000"/>
- <copyField source="studyAssayPlatform" dest="_text_" maxChars="3000"/>
<copyField source="studyAssayOtherPlatform" dest="_text_" maxChars="3000"/>
- <copyField source="studyAssayTechnologyType" dest="_text_" maxChars="3000"/>
<copyField source="studyAssayOtherTechnologyType" dest="_text_" maxChars="3000"/>
+ <copyField source="studyAssayPlatform" dest="_text_" maxChars="3000"/>
+ <copyField source="studyAssayTechnologyType" dest="_text_" maxChars="3000"/>
<copyField source="studyDesignType" dest="_text_" maxChars="3000"/>
- <copyField source="studyOtherDesignType" dest="_text_" maxChars="3000"/>
<copyField source="studyFactorType" dest="_text_" maxChars="3000"/>
+ <copyField source="studyOtherDesignType" dest="_text_" maxChars="3000"/>
<copyField source="studyOtherFactorType" dest="_text_" maxChars="3000"/>
<copyField source="subject" dest="_text_" maxChars="3000"/>
<copyField source="subtitle" dest="_text_" maxChars="3000"/>
@@ -644,10 +644,10 @@
<copyField source="timePeriodCoveredEnd" dest="_text_" maxChars="3000"/>
<copyField source="timePeriodCoveredStart" dest="_text_" maxChars="3000"/>
<copyField source="title" dest="_text_" maxChars="3000"/>
+ <copyField source="topicClassification" dest="_text_" maxChars="3000"/>
<copyField source="topicClassValue" dest="_text_" maxChars="3000"/>
<copyField source="topicClassVocab" dest="_text_" maxChars="3000"/>
<copyField source="topicClassVocabURI" dest="_text_" maxChars="3000"/>
- <copyField source="topicClassification" dest="_text_" maxChars="3000"/>
<copyField source="unitOfAnalysis" dest="_text_" maxChars="3000"/>
<copyField source="universe" dest="_text_" maxChars="3000"/>
<copyField source="weighting" dest="_text_" maxChars="3000"/>
@@ -814,8 +814,6 @@
<!-- KD-tree versions of date fields -->
<fieldType name="pdate" class="solr.DatePointField" docValues="true"/>
<fieldType name="pdates" class="solr.DatePointField" docValues="true" multiValued="true"/>
-
- <fieldType name="date_range" class="solr.DateRangeField"/>
<!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings -->
<fieldType name="binary" class="solr.BinaryField"/>
From 55e3c605090045a4a2bd1a4d87b1c70f11141a20 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Fri, 15 Nov 2024 16:25:11 -0500
Subject: [PATCH 12/43] update Solr schema.xml using update-fields.sh #10887
This reflects the following changes:
* Integer fields are indexed as `plong` instead of `text_en`
* Float fields are indexed as `pdouble` instead of `text_en`
* Date fields are indexed as `date_range` (`solr.DateRangeField`) instead of `text_en`
---
conf/solr/schema.xml | 46 ++++++++++++++++++++++----------------------
1 file changed, 23 insertions(+), 23 deletions(-)
diff --git a/conf/solr/schema.xml b/conf/solr/schema.xml
index e9a5364d6fd..0b95cc482d2 100644
--- a/conf/solr/schema.xml
+++ b/conf/solr/schema.xml
@@ -272,23 +272,23 @@
<field name="contributorType" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="controlOperations" type="text_en" multiValued="false" stored="true" indexed="true"/>
<field name="country" type="text_en" multiValued="true" stored="true" indexed="true"/>
- <field name="coverage.Depth" type="text_en" multiValued="false" stored="true" indexed="true"/>
- <field name="coverage.ObjectCount" type="text_en" multiValued="false" stored="true" indexed="true"/>
- <field name="coverage.ObjectDensity" type="text_en" multiValued="false" stored="true" indexed="true"/>
+ <field name="coverage.Depth" type="pdouble" multiValued="false" stored="true" indexed="true"/>
+ <field name="coverage.ObjectCount" type="plong" multiValued="false" stored="true" indexed="true"/>
+ <field name="coverage.ObjectDensity" type="pdouble" multiValued="false" stored="true" indexed="true"/>
<field name="coverage.Polarization" type="text_en" multiValued="false" stored="true" indexed="true"/>
- <field name="coverage.Redshift.MaximumValue" type="text_en" multiValued="true" stored="true" indexed="true"/>
- <field name="coverage.Redshift.MinimumValue" type="text_en" multiValued="true" stored="true" indexed="true"/>
- <field name="coverage.RedshiftValue" type="text_en" multiValued="true" stored="true" indexed="true"/>
- <field name="coverage.SkyFraction" type="text_en" multiValued="false" stored="true" indexed="true"/>
+ <field name="coverage.Redshift.MaximumValue" type="pdouble" multiValued="true" stored="true" indexed="true"/>
+ <field name="coverage.Redshift.MinimumValue" type="pdouble" multiValued="true" stored="true" indexed="true"/>
+ <field name="coverage.RedshiftValue" type="pdouble" multiValued="true" stored="true" indexed="true"/>
+ <field name="coverage.SkyFraction" type="pdouble" multiValued="false" stored="true" indexed="true"/>
<field name="coverage.Spatial" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="coverage.Spectral.Bandpass" type="text_en" multiValued="true" stored="true" indexed="true"/>
- <field name="coverage.Spectral.CentralWavelength" type="text_en" multiValued="true" stored="true" indexed="true"/>
- <field name="coverage.Spectral.MaximumWavelength" type="text_en" multiValued="true" stored="true" indexed="true"/>
- <field name="coverage.Spectral.MinimumWavelength" type="text_en" multiValued="true" stored="true" indexed="true"/>
+ <field name="coverage.Spectral.CentralWavelength" type="pdouble" multiValued="true" stored="true" indexed="true"/>
+ <field name="coverage.Spectral.MaximumWavelength" type="pdouble" multiValued="true" stored="true" indexed="true"/>
+ <field name="coverage.Spectral.MinimumWavelength" type="pdouble" multiValued="true" stored="true" indexed="true"/>
<field name="coverage.Spectral.Wavelength" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="coverage.Temporal" type="text_en" multiValued="true" stored="true" indexed="true"/>
- <field name="coverage.Temporal.StartTime" type="text_en" multiValued="true" stored="true" indexed="true"/>
- <field name="coverage.Temporal.StopTime" type="text_en" multiValued="true" stored="true" indexed="true"/>
+ <field name="coverage.Temporal.StartTime" type="date_range" multiValued="true" stored="true" indexed="true"/>
+ <field name="coverage.Temporal.StopTime" type="date_range" multiValued="true" stored="true" indexed="true"/>
<field name="dataCollectionSituation" type="text_en" multiValued="false" stored="true" indexed="true"/>
<field name="dataCollector" type="text_en" multiValued="false" stored="true" indexed="true"/>
<field name="datasetContact" type="text_en" multiValued="true" stored="true" indexed="true"/>
@@ -298,12 +298,12 @@
<field name="datasetLevelErrorNotes" type="text_en" multiValued="false" stored="true" indexed="true"/>
<field name="dataSources" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="dateOfCollection" type="text_en" multiValued="true" stored="true" indexed="true"/>
- <field name="dateOfCollectionEnd" type="text_en" multiValued="true" stored="true" indexed="true"/>
- <field name="dateOfCollectionStart" type="text_en" multiValued="true" stored="true" indexed="true"/>
- <field name="dateOfDeposit" type="text_en" multiValued="false" stored="true" indexed="true"/>
+ <field name="dateOfCollectionEnd" type="date_range" multiValued="true" stored="true" indexed="true"/>
+ <field name="dateOfCollectionStart" type="date_range" multiValued="true" stored="true" indexed="true"/>
+ <field name="dateOfDeposit" type="date_range" multiValued="false" stored="true" indexed="true"/>
<field name="depositor" type="text_en" multiValued="false" stored="true" indexed="true"/>
<field name="deviationsFromSampleDesign" type="text_en" multiValued="false" stored="true" indexed="true"/>
- <field name="distributionDate" type="text_en" multiValued="false" stored="true" indexed="true"/>
+ <field name="distributionDate" type="date_range" multiValued="false" stored="true" indexed="true"/>
<field name="distributor" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="distributorAbbreviation" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="distributorAffiliation" type="text_en" multiValued="true" stored="true" indexed="true"/>
@@ -311,7 +311,7 @@
<field name="distributorName" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="distributorURL" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="dsDescription" type="text_en" multiValued="true" stored="true" indexed="true"/>
- <field name="dsDescriptionDate" type="text_en" multiValued="true" stored="true" indexed="true"/>
+ <field name="dsDescriptionDate" type="date_range" multiValued="true" stored="true" indexed="true"/>
<field name="dsDescriptionValue" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="eastLongitude" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="frequencyOfDataCollection" type="text_en" multiValued="false" stored="true" indexed="true"/>
@@ -323,7 +323,7 @@
<field name="grantNumberValue" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="journalArticleType" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="journalIssue" type="text_en" multiValued="true" stored="true" indexed="true"/>
- <field name="journalPubDate" type="text_en" multiValued="true" stored="true" indexed="true"/>
+ <field name="journalPubDate" type="date_range" multiValued="true" stored="true" indexed="true"/>
<field name="journalVolume" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="journalVolumeIssue" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="keyword" type="text_en" multiValued="true" stored="true" indexed="true"/>
@@ -348,7 +348,7 @@
<field name="producerLogoURL" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="producerName" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="producerURL" type="text_en" multiValued="true" stored="true" indexed="true"/>
- <field name="productionDate" type="text_en" multiValued="false" stored="true" indexed="true"/>
+ <field name="productionDate" type="date_range" multiValued="false" stored="true" indexed="true"/>
<field name="productionPlace" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="publication" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="publicationCitation" type="text_en" multiValued="true" stored="true" indexed="true"/>
@@ -360,7 +360,7 @@
<field name="relatedDatasets" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="relatedMaterial" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="researchInstrument" type="text_en" multiValued="false" stored="true" indexed="true"/>
- <field name="resolution.Redshift" type="text_en" multiValued="false" stored="true" indexed="true"/>
+ <field name="resolution.Redshift" type="pdouble" multiValued="false" stored="true" indexed="true"/>
<field name="resolution.Spatial" type="text_en" multiValued="false" stored="true" indexed="true"/>
<field name="resolution.Spectral" type="text_en" multiValued="false" stored="true" indexed="true"/>
<field name="resolution.Temporal" type="text_en" multiValued="false" stored="true" indexed="true"/>
@@ -394,13 +394,13 @@
<field name="studyOtherFactorType" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="subject" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="subtitle" type="text_en" multiValued="false" stored="true" indexed="true"/>
- <field name="targetSampleActualSize" type="text_en" multiValued="false" stored="true" indexed="true"/>
+ <field name="targetSampleActualSize" type="plong" multiValued="false" stored="true" indexed="true"/>
<field name="targetSampleSize" type="text_en" multiValued="false" stored="true" indexed="true"/>
<field name="targetSampleSizeFormula" type="text_en" multiValued="false" stored="true" indexed="true"/>
<field name="timeMethod" type="text_en" multiValued="false" stored="true" indexed="true"/>
<field name="timePeriodCovered" type="text_en" multiValued="true" stored="true" indexed="true"/>
- <field name="timePeriodCoveredEnd" type="text_en" multiValued="true" stored="true" indexed="true"/>
- <field name="timePeriodCoveredStart" type="text_en" multiValued="true" stored="true" indexed="true"/>
+ <field name="timePeriodCoveredEnd" type="date_range" multiValued="true" stored="true" indexed="true"/>
+ <field name="timePeriodCoveredStart" type="date_range" multiValued="true" stored="true" indexed="true"/>
<field name="title" type="text_en" multiValued="false" stored="true" indexed="true"/>
<field name="topicClassification" type="text_en" multiValued="true" stored="true" indexed="true"/>
<field name="topicClassValue" type="text_en" multiValued="true" stored="true" indexed="true"/>
From 3d2218376e5b30dbe12467cf370781a58eda438b Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Fri, 15 Nov 2024 16:32:34 -0500
Subject: [PATCH 13/43] add date_range type #10887
---
conf/solr/schema.xml | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/conf/solr/schema.xml b/conf/solr/schema.xml
index 0b95cc482d2..1dac5bc8c76 100644
--- a/conf/solr/schema.xml
+++ b/conf/solr/schema.xml
@@ -814,7 +814,9 @@
<!-- KD-tree versions of date fields -->
<fieldType name="pdate" class="solr.DatePointField" docValues="true"/>
<fieldType name="pdates" class="solr.DatePointField" docValues="true" multiValued="true"/>
-
+
+ <fieldType name="date_range" class="solr.DateRangeField"/>
+
<!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings -->
<fieldType name="binary" class="solr.BinaryField"/>
From 2837abf4d840be5b87c9993046d195f7acf79d8a Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Fri, 15 Nov 2024 16:50:49 -0500
Subject: [PATCH 14/43] updating schema.xml is required #10887
---
doc/release-notes/10887-solr-field-types.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/doc/release-notes/10887-solr-field-types.md b/doc/release-notes/10887-solr-field-types.md
index ca5b210cb21..13ab8dc9b4c 100644
--- a/doc/release-notes/10887-solr-field-types.md
+++ b/doc/release-notes/10887-solr-field-types.md
@@ -6,6 +6,6 @@ This release enhances how numerical and date fields are indexed in Solr. Previou
This enables range queries via the search bar or API, such as `exampleIntegerField:[25 TO 50]` or `exampleDateField:[2000-11-01 TO 2014-12-01]`.
-To activate this feature, Dataverse administrators must update their Solr schema.xml (manually or by rerunning `update-fields.sh`) and reindex all datasets.
+Dataverse administrators must update their Solr schema.xml (manually or by rerunning `update-fields.sh`) and reindex all datasets.
-Additionally, search result highlighting is now more accurate, ensuring that only fields relevant to the query are highlighted in search results. If the query is specifically limited to certain fields, the highlighting is now limited to those fields as well.
\ No newline at end of file
+Additionally, search result highlighting is now more accurate, ensuring that only fields relevant to the query are highlighted in search results. If the query is specifically limited to certain fields, the highlighting is now limited to those fields as well.
From e011e537dcd562f35da1b1d49ca24729059811c1 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Fri, 15 Nov 2024 17:00:06 -0500
Subject: [PATCH 15/43] enumerate fields changed, use examples from blocks we
ship #10887
---
doc/release-notes/10887-solr-field-types.md | 28 ++++++++++++++++++++-
1 file changed, 27 insertions(+), 1 deletion(-)
diff --git a/doc/release-notes/10887-solr-field-types.md b/doc/release-notes/10887-solr-field-types.md
index 13ab8dc9b4c..93ff897f4c3 100644
--- a/doc/release-notes/10887-solr-field-types.md
+++ b/doc/release-notes/10887-solr-field-types.md
@@ -4,7 +4,33 @@ This release enhances how numerical and date fields are indexed in Solr. Previou
* Float fields are indexed as `pdouble`
* Date fields are indexed as `date_range` (`solr.DateRangeField`)
-This enables range queries via the search bar or API, such as `exampleIntegerField:[25 TO 50]` or `exampleDateField:[2000-11-01 TO 2014-12-01]`.
+Specifically, the following fields were updated:
+
+- coverage.Depth
+- coverage.ObjectCount
+- coverage.ObjectDensity
+- coverage.Redshift.MaximumValue
+- coverage.Redshift.MinimumValue
+- coverage.RedshiftValue
+- coverage.SkyFraction
+- coverage.Spectral.CentralWavelength
+- coverage.Spectral.MaximumWavelength
+- coverage.Spectral.MinimumWavelength
+- coverage.Temporal.StartTime
+- coverage.Temporal.StopTime
+- dateOfCollectionEnd
+- dateOfCollectionStart
+- dateOfDeposit
+- distributionDate
+- dsDescriptionDate
+- journalPubDate
+- productionDate
+- resolution.Redshift
+- targetSampleActualSize
+- timePeriodCoveredEnd
+- timePeriodCoveredStart
+
+This change enables range queries when searching from both the UI and the API, such as `dateOfDeposit:[2000-01-01 TO 2014-12-31]` or `targetSampleActualSize:[25 TO 50]`.
Dataverse administrators must update their Solr schema.xml (manually or by rerunning `update-fields.sh`) and reindex all datasets.
From 3f599cfa1935a5b421597590463284ba4b1dacef Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Thu, 21 Nov 2024 10:29:17 -0500
Subject: [PATCH 16/43] address review comments
---
.../edu/harvard/iq/dataverse/Dataverse.java | 11 +++++++++++
.../command/impl/PublishDatasetCommand.java | 19 +++++++------------
2 files changed, 18 insertions(+), 12 deletions(-)
diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java
index 5b6fbdee6ba..f5d935ad353 100644
--- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java
@@ -791,6 +791,17 @@ public List<Dataverse> getOwners() {
return owners;
}
+ public boolean getEffectiveRequireFilesToPublishDataset() {
+ Dataverse dv = this;
+ while (dv != null) {
+ if (dv.getRequireFilesToPublishDataset() != null) {
+ return dv.getRequireFilesToPublishDataset();
+ }
+ dv = dv.getOwner();
+ }
+ return false;
+ }
+
@Override
public boolean equals(Object object) {
// TODO: Warning - this method won't work in the case the id fields are not set
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java
index 50800f72271..54223ac63b6 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java
@@ -14,7 +14,6 @@
import edu.harvard.iq.dataverse.workflow.Workflow;
import edu.harvard.iq.dataverse.workflow.WorkflowContext.TriggerType;
-import java.util.List;
import java.util.Optional;
import java.util.logging.Logger;
import static java.util.stream.Collectors.joining;
@@ -223,22 +222,18 @@ private void verifyCommandArguments(CommandContext ctxt) throws IllegalCommandEx
throw new IllegalCommandException("Cannot release as minor version. Re-try as major release.", this);
}
- if (getDataset().getFiles().isEmpty() && requiresFilesToPublishDataset()) {
+ if (getDataset().getFiles().isEmpty() && getEffectiveRequireFilesToPublishDataset()) {
throw new IllegalCommandException(BundleUtil.getStringFromBundle("dataset.mayNotPublish.FilesRequired"), this);
}
}
}
- private boolean requiresFilesToPublishDataset() {
- if (!getUser().isSuperuser()) {
- Dataverse parent = getDataset().getOwner();
- while (parent != null) {
- if (parent.getRequireFilesToPublishDataset() != null) {
- return parent.getRequireFilesToPublishDataset();
- }
- parent = parent.getOwner();
- }
+ private boolean getEffectiveRequireFilesToPublishDataset() {
+ if (getUser().isSuperuser()) {
+ return false;
+ } else {
+ Dataverse dv = getDataset().getOwner();
+ return dv != null && dv.getEffectiveRequireFilesToPublishDataset();
}
- return false;
}
@Override
From a244f18eabff62070adf25f5cfc06a09b1b09859 Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Thu, 21 Nov 2024 10:43:30 -0500
Subject: [PATCH 17/43] address review comments
---
src/main/java/edu/harvard/iq/dataverse/Dataverse.java | 2 +-
.../dataverse/engine/command/impl/PublishDatasetCommand.java | 2 +-
.../java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java | 5 +----
.../resources/db/migration/{V6.5.0.1.sql => V6.4.0.3.sql} | 0
src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 2 +-
5 files changed, 4 insertions(+), 7 deletions(-)
rename src/main/resources/db/migration/{V6.5.0.1.sql => V6.4.0.3.sql} (100%)
diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java
index f5d935ad353..312f4aa13f1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java
@@ -791,7 +791,7 @@ public List<Dataverse> getOwners() {
return owners;
}
- public boolean getEffectiveRequireFilesToPublishDataset() {
+ public boolean getEffectiveRequiresFilesToPublishDataset() {
Dataverse dv = this;
while (dv != null) {
if (dv.getRequireFilesToPublishDataset() != null) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java
index 54223ac63b6..eccc69b95c6 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java
@@ -232,7 +232,7 @@ private boolean getEffectiveRequireFilesToPublishDataset() {
return false;
} else {
Dataverse dv = getDataset().getOwner();
- return dv != null && dv.getEffectiveRequireFilesToPublishDataset();
+ return dv != null && dv.getEffectiveRequiresFilesToPublishDataset();
}
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index fb4ef516f9b..dd8971d67a1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -17,7 +17,6 @@
import edu.harvard.iq.dataverse.authorization.users.User;
import edu.harvard.iq.dataverse.branding.BrandingUtil;
import edu.harvard.iq.dataverse.dataaccess.DataAccess;
-import edu.harvard.iq.dataverse.dataset.DatasetType;
import edu.harvard.iq.dataverse.dataset.DatasetUtil;
import edu.harvard.iq.dataverse.datavariable.CategoryMetadata;
import edu.harvard.iq.dataverse.datavariable.DataVariable;
@@ -294,9 +293,7 @@ public static JsonObjectBuilder json(Dataverse dv, Boolean hideEmail, Boolean re
if (dv.getFilePIDsEnabled() != null) {
bld.add("filePIDsEnabled", dv.getFilePIDsEnabled());
}
- if (dv.getRequireFilesToPublishDataset() != null) {
- bld.add("requireFilesToPublishDataset", dv.getRequireFilesToPublishDataset());
- }
+ bld.add("effectiveRequiresFilesToPublishDataset", dv.getEffectiveRequiresFilesToPublishDataset());
bld.add("isReleased", dv.isReleased());
List<DataverseFieldTypeInputLevel> inputLevels = dv.getDataverseFieldTypeInputLevels();
diff --git a/src/main/resources/db/migration/V6.5.0.1.sql b/src/main/resources/db/migration/V6.4.0.3.sql
similarity index 100%
rename from src/main/resources/db/migration/V6.5.0.1.sql
rename to src/main/resources/db/migration/V6.4.0.3.sql
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 1be1e498ccd..a33d077dc07 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -5193,7 +5193,7 @@ public void testRequireFilesToPublishDatasets() {
setDataverseAttributeResponse = UtilIT.setCollectionAttribute(ownerAlias, "requireFilesToPublishDataset", "true", apiTokenAdmin);
setDataverseAttributeResponse.prettyPrint();
setDataverseAttributeResponse.then().assertThat().statusCode(OK.getStatusCode());
- setDataverseAttributeResponse.then().assertThat().body("data.requireFilesToPublishDataset",equalTo(true));
+ setDataverseAttributeResponse.then().assertThat().body("data.effectiveRequiresFilesToPublishDataset",equalTo(true));
Response publishDataverseResponse = UtilIT.publishDataverseViaNativeApi(ownerAlias, apiTokenAdmin);
publishDataverseResponse.prettyPrint();
publishDataverseResponse.then().assertThat().statusCode(OK.getStatusCode());
From 5dce13b847be29ed0c32483cadba2b0ffe41523d Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Thu, 21 Nov 2024 10:50:31 -0500
Subject: [PATCH 18/43] address review comments
---
.../dataverse/engine/command/impl/PublishDatasetCommand.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java
index eccc69b95c6..2c32b1e8954 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java
@@ -222,12 +222,12 @@ private void verifyCommandArguments(CommandContext ctxt) throws IllegalCommandEx
throw new IllegalCommandException("Cannot release as minor version. Re-try as major release.", this);
}
- if (getDataset().getFiles().isEmpty() && getEffectiveRequireFilesToPublishDataset()) {
+ if (getDataset().getFiles().isEmpty() && getEffectiveRequiresFilesToPublishDataset()) {
throw new IllegalCommandException(BundleUtil.getStringFromBundle("dataset.mayNotPublish.FilesRequired"), this);
}
}
}
- private boolean getEffectiveRequireFilesToPublishDataset() {
+ private boolean getEffectiveRequiresFilesToPublishDataset() {
if (getUser().isSuperuser()) {
return false;
} else {
From 6186a6e1147bb48324670927afb124024d29c23c Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Thu, 21 Nov 2024 13:41:02 -0500
Subject: [PATCH 19/43] address review comments
---
src/main/java/propertyFiles/Bundle.properties | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index c82f9ab248d..30c1e96ee78 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -1542,7 +1542,7 @@ dataset.mayNotPublish.administrator= This dataset cannot be published until {0}
dataset.mayNotPublish.both= This dataset cannot be published until {0} is published. Would you like to publish both right now?
dataset.mayNotPublish.twoGenerations= This dataset cannot be published until {0} and {1} are published.
dataset.mayNotBePublished.both.button=Yes, Publish Both
-dataset.mayNotPublish.FilesRequired=This dataset cannot be published without uploaded files.
+dataset.mayNotPublish.FilesRequired=Published datasets should contain at least one data file.
dataset.viewVersion.unpublished=View Unpublished Version
dataset.viewVersion.published=View Published Version
dataset.link.title=Link Dataset
From d12d9f6bc11829b41330ba85aaeef5b3081b1140 Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Mon, 25 Nov 2024 10:16:18 -0500
Subject: [PATCH 20/43] fix bad merge
---
src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 65e984de919..1b2d7e9a431 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -5323,7 +5323,7 @@ public void testRequireFilesToPublishDatasets() {
setDataverseAttributeResponse = UtilIT.setCollectionAttribute(ownerAlias, "requireFilesToPublishDataset", "true", apiTokenAdmin);
setDataverseAttributeResponse.prettyPrint();
setDataverseAttributeResponse.then().assertThat().statusCode(OK.getStatusCode());
- setDataverseAttributeResponse.then().assertThat().body("data.requireFilesToPublishDataset", equalTo(true));
+ setDataverseAttributeResponse.then().assertThat().body("data.effectiveRequiresFilesToPublishDataset", equalTo(true));
Response publishDataverseResponse = UtilIT.publishDataverseViaNativeApi(ownerAlias, apiTokenAdmin);
publishDataverseResponse.prettyPrint();
publishDataverseResponse.then().assertThat().statusCode(OK.getStatusCode());
From d7cb845b8d549c0f51a23404a839525e90cfc254 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Wed, 20 Nov 2024 15:11:39 -0500
Subject: [PATCH 21/43] actions/checkout 2->3
---
.github/workflows/guides_build_sphinx.yml | 2 +-
.github/workflows/reviewdog_checkstyle.yml | 2 +-
.github/workflows/shellspec.yml | 6 +++---
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/guides_build_sphinx.yml b/.github/workflows/guides_build_sphinx.yml
index 86b59b11d35..50ca14d3f1b 100644
--- a/.github/workflows/guides_build_sphinx.yml
+++ b/.github/workflows/guides_build_sphinx.yml
@@ -10,7 +10,7 @@ jobs:
docs:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- uses: uncch-rdmc/sphinx-action@master
with:
docs-folder: "doc/sphinx-guides/"
diff --git a/.github/workflows/reviewdog_checkstyle.yml b/.github/workflows/reviewdog_checkstyle.yml
index 90a0dd7d06b..637691f8b16 100644
--- a/.github/workflows/reviewdog_checkstyle.yml
+++ b/.github/workflows/reviewdog_checkstyle.yml
@@ -10,7 +10,7 @@ jobs:
name: Checkstyle job
steps:
- name: Checkout
- uses: actions/checkout@v2
+ uses: actions/checkout@v3
- name: Run check style
uses: nikitasavinov/checkstyle-action@master
with:
diff --git a/.github/workflows/shellspec.yml b/.github/workflows/shellspec.yml
index 3320d9d08a4..2c73259b978 100644
--- a/.github/workflows/shellspec.yml
+++ b/.github/workflows/shellspec.yml
@@ -19,7 +19,7 @@ jobs:
steps:
- name: Install shellspec
run: curl -fsSL https://git.io/shellspec | sh -s ${{ env.SHELLSPEC_VERSION }} --yes
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Run Shellspec
run: |
cd tests/shell
@@ -30,7 +30,7 @@ jobs:
container:
image: rockylinux/rockylinux:9
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Install shellspec
run: |
curl -fsSL https://github.com/shellspec/shellspec/releases/download/${{ env.SHELLSPEC_VERSION }}/shellspec-dist.tar.gz | tar -xz -C /usr/share
@@ -47,7 +47,7 @@ jobs:
steps:
- name: Install shellspec
run: curl -fsSL https://git.io/shellspec | sh -s 0.28.1 --yes
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Run Shellspec
run: |
cd tests/shell
From 60698157abb2a1307e77f172f773eede54ebceef Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Mon, 25 Nov 2024 10:28:38 -0500
Subject: [PATCH 22/43] actions @v3 -> @v4
---
.github/workflows/container_app_pr.yml | 2 +-
.github/workflows/container_app_push.yml | 2 +-
.github/workflows/guides_build_sphinx.yml | 2 +-
.github/workflows/reviewdog_checkstyle.yml | 2 +-
.github/workflows/shellcheck.yml | 2 +-
.github/workflows/shellspec.yml | 6 +++---
.github/workflows/spi_release.yml | 10 +++++-----
7 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/.github/workflows/container_app_pr.yml b/.github/workflows/container_app_pr.yml
index c86d284e74b..4130506ba36 100644
--- a/.github/workflows/container_app_pr.yml
+++ b/.github/workflows/container_app_pr.yml
@@ -20,7 +20,7 @@ jobs:
if: ${{ github.repository_owner == 'IQSS' }}
steps:
# Checkout the pull request code as when merged
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
with:
ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge'
- uses: actions/setup-java@v3
diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml
index 3b7ce066d73..184b69583a5 100644
--- a/.github/workflows/container_app_push.yml
+++ b/.github/workflows/container_app_push.yml
@@ -68,7 +68,7 @@ jobs:
if: ${{ github.event_name != 'pull_request' && github.ref_name == 'develop' && github.repository_owner == 'IQSS' }}
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- uses: peter-evans/dockerhub-description@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
diff --git a/.github/workflows/guides_build_sphinx.yml b/.github/workflows/guides_build_sphinx.yml
index 50ca14d3f1b..fa3a876c418 100644
--- a/.github/workflows/guides_build_sphinx.yml
+++ b/.github/workflows/guides_build_sphinx.yml
@@ -10,7 +10,7 @@ jobs:
docs:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- uses: uncch-rdmc/sphinx-action@master
with:
docs-folder: "doc/sphinx-guides/"
diff --git a/.github/workflows/reviewdog_checkstyle.yml b/.github/workflows/reviewdog_checkstyle.yml
index 637691f8b16..804b04f696a 100644
--- a/.github/workflows/reviewdog_checkstyle.yml
+++ b/.github/workflows/reviewdog_checkstyle.yml
@@ -10,7 +10,7 @@ jobs:
name: Checkstyle job
steps:
- name: Checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
- name: Run check style
uses: nikitasavinov/checkstyle-action@master
with:
diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml
index 56f7d648dc4..fb9cf5a0a1f 100644
--- a/.github/workflows/shellcheck.yml
+++ b/.github/workflows/shellcheck.yml
@@ -21,7 +21,7 @@ jobs:
permissions:
pull-requests: write
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- name: shellcheck
uses: reviewdog/action-shellcheck@v1
with:
diff --git a/.github/workflows/shellspec.yml b/.github/workflows/shellspec.yml
index 2c73259b978..cc09992edac 100644
--- a/.github/workflows/shellspec.yml
+++ b/.github/workflows/shellspec.yml
@@ -19,7 +19,7 @@ jobs:
steps:
- name: Install shellspec
run: curl -fsSL https://git.io/shellspec | sh -s ${{ env.SHELLSPEC_VERSION }} --yes
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- name: Run Shellspec
run: |
cd tests/shell
@@ -30,7 +30,7 @@ jobs:
container:
image: rockylinux/rockylinux:9
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- name: Install shellspec
run: |
curl -fsSL https://github.com/shellspec/shellspec/releases/download/${{ env.SHELLSPEC_VERSION }}/shellspec-dist.tar.gz | tar -xz -C /usr/share
@@ -47,7 +47,7 @@ jobs:
steps:
- name: Install shellspec
run: curl -fsSL https://git.io/shellspec | sh -s 0.28.1 --yes
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- name: Run Shellspec
run: |
cd tests/shell
diff --git a/.github/workflows/spi_release.yml b/.github/workflows/spi_release.yml
index 8ad74b3e4bb..54718320d1e 100644
--- a/.github/workflows/spi_release.yml
+++ b/.github/workflows/spi_release.yml
@@ -37,8 +37,8 @@ jobs:
runs-on: ubuntu-latest
if: github.event_name == 'pull_request' && needs.check-secrets.outputs.available == 'true'
steps:
- - uses: actions/checkout@v3
- - uses: actions/setup-java@v3
+ - uses: actions/checkout@v4
+ - uses: actions/setup-java@v4
with:
java-version: '17'
distribution: 'adopt'
@@ -63,8 +63,8 @@ jobs:
runs-on: ubuntu-latest
if: github.event_name == 'push' && needs.check-secrets.outputs.available == 'true'
steps:
- - uses: actions/checkout@v3
- - uses: actions/setup-java@v3
+ - uses: actions/checkout@v4
+ - uses: actions/setup-java@v4
with:
java-version: '17'
distribution: 'adopt'
@@ -76,7 +76,7 @@ jobs:
# Running setup-java again overwrites the settings.xml - IT'S MANDATORY TO DO THIS SECOND SETUP!!!
- name: Set up Maven Central Repository
- uses: actions/setup-java@v3
+ uses: actions/setup-java@v4
with:
java-version: '17'
distribution: 'adopt'
From 514264394e0e56e7252477ed8c538d0a56978d9e Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Mon, 25 Nov 2024 10:45:58 -0500
Subject: [PATCH 23/43] more @3->@4
---
.github/workflows/container_app_pr.yml | 6 +++---
.github/workflows/container_app_push.yml | 4 ++--
.github/workflows/pr_comment_commands.yml | 2 +-
3 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/.github/workflows/container_app_pr.yml b/.github/workflows/container_app_pr.yml
index 4130506ba36..c3f9e7bdc0d 100644
--- a/.github/workflows/container_app_pr.yml
+++ b/.github/workflows/container_app_pr.yml
@@ -23,11 +23,11 @@ jobs:
- uses: actions/checkout@v4
with:
ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge'
- - uses: actions/setup-java@v3
+ - uses: actions/setup-java@v4
with:
java-version: "17"
distribution: 'adopt'
- - uses: actions/cache@v3
+ - uses: actions/cache@v4
with:
path: ~/.m2
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
@@ -87,7 +87,7 @@ jobs:
:ship: [See on GHCR](https://github.com/orgs/gdcc/packages/container). Use by referencing with full name as printed above, mind the registry name.
# Leave a note when things have gone sideways
- - uses: peter-evans/create-or-update-comment@v3
+ - uses: peter-evans/create-or-update-comment@v4
if: ${{ failure() }}
with:
issue-number: ${{ github.event.client_payload.pull_request.number }}
diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml
index 184b69583a5..afb4f6f874b 100644
--- a/.github/workflows/container_app_push.yml
+++ b/.github/workflows/container_app_push.yml
@@ -69,14 +69,14 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- - uses: peter-evans/dockerhub-description@v3
+ - uses: peter-evans/dockerhub-description@v4
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
repository: gdcc/dataverse
short-description: "Dataverse Application Container Image providing the executable"
readme-filepath: ./src/main/docker/README.md
- - uses: peter-evans/dockerhub-description@v3
+ - uses: peter-evans/dockerhub-description@v4
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
diff --git a/.github/workflows/pr_comment_commands.yml b/.github/workflows/pr_comment_commands.yml
index 5ff75def623..06b11b1ac5b 100644
--- a/.github/workflows/pr_comment_commands.yml
+++ b/.github/workflows/pr_comment_commands.yml
@@ -9,7 +9,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Dispatch
- uses: peter-evans/slash-command-dispatch@v3
+ uses: peter-evans/slash-command-dispatch@v4
with:
# This token belongs to @dataversebot and has sufficient scope.
token: ${{ secrets.GHCR_TOKEN }}
From 359335d24306b6a9e34360c61a5789e7f965d15f Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Thu, 5 Dec 2024 09:37:44 -0500
Subject: [PATCH 24/43] fix merge
---
src/main/resources/db/migration/V6.4.0.4.sql | 2 ++
1 file changed, 2 insertions(+)
create mode 100644 src/main/resources/db/migration/V6.4.0.4.sql
diff --git a/src/main/resources/db/migration/V6.4.0.4.sql b/src/main/resources/db/migration/V6.4.0.4.sql
new file mode 100644
index 00000000000..661924b54af
--- /dev/null
+++ b/src/main/resources/db/migration/V6.4.0.4.sql
@@ -0,0 +1,2 @@
+-- files are required to publish datasets
+ALTER TABLE dataverse ADD COLUMN IF NOT EXISTS requirefilestopublishdataset bool;
From 8fda98cd4fde8a949650ef3293c100c9af1d5dc5 Mon Sep 17 00:00:00 2001
From: Ludovic DANIEL <ludovic.daniel@smile.fr>
Date: Thu, 5 Dec 2024 16:01:16 +0100
Subject: [PATCH 25/43] #7961 - Update the guide to mention boolean possibility
in the Metadata Customization page
---
doc/sphinx-guides/source/admin/metadatacustomization.rst | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst
index e5326efebef..ffabb033c25 100644
--- a/doc/sphinx-guides/source/admin/metadatacustomization.rst
+++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst
@@ -259,9 +259,9 @@ Each of the three main sections own sets of properties:
| | | an existing #datasetField from |
| | | another metadata block.) |
+--------------+--------------------------------------------+-----------------------------------------+
-| Value | A short display string, representing | Free text |
-| | an enumerated value for this field. If | |
-| | the identifier property is empty, | |
+| Value | A short display string, representing | Free text. As boolean, values "True" |
+| | an enumerated value for this field. If | and "False" are recommended, "Unknown" |
+| | the identifier property is empty, | value is an option. |
| | this value is used as the identifier. | |
+--------------+--------------------------------------------+-----------------------------------------+
| identifier | A string used to encode the selected | Free text |
@@ -293,6 +293,9 @@ FieldType definitions
+---------------+------------------------------------+
| text | Any text other than newlines may |
| | be entered into this field. |
+| | The text fieldtype can be used |
+| | combined with |
+| | #controlledVocabulary as boolean. |
+---------------+------------------------------------+
| textbox | Any text may be entered. For |
| | input, the Dataverse Software |
From 57d1dd5781175a672b0f61011a1b148017ecbfaa Mon Sep 17 00:00:00 2001
From: Ludovic DANIEL <ludovic.daniel@smile.fr>
Date: Thu, 5 Dec 2024 16:24:28 +0100
Subject: [PATCH 26/43] added release note
---
doc/release-notes/11064-update-metadata-customization.md | 1 +
1 file changed, 1 insertion(+)
create mode 100644 doc/release-notes/11064-update-metadata-customization.md
diff --git a/doc/release-notes/11064-update-metadata-customization.md b/doc/release-notes/11064-update-metadata-customization.md
new file mode 100644
index 00000000000..bcead3497f8
--- /dev/null
+++ b/doc/release-notes/11064-update-metadata-customization.md
@@ -0,0 +1 @@
+Metadata Customization guide has been updated to explain how to implement a kind of boolean fieldtype (see [Metadata Customization Guide](https://guides.dataverse.org/en/latest/admin/metadatacustomization.html#controlledvocabulary-enumerated-properties))
\ No newline at end of file
From fa73fe048b0f9295204af4c88a98ace183b78db1 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Thu, 5 Dec 2024 15:07:04 -0500
Subject: [PATCH 27/43] add detailed upgrade instruction steps for Solr #10887
---
doc/release-notes/10887-solr-field-types.md | 45 +++++++++++++++++++++
1 file changed, 45 insertions(+)
diff --git a/doc/release-notes/10887-solr-field-types.md b/doc/release-notes/10887-solr-field-types.md
index 93ff897f4c3..2d8225172af 100644
--- a/doc/release-notes/10887-solr-field-types.md
+++ b/doc/release-notes/10887-solr-field-types.md
@@ -35,3 +35,48 @@ This change enables range queries when searching from both the UI and the API, s
Dataverse administrators must update their Solr schema.xml (manually or by rerunning `update-fields.sh`) and reindex all datasets.
Additionally, search result highlighting is now more accurate, ensuring that only fields relevant to the query are highlighted in search results. If the query is specifically limited to certain fields, the highlighting is now limited to those fields as well.
+
+## Upgrade Instructions
+
+7\. Update Solr schema.xml file. Start with the standard v6.5 schema.xml, then, if your installation uses any custom or experimental metadata blocks, update it to include the extra fields (step 7a).
+
+Stop Solr (usually `service solr stop`, depending on Solr installation/OS, see the [Installation Guide](https://guides.dataverse.org/en/6.5/installation/prerequisites.html#solr-init-script)).
+
+```shell
+service solr stop
+```
+
+Replace schema.xml
+
+```shell
+wget https://raw.githubusercontent.com/IQSS/dataverse/v6.5/conf/solr/schema.xml
+cp schema.xml /usr/local/solr/solr-9.4.1/server/solr/collection1/conf
+```
+
+Start Solr (but if you use any custom metadata blocks, perform the next step, 7a first).
+
+```shell
+service solr start
+```
+
+7a\. For installations with custom or experimental metadata blocks:
+
+Before starting Solr, update the schema to include all the extra metadata fields that your installation uses. We do this by collecting the output of the Dataverse schema API and feeding it to the `update-fields.sh` script that we supply, as in the example below (modify the command lines as needed to reflect the names of the directories, if different):
+
+```shell
+ wget https://raw.githubusercontent.com/IQSS/dataverse/v6.5/conf/solr/update-fields.sh
+ chmod +x update-fields.sh
+ curl "http://localhost:8080/api/admin/index/solr/schema" | ./update-fields.sh /usr/local/solr/solr-9.4.1/server/solr/collection1/conf/schema.xml
+```
+
+Now start Solr.
+
+8\. Reindex Solr
+
+Below is the simplest way to reindex Solr:
+
+```shell
+curl http://localhost:8080/api/admin/index
+```
+
+The API above rebuilds the existing index "in place". If you want to be absolutely sure that your index is up-to-date and consistent, you may consider wiping it clean and reindexing everything from scratch (see [the guides](https://guides.dataverse.org/en/latest/admin/solr-search-index.html)). Just note that, depending on the size of your database, a full reindex may take a while and the users will be seeing incomplete search results during that window.
From 6cae3ecb6ab14495690a6460382908f8848d42dc Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Mon, 9 Dec 2024 15:22:22 -0500
Subject: [PATCH 28/43] add new apis for marketplace external tool registration
---
.../10930-marketplace-external-tools-apis.md | 14 +++
.../source/admin/external-tools.rst | 14 ++-
.../iq/dataverse/api/ExternalToolsApi.java | 62 +++++++++++++
.../iq/dataverse/api/ExternalToolsIT.java | 89 ++++++++++++++++++-
.../edu/harvard/iq/dataverse/api/UtilIT.java | 36 ++++++++
5 files changed, 213 insertions(+), 2 deletions(-)
create mode 100644 doc/release-notes/10930-marketplace-external-tools-apis.md
create mode 100644 src/main/java/edu/harvard/iq/dataverse/api/ExternalToolsApi.java
diff --git a/doc/release-notes/10930-marketplace-external-tools-apis.md b/doc/release-notes/10930-marketplace-external-tools-apis.md
new file mode 100644
index 00000000000..9e20c908823
--- /dev/null
+++ b/doc/release-notes/10930-marketplace-external-tools-apis.md
@@ -0,0 +1,14 @@
+## New APIs for External Tools Registration for Marketplace
+
+New API base path /api/externalTools created that mimics the admin APIs /api/admin/externalTools. These new apis require an authenticated superuser token.
+
+Example:
+```
+ API_TOKEN='xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'
+ export TOOL_ID=1
+
+ curl -s -H "X-Dataverse-key:$API_TOKEN" http://localhost:8080/api/externalTools
+ curl -s -H "X-Dataverse-key:$API_TOKEN" http://localhost:8080/api/externalTools/$TOOL_ID
+ curl -s -H "X-Dataverse-key:$API_TOKEN" -X POST -H 'Content-type: application/json' http://localhost:8080/api/externalTools --upload-file fabulousFileTool.json
+ curl -s -H "X-Dataverse-key:$API_TOKEN" -X DELETE http://localhost:8080/api/externalTools/$TOOL_ID
+```
diff --git a/doc/sphinx-guides/source/admin/external-tools.rst b/doc/sphinx-guides/source/admin/external-tools.rst
index 346ca0b15ee..50c2ec63c44 100644
--- a/doc/sphinx-guides/source/admin/external-tools.rst
+++ b/doc/sphinx-guides/source/admin/external-tools.rst
@@ -35,7 +35,10 @@ Configure the tool with the curl command below, making sure to replace the ``fab
.. code-block:: bash
- curl -X POST -H 'Content-type: application/json' http://localhost:8080/api/admin/externalTools --upload-file fabulousFileTool.json
+ curl -X POST -H 'Content-type: application/json' http://localhost:8080/api/admin/externalTools --upload-file fabulousFileTool.json
+
+ This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
+ curl -s -H "X-Dataverse-key:$API_TOKEN" -X POST -H 'Content-type: application/json' http://localhost:8080/api/externalTools --upload-file fabulousFileTool.json
Listing All External Tools in a Dataverse Installation
++++++++++++++++++++++++++++++++++++++++++++++++++++++
@@ -46,6 +49,9 @@ To list all the external tools that are available in a Dataverse installation:
curl http://localhost:8080/api/admin/externalTools
+ This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
+ curl -s -H "X-Dataverse-key:$API_TOKEN" http://localhost:8080/api/externalTools
+
Showing an External Tool in a Dataverse Installation
++++++++++++++++++++++++++++++++++++++++++++++++++++
@@ -56,6 +62,9 @@ To show one of the external tools that are available in a Dataverse installation
export TOOL_ID=1
curl http://localhost:8080/api/admin/externalTools/$TOOL_ID
+ This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
+ curl -s -H "X-Dataverse-key:$API_TOKEN" http://localhost:8080/api/externalTools/$TOOL_ID
+
Removing an External Tool From a Dataverse Installation
+++++++++++++++++++++++++++++++++++++++++++++++++++++++
@@ -66,6 +75,9 @@ Assuming the external tool database id is "1", remove it with the following comm
export TOOL_ID=1
curl -X DELETE http://localhost:8080/api/admin/externalTools/$TOOL_ID
+ This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
+ curl -s -H "X-Dataverse-key:$API_TOKEN" -X DELETE http://localhost:8080/api/externalTools/$TOOL_ID
+
.. _testing-external-tools:
Testing External Tools
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/ExternalToolsApi.java b/src/main/java/edu/harvard/iq/dataverse/api/ExternalToolsApi.java
new file mode 100644
index 00000000000..bf5634e09a8
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/api/ExternalToolsApi.java
@@ -0,0 +1,62 @@
+package edu.harvard.iq.dataverse.api;
+
+import edu.harvard.iq.dataverse.api.auth.AuthRequired;
+import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
+import jakarta.inject.Inject;
+import jakarta.ws.rs.DELETE;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.container.ContainerRequestContext;
+import jakarta.ws.rs.core.Context;
+import jakarta.ws.rs.core.Response;
+
+@Path("externalTools")
+public class ExternalToolsApi extends AbstractApiBean {
+
+ @Inject
+ ExternalTools externalTools;
+
+ @GET
+ @AuthRequired
+ public Response getExternalTools(@Context ContainerRequestContext crc) {
+ Response notAuthorized = authorize(crc);
+ return notAuthorized == null ? externalTools.getExternalTools() : notAuthorized;
+ }
+
+ @GET
+ @AuthRequired
+ @Path("{id}")
+ public Response getExternalTool(@Context ContainerRequestContext crc, @PathParam("id") long externalToolIdFromUser) {
+ Response notAuthorized = authorize(crc);
+ return notAuthorized == null ? externalTools.getExternalTool(externalToolIdFromUser) : notAuthorized;
+ }
+
+ @POST
+ @AuthRequired
+ public Response addExternalTool(@Context ContainerRequestContext crc, String manifest) {
+ Response notAuthorized = authorize(crc);
+ return notAuthorized == null ? externalTools.addExternalTool(manifest) : notAuthorized;
+ }
+
+ @DELETE
+ @AuthRequired
+ @Path("{id}")
+ public Response deleteExternalTool(@Context ContainerRequestContext crc, @PathParam("id") long externalToolIdFromUser) {
+ Response notAuthorized = authorize(crc);
+ return notAuthorized == null ? externalTools.deleteExternalTool(externalToolIdFromUser) : notAuthorized;
+ }
+
+ private Response authorize(ContainerRequestContext crc) {
+ try {
+ AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc);
+ if (!user.isSuperuser()) {
+ return error(Response.Status.FORBIDDEN, "Superusers only.");
+ }
+ } catch (WrappedResponse ex) {
+ return error(Response.Status.FORBIDDEN, "Superusers only.");
+ }
+ return null;
+ }
+}
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java
index 22abf6fa2e3..a3e2cca329d 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java
@@ -11,11 +11,11 @@
import java.nio.file.Paths;
import jakarta.json.Json;
import jakarta.json.JsonArray;
-import jakarta.json.JsonObject;
import jakarta.json.JsonObjectBuilder;
import jakarta.json.JsonReader;
import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST;
import static jakarta.ws.rs.core.Response.Status.CREATED;
+import static jakarta.ws.rs.core.Response.Status.FORBIDDEN;
import static jakarta.ws.rs.core.Response.Status.OK;
import org.hamcrest.CoreMatchers;
import org.hamcrest.Matchers;
@@ -37,6 +37,93 @@ public void testGetExternalTools() {
getExternalTools.prettyPrint();
}
+ @Test
+ public void testExternalToolsNonAdminEndpoint() {
+ Response createUser = UtilIT.createRandomUser();
+ createUser.prettyPrint();
+ createUser.then().assertThat()
+ .statusCode(OK.getStatusCode());
+ String username = UtilIT.getUsernameFromResponse(createUser);
+ String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+ UtilIT.setSuperuserStatus(username, true);
+
+ Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+ createDataverseResponse.prettyPrint();
+ createDataverseResponse.then().assertThat()
+ .statusCode(CREATED.getStatusCode());
+
+ String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+ Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+ createDataset.prettyPrint();
+ createDataset.then().assertThat()
+ .statusCode(CREATED.getStatusCode());
+
+ Integer datasetId = JsonPath.from(createDataset.getBody().asString()).getInt("data.id");
+ String datasetPid = JsonPath.from(createDataset.getBody().asString()).getString("data.persistentId");
+
+ String toolManifest = """
+{
+ "displayName": "Dataset Configurator",
+ "description": "Slices! Dices! <a href='https://docs.datasetconfigurator.com' target='_blank'>More info</a>.",
+ "types": [
+ "configure"
+ ],
+ "scope": "dataset",
+ "toolUrl": "https://datasetconfigurator.com",
+ "toolParameters": {
+ "queryParameters": [
+ {
+ "datasetPid": "{datasetPid}"
+ },
+ {
+ "localeCode": "{localeCode}"
+ }
+ ]
+ }
+ }
+""";
+
+ Response addExternalTool = UtilIT.addExternalTool(JsonUtil.getJsonObject(toolManifest), apiToken);
+ addExternalTool.prettyPrint();
+ addExternalTool.then().assertThat()
+ .statusCode(OK.getStatusCode())
+ .body("data.displayName", CoreMatchers.equalTo("Dataset Configurator"));
+
+ Long toolId = JsonPath.from(addExternalTool.getBody().asString()).getLong("data.id");
+ Response getExternalToolsByDatasetId = UtilIT.getExternalToolForDatasetById(datasetId.toString(), "configure", apiToken, toolId.toString());
+ getExternalToolsByDatasetId.prettyPrint();
+ getExternalToolsByDatasetId.then().assertThat()
+ .body("data.displayName", CoreMatchers.equalTo("Dataset Configurator"))
+ .body("data.scope", CoreMatchers.equalTo("dataset"))
+ .body("data.types[0]", CoreMatchers.equalTo("configure"))
+ .body("data.toolUrlWithQueryParams", CoreMatchers.equalTo("https://datasetconfigurator.com?datasetPid=" + datasetPid))
+ .statusCode(OK.getStatusCode());
+
+ Response getExternalTools = UtilIT.getExternalTools(apiToken);
+ getExternalTools.prettyPrint();
+ getExternalTools.then().assertThat()
+ .statusCode(OK.getStatusCode());
+ Response getExternalTool = UtilIT.getExternalTool(toolId, apiToken);
+ getExternalTool.prettyPrint();
+ getExternalTool.then().assertThat()
+ .statusCode(OK.getStatusCode());
+
+ //Delete the tool added by this test...
+ Response deleteExternalTool = UtilIT.deleteExternalTool(toolId, apiToken);
+ deleteExternalTool.prettyPrint();
+ deleteExternalTool.then().assertThat()
+ .statusCode(OK.getStatusCode());
+
+ // non superuser has no access
+ UtilIT.setSuperuserStatus(username, false);
+ getExternalTools = UtilIT.getExternalTools(apiToken);
+ getExternalTools.prettyPrint();
+ getExternalTools.then().assertThat()
+ .statusCode(FORBIDDEN.getStatusCode())
+ .body("message", CoreMatchers.equalTo("Superusers only."));
+ }
+
@Test
public void testFileLevelTool1() {
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 1930610532a..c6762c83bac 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -2538,6 +2538,42 @@ static Response deleteExternalTool(long externalToolid) {
.delete("/api/admin/externalTools/" + externalToolid);
}
+// ExternalTools with token
+ static Response getExternalTools(String apiToken) {
+ RequestSpecification requestSpecification = given();
+ if (apiToken != null) {
+ requestSpecification.header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken);
+ }
+ return requestSpecification.get("/api/externalTools");
+ }
+
+ static Response getExternalTool(long id, String apiToken) {
+ RequestSpecification requestSpecification = given();
+ if (apiToken != null) {
+ requestSpecification.header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken);
+ }
+ return requestSpecification.get("/api/externalTools/" + id);
+ }
+
+ static Response addExternalTool(JsonObject jsonObject, String apiToken) {
+ RequestSpecification requestSpecification = given();
+ if (apiToken != null) {
+ requestSpecification.header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken);
+ }
+ return requestSpecification
+ .body(jsonObject.toString())
+ .contentType(ContentType.JSON)
+ .post("/api/externalTools");
+ }
+
+ static Response deleteExternalTool(long externalToolid, String apiToken) {
+ RequestSpecification requestSpecification = given();
+ if (apiToken != null) {
+ requestSpecification.header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken);
+ }
+ return requestSpecification.delete("/api/externalTools/" + externalToolid);
+ }
+
static Response getExternalToolsForDataset(String idOrPersistentIdOfDataset, String type, String apiToken) {
String idInPath = idOrPersistentIdOfDataset; // Assume it's a number.
String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path.
From 0c7aeb484d7afe69dbd2b9d7b3a61e08119eabf2 Mon Sep 17 00:00:00 2001
From: stevenferey <steven.ferey@gmail.com>
Date: Fri, 13 Dec 2024 11:10:17 +0100
Subject: [PATCH 29/43] Adapting HarvestingClient and tests
---
.../harvard/iq/dataverse/harvest/client/HarvestingClient.java | 4 ++--
.../edu/harvard/iq/dataverse/api/HarvestingClientsIT.java | 4 +++-
2 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java
index e80e53ff7a7..e73310650b4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java
@@ -297,7 +297,7 @@ public ClientHarvestRun getLastSuccessfulRun() {
int i = harvestHistory.size() - 1;
while (i > -1) {
- if (harvestHistory.get(i).isCompleted()) {
+ if (harvestHistory.get(i).isCompleted() || harvestHistory.get(i).isCompletedWithFailures()) {
return harvestHistory.get(i);
}
i--;
@@ -314,7 +314,7 @@ ClientHarvestRun getLastNonEmptyRun() {
int i = harvestHistory.size() - 1;
while (i > -1) {
- if (harvestHistory.get(i).isCompleted()) {
+ if (harvestHistory.get(i).isCompleted() || harvestHistory.get(i).isCompletedWithFailures()) {
if (harvestHistory.get(i).getHarvestedDatasetCount().longValue() > 0 ||
harvestHistory.get(i).getDeletedDatasetCount().longValue() > 0) {
return harvestHistory.get(i);
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java
index 340eab161bb..5030b98ebfb 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java
@@ -268,7 +268,7 @@ private void harvestingClientRun(boolean allowHarvestingMissingCVV) throws Inte
assertEquals("inActive", clientStatus, "Unexpected client status: "+clientStatus);
// b) Confirm that it has actually succeeded:
- assertEquals("SUCCESS", responseJsonPath.getString("data.lastResult"), "Last harvest not reported a success (took "+i+" seconds)");
+ assertTrue(responseJsonPath.getString("data.lastResult").contains("Completed"), "Last harvest not reported a success (took "+i+" seconds)");
String harvestTimeStamp = responseJsonPath.getString("data.lastHarvest");
assertNotNull(harvestTimeStamp);
@@ -288,6 +288,8 @@ private void harvestingClientRun(boolean allowHarvestingMissingCVV) throws Inte
// Let's give the asynchronous indexing an extra sec. to finish:
Thread.sleep(1000L);
+ // Requires the index-harvested-metadata-source Flag feature to be enabled to search on the nickName
+ // Otherwise, the search must be performed with metadataSource:Harvested
Response searchHarvestedDatasets = UtilIT.search("metadataSource:" + nickName, normalUserAPIKey);
searchHarvestedDatasets.then().assertThat().statusCode(OK.getStatusCode());
searchHarvestedDatasets.prettyPrint();
From 5bb6c002d575149b52da1842ee1f13776d2215ef Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Fri, 13 Dec 2024 14:16:10 -0500
Subject: [PATCH 30/43] reword #7961
---
.../source/admin/metadatacustomization.rst | 16 +++++++++-------
1 file changed, 9 insertions(+), 7 deletions(-)
diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst
index ffabb033c25..3112fdb44bd 100644
--- a/doc/sphinx-guides/source/admin/metadatacustomization.rst
+++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst
@@ -244,6 +244,8 @@ Each of the three main sections own sets of properties:
| | #metadataBlock) | | |
+---------------------------+--------------------------------------------------------+----------------------------------------------------------+-----------------------+
+.. _cvoc-props:
+
#controlledVocabulary (enumerated) properties
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -259,10 +261,10 @@ Each of the three main sections own sets of properties:
| | | an existing #datasetField from |
| | | another metadata block.) |
+--------------+--------------------------------------------+-----------------------------------------+
-| Value | A short display string, representing | Free text. As boolean, values "True" |
-| | an enumerated value for this field. If | and "False" are recommended, "Unknown" |
-| | the identifier property is empty, | value is an option. |
-| | this value is used as the identifier. | |
+| Value | A short display string, representing | Free text. When defining a boolean, the |
+| | an enumerated value for this field. If | values "True" and "False" are |
+| | the identifier property is empty, | recommended and "Unknown" can be added |
+| | this value is used as the identifier. | if needed. |
+--------------+--------------------------------------------+-----------------------------------------+
| identifier | A string used to encode the selected | Free text |
| | enumerated value of a field. If this | |
@@ -293,9 +295,9 @@ FieldType definitions
+---------------+------------------------------------+
| text | Any text other than newlines may |
| | be entered into this field. |
-| | The text fieldtype can be used |
-| | combined with |
-| | #controlledVocabulary as boolean. |
+| | The text fieldtype may used to |
+| | define a boolean (see "Value" |
+| | under :ref:`cvoc-props`). |
+---------------+------------------------------------+
| textbox | Any text may be entered. For |
| | input, the Dataverse Software |
From 179d08576168b7f87431a4486a947c978f20f3cc Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Mon, 16 Dec 2024 09:16:02 -0500
Subject: [PATCH 31/43] update db script to 6.5
---
src/main/resources/db/migration/V6.4.0.3.sql | 1 -
src/main/resources/db/migration/{V6.4.0.4.sql => V6.5.0.1.sql} | 0
2 files changed, 1 deletion(-)
rename src/main/resources/db/migration/{V6.4.0.4.sql => V6.5.0.1.sql} (100%)
diff --git a/src/main/resources/db/migration/V6.4.0.3.sql b/src/main/resources/db/migration/V6.4.0.3.sql
index c3639b3fb7f..307d8ed206c 100644
--- a/src/main/resources/db/migration/V6.4.0.3.sql
+++ b/src/main/resources/db/migration/V6.4.0.3.sql
@@ -1,3 +1,2 @@
-- Add this boolean flag to accommodate a new harvesting client feature
ALTER TABLE harvestingclient ADD COLUMN IF NOT EXISTS useOaiIdAsPid BOOLEAN DEFAULT FALSE;
-
diff --git a/src/main/resources/db/migration/V6.4.0.4.sql b/src/main/resources/db/migration/V6.5.0.1.sql
similarity index 100%
rename from src/main/resources/db/migration/V6.4.0.4.sql
rename to src/main/resources/db/migration/V6.5.0.1.sql
From 649e2afe7622e215bc72893744ca1c7d159ad373 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 16 Dec 2024 14:16:38 -0500
Subject: [PATCH 32/43] typo #7961
---
doc/sphinx-guides/source/admin/metadatacustomization.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst
index 3112fdb44bd..4c9dc693a0d 100644
--- a/doc/sphinx-guides/source/admin/metadatacustomization.rst
+++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst
@@ -295,7 +295,7 @@ FieldType definitions
+---------------+------------------------------------+
| text | Any text other than newlines may |
| | be entered into this field. |
-| | The text fieldtype may used to |
+| | The text fieldtype may be used to |
| | define a boolean (see "Value" |
| | under :ref:`cvoc-props`). |
+---------------+------------------------------------+
From 133b1db69b30da1d132c554e9b16f530925cc272 Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Tue, 17 Dec 2024 09:41:04 -0500
Subject: [PATCH 33/43] fix doc formatting
---
doc/sphinx-guides/source/admin/external-tools.rst | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/doc/sphinx-guides/source/admin/external-tools.rst b/doc/sphinx-guides/source/admin/external-tools.rst
index 50c2ec63c44..1669398e349 100644
--- a/doc/sphinx-guides/source/admin/external-tools.rst
+++ b/doc/sphinx-guides/source/admin/external-tools.rst
@@ -37,7 +37,8 @@ Configure the tool with the curl command below, making sure to replace the ``fab
curl -X POST -H 'Content-type: application/json' http://localhost:8080/api/admin/externalTools --upload-file fabulousFileTool.json
- This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
+This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
+.. code-block:: bash
curl -s -H "X-Dataverse-key:$API_TOKEN" -X POST -H 'Content-type: application/json' http://localhost:8080/api/externalTools --upload-file fabulousFileTool.json
Listing All External Tools in a Dataverse Installation
@@ -49,7 +50,8 @@ To list all the external tools that are available in a Dataverse installation:
curl http://localhost:8080/api/admin/externalTools
- This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
+This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
+.. code-block:: bash
curl -s -H "X-Dataverse-key:$API_TOKEN" http://localhost:8080/api/externalTools
Showing an External Tool in a Dataverse Installation
@@ -62,7 +64,8 @@ To show one of the external tools that are available in a Dataverse installation
export TOOL_ID=1
curl http://localhost:8080/api/admin/externalTools/$TOOL_ID
- This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
+This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
+.. code-block:: bash
curl -s -H "X-Dataverse-key:$API_TOKEN" http://localhost:8080/api/externalTools/$TOOL_ID
Removing an External Tool From a Dataverse Installation
@@ -75,7 +78,8 @@ Assuming the external tool database id is "1", remove it with the following comm
export TOOL_ID=1
curl -X DELETE http://localhost:8080/api/admin/externalTools/$TOOL_ID
- This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
+This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
+.. code-block:: bash
curl -s -H "X-Dataverse-key:$API_TOKEN" -X DELETE http://localhost:8080/api/externalTools/$TOOL_ID
.. _testing-external-tools:
From a8ee025eaa093be993394742f24c6395c27f8dd4 Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Tue, 17 Dec 2024 10:04:42 -0500
Subject: [PATCH 34/43] fix doc formatting
---
doc/sphinx-guides/source/admin/external-tools.rst | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/doc/sphinx-guides/source/admin/external-tools.rst b/doc/sphinx-guides/source/admin/external-tools.rst
index 1669398e349..d654bcd1e8d 100644
--- a/doc/sphinx-guides/source/admin/external-tools.rst
+++ b/doc/sphinx-guides/source/admin/external-tools.rst
@@ -39,6 +39,7 @@ Configure the tool with the curl command below, making sure to replace the ``fab
This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
.. code-block:: bash
+
curl -s -H "X-Dataverse-key:$API_TOKEN" -X POST -H 'Content-type: application/json' http://localhost:8080/api/externalTools --upload-file fabulousFileTool.json
Listing All External Tools in a Dataverse Installation
@@ -52,6 +53,7 @@ To list all the external tools that are available in a Dataverse installation:
This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
.. code-block:: bash
+
curl -s -H "X-Dataverse-key:$API_TOKEN" http://localhost:8080/api/externalTools
Showing an External Tool in a Dataverse Installation
@@ -66,6 +68,7 @@ To show one of the external tools that are available in a Dataverse installation
This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
.. code-block:: bash
+
curl -s -H "X-Dataverse-key:$API_TOKEN" http://localhost:8080/api/externalTools/$TOOL_ID
Removing an External Tool From a Dataverse Installation
@@ -80,6 +83,7 @@ Assuming the external tool database id is "1", remove it with the following comm
This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
.. code-block:: bash
+
curl -s -H "X-Dataverse-key:$API_TOKEN" -X DELETE http://localhost:8080/api/externalTools/$TOOL_ID
.. _testing-external-tools:
From a65dbde46453a3cda2f0f4dee4644be2c6464eb2 Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Tue, 17 Dec 2024 13:54:55 -0500
Subject: [PATCH 35/43] fix doc formatting
---
doc/sphinx-guides/source/admin/external-tools.rst | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/doc/sphinx-guides/source/admin/external-tools.rst b/doc/sphinx-guides/source/admin/external-tools.rst
index d654bcd1e8d..3d7c057bda5 100644
--- a/doc/sphinx-guides/source/admin/external-tools.rst
+++ b/doc/sphinx-guides/source/admin/external-tools.rst
@@ -38,6 +38,7 @@ Configure the tool with the curl command below, making sure to replace the ``fab
curl -X POST -H 'Content-type: application/json' http://localhost:8080/api/admin/externalTools --upload-file fabulousFileTool.json
This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
+
.. code-block:: bash
curl -s -H "X-Dataverse-key:$API_TOKEN" -X POST -H 'Content-type: application/json' http://localhost:8080/api/externalTools --upload-file fabulousFileTool.json
@@ -52,6 +53,7 @@ To list all the external tools that are available in a Dataverse installation:
curl http://localhost:8080/api/admin/externalTools
This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
+
.. code-block:: bash
curl -s -H "X-Dataverse-key:$API_TOKEN" http://localhost:8080/api/externalTools
@@ -67,6 +69,7 @@ To show one of the external tools that are available in a Dataverse installation
curl http://localhost:8080/api/admin/externalTools/$TOOL_ID
This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
+
.. code-block:: bash
curl -s -H "X-Dataverse-key:$API_TOKEN" http://localhost:8080/api/externalTools/$TOOL_ID
@@ -82,6 +85,7 @@ Assuming the external tool database id is "1", remove it with the following comm
curl -X DELETE http://localhost:8080/api/admin/externalTools/$TOOL_ID
This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
+
.. code-block:: bash
curl -s -H "X-Dataverse-key:$API_TOKEN" -X DELETE http://localhost:8080/api/externalTools/$TOOL_ID
From 4faadf6e9a881569bcafc44123c736bcfff353f8 Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Tue, 17 Dec 2024 14:39:29 -0500
Subject: [PATCH 36/43] open up get apis for non-superuser
---
.../10930-marketplace-external-tools-apis.md | 6 ++--
.../source/admin/external-tools.rst | 8 ++---
.../iq/dataverse/api/ExternalToolsApi.java | 12 +++----
.../iq/dataverse/api/ExternalToolsIT.java | 31 ++++++++++++++-----
4 files changed, 34 insertions(+), 23 deletions(-)
diff --git a/doc/release-notes/10930-marketplace-external-tools-apis.md b/doc/release-notes/10930-marketplace-external-tools-apis.md
index 9e20c908823..e3350a8b2d2 100644
--- a/doc/release-notes/10930-marketplace-external-tools-apis.md
+++ b/doc/release-notes/10930-marketplace-external-tools-apis.md
@@ -1,14 +1,14 @@
## New APIs for External Tools Registration for Marketplace
-New API base path /api/externalTools created that mimics the admin APIs /api/admin/externalTools. These new apis require an authenticated superuser token.
+New API base path /api/externalTools created that mimics the admin APIs /api/admin/externalTools. These new add and delete apis require an authenticated superuser token.
Example:
```
API_TOKEN='xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'
export TOOL_ID=1
- curl -s -H "X-Dataverse-key:$API_TOKEN" http://localhost:8080/api/externalTools
- curl -s -H "X-Dataverse-key:$API_TOKEN" http://localhost:8080/api/externalTools/$TOOL_ID
+ curl http://localhost:8080/api/externalTools
+ curl http://localhost:8080/api/externalTools/$TOOL_ID
curl -s -H "X-Dataverse-key:$API_TOKEN" -X POST -H 'Content-type: application/json' http://localhost:8080/api/externalTools --upload-file fabulousFileTool.json
curl -s -H "X-Dataverse-key:$API_TOKEN" -X DELETE http://localhost:8080/api/externalTools/$TOOL_ID
```
diff --git a/doc/sphinx-guides/source/admin/external-tools.rst b/doc/sphinx-guides/source/admin/external-tools.rst
index 3d7c057bda5..c3e71c13ac6 100644
--- a/doc/sphinx-guides/source/admin/external-tools.rst
+++ b/doc/sphinx-guides/source/admin/external-tools.rst
@@ -52,11 +52,11 @@ To list all the external tools that are available in a Dataverse installation:
curl http://localhost:8080/api/admin/externalTools
-This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
+This API is open to any user. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
.. code-block:: bash
- curl -s -H "X-Dataverse-key:$API_TOKEN" http://localhost:8080/api/externalTools
+ curl http://localhost:8080/api/externalTools
Showing an External Tool in a Dataverse Installation
++++++++++++++++++++++++++++++++++++++++++++++++++++
@@ -68,11 +68,11 @@ To show one of the external tools that are available in a Dataverse installation
export TOOL_ID=1
curl http://localhost:8080/api/admin/externalTools/$TOOL_ID
-This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
+This API is open to any user. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools).
.. code-block:: bash
- curl -s -H "X-Dataverse-key:$API_TOKEN" http://localhost:8080/api/externalTools/$TOOL_ID
+ curl http://localhost:8080/api/externalTools/$TOOL_ID
Removing an External Tool From a Dataverse Installation
+++++++++++++++++++++++++++++++++++++++++++++++++++++++
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/ExternalToolsApi.java b/src/main/java/edu/harvard/iq/dataverse/api/ExternalToolsApi.java
index bf5634e09a8..92139d86caf 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/ExternalToolsApi.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/ExternalToolsApi.java
@@ -19,18 +19,14 @@ public class ExternalToolsApi extends AbstractApiBean {
ExternalTools externalTools;
@GET
- @AuthRequired
- public Response getExternalTools(@Context ContainerRequestContext crc) {
- Response notAuthorized = authorize(crc);
- return notAuthorized == null ? externalTools.getExternalTools() : notAuthorized;
+ public Response getExternalTools() {
+ return externalTools.getExternalTools();
}
@GET
- @AuthRequired
@Path("{id}")
- public Response getExternalTool(@Context ContainerRequestContext crc, @PathParam("id") long externalToolIdFromUser) {
- Response notAuthorized = authorize(crc);
- return notAuthorized == null ? externalTools.getExternalTool(externalToolIdFromUser) : notAuthorized;
+ public Response getExternalTool(@PathParam("id") long externalToolIdFromUser) {
+ return externalTools.getExternalTool(externalToolIdFromUser);
}
@POST
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java
index a3e2cca329d..1956e0eb8df 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java
@@ -109,19 +109,34 @@ public void testExternalToolsNonAdminEndpoint() {
getExternalTool.then().assertThat()
.statusCode(OK.getStatusCode());
- //Delete the tool added by this test...
- Response deleteExternalTool = UtilIT.deleteExternalTool(toolId, apiToken);
- deleteExternalTool.prettyPrint();
- deleteExternalTool.then().assertThat()
- .statusCode(OK.getStatusCode());
-
- // non superuser has no access
+ // non superuser can only view tools
UtilIT.setSuperuserStatus(username, false);
getExternalTools = UtilIT.getExternalTools(apiToken);
- getExternalTools.prettyPrint();
getExternalTools.then().assertThat()
+ .statusCode(OK.getStatusCode());
+ getExternalToolsByDatasetId = UtilIT.getExternalToolForDatasetById(datasetId.toString(), "configure", apiToken, toolId.toString());
+ getExternalToolsByDatasetId.prettyPrint();
+ getExternalToolsByDatasetId.then().assertThat()
+ .statusCode(OK.getStatusCode());
+
+ //Add by non-superuser will fail
+ addExternalTool = UtilIT.addExternalTool(JsonUtil.getJsonObject(toolManifest), apiToken);
+ addExternalTool.then().assertThat()
+ .statusCode(FORBIDDEN.getStatusCode())
+ .body("message", CoreMatchers.equalTo("Superusers only."));
+
+ //Delete by non-superuser will fail
+ Response deleteExternalTool = UtilIT.deleteExternalTool(toolId, apiToken);
+ deleteExternalTool.then().assertThat()
.statusCode(FORBIDDEN.getStatusCode())
.body("message", CoreMatchers.equalTo("Superusers only."));
+
+ //Delete the tool added by this test...
+ UtilIT.setSuperuserStatus(username, true);
+ deleteExternalTool = UtilIT.deleteExternalTool(toolId, apiToken);
+ deleteExternalTool.prettyPrint();
+ deleteExternalTool.then().assertThat()
+ .statusCode(OK.getStatusCode());
}
@Test
From 106ebe46105b213468460faa102b8dc33990198d Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@gmail.com>
Date: Thu, 19 Dec 2024 10:33:34 -0500
Subject: [PATCH 37/43] #10707 document S3 RBAC preference on v5.14+
---
doc/sphinx-guides/source/installation/config.rst | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index 30a36da9499..009d7775a13 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -1093,6 +1093,8 @@ The Dataverse Software S3 driver supports multi-part upload for large files (ove
First: Set Up Accounts and Access Credentials
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+**Note:** As of version 5.14, if Dataverse is running in an EC2 instance it will prefer RBAC for S3, even if administrators configure Dataverse with programmatic access keys. This is preferential from a security perspective as there are no keys to rotate or have stolen. If you intend to assign a role to your EC2 instance, you will still need the ``~/.aws/config`` file to specify the region but you need not generate credentials. For more information please see https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html
+
The Dataverse Software and the AWS SDK make use of the "AWS credentials profile file" and "AWS config profile file" located in
``~/.aws/`` where ``~`` is the home directory of the user you run Payara as. This file can be generated via either
of two methods described below:
@@ -1116,13 +1118,6 @@ To **create a user** with full S3 access and nothing more for security reasons,
for more info on this process.
To use programmatic access, **Generate the user keys** needed for a Dataverse installation afterwards by clicking on the created user.
-(You can skip this step when running on EC2, see below.)
-
-.. TIP::
- If you are hosting your Dataverse installation on an AWS EC2 instance alongside storage in S3, it is possible to use IAM Roles instead
- of the credentials file (the file at ``~/.aws/credentials`` mentioned below). Please note that you will still need the
- ``~/.aws/config`` file to specify the region. For more information on this option, see
- https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html
Preparation When Using Custom S3-Compatible Service
###################################################
From 188f8dcc1b06467610d903f817f07efe72c96183 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Thu, 19 Dec 2024 14:35:02 -0500
Subject: [PATCH 38/43] add space after link to prevent it from breaking #10384
---
src/main/java/propertyFiles/Bundle.properties | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 81f564838d7..d4057c5db0c 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -804,7 +804,7 @@ notification.email.greeting.html=Hello, <br>
# Bundle file editors, please note that "notification.email.welcome" is used in a unit test
notification.email.welcome=Welcome to {0}! Get started by adding or finding data. Have questions? Check out the User Guide at {1}/{2}/user or contact {3} at {4} for assistance.
notification.email.welcomeConfirmEmailAddOn=\n\nPlease verify your email address at {0} . Note, the verify link will expire after {1}. Send another verification email by visiting your account page.
-notification.email.requestFileAccess=File access requested for dataset: {0} by {1} ({2}). Manage permissions at {3}.
+notification.email.requestFileAccess=File access requested for dataset: {0} by {1} ({2}). Manage permissions at {3} .
notification.email.requestFileAccess.guestbookResponse=<br><br>Guestbook Response:<br><br>{0}
notification.email.grantFileAccess=Access granted for files in dataset: {0} (view at {1} ).
notification.email.rejectFileAccess=Your request for access was rejected for the requested files in the dataset: {0} (view at {1} ). If you have any questions about why your request was rejected, you may reach the dataset owner using the "Contact" link on the upper right corner of the dataset page.
From 5abbc8d7bce684cba386b08a69d1b53b8c72f00e Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Thu, 19 Dec 2024 14:44:59 -0500
Subject: [PATCH 39/43] add release note #10384
---
doc/release-notes/10384-link.md | 3 +++
1 file changed, 3 insertions(+)
create mode 100644 doc/release-notes/10384-link.md
diff --git a/doc/release-notes/10384-link.md b/doc/release-notes/10384-link.md
new file mode 100644
index 00000000000..7092241adf4
--- /dev/null
+++ b/doc/release-notes/10384-link.md
@@ -0,0 +1,3 @@
+### Broken Link in Email When Users Request Access to Files
+
+When users request access to a files, the people who have permission to grant access receive an email with a link in it that didn't work due to a trailing period (full stop) right next to the link (e.g. `https://demo.dataverse.org/permissions-manage-files.xhtml?id=9.`) A space has been added to fix this. See #10384 and #11115.
From b77a7a1cc9c6719dc72ca0e7144f73f72535a793 Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@gmail.com>
Date: Fri, 20 Dec 2024 11:10:29 -0500
Subject: [PATCH 40/43] #10707 Jim points out that this applies only to the
default profile
---
doc/sphinx-guides/source/installation/config.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index 009d7775a13..b6d0287a88d 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -1093,7 +1093,7 @@ The Dataverse Software S3 driver supports multi-part upload for large files (ove
First: Set Up Accounts and Access Credentials
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-**Note:** As of version 5.14, if Dataverse is running in an EC2 instance it will prefer RBAC for S3, even if administrators configure Dataverse with programmatic access keys. This is preferential from a security perspective as there are no keys to rotate or have stolen. If you intend to assign a role to your EC2 instance, you will still need the ``~/.aws/config`` file to specify the region but you need not generate credentials. For more information please see https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html
+**Note:** As of version 5.14, if Dataverse is running in an EC2 instance it will prefer RBAC for the S3 default profile, even if administrators configure Dataverse with programmatic access keys. This is preferential from a security perspective as there are no keys to rotate or have stolen. If you intend to assign a role to your EC2 instance, you will still need the ``~/.aws/config`` file to specify the region but you need not generate credentials for the default profile. For more information please see https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html
The Dataverse Software and the AWS SDK make use of the "AWS credentials profile file" and "AWS config profile file" located in
``~/.aws/`` where ``~`` is the home directory of the user you run Payara as. This file can be generated via either
From 8b3f0e19ace18afaf22aeadbc04e9ed477392cca Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@gmail.com>
Date: Fri, 20 Dec 2024 11:32:16 -0500
Subject: [PATCH 41/43] #10707 make named profiles more explicit per qqmyers
---
doc/sphinx-guides/source/installation/config.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index b6d0287a88d..3910580de9a 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -1093,7 +1093,7 @@ The Dataverse Software S3 driver supports multi-part upload for large files (ove
First: Set Up Accounts and Access Credentials
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-**Note:** As of version 5.14, if Dataverse is running in an EC2 instance it will prefer RBAC for the S3 default profile, even if administrators configure Dataverse with programmatic access keys. This is preferential from a security perspective as there are no keys to rotate or have stolen. If you intend to assign a role to your EC2 instance, you will still need the ``~/.aws/config`` file to specify the region but you need not generate credentials for the default profile. For more information please see https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html
+**Note:** As of version 5.14, if Dataverse is running in an EC2 instance it will prefer RBAC for the S3 default profile, even if administrators configure Dataverse with programmatic access keys. Named profiles can still be used to override RBAC for specific datastores. This is preferential from a security perspective as there are no keys to rotate or have stolen. If you intend to assign a role to your EC2 instance, you will still need the ``~/.aws/config`` file to specify the region but you need not generate credentials for the default profile. For more information please see https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html
The Dataverse Software and the AWS SDK make use of the "AWS credentials profile file" and "AWS config profile file" located in
``~/.aws/`` where ``~`` is the home directory of the user you run Payara as. This file can be generated via either
From b9f99643dc370707b35d36b8df8b430ccc6fd7ed Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 20 Dec 2024 17:06:36 +0000
Subject: [PATCH 42/43] Bump actions/cache from 2 to 4
Bumps [actions/cache](https://github.com/actions/cache) from 2 to 4.
- [Release notes](https://github.com/actions/cache/releases)
- [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md)
- [Commits](https://github.com/actions/cache/compare/v2...v4)
---
updated-dependencies:
- dependency-name: actions/cache
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <support@github.com>
---
.github/workflows/spi_release.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/spi_release.yml b/.github/workflows/spi_release.yml
index 54718320d1e..6398edca412 100644
--- a/.github/workflows/spi_release.yml
+++ b/.github/workflows/spi_release.yml
@@ -45,7 +45,7 @@ jobs:
server-id: ossrh
server-username: MAVEN_USERNAME
server-password: MAVEN_PASSWORD
- - uses: actions/cache@v2
+ - uses: actions/cache@v4
with:
path: ~/.m2
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
@@ -68,7 +68,7 @@ jobs:
with:
java-version: '17'
distribution: 'adopt'
- - uses: actions/cache@v2
+ - uses: actions/cache@v4
with:
path: ~/.m2
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
From ac60f4bcc895465e1d13965e9e54645646dbd7dc Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@gmail.com>
Date: Fri, 20 Dec 2024 15:27:39 -0500
Subject: [PATCH 43/43] #10707 final round of corrections per qqmyers
---
doc/sphinx-guides/source/installation/config.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index 3910580de9a..a2cbd36e694 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -1093,7 +1093,7 @@ The Dataverse Software S3 driver supports multi-part upload for large files (ove
First: Set Up Accounts and Access Credentials
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-**Note:** As of version 5.14, if Dataverse is running in an EC2 instance it will prefer RBAC for the S3 default profile, even if administrators configure Dataverse with programmatic access keys. Named profiles can still be used to override RBAC for specific datastores. This is preferential from a security perspective as there are no keys to rotate or have stolen. If you intend to assign a role to your EC2 instance, you will still need the ``~/.aws/config`` file to specify the region but you need not generate credentials for the default profile. For more information please see https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html
+**Note:** As of version 5.14, if Dataverse is running in an EC2 instance it will prefer Role-Based Access Control over the S3 default profile, even if administrators configure Dataverse with programmatic access keys. Named profiles can still be used to override RBAC for specific datastores. RBAC is preferential from a security perspective as there are no keys to rotate or have stolen. If you intend to assign a role to your EC2 instance, you will still need the ``~/.aws/config`` file to specify the region but you need not generate credentials for the default profile. For more information please see https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html
The Dataverse Software and the AWS SDK make use of the "AWS credentials profile file" and "AWS config profile file" located in
``~/.aws/`` where ``~`` is the home directory of the user you run Payara as. This file can be generated via either