Merge branch 'develop' into 10542-signposting #10542

conf/solr/schema.xml

@@ -234,6 +234,7 @@
     <field name="datasetValid" type="boolean" stored="true" indexed="true" multiValued="false"/>
 
     <field name="license" type="string" stored="true" indexed="true" multiValued="false"/>
+    <field name="fileCount" type="plong" stored="true" indexed="true" multiValued="false"/>
 
     <!--
         METADATA SCHEMA FIELDS

doc/release-notes/8941-adding-fileCount-in-solr.md

@@ -0,0 +1,15 @@
+## Release Highlights
+
+### Adding fileCount as SOLR field
+
+A new search field called `fileCount` can be searched to discover the number of files per dataset. (#10598)
+
+## Upgrade Instructions
+
+1. Update your Solr `schema.xml` to include the new field.
+For details, please see https://guides.dataverse.org/en/latest/admin/metadatacustomization.html#updating-the-solr-schema
+
+2. Reindex Solr.
+Once the schema.xml is updated, Solr must be restarted and a reindex initiated.
+For details, see https://guides.dataverse.org/en/latest/admin/solr-search-index.html but here is the reindex command:
+`curl http://localhost:8080/api/admin/index`

doc/release-notes/expose-export-formats.md

@@ -0,0 +1,2 @@
+# New API method for listing the available exporters
+Found at `/api/info/exportFormats`, produces an object with available format names as keys, and as values an object with various info about the exporter. See also #10739.

doc/sphinx-guides/source/api/native-api.rst

@@ -4889,6 +4889,35 @@ The fully expanded example above (without environment variables) looks like this
 
   curl "https://demo.dataverse.org/api/info/settings/:MaxEmbargoDurationInMonths"
 
+Get Export Formats
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Get the available export formats, including custom formats.
+
+The response contains an object with available format names as keys, and as values an object with the following properties:
+
+* ``displayName``
+* ``mediaType``
+* ``isHarvestable``
+* ``isVisibleInUserInterface`` (corresponds to isAvailableToUsers)
+* ``XMLNameSpace`` (only for XML exporters)
+* ``XMLSchemaLocation`` (only for XML exporters)
+* ``XMLSchemaVersion`` (only for XML exporters)
+
+.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below.
+
+.. code-block:: bash
+
+  export SERVER_URL=https://demo.dataverse.org
+
+  curl "$SERVER_URL/api/info/exportFormats"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/info/exportFormats"
+
 .. _metadata-blocks-api:
 
 Metadata Blocks

src/main/java/edu/harvard/iq/dataverse/api/Info.java

@@ -12,12 +12,17 @@
 import jakarta.ws.rs.Produces;
 import org.apache.commons.io.IOUtils;
 
+import edu.harvard.iq.dataverse.export.ExportService;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
+import io.gdcc.spi.export.Exporter;
+import io.gdcc.spi.export.ExportException;
+import io.gdcc.spi.export.XMLExporter;
 import jakarta.ejb.EJB;
 import jakarta.json.Json;
+import jakarta.json.JsonObjectBuilder;
 import jakarta.json.JsonValue;
 import jakarta.ws.rs.GET;
 import jakarta.ws.rs.Path;
@@ -92,6 +97,32 @@ public Response getZipDownloadLimit() {
         return ok(zipDownloadLimit);
     }
 
+    @GET
+    @Path("exportFormats")
+    public Response getExportFormats() {
+        JsonObjectBuilder responseModel = Json.createObjectBuilder();
+        ExportService instance = ExportService.getInstance();
+        for (String[] labels : instance.getExportersLabels()) {
+            try {
+                Exporter exporter = instance.getExporter(labels[1]);
+                JsonObjectBuilder exporterObject = Json.createObjectBuilder().add("displayName", labels[0])
+                        .add("mediaType", exporter.getMediaType()).add("isHarvestable", exporter.isHarvestable())
+                        .add("isVisibleInUserInterface", exporter.isAvailableToUsers());
+                if (exporter instanceof XMLExporter xmlExporter) {
+                    exporterObject.add("XMLNameSpace", xmlExporter.getXMLNameSpace())
+                            .add("XMLSchemaLocation", xmlExporter.getXMLSchemaLocation())
+                            .add("XMLSchemaVersion", xmlExporter.getXMLSchemaVersion());
+                }
+                responseModel.add(labels[1], exporterObject);
+            }
+            catch (ExportException ex){
+                logger.warning("Failed to get: " + labels[1]);
+                logger.warning(ex.getLocalizedMessage());
+            }
+        }
+        return ok(responseModel);
+    }
+
     private Response getSettingResponseByKey(SettingsServiceBean.Key key) {
         String setting = settingsService.getValueForKey(key);
         if (setting != null) {

src/main/java/edu/harvard/iq/dataverse/api/Search.java

@@ -175,7 +175,7 @@ public Response search(
             JsonArrayBuilder itemsArrayBuilder = Json.createArrayBuilder();
             List<SolrSearchResult> solrSearchResults = solrQueryResponse.getSolrSearchResults();
             for (SolrSearchResult solrSearchResult : solrSearchResults) {
-                itemsArrayBuilder.add(solrSearchResult.json(showRelevance, showEntityIds, showApiUrls, metadataFields, getDatasetFileCount(solrSearchResult)));
+                itemsArrayBuilder.add(solrSearchResult.json(showRelevance, showEntityIds, showApiUrls, metadataFields));
             }
 
             JsonObjectBuilder spelling_alternatives = Json.createObjectBuilder();
@@ -229,15 +229,6 @@ public Response search(
         }
     }
 
-    private Long getDatasetFileCount(SolrSearchResult solrSearchResult) {
-        DvObject dvObject = solrSearchResult.getEntity();
-        if (dvObject.isInstanceofDataset()) {
-            DatasetVersion datasetVersion = ((Dataset) dvObject).getVersionFromId(solrSearchResult.getDatasetVersionId());
-            return datasetVersionFilesServiceBean.getFileMetadataCount(datasetVersion);
-        }
-        return null;
-    }
-
     private User getUser(ContainerRequestContext crc) throws WrappedResponse {
         User userToExecuteSearchAs = GuestUser.get();
         try {

src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java

@@ -135,6 +135,9 @@ public class IndexServiceBean {
     @EJB
     DatasetFieldServiceBean datasetFieldService;
 
+    @Inject
+    DatasetVersionFilesServiceBean datasetVersionFilesServiceBean;
+
     public static final String solrDocIdentifierDataverse = "dataverse_";
     public static final String solrDocIdentifierFile = "datafile_";
     public static final String solrDocIdentifierDataset = "dataset_";
@@ -1018,6 +1021,8 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
             solrInputDocument.addField(SearchFields.DATASET_CITATION, datasetVersion.getCitation(false));
             solrInputDocument.addField(SearchFields.DATASET_CITATION_HTML, datasetVersion.getCitation(true));
 
+            solrInputDocument.addField(SearchFields.FILE_COUNT, datasetVersionFilesServiceBean.getFileMetadataCount(datasetVersion));
+
             if (datasetVersion.isInReview()) {
                 solrInputDocument.addField(SearchFields.PUBLICATION_STATUS, IN_REVIEW_STRING);
             }

src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java

@@ -291,5 +291,6 @@ more targeted results for just datasets. The format is YYYY (i.e.
     public static final String DATASET_VALID = "datasetValid";
 
     public static final String DATASET_LICENSE = "license";
+    public static final String FILE_COUNT = "fileCount";
 
 }

src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java

@@ -497,7 +497,8 @@ public SolrQueryResponse search(
             Long retentionEndDate = (Long) solrDocument.getFieldValue(SearchFields.RETENTION_END_DATE);
             //
             Boolean datasetValid = (Boolean) solrDocument.getFieldValue(SearchFields.DATASET_VALID);
-
+            Long fileCount = (Long) solrDocument.getFieldValue(SearchFields.FILE_COUNT);
+
             List<String> matchedFields = new ArrayList<>();
 
             SolrSearchResult solrSearchResult = new SolrSearchResult(query, name);
@@ -570,6 +571,7 @@ public SolrQueryResponse search(
             solrSearchResult.setDeaccessionReason(deaccessionReason);
             solrSearchResult.setDvTree(dvTree);
             solrSearchResult.setDatasetValid(datasetValid);
+            solrSearchResult.setFileCount(fileCount);
 
             if (Boolean.TRUE.equals((Boolean) solrDocument.getFieldValue(SearchFields.IS_HARVESTED))) {
                 solrSearchResult.setHarvested(true);

src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java

@@ -78,6 +78,10 @@ public class SolrSearchResult {
     private String citation;
     private String citationHtml;
     private String datasetType;
+    /**
+    * Only Dataset can have a file count
+    */
+    private Long fileCount;
     /**
      * Files and datasets might have a UNF. Dataverses don't.
      */
@@ -456,10 +460,10 @@ public JsonObjectBuilder getJsonForMyData(boolean isValid) {
     } // getJsonForMydata
 
     public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, boolean showApiUrls) {
-        return json(showRelevance, showEntityIds, showApiUrls, null, null);
+        return json(showRelevance, showEntityIds, showApiUrls, null);
     }
 
-    public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, boolean showApiUrls, List<String> metadataFields, Long datasetFileCount) {
+    public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, boolean showApiUrls, List<String> metadataFields) {
         if (this.type == null) {
             return jsonObjectBuilder();
         }
@@ -597,7 +601,7 @@ public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, bool
                     subjects.add(subject);
                 }
                 nullSafeJsonBuilder.add("subjects", subjects);
-                nullSafeJsonBuilder.add("fileCount", datasetFileCount);
+                nullSafeJsonBuilder.add("fileCount", this.fileCount);
                 nullSafeJsonBuilder.add("versionId", dv.getId());
                 nullSafeJsonBuilder.add("versionState", dv.getVersionState().toString());
                 if (this.isPublishedState()) {
@@ -1348,4 +1352,12 @@ public boolean isValid(Predicate<SolrSearchResult> canUpdateDataset) {
         }
         return !canUpdateDataset.test(this);
     }
+
+    public Long getFileCount() {
+        return fileCount;
+    }
+
+    public void setFileCount(Long fileCount) {
+        this.fileCount = fileCount;
+    }
 }

src/test/java/edu/harvard/iq/dataverse/api/InfoIT.java

@@ -6,11 +6,14 @@
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
-import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST;
 import static jakarta.ws.rs.core.Response.Status.NOT_FOUND;
 import static jakarta.ws.rs.core.Response.Status.OK;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Paths;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.notNullValue;
+import org.skyscreamer.jsonassert.JSONAssert;
 
 public class InfoIT {
 
@@ -81,6 +84,22 @@ public void testGetZipDownloadLimit() {
                 .body("data", notNullValue());
     }
 
+    @Test
+    public void testGetExportFormats() throws IOException {
+        Response response = given().urlEncodingEnabled(false)
+                .get("/api/info/exportFormats");
+        response.prettyPrint();
+        response.then().assertThat().statusCode(OK.getStatusCode());
+
+        String actual = response.getBody().asString();
+        String expected =
+                java.nio.file.Files.readString(
+                        Paths.get("src/test/resources/json/export-formats.json"),
+                        StandardCharsets.UTF_8);
+        JSONAssert.assertEquals(expected, actual, true);
+
+    }
+
 
     private void testSettingEndpoint(SettingsServiceBean.Key settingKey, String testSettingValue) {
         String endpoint =  "/api/info/settings/" + settingKey;

src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java

@@ -53,6 +53,7 @@ public void setUp() {
         indexService.dataverseService = Mockito.mock(DataverseServiceBean.class);
         indexService.datasetFieldService = Mockito.mock(DatasetFieldServiceBean.class);
         indexService.datasetVersionService = Mockito.mock(DatasetVersionServiceBean.class);
+        indexService.datasetVersionFilesServiceBean = Mockito.mock(DatasetVersionFilesServiceBean.class);
         BrandingUtil.injectServices(indexService.dataverseService, indexService.settingsService);
 
         Mockito.when(indexService.dataverseService.findRootDataverse()).thenReturn(dataverse);

src/test/resources/json/export-formats.json

@@ -0,0 +1,83 @@
+{
+  "status": "OK",
+  "data": {
+    "OAI_ORE": {
+      "displayName": "OAI_ORE",
+      "mediaType": "application/json",
+      "isHarvestable": false,
+      "isVisibleInUserInterface": true
+    },
+    "Datacite": {
+      "displayName": "DataCite",
+      "mediaType": "application/xml",
+      "isHarvestable": true,
+      "isVisibleInUserInterface": true,
+      "XMLNameSpace": "http://datacite.org/schema/kernel-4",
+      "XMLSchemaLocation": "http://datacite.org/schema/kernel-4 http://schema.datacite.org/meta/kernel-4.5/metadata.xsd",
+      "XMLSchemaVersion": "4.5"
+    },
+    "oai_dc": {
+      "displayName": "Dublin Core",
+      "mediaType": "application/xml",
+      "isHarvestable": true,
+      "isVisibleInUserInterface": false,
+      "XMLNameSpace": "http://www.openarchives.org/OAI/2.0/oai_dc/",
+      "XMLSchemaLocation": "http://www.openarchives.org/OAI/2.0/oai_dc.xsd",
+      "XMLSchemaVersion": "2.0"
+    },
+    "oai_datacite": {
+      "displayName": "OpenAIRE",
+      "mediaType": "application/xml",
+      "isHarvestable": true,
+      "isVisibleInUserInterface": true,
+      "XMLNameSpace": "http://datacite.org/schema/kernel-4",
+      "XMLSchemaLocation": "http://schema.datacite.org/meta/kernel-4.1/metadata.xsd",
+      "XMLSchemaVersion": "4.1"
+    },
+    "schema.org": {
+      "displayName": "Schema.org JSON-LD",
+      "mediaType": "application/json",
+      "isHarvestable": false,
+      "isVisibleInUserInterface": true
+    },
+    "ddi": {
+      "displayName": "DDI Codebook v2",
+      "mediaType": "application/xml",
+      "isHarvestable": false,
+      "isVisibleInUserInterface": true,
+      "XMLNameSpace": "ddi:codebook:2_5",
+      "XMLSchemaLocation": "https://ddialliance.org/Specification/DDI-Codebook/2.5/XMLSchema/codebook.xsd",
+      "XMLSchemaVersion": "2.5"
+    },
+    "dcterms": {
+      "displayName": "Dublin Core",
+      "mediaType": "application/xml",
+      "isHarvestable": false,
+      "isVisibleInUserInterface": true,
+      "XMLNameSpace": "http://purl.org/dc/terms/",
+      "XMLSchemaLocation": "http://dublincore.org/schemas/xmls/qdc/dcterms.xsd",
+      "XMLSchemaVersion": "2.0"
+    },
+    "html": {
+      "displayName": "DDI HTML Codebook",
+      "mediaType": "text/html",
+      "isHarvestable": false,
+      "isVisibleInUserInterface": true
+    },
+    "dataverse_json": {
+      "displayName": "JSON",
+      "mediaType": "application/json",
+      "isHarvestable": true,
+      "isVisibleInUserInterface": true
+    },
+    "oai_ddi": {
+      "displayName": "DDI Codebook v2",
+      "mediaType": "application/xml",
+      "isHarvestable": true,
+      "isVisibleInUserInterface": false,
+      "XMLNameSpace": "ddi:codebook:2_5",
+      "XMLSchemaLocation": "https://ddialliance.org/Specification/DDI-Codebook/2.5/XMLSchema/codebook.xsd",
+      "XMLSchemaVersion": "2.5"
+    }
+  }
+}