From 385174065ed4b095bbcc6ab36873c9d036bf56b7 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 31 Jan 2019 18:31:18 -0500 Subject: [PATCH 001/970] IQSS-5505 --- pom.xml | 5 +++ .../AbstractGlobalIdServiceBean.java | 4 +++ .../dataverse/DOIDataCiteRegisterService.java | 35 +++++++++++++++++++ .../iq/dataverse/GlobalIdServiceBean.java | 2 ++ .../harvard/iq/dataverse/api/Datasets.java | 12 +++++-- .../UpdateDvObjectPIDMetadataCommand.java | 4 +-- 6 files changed, 58 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 380fbbbe402..224ab957fd7 100644 --- a/pom.xml +++ b/pom.xml @@ -585,6 +585,11 @@ tika-parsers 1.19 + + org.xmlunit + xmlunit-core + 2.6.2 + diff --git a/src/main/java/edu/harvard/iq/dataverse/AbstractGlobalIdServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/AbstractGlobalIdServiceBean.java index 98310a136b5..4e632dce27f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/AbstractGlobalIdServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/AbstractGlobalIdServiceBean.java @@ -428,5 +428,9 @@ public String getMetadataFromDvObject(String identifier, Map met logger.log(Level.FINE, "XML to send to DataCite: {0}", xmlMetadata); return xmlMetadata; } + + public boolean updateIdentifier(DvObject dvObject) { + return publicizeIdentifier(dvObject); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java index 50f92f81fb5..99cf4f3694d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java +++ b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java @@ -23,10 +23,17 @@ import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; import javax.persistence.TypedQuery; +import javax.xml.transform.Source; + import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import org.xmlunit.builder.DiffBuilder; +import org.xmlunit.builder.Input; +import org.xmlunit.builder.Input.Builder; +import org.xmlunit.diff.Diff; +import org.xmlunit.diff.Difference; /** * @@ -114,6 +121,34 @@ public String registerIdentifier(String identifier, Map metadata } return retString; } + + public String reRegisterIdentifier(String identifier, Map metadata, DvObject dvObject) throws IOException { + String retString = ""; + String numericIdentifier = identifier.substring(identifier.indexOf(":") + 1); + String xmlMetadata = getMetadataFromDvObject(identifier, metadata, dvObject); + String target = metadata.get("_target"); + DataCiteRESTfullClient client = getClient(); + String currentMetadata = client.getMetadata(numericIdentifier); + Diff myDiff = DiffBuilder.compare(xmlMetadata) + .withTest(currentMetadata).ignoreWhitespace().checkForSimilar() + .build(); + + if (myDiff.hasDifferences()) { + for(Difference d : myDiff.getDifferences()) { + + logger.fine(d.toString()); + } + retString = "metadata:\\r" + client.postMetadata(xmlMetadata) + "\\r"; + } + if (!target.equals(client.getUrl(numericIdentifier))) { + logger.fine("updating target URl to " + target); + client.postUrl(numericIdentifier, target); + retString = retString + "url:\\r" + target; + + } + + return retString; + } public String deactivateIdentifier(String identifier, HashMap metadata, DvObject dvObject) { String retString = ""; diff --git a/src/main/java/edu/harvard/iq/dataverse/GlobalIdServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/GlobalIdServiceBean.java index 0d64c1050b8..cd0c2c04c73 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GlobalIdServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/GlobalIdServiceBean.java @@ -50,6 +50,8 @@ public interface GlobalIdServiceBean { boolean publicizeIdentifier(DvObject studyIn); + boolean updateIdentifier(DvObject dvObject); + static GlobalIdServiceBean getBean(String protocol, CommandContext ctxt) { final Function protocolHandler = BeanDispatcher.DISPATCHER.get(protocol); if ( protocolHandler != null ) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 4f868d90ae7..42f01f67ace 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -386,8 +386,9 @@ public Response updateDatasetPIDMetadata(@PathParam("id") String id) { } return response(req -> { - execCommand(new UpdateDvObjectPIDMetadataCommand(findDatasetOrDie(id), req)); - List args = Arrays.asList(id); + Dataset dataset = findDatasetOrDie(id); + execCommand(new UpdateDvObjectPIDMetadataCommand(dataset, req)); + List args = Arrays.asList(dataset.getIdentifier()); return ok(BundleUtil.getStringFromBundle("datasets.api.updatePIDMetadata.success.for.single.dataset", args)); }); } @@ -398,7 +399,14 @@ public Response updateDatasetPIDMetadataAll() { return response( req -> { datasetService.findAll().forEach( ds -> { try { + logger.fine("ReRegistering: " + ds.getId() + " : " + ds.getIdentifier()); + if (!ds.isReleased() || (!ds.isIdentifierRegistered() || (ds.getIdentifier() == null))) { + if (ds.isReleased()) { + logger.warning("Dataset id=" + ds.getId() + " is in an inconsistent state (publicationdate but no identifier/identifier not registered"); + } + } else { execCommand(new UpdateDvObjectPIDMetadataCommand(findDatasetOrDie(ds.getId().toString()), req)); + } } catch (WrappedResponse ex) { Logger.getLogger(Datasets.class.getName()).log(Level.SEVERE, null, ex); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDvObjectPIDMetadataCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDvObjectPIDMetadataCommand.java index e36fe06b863..99d0a183c9d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDvObjectPIDMetadataCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDvObjectPIDMetadataCommand.java @@ -47,13 +47,13 @@ protected void executeImpl(CommandContext ctxt) throws CommandException { } GlobalIdServiceBean idServiceBean = GlobalIdServiceBean.getBean(target.getProtocol(), ctxt); try { - Boolean doiRetString = idServiceBean.publicizeIdentifier(target); + Boolean doiRetString = idServiceBean.updateIdentifier(target); if (doiRetString) { target.setGlobalIdCreateTime(new Timestamp(new Date().getTime())); ctxt.em().merge(target); ctxt.em().flush(); for (DataFile df : target.getFiles()) { - doiRetString = idServiceBean.publicizeIdentifier(df); + doiRetString = idServiceBean.updateIdentifier(df); if (doiRetString) { df.setGlobalIdCreateTime(new Timestamp(new Date().getTime())); ctxt.em().merge(df); From 8195b1ed98e4312b93e28c7577c22b61aac13991 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 11 Mar 2019 13:55:17 -0400 Subject: [PATCH 002/970] use dataset thumbnail if available --- src/main/webapp/dataset.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 0cb0f9d80e4..133bb263b7f 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -43,7 +43,7 @@ - + From d5dec010424435bdcea65c76dd2b1e524473d8a9 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 22 Jun 2020 17:56:28 -0400 Subject: [PATCH 003/970] xml escape dataset description --- .../edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java index f9107d8565b..fa04b710819 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java +++ b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java @@ -228,7 +228,8 @@ public static String getMetadataFromDvObject(String identifier, Map from HTML, it leaves '&' (at least so we need to xml escape as well + String description = StringEscapeUtils.escapeXml(dataset.getLatestVersion().getDescriptionPlainText()); if (description.isEmpty() || description.equals(DatasetField.NA_VALUE)) { description = AbstractGlobalIdServiceBean.UNAVAILABLE; } From 36098f245264e11516cd31f161b0e3c5f1117cdf Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 1 Jul 2020 13:34:14 -0400 Subject: [PATCH 004/970] fix mimetype of error pages --- src/main/webapp/WEB-INF/web.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index ce3e7d6e37b..9e007d48a11 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -185,6 +185,11 @@ webmanifest application/manifest+json + + xhtml + text/html + + From 95e60b385f7e940fac46dc59683e4485ca52e7c7 Mon Sep 17 00:00:00 2001 From: bencomp Date: Mon, 3 Oct 2022 01:58:47 +0200 Subject: [PATCH 005/970] Overload DatasetField method to remove type check I spotted a TODO near a type check based on a string comparison of the class name. --- .../edu/harvard/iq/dataverse/DatasetField.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java index 31d08f84c02..772538aa5cb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java @@ -54,15 +54,18 @@ public int compare(DatasetField o1, DatasetField o2) { o2.getDatasetFieldType().getDisplayOrder() ); }}; - public static DatasetField createNewEmptyDatasetField(DatasetFieldType dsfType, Object dsv) { + public static DatasetField createNewEmptyDatasetField(DatasetFieldType dsfType, DatasetVersion dsv) { DatasetField dsfv = createNewEmptyDatasetField(dsfType); - //TODO - a better way to handle this? - if (dsv.getClass().getName().equals("edu.harvard.iq.dataverse.DatasetVersion")){ - dsfv.setDatasetVersion((DatasetVersion)dsv); - } else { - dsfv.setTemplate((Template)dsv); - } + dsfv.setDatasetVersion(dsv); + + return dsfv; + } + + public static DatasetField createNewEmptyDatasetField(DatasetFieldType dsfType, Template dsv) { + + DatasetField dsfv = createNewEmptyDatasetField(dsfType); + dsfv.setTemplate(dsv); return dsfv; } From daccc0e3f4d4efee653489f249f8a1db9e3828bd Mon Sep 17 00:00:00 2001 From: bencomp Date: Mon, 3 Oct 2022 12:32:13 +0200 Subject: [PATCH 006/970] Compare types using instanceof --- src/main/java/edu/harvard/iq/dataverse/DatasetField.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java index 772538aa5cb..e6a4ca21fea 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java @@ -563,7 +563,7 @@ private DatasetField copy(Object version, DatasetFieldCompoundValue parent) { dsf.setDatasetFieldType(datasetFieldType); if (version != null) { - if (version.getClass().getName().equals("edu.harvard.iq.dataverse.DatasetVersion")) { + if (version instanceof DatasetVersion) { dsf.setDatasetVersion((DatasetVersion) version); } else { dsf.setTemplate((Template) version); From bdb62163d8ad4ed01b94943ad3c45f13406a6dae Mon Sep 17 00:00:00 2001 From: bencomp Date: Mon, 3 Oct 2022 13:07:46 +0200 Subject: [PATCH 007/970] Create type-specific copy methods in DatasetField Both `copy(DatasetVersion)` and `copy(Template)` still refer to `copy(Object, DatasetFieldCompoundValue)`, because copying that method would make the code less DRY. --- .../edu/harvard/iq/dataverse/DatasetField.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java index e6a4ca21fea..35a8184e45b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java @@ -548,9 +548,12 @@ public String toString() { return "edu.harvard.iq.dataverse.DatasetField[ id=" + id + " ]"; } - public DatasetField copy(Object version) { + public DatasetField copy(DatasetVersion version) { return copy(version, null); } + public DatasetField copy(Template template) { + return copy(template, null); + } // originally this was an overloaded method, but we renamed it to get around an issue with Bean Validation // (that looked t overloaded methods, when it meant to look at overriden methods @@ -558,15 +561,15 @@ public DatasetField copyChild(DatasetFieldCompoundValue parent) { return copy(null, parent); } - private DatasetField copy(Object version, DatasetFieldCompoundValue parent) { + private DatasetField copy(Object versionOrTemplate, DatasetFieldCompoundValue parent) { DatasetField dsf = new DatasetField(); dsf.setDatasetFieldType(datasetFieldType); - if (version != null) { - if (version instanceof DatasetVersion) { - dsf.setDatasetVersion((DatasetVersion) version); + if (versionOrTemplate != null) { + if (versionOrTemplate instanceof DatasetVersion) { + dsf.setDatasetVersion((DatasetVersion) versionOrTemplate); } else { - dsf.setTemplate((Template) version); + dsf.setTemplate((Template) versionOrTemplate); } } From 53535b4c3830c1cecd3def374c3d8d353e9a46f0 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 28 Feb 2023 08:58:14 +0100 Subject: [PATCH 008/970] Add JProfile acknowledgement for free license --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index d40e5f228f7..36fa2de67bf 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ We love contributors! Please see our [Contributing Guide][] for ways you can hel Dataverse is a trademark of President and Fellows of Harvard College and is registered in the United States. +We thank EJ Technologies for granting us a free open source project license for their Java profiler [JProfiler][]. + [![Dataverse Project logo](src/main/webapp/resources/images/dataverseproject_logo.jpg?raw=true "Dataverse Project")](http://dataverse.org) [![API Test Status](https://jenkins.dataverse.org/buildStatus/icon?job=IQSS-dataverse-develop&subject=API%20Test%20Status)](https://jenkins.dataverse.org/job/IQSS-dataverse-develop/) @@ -38,3 +40,4 @@ Dataverse is a trademark of President and Fellows of Harvard College and is regi [chat.dataverse.org]: http://chat.dataverse.org [Dataverse Community Meeting]: https://dataverse.org/events [open source]: LICENSE.md +[JProfiler]: https://www.ej-technologies.com/products/jprofiler/overview.html From 3a337e9405dee42ece3868c75930f695fb0125fe Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 28 Feb 2023 09:04:58 +0100 Subject: [PATCH 009/970] Add JProfiler acknowledgement to dev guide intro --- doc/sphinx-guides/source/developers/intro.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/sphinx-guides/source/developers/intro.rst b/doc/sphinx-guides/source/developers/intro.rst index 7f4e8c1ba34..4eed89a28ef 100755 --- a/doc/sphinx-guides/source/developers/intro.rst +++ b/doc/sphinx-guides/source/developers/intro.rst @@ -32,6 +32,8 @@ We make use of a variety of Jakarta EE technologies such as JPA, JAX-RS, JMS, an In addition, we start to adopt parts of Eclipse MicroProfile, namely `MicroProfile Config `_. +We thank EJ Technologies for granting us a free open source project license for their Java profiler `JProfiler `_. + Roadmap ------- From fc4cd59dfc1aecc51a319a9034360aa7d7b05166 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 28 Feb 2023 09:12:18 +0100 Subject: [PATCH 010/970] Add JProfiler and acknowledgement to tools page of dev guide --- doc/sphinx-guides/source/developers/tools.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/sphinx-guides/source/developers/tools.rst b/doc/sphinx-guides/source/developers/tools.rst index cbd27d6e8d2..4ef52f25f43 100755 --- a/doc/sphinx-guides/source/developers/tools.rst +++ b/doc/sphinx-guides/source/developers/tools.rst @@ -147,6 +147,14 @@ For example... would be consistent with a file descriptor leak on the dataset page. +JProfiler ++++++++++ + +Tracking down resource drainage, bottlenecks etc gets easier using a profiler. + +We thank EJ Technologies for granting us a free open source project license for their Java profiler +`JProfiler `_. + jmap and jstat ++++++++++++++ From 305739fb73360118997d0514325e109d10c4db53 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 1 Mar 2023 14:30:14 +0100 Subject: [PATCH 011/970] Revert "Add JProfile acknowledgement for free license" This reverts commit 53535b4c3830c1cecd3def374c3d8d353e9a46f0. --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 36fa2de67bf..d40e5f228f7 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,6 @@ We love contributors! Please see our [Contributing Guide][] for ways you can hel Dataverse is a trademark of President and Fellows of Harvard College and is registered in the United States. -We thank EJ Technologies for granting us a free open source project license for their Java profiler [JProfiler][]. - [![Dataverse Project logo](src/main/webapp/resources/images/dataverseproject_logo.jpg?raw=true "Dataverse Project")](http://dataverse.org) [![API Test Status](https://jenkins.dataverse.org/buildStatus/icon?job=IQSS-dataverse-develop&subject=API%20Test%20Status)](https://jenkins.dataverse.org/job/IQSS-dataverse-develop/) @@ -40,4 +38,3 @@ We thank EJ Technologies for granting us a free open source project license for [chat.dataverse.org]: http://chat.dataverse.org [Dataverse Community Meeting]: https://dataverse.org/events [open source]: LICENSE.md -[JProfiler]: https://www.ej-technologies.com/products/jprofiler/overview.html From e4d12a14ef631beefdc2d7287f8b9ef49227e9df Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 1 Mar 2023 14:30:17 +0100 Subject: [PATCH 012/970] Revert "Add JProfiler acknowledgement to dev guide intro" This reverts commit 3a337e9405dee42ece3868c75930f695fb0125fe. --- doc/sphinx-guides/source/developers/intro.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/sphinx-guides/source/developers/intro.rst b/doc/sphinx-guides/source/developers/intro.rst index 4eed89a28ef..7f4e8c1ba34 100755 --- a/doc/sphinx-guides/source/developers/intro.rst +++ b/doc/sphinx-guides/source/developers/intro.rst @@ -32,8 +32,6 @@ We make use of a variety of Jakarta EE technologies such as JPA, JAX-RS, JMS, an In addition, we start to adopt parts of Eclipse MicroProfile, namely `MicroProfile Config `_. -We thank EJ Technologies for granting us a free open source project license for their Java profiler `JProfiler `_. - Roadmap ------- From 72fca2c7e30cce95c8c4446399d32c6a418630f0 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Mon, 27 Mar 2023 13:26:21 -0400 Subject: [PATCH 013/970] Pdf export function --- pom.xml | 6 +++ .../dataverse/export/ddi/DdiExportUtil.java | 46 ++++++++++++++++++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e028771d24b..780bb7bdc0c 100644 --- a/pom.xml +++ b/pom.xml @@ -429,6 +429,12 @@ commons-compress + + + org.apache.xmlgraphics + fop + 2.8 + org.duracloud common diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index c78bb02d5c5..ee9231a9d53 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -74,7 +74,19 @@ import javax.xml.transform.stream.StreamResult; import java.io.File; import java.io.InputStream; -import java.io.InputStreamReader; + +import java.io.OutputStream; +import javax.xml.transform.Result; +import javax.xml.transform.Source; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.sax.SAXResult; +import javax.xml.transform.stream.StreamSource; + +import org.apache.fop.apps.FOUserAgent; +import org.apache.fop.apps.Fop; +import org.apache.fop.apps.FopFactory; +import org.apache.fop.apps.MimeConstants; public class DdiExportUtil { @@ -1948,6 +1960,38 @@ private static boolean checkParentElement(XMLStreamWriter xmlw, String elementNa return true; } + public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { + try { + File xsltfile = new File("ddi-to-fo.xsl"); + + final FopFactory fopFactory = FopFactory.newInstance(new File(".").toURI)); + FOUserAgent foUserAgent = fopFactory.newFOUserAgent(); + + try { + Fop fop = fopFactory.newFop(MimeConstants.MIME_PDF, foUserAgent, outputStream); + // Setup XSLT + TransformerFactory factory = TransformerFactory.newInstance(); + Transformer transformer = factory.newTransformer(new StreamSource(xsltfile)); + + // Set the value of a in the stylesheet + transformer.setParameter("versionParam", "2.0"); + + // Setup input for XSLT transformation + Source src = new StreamSource(datafile); + + // Resulting SAX events (the generated FO) must be piped through to FOP + Result res = new SAXResult(fop.getDefaultHandler()); + + // Start XSLT transformation and FOP processing + transformer.transform(src, res); + } finally { + outputStream.close(); + } + } catch (Exception e) { + logger.severe(e.getMessage()); + } + } + public static void datasetHtmlDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); From a5821b9fd0d53137918c329fb79590bc6431dbe1 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Mon, 3 Apr 2023 17:35:28 -0400 Subject: [PATCH 014/970] pdf codebook --- .../harvard/iq/dataverse/api/Datasets.java | 2 +- .../dataverse/export/PdfCodeBookExporter.java | 91 + .../dataverse/export/ddi/DdiExportUtil.java | 7 +- src/main/java/propertyFiles/Bundle.properties | 1 + .../from-ddi-2.5/ddi-pdf/i18n.inc.xslt | 5 + .../ddi-pdf/messages_en.properties.xml | 174 + .../ddi-pdf/messages_es.properties.xml | 170 + .../ddi-pdf/messages_fr.properties.xml | 173 + .../ddi-pdf/messages_ja.properties.xml | 161 + .../ddi-pdf/messages_nn.properties.xml | 174 + .../ddi-pdf/messages_no.properties.xml | 174 + .../ddi-pdf/messages_ru.properties.xml | 169 + .../iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl | 4473 +++++++++++++++++ 13 files changed, 5771 insertions(+), 3 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index d40bc153141..3b3326611dc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -272,7 +272,7 @@ public Response getDataset(@Context ContainerRequestContext crc, @PathParam("id" @GET @Path("/export") - @Produces({"application/xml", "application/json", "application/html" }) + @Produces({"application/xml", "application/json", "application/html", "*/*" }) public Response exportDataset(@QueryParam("persistentId") String persistentId, @QueryParam("exporter") String exporter, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) { try { diff --git a/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java new file mode 100644 index 00000000000..8f3bb78f8d3 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java @@ -0,0 +1,91 @@ +package edu.harvard.iq.dataverse.export; + +import com.google.auto.service.AutoService; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.export.ddi.DdiExportUtil; +import edu.harvard.iq.dataverse.export.spi.Exporter; +import edu.harvard.iq.dataverse.util.BundleUtil; + +import javax.json.JsonObject; +import javax.ws.rs.core.MediaType; +import javax.xml.stream.XMLStreamException; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.nio.file.Paths; + +@AutoService(Exporter.class) +public class PdfCodeBookExporter implements Exporter { + + @Override + public String getProviderName() { + return "pdf"; + } + + @Override + public String getDisplayName() { + return BundleUtil.getStringFromBundle("dataset.exportBtn.itemLabel.pdf") != null ? BundleUtil.getStringFromBundle("dataset.exportBtn.itemLabel.pdf") : "DDI PDF Codebook"; + } + + @Override + public void exportDataset(DatasetVersion version, JsonObject json, OutputStream outputStream) throws ExportException { + try { + InputStream ddiInputStream; + try { + ddiInputStream = ExportService.getInstance().getExport(version.getDataset(), "ddi"); + } catch(ExportException | IOException e) { + throw new ExportException ("Cannot open export_ddi cached file"); + } + DdiExportUtil.datasetPdfDDI(ddiInputStream, outputStream); + } catch (XMLStreamException xse) { + throw new ExportException ("Caught XMLStreamException performing DDI export"); + } + } + + @Override + public Boolean isXMLFormat() { + return false; + } + + @Override + public Boolean isHarvestable() { + // No, we don't want this format to be harvested! + // For datasets with tabular data the portions of the DDIs + // become huge and expensive to parse; even as they don't contain any + // metadata useful to remote harvesters. -- L.A. 4.5 + return false; + } + + @Override + public Boolean isAvailableToUsers() { + return true; + } + + @Override + public String getXMLNameSpace() throws ExportException { + return null; + } + + @Override + public String getXMLSchemaLocation() throws ExportException { + return null; + } + + @Override + public String getXMLSchemaVersion() throws ExportException { + return null; + } + + @Override + public void setParam(String name, Object value) { + // this exporter does not uses or supports any parameters as of now. + } + + @Override + public String getMediaType() { + return MediaType.WILDCARD; + }; +} diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index ee9231a9d53..3912b28a886 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -1962,9 +1962,12 @@ private static boolean checkParentElement(XMLStreamWriter xmlw, String elementNa public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { try { - File xsltfile = new File("ddi-to-fo.xsl"); + File xsltfile = new File("/home/victoria/ddi-to-fo.xsl"); + logger.info("start datasetPdfDDI"); + //InputStream xsltfile = DdiExportUtil.class.getClassLoader().getResourceAsStream( + // "edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl"); - final FopFactory fopFactory = FopFactory.newInstance(new File(".").toURI)); + final FopFactory fopFactory = FopFactory.newInstance(new File(".").toURI()); FOUserAgent foUserAgent = fopFactory.newFOUserAgent(); try { diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 45807dc7cde..3ba8d3fc8dd 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1395,6 +1395,7 @@ dataset.exportBtn.itemLabel.json=JSON dataset.exportBtn.itemLabel.oai_ore=OAI_ORE dataset.exportBtn.itemLabel.dataciteOpenAIRE=OpenAIRE dataset.exportBtn.itemLabel.html=DDI HTML Codebook +dataset.exportBtn.itemLabel.pdf=DDI PDF Codebook license.custom=Custom Dataset Terms license.custom.description=Custom terms specific to this dataset metrics.title=Metrics diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt new file mode 100644 index 00000000000..edf876f3b04 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt @@ -0,0 +1,5 @@ + + + + + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml new file mode 100644 index 00000000000..d8e98dfd3c6 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml @@ -0,0 +1,174 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 +Valid +Frequency table not shown +Derivation +discrete +Data Collection Mode +Other Processing +Other Acknowledgment(s) +Untitled +Identification +Click here to access/export data files from Nesstar format +Value +Percentage +Weighting +Primary Investigator(s) +This document was generated using the +Sampling +Cases +Access Conditions +Source +Modalities +Rights & Disclaimer +Definition +Estimates of Sampling Error +Data Files +Geographic Coverage +April +Mean +Metadata Production +Data Collection +Missing Data +Scripts and programs +Variable(s) +Interviewer instructions +Funding Agency/ies +November +Missing +Version +Universe +Contributor(s) +Access Authority +Data Processing & Appraisal +Scope +Administrative documents +StdDev +Contact(s) +Label +Technical documents +Decimal(s) +Type +Literal question +Concepts +Range +Abstract +June +Supervision +Other Forms of Data Appraisal +References +Accessibility +Data Collection Dates +Data Editing +Questionnaires +Valid case(s) +Reports and analytical documents +Copyright +Documentation +Deviations from Sample Design +Publisher(s) +February +Dataset contains +Acknowledgment(s) +Continuous +Standard deviation +Variables Description +Producer +Production Date + +The Explorer allows you to view data files and export them to common statistical formats +Discrete +Group +July +Filename +Cases +Name +Warning: these figures indicate the number of cases found in the data file. They cannot be interpreted as summary statistics of the population of interest. +Statistical tables +December +Subjects +Processing Checks +software +Interviewer's instructions +Table of Contents +Document Information +Subgroup(s) +Keywords +group(s) +W +Weight +Files Description +Notes +Data Collection Notes +file(s) +continuous +Disclaimer +Content +variable(s) +Other Producer(s) +Producers & Sponsors +Data Cleaning Notes +Distributor(s) +Overview +Citation Requirements +September +Category +Confidentiality +Statistics +May +Undetermined +Structure +file +Pre-question +Response Rate +Width +Recoding and Derivation +Series +October +Unit of Analysis +Data Processing Notes +Kind of Data +File +Time Period(s) +File Content +Invalid +Vars +cont. +Key(s) +Question +Source of information +Imputation +Security +To open this file, you will need the free +Other resources +Data Dictionnary +Information +January +Other documents +Minimum +Scope & Coverage +Metadata Producer(s) +Show more info +Data Collector(s) +Post-question +Topics +Sampling Procedure +File Structure +Variables List +Format +Sampling Notes +Variables Group(s) +Description +Categories +Maximum +Depositor(s) +August +NW +Cover Page +Weighted +March + total - showing a subset of +Countries +question details + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml new file mode 100644 index 00000000000..9cfcdaf6e7e --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml @@ -0,0 +1,170 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 +Válido +No se presentan las tablas de frecuencias +Derivación +discreta +Método de Recolección +Otros relacionados al procesamiento +Otros Reconocimientos +Sin título +Identificación +Presione aquí para acceder/exportar al archivo(s) de datos +Valor +Porcentaje +Ponderando +Investigadores Principales +Este documento fue producido utilizando el +Muestreo +Casos +Condiciones de uso +Fuente +Modalidades +Derechos y Notas Legales +Definición +Estimaciones del Error Muestral +Archivo de Datos +Cobertura Geográfica +Abril +Media +Producción de los Metadatos +Recolección de Datos +Datos perdidos +Programas informáticos +Variable(s) +Manual del encuestador +Agencias Auspiciadoras +Noviembre +Valores perdidos +Versión +Universo +Contribuidor(es) +Institución Propietaria +Tratamiento y Validación de Datos +Dominio Temático +Documentos Administrativos +Desviación +Contacto(s) +Etiqueta +Documentos Técnicos +Decimal(es) +Tipo +Pregunta textual +Conceptos +Rango +Resumen +Junio +Supervisión +Otras Formas de Validación de los Datos +Referencias +Accesibilidad +Fechas de Recolección de Datos +Procesamiento de Datos +Cuestionarios +Casos válidos +Reportes y documentos analíticos +Derechos de Autor +Documentación +Modificaciones al Diseño Muestral +Editor(es) +Febrero +Contenido de la Base de Datos +Reconocimiento(s) +Contínua +Desviación estándar +Descripción de la variable +Productor +Fecha de Producción +El Explorador NESSTAR permite visualizar los archivos de datos y exportarlos a diferentes formatos estadísticos +Discreta +Grupo +Julio +Nombre del Archivo +Casos +Nombre +Cuadros estadísticos +Diciembre +Temas +Controles de Tratamiento +software +Manual del encuestador +Indice +Información acerca de la Documentación +Subgrupo(s) +Palabra Clave +grupo(s) +P +Ponderador +Descripción de los Archivos +Notas +Notas sobre la Recolección de Datos +archivo(s) +continua +Nota Legal +Contenido +variable(s) +Otros Productores +Productores y Auspiciadores +Notas acerca de la Depuración de los Datos +Distribuidor(es) +Resumen General +Forma de citar +Septiembre +Categoría +Confidencialidad +Estadística + +Mayo +Indeterminado +Estructura +archivo +Pre-pregunta +Tasa de Respuesta +Ancho +Recodificación y Derivación +Series +Octubre +Unidad de Análisis +Notas sobre el Procesamiento de Datos +Tipo de Datos +Archivo +Periodo de Referencia +Contenido del Archivo +Inválido +Vars. +cont. +Clave(s) +Pregunta +Fuente de información +Imputación +Seguridad +Para abrir este archivo se necesita el software gratuito +Otros recursos +Diccionario de Datos +Información +Enero +Otros documentos +Mínimo +Cobertura y Dominio Temático +Productor de los Metadatos +Mostrar más información +Entrevistador(es) +Pos-pregunta +Temas +Procedimiento de Muestreo +Estructura del Archivo +Lista de variables +Formato +Notas sobre el Muestreo +Grupo(s) de Variables +Descripción +Categorías +Máximo +Depositante(s) +Agosto +NP +Carátula +Ponderado +Marzo + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml new file mode 100644 index 00000000000..9fa4d2178b1 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml @@ -0,0 +1,173 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 +Valide +Tableau de fréquences non-affiché +Mode de calcul +discrète +Méthode de collecte +Autre traitement +Autre(s) remerciement(s) +Sans titre +Identification +Cliquer ici pour accéder/exporter les fichiers de données du format Nesstar +Valeur +Pourcentage +Pondération +Enquêteur(s) principal/aux +Ce document a été généré à l'aide du +Echantillonage +Enregistrements +Conditions d'accès +Source +Modalités +Responsabilité et droits d'auteurs +Définition +Estimation des erreurs d'échantillonage +Fichiers de données +Couverture géographique +Avril +Moyenne +Production des méta-données +Collecte des données +Valeures manquantes +Programmes informatiques +Variable(s) +Instructions aux enquêteurs +Financement +Novembre +Manquant +Version +Univers +Contributeur(s) +Agence propriétaire +Traitement et évaluation des données +Domaines thématiques +Documents administratifs +Ecart type +Contact(s) +Libellé +Documents techniques +Décimale(s) +Type +Formulation de la question +Concepts +Gamme +Résumé +Juin +Supervision +Autres formes d'évaluation des données +Références +Accessibilité +Dates de collecte +Edition des données +Questionnaires +Cas valide(s) +Rapports et documents analytiques +Droits d'auteurs +Documentation +Déviations par rapport à l'échantillon initial +Editeur(s) +Février +Le jeu de données contient +Remerciement(s) +Continue +Ecart type +Description des variables +Producteur +Date de production + +L'Explorer vous permet d'accéder aux données et de les exporter vers les formats statistiques les plus courants +Discrète +Groupe +Juillet +Nom du fichier +Enreg. +Nom +Avertissement: Ces chiffres indiquent le nombre de cas identifiés dans le fichier de données. Ils ne peuvent pas être interpretés comme étant représentatifs de la population concernée. +Tableaux statistiques +Décembre +Sujets +Contrôles de traitement + +Instructions aux enquêteurs +Table des matières +Informations sur le document +Sous-groupe(s) +Mots-clé +groupe(s) +P +Pondération +Description des fichiers +Notes +Notes sur la collecte +fichier(s) +continue +Responsabilité(s) +Contenu +variable(s) +Autre(s) producteur(s) +Producteurs et sponsors +Notes sur l'apurement des données +Distributeur(s) +Aperçu +Citation +Septembre +Catégorie +Confidentialité +Statistiques +Mai +Indéterminé +Structure +fichier +Pré-question +Taux de réponse +Taille +Formulation de la question +Recodage et dérivation +Série +Octobre +Unité d'analyse +Notes sur le traitement des données +Type d'étude +Fichier +Période(s) de référence +Contenu du fichier +Non-valide +Vars +suite +Clé(s) +Question +Source d'information +Imputation +Sécurité +Pour ouvrir ce fichier, vous avez besoin du logiciel gratuit +Autres resources +Dictionnaire des variables +Information +Janvier +Autres documents +Minimum +Domaines thématiques et couverture +Producteur(s) des méta-données +Information complémentaire +Enquêteurs +Post-question +Thèmes +Procédure d'échantillonage +Structure du fichier +Liste des variables +Format +Notes sur l'échantillonage +Groupe(s) de variables +Description +Catégories +Maximum +Dépositaire(s) +Août +NP +Couverture +Pondéré +Mars +question details + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml new file mode 100644 index 00000000000..bc5dbb06154 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml @@ -0,0 +1,161 @@ + + +Generated by Properties2Xml on Tue Feb 13 13:55:43 EST 2007 +有効な +度数表(Frequency table)は表示されません +由来 +不連続な +データ収集モード +その他の確認事項 +識別番号 +データファイルにアクセスするにはここをクリックしてください +無題 +値 +割合 +ウェイティング +第一次調査官 +この文書はToolkitを使用して作られました +サンプリング +ケース +アクセス条件 +情報源 +様相 +権利及び声明文 +定義 +サンプルエラーの見積もり +データファイル +地理的な適用範囲 +４月 +平均 +メタデータ製作 +データ収集 +損失データ +スクリプトおよびプログラム +可変的 +面接者の指示 +出資機関 +１１月 +バージョン +共通の +貢献者 +アクセス権限 +データ処理、評価 +範囲, 領域 +管理用文章 +連絡先 +ラベル +技術的な文書 +小数点 +タイプ +文字の質問 +概念 +範囲 +要約 +６月 +監督 +その他ファーマットのデータ評価 +参照 +アクセス、入手法 +データ収集日 +データ編集 +質問 +レポートおよび分析的な文書 +有効な場合 +コピーライト +書類 +サンプルデザインによる偏差 +発行者 +２月 +データセットに含まれる +確認事項 +連続的な +標準偏差 +変数の記述 +製作者 +製作日 +” Explorer”によってデータファイルを参照することも一般的に使えわれている統計データフォーマットに変換。抽出することも可能です +不連続性 +グループ +７月 +ファイルの名前 +ケース +名前 +統計表 +１２月 +主題, 内容 +工程監査 +ソフト +面接者への指示 +目録 +書類の情報 +サブグループ +キーワード + +グループ +ウェイト +ファイルの詳細 +メモ +データ収集メモ +ファイル +継続的な +声明文 +内容 +変数 +その他の製作者 +製作者とスポンサー +データクリーニングメモ +分配者 +概略 +引用する場合の必要条件 +９月 +カテゴリー +機密性、コンフィデンシャリティー +５月 +未定 +構造 +ファイル +調査前の質問 +回答比率 +幅 +記録と誘導 +シリー +１０月 +分析の単位 +データ処理メモ +データの種類 + +ファイル +期間 +ファイルの内容 +無効 +キー +情報源 +非難 +セキュリティー +このファイルを開けるには、無料で配布されているNesstar Explorer が必要です。 +その他の資料 +データ辞典 +情報 +１月 +その他の書類 +最小値 +規模及び適用範囲 +メタデータ製作者 +さらにインフォメーションを表示 +データ収集者 +調査後の質問 +サンプリングの手順 +ファイルの構造 +変数のリスト +フォーマット +サンプリングメモ +変数のグループ +詳細 +カテゴリー +最大値 +デポジター、提供者、供託者 +８月 +表紙 +ウェイトされた +３月 + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml new file mode 100644 index 00000000000..fdf14f5dfcd --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml @@ -0,0 +1,174 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 +Gyldige +Frekvenstabell ikke vist +Avledning +diskret +Type datainnsamling +Annen prosessering +Andre identifikatorer og krediteringer +Uten tittel +Identifisering +Click here to access/export data files from Nesstar format +Verdi +Prosent +Vekting +Primary Investigator(s) +Dette dokumentet ble generert av +Utvalg +Enheter +Tilgangsbetingelser +Kilde +Modaliteter +Rights & Disclaimer +Definisjon +Estimert utvalgsfeil +Datafiler +Geografisk omfang +April +Mean +Metadata-produksjon +Datainnsamling +Manglende data +Script og programmer +Variable(r) +Instruksjoner til intervjueren +Sponsor/finansierende institusjon(er) +November +Missing +Versjon +Univers +Bidragsyter(e) +Tilgangskontrollør +Dataprosessering og -evaluering +Omfang +Administrative dokumenter +Standardavvik +Kontaktperson(er) +Merkelapp +Tekniske dokumenter +Desimal(er) +Type +Spørsmålstekst +Begrep(er) +Rekkevidde +Sammendrag +Juni +Supervision +Andre former for dataevaluering +Referanser +Tilgjengelighet +Datainnsamlingsdatoer +Dataredigering +Spørreskjema +Gyldige enheter +Rapporter og analysedokumenter +Copyright +Dokumentasjon +Avvik fra utvalgsdesign +Utgiver(e) +Februar +Datasettet inneholder +Krediteringer +Kontinuerlig +Standardavvik +Variabelbeskrivelse +Produsent +Produksjonsdato + +The Explorer allows you to view data files and export them to common statistical formats +Diskret +Gruppe +Juli +Filnavn +Enheter +Navn +Advarsel: disse tallene indikerer antall enheter (cases) i datafilen. De kan ikke tolkes som oppsummert statistikk for populasjonen. +Statistiske tabeller +Desember +Emner +Prosesseringssjekk +programvare +Instruksjoner til intervjueren +Innholdsfortegnelse +Dokumentinformasjon +Undergruppe(r) +Nøkkelord +gruppe(r) +W +Vekt +Filbeskrivelse +Kommentarer +Datainnsamlingskommentarer +file(r) +kontinuerlig +Fraskrivelse +Innhold +variable(r) +Andre produsenter +Produsenter og sponsorer +Kommentarer om datarensing +Distributør(er) +Oversikt +Sitatkrav +September +Kategori +Konfidensialitet +Statistikk +Mai +Uavklart +Struktur +fil +Tekst før spørsmål +Responsrate +Bredde +Omkodinger og utledninger +Serie +Oktober +Analyseenhet +Dataprosesseringskommentarer +Datatype +Fil +Tidsperiode(r) +Filinnhold +Ugyldig +Variabler +kont. +Nøkler +Spørsmål +Kilde for informasjon +Imputasjon +Sikkerhet +For å åpne denne filen trenger du følgende gratisverktøy +Andre ressurser +Dataordbok +Informasjon +Januar +Andre dokumenter +Minimum +Omfang +Metadataprodusenter +Vis mer informasjon +Datainnsamler(e) +Tekst etter spørsmål +Emner +Utvalgsprosedyre +Filstruktur +Variabelliste +Format +Utvalgskommentarer +Variabelgrupper +Beskrivelse +Kategorier +Maksimum +Utgiver(e) +August +NW +Forside +Vektet +Mars + total - viser et utvalg av +Land +spørsmålsdetaljer + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml new file mode 100644 index 00000000000..fdf14f5dfcd --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml @@ -0,0 +1,174 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 +Gyldige +Frekvenstabell ikke vist +Avledning +diskret +Type datainnsamling +Annen prosessering +Andre identifikatorer og krediteringer +Uten tittel +Identifisering +Click here to access/export data files from Nesstar format +Verdi +Prosent +Vekting +Primary Investigator(s) +Dette dokumentet ble generert av +Utvalg +Enheter +Tilgangsbetingelser +Kilde +Modaliteter +Rights & Disclaimer +Definisjon +Estimert utvalgsfeil +Datafiler +Geografisk omfang +April +Mean +Metadata-produksjon +Datainnsamling +Manglende data +Script og programmer +Variable(r) +Instruksjoner til intervjueren +Sponsor/finansierende institusjon(er) +November +Missing +Versjon +Univers +Bidragsyter(e) +Tilgangskontrollør +Dataprosessering og -evaluering +Omfang +Administrative dokumenter +Standardavvik +Kontaktperson(er) +Merkelapp +Tekniske dokumenter +Desimal(er) +Type +Spørsmålstekst +Begrep(er) +Rekkevidde +Sammendrag +Juni +Supervision +Andre former for dataevaluering +Referanser +Tilgjengelighet +Datainnsamlingsdatoer +Dataredigering +Spørreskjema +Gyldige enheter +Rapporter og analysedokumenter +Copyright +Dokumentasjon +Avvik fra utvalgsdesign +Utgiver(e) +Februar +Datasettet inneholder +Krediteringer +Kontinuerlig +Standardavvik +Variabelbeskrivelse +Produsent +Produksjonsdato + +The Explorer allows you to view data files and export them to common statistical formats +Diskret +Gruppe +Juli +Filnavn +Enheter +Navn +Advarsel: disse tallene indikerer antall enheter (cases) i datafilen. De kan ikke tolkes som oppsummert statistikk for populasjonen. +Statistiske tabeller +Desember +Emner +Prosesseringssjekk +programvare +Instruksjoner til intervjueren +Innholdsfortegnelse +Dokumentinformasjon +Undergruppe(r) +Nøkkelord +gruppe(r) +W +Vekt +Filbeskrivelse +Kommentarer +Datainnsamlingskommentarer +file(r) +kontinuerlig +Fraskrivelse +Innhold +variable(r) +Andre produsenter +Produsenter og sponsorer +Kommentarer om datarensing +Distributør(er) +Oversikt +Sitatkrav +September +Kategori +Konfidensialitet +Statistikk +Mai +Uavklart +Struktur +fil +Tekst før spørsmål +Responsrate +Bredde +Omkodinger og utledninger +Serie +Oktober +Analyseenhet +Dataprosesseringskommentarer +Datatype +Fil +Tidsperiode(r) +Filinnhold +Ugyldig +Variabler +kont. +Nøkler +Spørsmål +Kilde for informasjon +Imputasjon +Sikkerhet +For å åpne denne filen trenger du følgende gratisverktøy +Andre ressurser +Dataordbok +Informasjon +Januar +Andre dokumenter +Minimum +Omfang +Metadataprodusenter +Vis mer informasjon +Datainnsamler(e) +Tekst etter spørsmål +Emner +Utvalgsprosedyre +Filstruktur +Variabelliste +Format +Utvalgskommentarer +Variabelgrupper +Beskrivelse +Kategorier +Maksimum +Utgiver(e) +August +NW +Forside +Vektet +Mars + total - viser et utvalg av +Land +spørsmålsdetaljer + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml new file mode 100644 index 00000000000..06fde85af5e --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml @@ -0,0 +1,169 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 +Валидный +Частотная таблица не выводится +Расчет +дискретная +Способ сбора данных +Прочая обработка +Другие участники +Безымянный +Индентификация +Щелкните здесь, чтобы получить доступ к файлам или экспортировать их +Значение +Процент +Взвешивание +Первичный(е) исследователь(и) +Документ был сгенерирован с использованием +Выборка +Наблюдения +Условия доступа +Источник +Модальности +Авторские права и ограничения ответственности +Определение +Оценки ошибок выборки +Файлы данных +Географический охват +Апрель +Среднее +Разработка метаданных +Сбор данных +Пропущенные данные +Скрипты и программы +Переменная(ые) +Инструкции интервьюеру +Кто финансировал +Ноябрь +Пропущенные +Версия +Генеральная совокупность +Участник(и) +Права доступа +Обработка и инспекция данных +Охват +Административные документы +СтдОткл +Контак(ы) +Метка +Технические документы +Десятичные +Тип +Текст вопроса +Концепции +Диапазон +Резюме +Июнь +Контроль +Другие формы инспекции данных +Установки +Доступность +Даты сбора данных +Редактирование данных +Вопросники +Валидное(ые) наблюдение(я) +Отчеты и аналитические документы +Авторские права +Документация +Отклонения от дизайна выборки +Издатель(и) +Февраль +Набор данных содержит +Участник(и) +Непрерывная +Стандартное отклонение +Описание переменных +Разработчик +Дата разработки +Проводник позволяет просматривать файлы данных и экспортировать их в распространенные статистические форматы +Дикретная +Группа +Июль +Имя файла +Наблюдения +Имя +Статистичсекие таблицы +Декабрь +Темы +Контроль обработки +программное обеспечение +Инструкции интервьюеру +Оглавление +Информация о документе +Подгруппа(ы) +Ключевые слова +группа(ы) +B +Вес +Описание файла +Примечания +Примечания по сбору данных +файл(ы) +непрерывная +Ограничения ответственности +Содержание +переменная(ые) +Другие разработчики +Разработчики и спонсоры +Примечания по чистке данных +Дистрибьютор(ы) +Обзор +Требования по цитированию +Сентябрь +Категория +Конфиденциальность +Статистики +Май +Неопределенный +Структура +файл +Текст, предваряющий вопрос +Доля ответов +Ширина +Перекодировка и расчеты +Серия +Октябрь +Единица анализа +Примечания по обработке данных +Тип данных +Файл +Период(ы) времени +Содержание файла +Некорректный +Переменные +непр. +Ключ(и) +Вопрос +Источник информации +Импутация +Безопасность +Чтобы открыть этот файл, необходимо иметь свободным +Прочие источники +Словарь данных +Информация +Январь +Прочие документы +Минимум +Охват и покрытие +Разработчик(и) метаданных +Показать дополнительную информацию +Кто собирал данные +Текст после вопроса +Разделы +Процедура выборки +Структура файла +Список переменных +Формат +Примечания по выборке +Группа(ы) переменных +Описание +Категории +Максимум +Депозитор(ы) +Август +HB +Титульный лист +взвешенные +Март + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl new file mode 100644 index 00000000000..64849846481 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl @@ -0,0 +1,4473 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + default-page + + + + + + + + + + + + + + + + + + + + + + Times + + + + + + + + + + + + + + + + + + + + + + + + + 50 + + + + + + + 0 + 0 + 0 + 1 + + + + + + + 1 + 0 + + + + + + + 0 + 1 + + + + + + ddp + toolkit + toolkit + toolkit + + + + + + + + () + + + + + + , + + + + + + + + + + + + + + + + - + + + + + + + + + + + + 1 + 1 + 1 + 1 + 0 + + + + + 1 + 1 + 1 + 1 + 0 + + + + + 1 + 1 + 1 + 1 + 1 + 0 + + + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 0 + + + + + 1 + 1 + 1 + 0 + + + + + 1 + 1 + 1 + 1 + 1 + 1 + 0 + + + + + 1 + 1 + 0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + : + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + : + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + *** + + **** + + + + + + + + + + + + + + + , + + + + + + + + : + + + + + + + + + + + + + () + , + + + + , + + + + + + + + + + + () + , + + + + + + + : + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + # + + + + + + + + + + + # + + + + + + + + + + + + + + + + + : + + + + + + :  + + + + + ,  + + +  () + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + : + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + () + , + + + + + + + + + + , + + , + + + + + + + + + + () + , + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + .. + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + - + + + + - + + + + . + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + +   + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + # + + + + + : + + + + + + + + + + + + + + + + + + + + + [= + + + + + + ] + + + + [= + + - + + + + . + + + ] + + + + [= - + ] + + + [ + + =* + + / + + + ] + + + + + + + + + + + + + + [ + + / + + ] + + + + + + + + + + + + + + + + + + + + + + + + + + + [ + + = + + + / + + + + + + - + + + ] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + () + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ( + ) + + + + + + + + + + + + + + + + + + + + : + + + + + + + + + + + + + + + + + + *** + *** + + + + + , + + + + + + + , + + + + + + , + + + + + + + + + + + + + , + + + + + , + + + + + , " + + " + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + # + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <h2> + + + </h2> + + + <br/> + + + + + + + + + + + + + + + + + + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Z + + + + + + + + + + + + + + + + + + + - + + + + + + + + NaN + + + + + + + + + + + From 84caa2a85e2f86d1c6f82725673bb4a60ec19f3d Mon Sep 17 00:00:00 2001 From: lubitchv Date: Mon, 12 Jun 2023 17:52:04 -0400 Subject: [PATCH 015/970] q --- .../dataverse/export/PdfCodeBookExporter.java | 88 +++++++++---------- 1 file changed, 40 insertions(+), 48 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java index 8f3bb78f8d3..c4f1d3ff2cc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java @@ -1,55 +1,58 @@ + + package edu.harvard.iq.dataverse.export; import com.google.auto.service.AutoService; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.export.ddi.DdiExportUtil; -import edu.harvard.iq.dataverse.export.spi.Exporter; -import edu.harvard.iq.dataverse.util.BundleUtil; + import edu.harvard.iq.dataverse.export.ddi.DdiExportUtil; + import io.gdcc.spi.export.ExportDataProvider; + import io.gdcc.spi.export.ExportException; + import io.gdcc.spi.export.Exporter; + import edu.harvard.iq.dataverse.util.BundleUtil; -import javax.json.JsonObject; -import javax.ws.rs.core.MediaType; -import javax.xml.stream.XMLStreamException; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Path; -import java.nio.file.Paths; + import javax.json.JsonObject; + import javax.ws.rs.core.MediaType; + import javax.xml.stream.XMLStreamException; + import java.io.File; + import java.io.IOException; + import java.io.InputStream; + import java.io.OutputStream; + import java.nio.file.Path; + import java.nio.file.Paths; + import java.util.Locale; + import java.util.Optional; @AutoService(Exporter.class) public class PdfCodeBookExporter implements Exporter { @Override - public String getProviderName() { + public String getFormatName() { return "pdf"; } @Override - public String getDisplayName() { - return BundleUtil.getStringFromBundle("dataset.exportBtn.itemLabel.pdf") != null ? BundleUtil.getStringFromBundle("dataset.exportBtn.itemLabel.pdf") : "DDI PDF Codebook"; + public String getDisplayName(Locale locale) { + String displayName = BundleUtil.getStringFromBundle("dataset.exportBtn.itemLabel.pdf", locale); + return Optional.ofNullable(displayName).orElse("DDI pdf codebook"); } @Override - public void exportDataset(DatasetVersion version, JsonObject json, OutputStream outputStream) throws ExportException { - try { - InputStream ddiInputStream; - try { - ddiInputStream = ExportService.getInstance().getExport(version.getDataset(), "ddi"); - } catch(ExportException | IOException e) { - throw new ExportException ("Cannot open export_ddi cached file"); + public void exportDataset(ExportDataProvider dataProvider, OutputStream outputStream) throws ExportException { + Optional ddiInputStreamOptional = dataProvider.getPrerequisiteInputStream(); + if (ddiInputStreamOptional.isPresent()) { + try (InputStream ddiInputStream = ddiInputStreamOptional.get()) { + DdiExportUtil.datasetPdfDDI(ddiInputStream, outputStream); + } catch (IOException e) { + throw new ExportException("Cannot open export_ddi cached file"); + } catch (XMLStreamException xse) { + throw new ExportException("Caught XMLStreamException performing DDI export"); } - DdiExportUtil.datasetPdfDDI(ddiInputStream, outputStream); - } catch (XMLStreamException xse) { - throw new ExportException ("Caught XMLStreamException performing DDI export"); + } else { + throw new ExportException("No prerequisite input stream found"); } } - @Override - public Boolean isXMLFormat() { - return false; - } - @Override public Boolean isHarvestable() { // No, we don't want this format to be harvested! @@ -65,27 +68,16 @@ public Boolean isAvailableToUsers() { } @Override - public String getXMLNameSpace() throws ExportException { - return null; - } - - @Override - public String getXMLSchemaLocation() throws ExportException { - return null; - } - - @Override - public String getXMLSchemaVersion() throws ExportException { - return null; - } - - @Override - public void setParam(String name, Object value) { - // this exporter does not uses or supports any parameters as of now. + public Optional getPrerequisiteFormatName() { + //This exporter relies on being able to get the output of the ddi exporter + return Optional.of("ddi"); } @Override public String getMediaType() { - return MediaType.WILDCARD; + return MediaType.TEXT_HTML; }; } + + + From cb85fd678945ed206cbcca7fc378bbb7d1771394 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Thu, 15 Jun 2023 17:54:09 -0400 Subject: [PATCH 016/970] english --- .../dataverse/export/PdfCodeBookExporter.java | 2 +- .../dataverse/export/ddi/DdiExportUtil.java | 15 +- .../{from-ddi-2.5 => }/ddi-to-fo.xsl | 444 +++++++++--------- .../from-ddi-2.5/ddi-pdf/i18n.inc.xslt | 5 - .../ddi-pdf/messages_en.properties.xml | 174 ------- .../ddi-pdf/messages_es.properties.xml | 170 ------- .../ddi-pdf/messages_fr.properties.xml | 173 ------- .../ddi-pdf/messages_ja.properties.xml | 161 ------- .../ddi-pdf/messages_nn.properties.xml | 174 ------- .../ddi-pdf/messages_no.properties.xml | 174 ------- .../ddi-pdf/messages_ru.properties.xml | 169 ------- 11 files changed, 234 insertions(+), 1427 deletions(-) rename src/main/resources/edu/harvard/iq/dataverse/{from-ddi-2.5 => }/ddi-to-fo.xsl (91%) delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml diff --git a/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java index c4f1d3ff2cc..e0d5171e30c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java @@ -75,7 +75,7 @@ public Optional getPrerequisiteFormatName() { @Override public String getMediaType() { - return MediaType.TEXT_HTML; + return MediaType.WILDCARD; }; } diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 00ec3c4d5c9..c77be3515f7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -43,6 +43,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -2110,7 +2111,10 @@ private static boolean checkParentElement(XMLStreamWriter xmlw, String elementNa public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { try { - File xsltfile = new File("/home/victoria/ddi-to-fo.xsl"); + //File xsltfile = new File("/home/victoria/fop-2.8/fop/ddi-to-fo.xsl"); + //URL resource = DdiExportUtil.class.getResource("edu/harvard/iq/dataverse/ddi-to-fo.xsl"); + //File xsltfile = new File(resource.toURI()); + InputStream styleSheetInput = DdiExportUtil.class.getClassLoader().getResourceAsStream("edu/harvard/iq/dataverse/ddi-to-fo.xsl"); logger.info("start datasetPdfDDI"); //InputStream xsltfile = DdiExportUtil.class.getClassLoader().getResourceAsStream( // "edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl"); @@ -2122,7 +2126,7 @@ public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream Fop fop = fopFactory.newFop(MimeConstants.MIME_PDF, foUserAgent, outputStream); // Setup XSLT TransformerFactory factory = TransformerFactory.newInstance(); - Transformer transformer = factory.newTransformer(new StreamSource(xsltfile)); + Transformer transformer = factory.newTransformer(new StreamSource(styleSheetInput)); // Set the value of a in the stylesheet transformer.setParameter("versionParam", "2.0"); @@ -2135,10 +2139,13 @@ public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream // Start XSLT transformation and FOP processing transformer.transform(src, res); - } finally { - outputStream.close(); + + } catch (Exception e) { + logger.info("First try"); + logger.severe(e.getMessage()); } } catch (Exception e) { + logger.info("Second try"); logger.severe(e.getMessage()); } } diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl similarity index 91% rename from src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl rename to src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl index 64849846481..1c03d5caf34 100644 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl @@ -66,7 +66,8 @@ Report optional text --> - + + @@ -123,7 +124,6 @@ Times - - + - + - + - + - + @@ -437,21 +437,21 @@ + select="'Data_Processing_and_Appraisal'"/> - + - + @@ -461,7 +461,7 @@ - + @@ -486,7 +486,7 @@ - + @@ -503,7 +503,7 @@ - + @@ -528,7 +528,7 @@ - + @@ -569,7 +569,7 @@ - + @@ -642,7 +642,7 @@ - + @@ -657,7 +657,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="'Metadata_Producers'"/> + select="'Production_Date'"/> + select="'Version'"/> + select="'Identification'"/> - + @@ -745,7 +745,7 @@ - + @@ -764,14 +764,14 @@ - + - + @@ -782,7 +782,7 @@ + select="'Scope_and_Coverage'"/> @@ -793,7 +793,7 @@ + select="'Producers_and_Sponsors'"/> @@ -803,7 +803,7 @@ - + @@ -813,7 +813,7 @@ - + @@ -825,7 +825,7 @@ internal-destination="data-processing-and-appraisal" text-decoration="underline" color="blue"> + select="'Data_Processing_and_Appraisal'"/> @@ -836,7 +836,7 @@ - + @@ -847,7 +847,7 @@ + select="'Rights_and_Disclaimer'"/> @@ -858,7 +858,7 @@ + select="'Files_Description'"/> @@ -891,7 +891,7 @@ - + @@ -924,7 +924,7 @@ - + @@ -949,7 +949,7 @@ + select="'Variables_Description'"/> @@ -982,7 +982,7 @@ - + @@ -1002,7 +1002,7 @@ - + @@ -1065,7 +1065,7 @@ - + @@ -1078,7 +1078,7 @@ - + + select="'Identification'"/> - @@ -1130,7 +1130,7 @@ select="/ddi:codeBook/ddi:stdyDscr/ddi:citation/ddi:verStmt/ddi:version"> : + select="'Production_Date'"/>: @@ -1139,7 +1139,7 @@ - @@ -1156,7 +1156,7 @@ - + - + select="'Kind_of_Data'"/> + select="'Unit_of_Analysis'"/> + select="'Scope_and_Coverage'"/> @@ -1257,7 +1257,7 @@ - + - @@ -1299,7 +1299,7 @@ - + - + @@ -1344,7 +1344,7 @@ - + @@ -1361,7 +1361,7 @@ border="{$default-border}" padding="{$cell-padding}"> - @@ -1422,7 +1422,7 @@ padding="{$cell-padding}"> @@ -1444,7 +1444,7 @@ padding="{$cell-padding}"> + select="'Other_Producers'"/> + select="'Funding_Agencies'"/> @@ -1514,7 +1514,7 @@ - @@ -1534,7 +1534,7 @@ padding="{$cell-padding}"> + select="'Sampling'"/> + select="'Sampling_Notes'"/> + select="'Sampling_Procedure'"/> + select="'Response_Rate'"/> + select="'Weighting'"/> + select="'Data_Collection'"/> @@ -1672,7 +1672,7 @@ padding="{$cell-padding}"> @@ -1694,7 +1694,7 @@ padding="{$cell-padding}"> + select="'Time_Periods'"/> @@ -1738,7 +1738,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="'Questionnaires'"/> + select="'Data_Collectors'"/> + select="'Supervision'"/> - + @@ -1885,7 +1885,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="'Data_Editing'"/> + select="'Other_Processing'"/> + select="'Accessibility'"/> @@ -1980,7 +1980,7 @@ padding="{$cell-padding}"> + select="'Access_Authority'"/> - @@ -2022,7 +2022,7 @@ padding="{$cell-padding}"> + select="'Distributors'"/> + select="'Depositors'"/> + select="'Confidentiality'"/> + select="'Access_Conditions'"/> @@ -2142,7 +2142,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="'Disclaimer'"/> + select="'Copyright'"/> - + @@ -2194,15 +2194,15 @@ - + - + - + @@ -2218,7 +2218,7 @@ - + @@ -2227,16 +2227,16 @@ - + - + - - + + - + @@ -2262,17 +2262,17 @@ - + - + - + - + @@ -2290,7 +2290,7 @@ - + @@ -2299,17 +2299,17 @@ - + - + - + - + @@ -2327,7 +2327,7 @@ - + @@ -2336,7 +2336,7 @@ - + @@ -2372,7 +2372,7 @@ + select="'Reports_and_analytical_documents'"/> @@ -2389,7 +2389,7 @@ space-after="0.03in"> - + @@ -2407,7 +2407,7 @@ + select="'Technical_documents'"/> @@ -2425,7 +2425,7 @@ + select="'Administrative_documents'"/> @@ -2442,7 +2442,7 @@ space-after="0.03in"> - + @@ -2459,7 +2459,7 @@ space-after="0.03in"> - + @@ -2477,7 +2477,7 @@ + select="'Statistical_tables'"/> @@ -2495,7 +2495,7 @@ + select="'Scripts_and_programs'"/> @@ -2512,7 +2512,7 @@ space-after="0.03in"> - + @@ -2532,7 +2532,7 @@ + select="'Reports_and_analytical_documents'"/> - + - + - - + - + - + @@ -2602,7 +2602,7 @@ - + @@ -2611,7 +2611,7 @@ - + *** - + **** @@ -2749,7 +2749,7 @@ - # # @@ -2762,7 +2762,7 @@ - # # @@ -2776,18 +2776,18 @@ - + - : + : - :  + :  @@ -2811,7 +2811,7 @@ - + @@ -2823,7 +2823,7 @@ - + @@ -2835,7 +2835,7 @@ - + @@ -2847,7 +2847,7 @@ - + @@ -2859,7 +2859,7 @@ - + @@ -2871,7 +2871,7 @@ - + @@ -2905,10 +2905,10 @@ - + - + @@ -2955,7 +2955,7 @@ - + : @@ -3059,7 +3059,7 @@ - + @@ -3080,7 +3080,7 @@ - + @@ -3092,7 +3092,7 @@ - + @@ -3104,7 +3104,7 @@ - + @@ -3116,7 +3116,7 @@ - + @@ -3239,13 +3239,13 @@ - + - + - + @@ -3322,7 +3322,7 @@   - + @@ -3369,7 +3369,7 @@ : - + @@ -3382,7 +3382,7 @@ - + [= + select="'Type'"/>= + select="'discrete'"/> + select="'continuous'"/> ] @@ -3404,7 +3404,7 @@ [== - @@ -3419,13 +3419,13 @@ [= = - ] [ - + =* / @@ -3437,7 +3437,7 @@ @@ -3452,12 +3452,12 @@ - + [ + select="'Abbrev_NotWeighted'"/> / - + ] @@ -3470,18 +3470,18 @@ - + select="'Invalid'"/> - + - @@ -3517,7 +3517,7 @@ - + - + - + - + - + - + @@ -3638,7 +3638,7 @@ - + + select="'Recoding_and_Derivation'"/> - + - + - + - - + + - @@ -3798,7 +3798,7 @@ padding="{$cell-padding}" text-align="center"> + select="'Cases_Abbreviation'"/> @@ -3806,7 +3806,7 @@ padding="{$cell-padding}" text-align="center"> + select="'Weighted'"/> @@ -3814,9 +3814,9 @@ padding="{$cell-padding}" text-align="center"> + select="'Percentage'"/> () + select="'Weighted'"/>) @@ -3950,16 +3950,16 @@ + select="'SumStat_Warning'"/> - + ( - ) + ) @@ -4003,7 +4003,7 @@ - *** + *** *** @@ -4055,7 +4055,7 @@ - + @@ -4070,7 +4070,7 @@ - + @@ -4085,7 +4085,7 @@ - + @@ -4100,7 +4100,7 @@ - + @@ -4162,38 +4162,38 @@ - + - + - + - + - + - + - + @@ -4257,40 +4257,40 @@ - + - + - + - + - + - + - + - + - + - + - + - + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt deleted file mode 100644 index edf876f3b04..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml deleted file mode 100644 index d8e98dfd3c6..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml +++ /dev/null @@ -1,174 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 -Valid -Frequency table not shown -Derivation -discrete -Data Collection Mode -Other Processing -Other Acknowledgment(s) -Untitled -Identification -Click here to access/export data files from Nesstar format -Value -Percentage -Weighting -Primary Investigator(s) -This document was generated using the -Sampling -Cases -Access Conditions -Source -Modalities -Rights & Disclaimer -Definition -Estimates of Sampling Error -Data Files -Geographic Coverage -April -Mean -Metadata Production -Data Collection -Missing Data -Scripts and programs -Variable(s) -Interviewer instructions -Funding Agency/ies -November -Missing -Version -Universe -Contributor(s) -Access Authority -Data Processing & Appraisal -Scope -Administrative documents -StdDev -Contact(s) -Label -Technical documents -Decimal(s) -Type -Literal question -Concepts -Range -Abstract -June -Supervision -Other Forms of Data Appraisal -References -Accessibility -Data Collection Dates -Data Editing -Questionnaires -Valid case(s) -Reports and analytical documents -Copyright -Documentation -Deviations from Sample Design -Publisher(s) -February -Dataset contains -Acknowledgment(s) -Continuous -Standard deviation -Variables Description -Producer -Production Date - -The Explorer allows you to view data files and export them to common statistical formats -Discrete -Group -July -Filename -Cases -Name -Warning: these figures indicate the number of cases found in the data file. They cannot be interpreted as summary statistics of the population of interest. -Statistical tables -December -Subjects -Processing Checks -software -Interviewer's instructions -Table of Contents -Document Information -Subgroup(s) -Keywords -group(s) -W -Weight -Files Description -Notes -Data Collection Notes -file(s) -continuous -Disclaimer -Content -variable(s) -Other Producer(s) -Producers & Sponsors -Data Cleaning Notes -Distributor(s) -Overview -Citation Requirements -September -Category -Confidentiality -Statistics -May -Undetermined -Structure -file -Pre-question -Response Rate -Width -Recoding and Derivation -Series -October -Unit of Analysis -Data Processing Notes -Kind of Data -File -Time Period(s) -File Content -Invalid -Vars -cont. -Key(s) -Question -Source of information -Imputation -Security -To open this file, you will need the free -Other resources -Data Dictionnary -Information -January -Other documents -Minimum -Scope & Coverage -Metadata Producer(s) -Show more info -Data Collector(s) -Post-question -Topics -Sampling Procedure -File Structure -Variables List -Format -Sampling Notes -Variables Group(s) -Description -Categories -Maximum -Depositor(s) -August -NW -Cover Page -Weighted -March - total - showing a subset of -Countries -question details - diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml deleted file mode 100644 index 9cfcdaf6e7e..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml +++ /dev/null @@ -1,170 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 -Válido -No se presentan las tablas de frecuencias -Derivación -discreta -Método de Recolección -Otros relacionados al procesamiento -Otros Reconocimientos -Sin título -Identificación -Presione aquí para acceder/exportar al archivo(s) de datos -Valor -Porcentaje -Ponderando -Investigadores Principales -Este documento fue producido utilizando el -Muestreo -Casos -Condiciones de uso -Fuente -Modalidades -Derechos y Notas Legales -Definición -Estimaciones del Error Muestral -Archivo de Datos -Cobertura Geográfica -Abril -Media -Producción de los Metadatos -Recolección de Datos -Datos perdidos -Programas informáticos -Variable(s) -Manual del encuestador -Agencias Auspiciadoras -Noviembre -Valores perdidos -Versión -Universo -Contribuidor(es) -Institución Propietaria -Tratamiento y Validación de Datos -Dominio Temático -Documentos Administrativos -Desviación -Contacto(s) -Etiqueta -Documentos Técnicos -Decimal(es) -Tipo -Pregunta textual -Conceptos -Rango -Resumen -Junio -Supervisión -Otras Formas de Validación de los Datos -Referencias -Accesibilidad -Fechas de Recolección de Datos -Procesamiento de Datos -Cuestionarios -Casos válidos -Reportes y documentos analíticos -Derechos de Autor -Documentación -Modificaciones al Diseño Muestral -Editor(es) -Febrero -Contenido de la Base de Datos -Reconocimiento(s) -Contínua -Desviación estándar -Descripción de la variable -Productor -Fecha de Producción -El Explorador NESSTAR permite visualizar los archivos de datos y exportarlos a diferentes formatos estadísticos -Discreta -Grupo -Julio -Nombre del Archivo -Casos -Nombre -Cuadros estadísticos -Diciembre -Temas -Controles de Tratamiento -software -Manual del encuestador -Indice -Información acerca de la Documentación -Subgrupo(s) -Palabra Clave -grupo(s) -P -Ponderador -Descripción de los Archivos -Notas -Notas sobre la Recolección de Datos -archivo(s) -continua -Nota Legal -Contenido -variable(s) -Otros Productores -Productores y Auspiciadores -Notas acerca de la Depuración de los Datos -Distribuidor(es) -Resumen General -Forma de citar -Septiembre -Categoría -Confidencialidad -Estadística - -Mayo -Indeterminado -Estructura -archivo -Pre-pregunta -Tasa de Respuesta -Ancho -Recodificación y Derivación -Series -Octubre -Unidad de Análisis -Notas sobre el Procesamiento de Datos -Tipo de Datos -Archivo -Periodo de Referencia -Contenido del Archivo -Inválido -Vars. -cont. -Clave(s) -Pregunta -Fuente de información -Imputación -Seguridad -Para abrir este archivo se necesita el software gratuito -Otros recursos -Diccionario de Datos -Información -Enero -Otros documentos -Mínimo -Cobertura y Dominio Temático -Productor de los Metadatos -Mostrar más información -Entrevistador(es) -Pos-pregunta -Temas -Procedimiento de Muestreo -Estructura del Archivo -Lista de variables -Formato -Notas sobre el Muestreo -Grupo(s) de Variables -Descripción -Categorías -Máximo -Depositante(s) -Agosto -NP -Carátula -Ponderado -Marzo - diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml deleted file mode 100644 index 9fa4d2178b1..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml +++ /dev/null @@ -1,173 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 -Valide -Tableau de fréquences non-affiché -Mode de calcul -discrète -Méthode de collecte -Autre traitement -Autre(s) remerciement(s) -Sans titre -Identification -Cliquer ici pour accéder/exporter les fichiers de données du format Nesstar -Valeur -Pourcentage -Pondération -Enquêteur(s) principal/aux -Ce document a été généré à l'aide du -Echantillonage -Enregistrements -Conditions d'accès -Source -Modalités -Responsabilité et droits d'auteurs -Définition -Estimation des erreurs d'échantillonage -Fichiers de données -Couverture géographique -Avril -Moyenne -Production des méta-données -Collecte des données -Valeures manquantes -Programmes informatiques -Variable(s) -Instructions aux enquêteurs -Financement -Novembre -Manquant -Version -Univers -Contributeur(s) -Agence propriétaire -Traitement et évaluation des données -Domaines thématiques -Documents administratifs -Ecart type -Contact(s) -Libellé -Documents techniques -Décimale(s) -Type -Formulation de la question -Concepts -Gamme -Résumé -Juin -Supervision -Autres formes d'évaluation des données -Références -Accessibilité -Dates de collecte -Edition des données -Questionnaires -Cas valide(s) -Rapports et documents analytiques -Droits d'auteurs -Documentation -Déviations par rapport à l'échantillon initial -Editeur(s) -Février -Le jeu de données contient -Remerciement(s) -Continue -Ecart type -Description des variables -Producteur -Date de production - -L'Explorer vous permet d'accéder aux données et de les exporter vers les formats statistiques les plus courants -Discrète -Groupe -Juillet -Nom du fichier -Enreg. -Nom -Avertissement: Ces chiffres indiquent le nombre de cas identifiés dans le fichier de données. Ils ne peuvent pas être interpretés comme étant représentatifs de la population concernée. -Tableaux statistiques -Décembre -Sujets -Contrôles de traitement - -Instructions aux enquêteurs -Table des matières -Informations sur le document -Sous-groupe(s) -Mots-clé -groupe(s) -P -Pondération -Description des fichiers -Notes -Notes sur la collecte -fichier(s) -continue -Responsabilité(s) -Contenu -variable(s) -Autre(s) producteur(s) -Producteurs et sponsors -Notes sur l'apurement des données -Distributeur(s) -Aperçu -Citation -Septembre -Catégorie -Confidentialité -Statistiques -Mai -Indéterminé -Structure -fichier -Pré-question -Taux de réponse -Taille -Formulation de la question -Recodage et dérivation -Série -Octobre -Unité d'analyse -Notes sur le traitement des données -Type d'étude -Fichier -Période(s) de référence -Contenu du fichier -Non-valide -Vars -suite -Clé(s) -Question -Source d'information -Imputation -Sécurité -Pour ouvrir ce fichier, vous avez besoin du logiciel gratuit -Autres resources -Dictionnaire des variables -Information -Janvier -Autres documents -Minimum -Domaines thématiques et couverture -Producteur(s) des méta-données -Information complémentaire -Enquêteurs -Post-question -Thèmes -Procédure d'échantillonage -Structure du fichier -Liste des variables -Format -Notes sur l'échantillonage -Groupe(s) de variables -Description -Catégories -Maximum -Dépositaire(s) -Août -NP -Couverture -Pondéré -Mars -question details - diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml deleted file mode 100644 index bc5dbb06154..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml +++ /dev/null @@ -1,161 +0,0 @@ - - -Generated by Properties2Xml on Tue Feb 13 13:55:43 EST 2007 -有効な -度数表(Frequency table)は表示されません -由来 -不連続な -データ収集モード -その他の確認事項 -識別番号 -データファイルにアクセスするにはここをクリックしてください -無題 -値 -割合 -ウェイティング -第一次調査官 -この文書はToolkitを使用して作られました -サンプリング -ケース -アクセス条件 -情報源 -様相 -権利及び声明文 -定義 -サンプルエラーの見積もり -データファイル -地理的な適用範囲 -４月 -平均 -メタデータ製作 -データ収集 -損失データ -スクリプトおよびプログラム -可変的 -面接者の指示 -出資機関 -１１月 -バージョン -共通の -貢献者 -アクセス権限 -データ処理、評価 -範囲, 領域 -管理用文章 -連絡先 -ラベル -技術的な文書 -小数点 -タイプ -文字の質問 -概念 -範囲 -要約 -６月 -監督 -その他ファーマットのデータ評価 -参照 -アクセス、入手法 -データ収集日 -データ編集 -質問 -レポートおよび分析的な文書 -有効な場合 -コピーライト -書類 -サンプルデザインによる偏差 -発行者 -２月 -データセットに含まれる -確認事項 -連続的な -標準偏差 -変数の記述 -製作者 -製作日 -” Explorer”によってデータファイルを参照することも一般的に使えわれている統計データフォーマットに変換。抽出することも可能です -不連続性 -グループ -７月 -ファイルの名前 -ケース -名前 -統計表 -１２月 -主題, 内容 -工程監査 -ソフト -面接者への指示 -目録 -書類の情報 -サブグループ -キーワード - -グループ -ウェイト -ファイルの詳細 -メモ -データ収集メモ -ファイル -継続的な -声明文 -内容 -変数 -その他の製作者 -製作者とスポンサー -データクリーニングメモ -分配者 -概略 -引用する場合の必要条件 -９月 -カテゴリー -機密性、コンフィデンシャリティー -５月 -未定 -構造 -ファイル -調査前の質問 -回答比率 -幅 -記録と誘導 -シリー -１０月 -分析の単位 -データ処理メモ -データの種類 - -ファイル -期間 -ファイルの内容 -無効 -キー -情報源 -非難 -セキュリティー -このファイルを開けるには、無料で配布されているNesstar Explorer が必要です。 -その他の資料 -データ辞典 -情報 -１月 -その他の書類 -最小値 -規模及び適用範囲 -メタデータ製作者 -さらにインフォメーションを表示 -データ収集者 -調査後の質問 -サンプリングの手順 -ファイルの構造 -変数のリスト -フォーマット -サンプリングメモ -変数のグループ -詳細 -カテゴリー -最大値 -デポジター、提供者、供託者 -８月 -表紙 -ウェイトされた -３月 - diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml deleted file mode 100644 index fdf14f5dfcd..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml +++ /dev/null @@ -1,174 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 -Gyldige -Frekvenstabell ikke vist -Avledning -diskret -Type datainnsamling -Annen prosessering -Andre identifikatorer og krediteringer -Uten tittel -Identifisering -Click here to access/export data files from Nesstar format -Verdi -Prosent -Vekting -Primary Investigator(s) -Dette dokumentet ble generert av -Utvalg -Enheter -Tilgangsbetingelser -Kilde -Modaliteter -Rights & Disclaimer -Definisjon -Estimert utvalgsfeil -Datafiler -Geografisk omfang -April -Mean -Metadata-produksjon -Datainnsamling -Manglende data -Script og programmer -Variable(r) -Instruksjoner til intervjueren -Sponsor/finansierende institusjon(er) -November -Missing -Versjon -Univers -Bidragsyter(e) -Tilgangskontrollør -Dataprosessering og -evaluering -Omfang -Administrative dokumenter -Standardavvik -Kontaktperson(er) -Merkelapp -Tekniske dokumenter -Desimal(er) -Type -Spørsmålstekst -Begrep(er) -Rekkevidde -Sammendrag -Juni -Supervision -Andre former for dataevaluering -Referanser -Tilgjengelighet -Datainnsamlingsdatoer -Dataredigering -Spørreskjema -Gyldige enheter -Rapporter og analysedokumenter -Copyright -Dokumentasjon -Avvik fra utvalgsdesign -Utgiver(e) -Februar -Datasettet inneholder -Krediteringer -Kontinuerlig -Standardavvik -Variabelbeskrivelse -Produsent -Produksjonsdato - -The Explorer allows you to view data files and export them to common statistical formats -Diskret -Gruppe -Juli -Filnavn -Enheter -Navn -Advarsel: disse tallene indikerer antall enheter (cases) i datafilen. De kan ikke tolkes som oppsummert statistikk for populasjonen. -Statistiske tabeller -Desember -Emner -Prosesseringssjekk -programvare -Instruksjoner til intervjueren -Innholdsfortegnelse -Dokumentinformasjon -Undergruppe(r) -Nøkkelord -gruppe(r) -W -Vekt -Filbeskrivelse -Kommentarer -Datainnsamlingskommentarer -file(r) -kontinuerlig -Fraskrivelse -Innhold -variable(r) -Andre produsenter -Produsenter og sponsorer -Kommentarer om datarensing -Distributør(er) -Oversikt -Sitatkrav -September -Kategori -Konfidensialitet -Statistikk -Mai -Uavklart -Struktur -fil -Tekst før spørsmål -Responsrate -Bredde -Omkodinger og utledninger -Serie -Oktober -Analyseenhet -Dataprosesseringskommentarer -Datatype -Fil -Tidsperiode(r) -Filinnhold -Ugyldig -Variabler -kont. -Nøkler -Spørsmål -Kilde for informasjon -Imputasjon -Sikkerhet -For å åpne denne filen trenger du følgende gratisverktøy -Andre ressurser -Dataordbok -Informasjon -Januar -Andre dokumenter -Minimum -Omfang -Metadataprodusenter -Vis mer informasjon -Datainnsamler(e) -Tekst etter spørsmål -Emner -Utvalgsprosedyre -Filstruktur -Variabelliste -Format -Utvalgskommentarer -Variabelgrupper -Beskrivelse -Kategorier -Maksimum -Utgiver(e) -August -NW -Forside -Vektet -Mars - total - viser et utvalg av -Land -spørsmålsdetaljer - diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml deleted file mode 100644 index fdf14f5dfcd..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml +++ /dev/null @@ -1,174 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 -Gyldige -Frekvenstabell ikke vist -Avledning -diskret -Type datainnsamling -Annen prosessering -Andre identifikatorer og krediteringer -Uten tittel -Identifisering -Click here to access/export data files from Nesstar format -Verdi -Prosent -Vekting -Primary Investigator(s) -Dette dokumentet ble generert av -Utvalg -Enheter -Tilgangsbetingelser -Kilde -Modaliteter -Rights & Disclaimer -Definisjon -Estimert utvalgsfeil -Datafiler -Geografisk omfang -April -Mean -Metadata-produksjon -Datainnsamling -Manglende data -Script og programmer -Variable(r) -Instruksjoner til intervjueren -Sponsor/finansierende institusjon(er) -November -Missing -Versjon -Univers -Bidragsyter(e) -Tilgangskontrollør -Dataprosessering og -evaluering -Omfang -Administrative dokumenter -Standardavvik -Kontaktperson(er) -Merkelapp -Tekniske dokumenter -Desimal(er) -Type -Spørsmålstekst -Begrep(er) -Rekkevidde -Sammendrag -Juni -Supervision -Andre former for dataevaluering -Referanser -Tilgjengelighet -Datainnsamlingsdatoer -Dataredigering -Spørreskjema -Gyldige enheter -Rapporter og analysedokumenter -Copyright -Dokumentasjon -Avvik fra utvalgsdesign -Utgiver(e) -Februar -Datasettet inneholder -Krediteringer -Kontinuerlig -Standardavvik -Variabelbeskrivelse -Produsent -Produksjonsdato - -The Explorer allows you to view data files and export them to common statistical formats -Diskret -Gruppe -Juli -Filnavn -Enheter -Navn -Advarsel: disse tallene indikerer antall enheter (cases) i datafilen. De kan ikke tolkes som oppsummert statistikk for populasjonen. -Statistiske tabeller -Desember -Emner -Prosesseringssjekk -programvare -Instruksjoner til intervjueren -Innholdsfortegnelse -Dokumentinformasjon -Undergruppe(r) -Nøkkelord -gruppe(r) -W -Vekt -Filbeskrivelse -Kommentarer -Datainnsamlingskommentarer -file(r) -kontinuerlig -Fraskrivelse -Innhold -variable(r) -Andre produsenter -Produsenter og sponsorer -Kommentarer om datarensing -Distributør(er) -Oversikt -Sitatkrav -September -Kategori -Konfidensialitet -Statistikk -Mai -Uavklart -Struktur -fil -Tekst før spørsmål -Responsrate -Bredde -Omkodinger og utledninger -Serie -Oktober -Analyseenhet -Dataprosesseringskommentarer -Datatype -Fil -Tidsperiode(r) -Filinnhold -Ugyldig -Variabler -kont. -Nøkler -Spørsmål -Kilde for informasjon -Imputasjon -Sikkerhet -For å åpne denne filen trenger du følgende gratisverktøy -Andre ressurser -Dataordbok -Informasjon -Januar -Andre dokumenter -Minimum -Omfang -Metadataprodusenter -Vis mer informasjon -Datainnsamler(e) -Tekst etter spørsmål -Emner -Utvalgsprosedyre -Filstruktur -Variabelliste -Format -Utvalgskommentarer -Variabelgrupper -Beskrivelse -Kategorier -Maksimum -Utgiver(e) -August -NW -Forside -Vektet -Mars - total - viser et utvalg av -Land -spørsmålsdetaljer - diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml deleted file mode 100644 index 06fde85af5e..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml +++ /dev/null @@ -1,169 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 -Валидный -Частотная таблица не выводится -Расчет -дискретная -Способ сбора данных -Прочая обработка -Другие участники -Безымянный -Индентификация -Щелкните здесь, чтобы получить доступ к файлам или экспортировать их -Значение -Процент -Взвешивание -Первичный(е) исследователь(и) -Документ был сгенерирован с использованием -Выборка -Наблюдения -Условия доступа -Источник -Модальности -Авторские права и ограничения ответственности -Определение -Оценки ошибок выборки -Файлы данных -Географический охват -Апрель -Среднее -Разработка метаданных -Сбор данных -Пропущенные данные -Скрипты и программы -Переменная(ые) -Инструкции интервьюеру -Кто финансировал -Ноябрь -Пропущенные -Версия -Генеральная совокупность -Участник(и) -Права доступа -Обработка и инспекция данных -Охват -Административные документы -СтдОткл -Контак(ы) -Метка -Технические документы -Десятичные -Тип -Текст вопроса -Концепции -Диапазон -Резюме -Июнь -Контроль -Другие формы инспекции данных -Установки -Доступность -Даты сбора данных -Редактирование данных -Вопросники -Валидное(ые) наблюдение(я) -Отчеты и аналитические документы -Авторские права -Документация -Отклонения от дизайна выборки -Издатель(и) -Февраль -Набор данных содержит -Участник(и) -Непрерывная -Стандартное отклонение -Описание переменных -Разработчик -Дата разработки -Проводник позволяет просматривать файлы данных и экспортировать их в распространенные статистические форматы -Дикретная -Группа -Июль -Имя файла -Наблюдения -Имя -Статистичсекие таблицы -Декабрь -Темы -Контроль обработки -программное обеспечение -Инструкции интервьюеру -Оглавление -Информация о документе -Подгруппа(ы) -Ключевые слова -группа(ы) -B -Вес -Описание файла -Примечания -Примечания по сбору данных -файл(ы) -непрерывная -Ограничения ответственности -Содержание -переменная(ые) -Другие разработчики -Разработчики и спонсоры -Примечания по чистке данных -Дистрибьютор(ы) -Обзор -Требования по цитированию -Сентябрь -Категория -Конфиденциальность -Статистики -Май -Неопределенный -Структура -файл -Текст, предваряющий вопрос -Доля ответов -Ширина -Перекодировка и расчеты -Серия -Октябрь -Единица анализа -Примечания по обработке данных -Тип данных -Файл -Период(ы) времени -Содержание файла -Некорректный -Переменные -непр. -Ключ(и) -Вопрос -Источник информации -Импутация -Безопасность -Чтобы открыть этот файл, необходимо иметь свободным -Прочие источники -Словарь данных -Информация -Январь -Прочие документы -Минимум -Охват и покрытие -Разработчик(и) метаданных -Показать дополнительную информацию -Кто собирал данные -Текст после вопроса -Разделы -Процедура выборки -Структура файла -Список переменных -Формат -Примечания по выборке -Группа(ы) переменных -Описание -Категории -Максимум -Депозитор(ы) -Август -HB -Титульный лист -взвешенные -Март - From 2968ba2d9a1f84773939a41b6867bab686d7a604 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Wed, 28 Jun 2023 10:34:22 +0200 Subject: [PATCH 017/970] combined query for retrieving datasets with API --- .../iq/dataverse/DatasetServiceBean.java | 1 + .../iq/dataverse/api/AbstractApiBean.java | 35 +++++++++++-------- .../harvard/iq/dataverse/api/Datasets.java | 2 +- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index 305afd2ed30..9bf506e6c1f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -127,6 +127,7 @@ public Dataset findDeep(Object pk) { .setHint("eclipselink.left-join-fetch", "o.files.dataFileTags") .setHint("eclipselink.left-join-fetch", "o.files.fileMetadatas") .setHint("eclipselink.left-join-fetch", "o.files.fileMetadatas.fileCategories") + .setHint("eclipselink.left-join-fetch", "o.files.fileMetadatas.varGroups") .setHint("eclipselink.left-join-fetch", "o.files.guestbookResponses") .setHint("eclipselink.left-join-fetch", "o.files.embargo") .setHint("eclipselink.left-join-fetch", "o.files.fileAccessRequests") diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 6b3e27becb7..fa281a531e8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -41,6 +41,7 @@ import edu.harvard.iq.dataverse.externaltools.ExternalToolServiceBean; import edu.harvard.iq.dataverse.license.LicenseServiceBean; import edu.harvard.iq.dataverse.metrics.MetricsServiceBean; +import edu.harvard.iq.dataverse.pidproviders.PidUtil; import edu.harvard.iq.dataverse.locality.StorageSiteServiceBean; import edu.harvard.iq.dataverse.search.savedsearch.SavedSearchServiceBean; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; @@ -390,6 +391,11 @@ protected DataverseLinkingDataverse findDataverseLinkingDataverseOrDie(String da } protected Dataset findDatasetOrDie(String id) throws WrappedResponse { + return findDatasetOrDie(id, false); + } + + protected Dataset findDatasetOrDie(String id, boolean deep) throws WrappedResponse { + Long datasetId; Dataset dataset; if (id.equals(PERSISTENT_ID_KEY)) { String persistentId = getRequestParameter(PERSISTENT_ID_KEY.substring(1)); @@ -397,23 +403,24 @@ protected Dataset findDatasetOrDie(String id) throws WrappedResponse { throw new WrappedResponse( badRequest(BundleUtil.getStringFromBundle("find.dataset.error.dataset_id_is_null", Collections.singletonList(PERSISTENT_ID_KEY.substring(1))))); } - dataset = datasetSvc.findByGlobalId(persistentId); + datasetId = dvObjSvc.findIdByGlobalId(PidUtil.parseAsGlobalID(persistentId), DvObject.DType.Dataset); + } else { + datasetId = Long.parseLong(id); + } + + try { + if (deep) { + dataset = datasetSvc.findDeep(datasetId); + } else { + dataset = datasetSvc.find(datasetId); + } if (dataset == null) { - throw new WrappedResponse(notFound(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.persistentId", Collections.singletonList(persistentId)))); + throw new WrappedResponse(notFound(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.id", Collections.singletonList(id)))); } return dataset; - - } else { - try { - dataset = datasetSvc.find(Long.parseLong(id)); - if (dataset == null) { - throw new WrappedResponse(notFound(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.id", Collections.singletonList(id)))); - } - return dataset; - } catch (NumberFormatException nfe) { - throw new WrappedResponse( - badRequest(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.bad.id", Collections.singletonList(id)))); - } + } catch (NumberFormatException nfe) { + throw new WrappedResponse( + badRequest(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.bad.id", Collections.singletonList(id)))); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 8c1390b597e..7a527881b2b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -257,7 +257,7 @@ public interface DsVersionHandler { @Path("{id}") public Response getDataset(@Context ContainerRequestContext crc, @PathParam("id") String id, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) { return response( req -> { - final Dataset retrieved = execCommand(new GetDatasetCommand(req, findDatasetOrDie(id))); + final Dataset retrieved = execCommand(new GetDatasetCommand(req, findDatasetOrDie(id, true))); final DatasetVersion latest = execCommand(new GetLatestAccessibleDatasetVersionCommand(req, retrieved)); final JsonObjectBuilder jsonbuilder = json(retrieved); //Report MDC if this is a released version (could be draft if user has access, or user may not have access at all and is not getting metadata beyond the minimum) From 6083ead476ef195501fd44276db8702fb455cc9a Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Thu, 29 Jun 2023 11:54:39 +0200 Subject: [PATCH 018/970] better error handling - should fix failed integration test --- .../iq/dataverse/api/AbstractApiBean.java | 42 ++++++++++++------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index fa281a531e8..5a9720639d1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -17,6 +17,7 @@ import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.DvObjectServiceBean; import edu.harvard.iq.dataverse.EjbDataverseEngine; +import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.GuestbookResponseServiceBean; import edu.harvard.iq.dataverse.MetadataBlock; import edu.harvard.iq.dataverse.MetadataBlockServiceBean; @@ -403,25 +404,36 @@ protected Dataset findDatasetOrDie(String id, boolean deep) throws WrappedRespon throw new WrappedResponse( badRequest(BundleUtil.getStringFromBundle("find.dataset.error.dataset_id_is_null", Collections.singletonList(PERSISTENT_ID_KEY.substring(1))))); } - datasetId = dvObjSvc.findIdByGlobalId(PidUtil.parseAsGlobalID(persistentId), DvObject.DType.Dataset); + GlobalId globalId; + try { + globalId = PidUtil.parseAsGlobalID(persistentId); + } catch (IllegalArgumentException e) { + throw new WrappedResponse( + badRequest(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.bad.id", Collections.singletonList(persistentId)))); + } + datasetId = dvObjSvc.findIdByGlobalId(globalId, DvObject.DType.Dataset); + if (datasetId == null) { + throw new WrappedResponse( + badRequest(BundleUtil.getStringFromBundle("find.dataset.error.dataset_id_is_null", Collections.singletonList(PERSISTENT_ID_KEY.substring(1))))); + } } else { - datasetId = Long.parseLong(id); + try { + datasetId = Long.parseLong(id); + } catch (NumberFormatException nfe) { + throw new WrappedResponse( + badRequest(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.bad.id", Collections.singletonList(id)))); + } } - try { - if (deep) { - dataset = datasetSvc.findDeep(datasetId); - } else { - dataset = datasetSvc.find(datasetId); - } - if (dataset == null) { - throw new WrappedResponse(notFound(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.id", Collections.singletonList(id)))); - } - return dataset; - } catch (NumberFormatException nfe) { - throw new WrappedResponse( - badRequest(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.bad.id", Collections.singletonList(id)))); + if (deep) { + dataset = datasetSvc.findDeep(datasetId); + } else { + dataset = datasetSvc.find(datasetId); + } + if (dataset == null) { + throw new WrappedResponse(notFound(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.id", Collections.singletonList(id)))); } + return dataset; } protected DataFile findDataFileOrDie(String id) throws WrappedResponse { From 8e75241e6fe3c2c4595f1d11fe658d354f7b2f4a Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Tue, 4 Jul 2023 14:01:47 +0200 Subject: [PATCH 019/970] find deep in file listing --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 7a527881b2b..7229444e088 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -495,7 +495,7 @@ public Response getVersion(@Context ContainerRequestContext crc, @PathParam("id" @Path("{id}/versions/{versionId}/files") public Response getVersionFiles(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers) { return response( req -> ok( jsonFileMetadatas( - getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers).getFileMetadatas())), getRequestUser(crc)); + getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId, true), uriInfo, headers).getFileMetadatas())), getRequestUser(crc)); } @GET From 15e200295818ca3c136ac3a0c57411191038083d Mon Sep 17 00:00:00 2001 From: Ludovic DANIEL Date: Wed, 26 Jul 2023 16:42:36 +0200 Subject: [PATCH 020/970] #8655 Re-add value 'cell counting' to Life Science metadatablock's Measurement Type field vocabulary --- scripts/api/data/metadatablocks/biomedical.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/api/data/metadatablocks/biomedical.tsv b/scripts/api/data/metadatablocks/biomedical.tsv index 28d59130c34..f26a25752e6 100644 --- a/scripts/api/data/metadatablocks/biomedical.tsv +++ b/scripts/api/data/metadatablocks/biomedical.tsv @@ -45,6 +45,7 @@ studyFactorType Treatment Compound EFO_0000369 17 studyFactorType Treatment Type EFO_0000727 18 studyFactorType Other OTHER_FACTOR 19 + studyAssayMeasurementType cell counting ERO_0001899 0 studyAssayMeasurementType cell sorting CHMO_0001085 1 studyAssayMeasurementType clinical chemistry analysis OBI_0000520 2 studyAssayMeasurementType copy number variation profiling OBI_0000537 3 From d9bb9a343de06cf0c849ee728e2d0b6ee57ccae9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20ROUCOU?= Date: Wed, 2 Aug 2023 17:45:46 +0200 Subject: [PATCH 021/970] i18n into MyData page --- .../iq/dataverse/mydata/DataRetrieverAPI.java | 29 +++++++-------- .../dataverse/mydata/MyDataFilterParams.java | 24 ++++++------- .../iq/dataverse/mydata/MyDataFinder.java | 35 ++++++++++--------- src/main/java/propertyFiles/Bundle.properties | 15 ++++++++ .../iq/dataverse/api/DataRetrieverApiIT.java | 20 +++++++++-- 5 files changed, 78 insertions(+), 45 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java b/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java index d244021d01a..51e1cb70bc7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java +++ b/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java @@ -25,13 +25,10 @@ import edu.harvard.iq.dataverse.search.SearchException; import edu.harvard.iq.dataverse.search.SearchFields; import edu.harvard.iq.dataverse.search.SortBy; -import java.math.BigDecimal; import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Random; import java.util.logging.Logger; -import java.util.Locale; import javax.ejb.EJB; import javax.inject.Inject; import javax.json.Json; @@ -41,7 +38,6 @@ import javax.ws.rs.Path; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; -import javax.ws.rs.DefaultValue; import javax.ws.rs.container.ContainerRequestContext; import javax.ws.rs.core.Context; @@ -272,9 +268,7 @@ public String retrieveMyDataAsJsonString( @QueryParam("dataset_valid") List datasetValidities) { boolean OTHER_USER = false; - String localeCode = session.getLocaleCode(); - String noMsgResultsFound = BundleUtil.getStringFromPropertyFile("dataretrieverAPI.noMsgResultsFound", - "Bundle", new Locale(localeCode)); + String noMsgResultsFound = BundleUtil.getStringFromBundle("dataretrieverAPI.noMsgResultsFound"); if ((session.getUser() != null) && (session.getUser().isAuthenticated())) { authUser = (AuthenticatedUser) session.getUser(); @@ -282,7 +276,10 @@ public String retrieveMyDataAsJsonString( try { authUser = getRequestAuthenticatedUserOrDie(crc); } catch (WrappedResponse e) { - return this.getJSONErrorString("Requires authentication. Please login.", "retrieveMyDataAsJsonString. User not found! Shouldn't be using this anyway"); + return this.getJSONErrorString( + BundleUtil.getStringFromBundle("dataretrieverAPI.authentication.required"), + BundleUtil.getStringFromBundle("dataretrieverAPI.authentication.required.opt") + ); } } @@ -295,7 +292,9 @@ public String retrieveMyDataAsJsonString( authUser = searchUser; OTHER_USER = true; } else { - return this.getJSONErrorString("No user found for: \"" + userIdentifier + "\"", null); + return this.getJSONErrorString( + BundleUtil.getStringFromBundle("dataretrieverAPI.user.not.found", Arrays.asList(userIdentifier)), + null); } } @@ -335,8 +334,7 @@ public String retrieveMyDataAsJsonString( myDataFinder = new MyDataFinder(rolePermissionHelper, roleAssigneeService, dvObjectServiceBean, - groupService, - noMsgResultsFound); + groupService); this.myDataFinder.runFindDataSteps(filterParams); if (myDataFinder.hasError()){ return this.getJSONErrorString(myDataFinder.getErrorMessage(), myDataFinder.getErrorMessage()); @@ -391,11 +389,14 @@ public String retrieveMyDataAsJsonString( } catch (SearchException ex) { solrQueryResponse = null; - this.logger.severe("Solr SearchException: " + ex.getMessage()); + logger.severe("Solr SearchException: " + ex.getMessage()); } - if (solrQueryResponse==null){ - return this.getJSONErrorString("Sorry! There was an error with the search service.", "Sorry! There was a SOLR Error"); + if (solrQueryResponse == null) { + return this.getJSONErrorString( + BundleUtil.getStringFromBundle("dataretrieverAPI.solr.error"), + BundleUtil.getStringFromBundle("dataretrieverAPI.solr.error.opt") + ); } // --------------------------------- diff --git a/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFilterParams.java b/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFilterParams.java index c378034f951..15d83fe4296 100644 --- a/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFilterParams.java +++ b/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFilterParams.java @@ -12,6 +12,7 @@ import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.search.SearchConstants; import edu.harvard.iq.dataverse.search.SearchFields; +import edu.harvard.iq.dataverse.util.BundleUtil; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -178,26 +179,25 @@ public List getRoleIds(){ } - - private void checkParams(){ - - if ((this.userIdentifier == null)||(this.userIdentifier.isEmpty())){ - this.addError("Sorry! No user was found!"); + private void checkParams() { + if ((this.userIdentifier == null) || (this.userIdentifier.isEmpty())) { + this.addError(BundleUtil.getStringFromBundle("myDataFilterParams.error.no.user")); return; } - if ((this.roleIds == null)||(this.roleIds.isEmpty())){ - this.addError("No results. Please select at least one Role."); + if ((this.roleIds == null) || (this.roleIds.isEmpty())) { + this.addError(BundleUtil.getStringFromBundle("myDataFilterParams.error.result.no.role")); return; } - if ((this.dvObjectTypes == null)||(this.dvObjectTypes.isEmpty())){ - this.addError("No results. Please select one of Dataverses, Datasets, Files."); + if ((this.dvObjectTypes == null) || (this.dvObjectTypes.isEmpty())) { + this.addError(BundleUtil.getStringFromBundle("myDataFilterParams.error.result.no.dvobject")); return; } - - if ((this.publicationStatuses == null)||(this.publicationStatuses.isEmpty())){ - this.addError("No results. Please select one of " + StringUtils.join(MyDataFilterParams.defaultPublishedStates, ", ") + "."); + + if ((this.publicationStatuses == null) || (this.publicationStatuses.isEmpty())) { + this.addError(BundleUtil.getStringFromBundle("dataretrieverAPI.user.not.found", + Arrays.asList(StringUtils.join(MyDataFilterParams.defaultPublishedStates, ", ")))); return; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFinder.java b/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFinder.java index 6acdfd9cdde..8ed2ef830e5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFinder.java +++ b/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFinder.java @@ -11,7 +11,9 @@ import edu.harvard.iq.dataverse.authorization.DataverseRolePermissionHelper; import edu.harvard.iq.dataverse.authorization.groups.GroupServiceBean; import edu.harvard.iq.dataverse.search.SearchFields; +import edu.harvard.iq.dataverse.util.BundleUtil; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -47,7 +49,6 @@ public class MyDataFinder { private RoleAssigneeServiceBean roleAssigneeService; private DvObjectServiceBean dvObjectServiceBean; private GroupServiceBean groupService; - private String noMsgResultsFound; //private RoleAssigneeServiceBean roleService = new RoleAssigneeServiceBean(); //private MyDataQueryHelperServiceBean myDataQueryHelperService; // -------------------- @@ -86,12 +87,11 @@ public class MyDataFinder { private List fileGrandparentFileIds = new ArrayList<>(); // dataverse has file permissions - public MyDataFinder(DataverseRolePermissionHelper rolePermissionHelper, RoleAssigneeServiceBean roleAssigneeService, DvObjectServiceBean dvObjectServiceBean, GroupServiceBean groupService, String _noMsgResultsFound) { + public MyDataFinder(DataverseRolePermissionHelper rolePermissionHelper, RoleAssigneeServiceBean roleAssigneeService, DvObjectServiceBean dvObjectServiceBean, GroupServiceBean groupService) { this.rolePermissionHelper = rolePermissionHelper; this.roleAssigneeService = roleAssigneeService; this.dvObjectServiceBean = dvObjectServiceBean; this.groupService = groupService; - this.noMsgResultsFound = _noMsgResultsFound; this.loadHarvestedDataverseIds(); } @@ -213,7 +213,7 @@ private List getSolrFilterQueries(boolean totalCountsOnly){ // ----------------------------------------------------------------- String dvObjectFQ = this.getSolrDvObjectFilterQuery(); if (dvObjectFQ ==null){ - this.addErrorMessage(noMsgResultsFound); + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.empty")); return null; } filterQueries.add(dvObjectFQ); @@ -286,7 +286,7 @@ public String getSolrDvObjectFilterQuery(){ if ((distinctEntityIds.isEmpty()) && (distinctParentIds.isEmpty())) { - this.addErrorMessage(noMsgResultsFound); + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.empty")); return null; } @@ -430,24 +430,25 @@ public JsonArrayBuilder getListofSelectedRoles(){ } - private boolean runStep1RoleAssignments(){ + private boolean runStep1RoleAssignments() { List results = this.roleAssigneeService.getAssigneeAndRoleIdListFor(filterParams); //logger.info("runStep1RoleAssignments results: " + results.toString()); - if (results == null){ - this.addErrorMessage("Sorry, the EntityManager isn't working (still)."); + if (results == null) { + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.null")); return false; - }else if (results.isEmpty()){ + } else if (results.isEmpty()) { List roleNames = this.rolePermissionHelper.getRoleNamesByIdList(this.filterParams.getRoleIds()); - if ((roleNames == null)||(roleNames.isEmpty())){ - this.addErrorMessage("Sorry, you have no assigned roles."); - }else{ - if (roleNames.size()==1){ - this.addErrorMessage("Sorry, nothing was found for this role: " + StringUtils.join(roleNames, ", ")); - }else{ - this.addErrorMessage("Sorry, nothing was found for these roles: " + StringUtils.join(roleNames, ", ")); + if ((roleNames == null) || (roleNames.isEmpty())) { + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.no.role")); + } else { + final List args = Arrays.asList(StringUtils.join(roleNames, ", ")); + if (roleNames.size() == 1) { + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.role.empty", args)); + } else { + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.roles.empty", args)); } } return false; @@ -497,7 +498,7 @@ private boolean runStep2DirectAssignments(){ List results = this.dvObjectServiceBean.getDvObjectInfoForMyData(directDvObjectIds); //List results = this.roleAssigneeService.getAssignmentsFor(this.userIdentifier); if (results.isEmpty()){ - this.addErrorMessage("Sorry, you have no assigned Dataverses, Datasets, or Files."); + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.no.dvobject")); return false; } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 64a1c626f13..a01d175d903 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2781,7 +2781,22 @@ passwdVal.passwdReq.lowercase=lowercase passwdVal.passwdReq.letter=letter passwdVal.passwdReq.numeral=numeral passwdVal.passwdReq.special=special +#mydata API (DataRetriverAPI.java and MyDataFinder.java) dataretrieverAPI.noMsgResultsFound=Sorry, no results were found. +dataretrieverAPI.authentication.required=Requires authentication. Please login. +dataretrieverAPI.authentication.required.opt=retrieveMyDataAsJsonString. User not found! Shouldn't be using this anyway. +dataretrieverAPI.user.not.found=No user found for: "{0}" +dataretrieverAPI.solr.error=Sorry! There was an error with the search service. +dataretrieverAPI.solr.error.opt=Sorry! There was a SOLR Error. +myDataFilterParams.error.no.user=Sorry! No user was found! +myDataFilterParams.error.result.no.role=No results. Please select at least one Role. +myDataFilterParams.error.result.no.dvobject=No results. Please select one of Dataverses, Datasets, Files. +myDataFilterParams.error.result.no.publicationStatus=No results. Please select one of {0}. +myDataFinder.error.result.null=Sorry, the EntityManager isn't working (still). +myDataFinder.error.result.no.role=Sorry, you have no assigned roles. +myDataFinder.error.result.role.empty=Sorry, nothing was found for this role: {0} +myDataFinder.error.result.roles.empty=Sorry, nothing was found for these roles: {0} +myDataFinder.error.result.no.dvobject=Sorry, you have no assigned Dataverses, Datasets, or Files. #xlsxfilereader.java xlsxfilereader.ioexception.parse=Could not parse Excel/XLSX spreadsheet. {0} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataRetrieverApiIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataRetrieverApiIT.java index 69d5db08744..fd0952fd336 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataRetrieverApiIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataRetrieverApiIT.java @@ -3,10 +3,14 @@ import com.jayway.restassured.RestAssured; import com.jayway.restassured.response.Response; import edu.harvard.iq.dataverse.api.auth.ApiKeyAuthMechanism; +import edu.harvard.iq.dataverse.util.BundleUtil; + import org.junit.BeforeClass; import org.junit.Test; import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import static javax.ws.rs.core.Response.Status.OK; import static javax.ws.rs.core.Response.Status.UNAUTHORIZED; @@ -15,6 +19,8 @@ public class DataRetrieverApiIT { + private static final String ERR_MSG_FORMAT = "{\"success\":false,\"error_message\":\"%s\"}"; + @BeforeClass public static void setUpClass() { RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); @@ -35,14 +41,24 @@ public void testRetrieveMyDataAsJsonString() { String badUserIdentifier = "bad-identifier"; Response invalidUserIdentifierResponse = UtilIT.retrieveMyDataAsJsonString(superUserApiToken, badUserIdentifier, emptyRoleIdsList); - assertEquals("{\"success\":false,\"error_message\":\"No user found for: \\\"" + badUserIdentifier + "\\\"\"}", invalidUserIdentifierResponse.prettyPrint()); + assertEquals(prettyPrintError("dataretrieverAPI.user.not.found", Arrays.asList(badUserIdentifier)), invalidUserIdentifierResponse.prettyPrint()); assertEquals(OK.getStatusCode(), invalidUserIdentifierResponse.getStatusCode()); // Call as superuser with valid user identifier Response createSecondUserResponse = UtilIT.createRandomUser(); String userIdentifier = UtilIT.getUsernameFromResponse(createSecondUserResponse); Response validUserIdentifierResponse = UtilIT.retrieveMyDataAsJsonString(superUserApiToken, userIdentifier, emptyRoleIdsList); - assertEquals("{\"success\":false,\"error_message\":\"Sorry, you have no assigned roles.\"}", validUserIdentifierResponse.prettyPrint()); + assertEquals(prettyPrintError("myDataFinder.error.result.no.role", null), validUserIdentifierResponse.prettyPrint()); assertEquals(OK.getStatusCode(), validUserIdentifierResponse.getStatusCode()); } + + private static String prettyPrintError(String resourceBundleKey, List params) { + final String errorMessage; + if (params == null || params.isEmpty()) { + errorMessage = BundleUtil.getStringFromBundle(resourceBundleKey); + } else { + errorMessage = BundleUtil.getStringFromBundle(resourceBundleKey, params); + } + return String.format(ERR_MSG_FORMAT, errorMessage.replaceAll("\"", "\\\\\"")); + } } From 697f081c7dc896344199818bf0ea09e4cc30558d Mon Sep 17 00:00:00 2001 From: Steven Ferey Date: Wed, 23 Aug 2023 16:39:44 +0200 Subject: [PATCH 022/970] Update src/main/java/propertyFiles/Bundle.properties Co-authored-by: Philip Durbin --- src/main/java/propertyFiles/Bundle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index b9651d0c36e..892dbd361da 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2800,7 +2800,7 @@ dataretrieverAPI.authentication.required=Requires authentication. Please login. dataretrieverAPI.authentication.required.opt=retrieveMyDataAsJsonString. User not found! Shouldn't be using this anyway. dataretrieverAPI.user.not.found=No user found for: "{0}" dataretrieverAPI.solr.error=Sorry! There was an error with the search service. -dataretrieverAPI.solr.error.opt=Sorry! There was a SOLR Error. +dataretrieverAPI.solr.error.opt=Sorry! There was a Solr Error. myDataFilterParams.error.no.user=Sorry! No user was found! myDataFilterParams.error.result.no.role=No results. Please select at least one Role. myDataFilterParams.error.result.no.dvobject=No results. Please select one of Dataverses, Datasets, Files. From f146f08d5cbea067572eb25350b10dcca4c0ee8d Mon Sep 17 00:00:00 2001 From: sferey Date: Wed, 23 Aug 2023 17:01:03 +0200 Subject: [PATCH 023/970] changed translation myDataFinder.error.result.null --- src/main/java/propertyFiles/Bundle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 892dbd361da..bdebc4a20f0 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2805,7 +2805,7 @@ myDataFilterParams.error.no.user=Sorry! No user was found! myDataFilterParams.error.result.no.role=No results. Please select at least one Role. myDataFilterParams.error.result.no.dvobject=No results. Please select one of Dataverses, Datasets, Files. myDataFilterParams.error.result.no.publicationStatus=No results. Please select one of {0}. -myDataFinder.error.result.null=Sorry, the EntityManager isn't working (still). +myDataFinder.error.result.null=Sorry, the authenticated user ID could not be retrieved. myDataFinder.error.result.no.role=Sorry, you have no assigned roles. myDataFinder.error.result.role.empty=Sorry, nothing was found for this role: {0} myDataFinder.error.result.roles.empty=Sorry, nothing was found for these roles: {0} From 685bc8be429e6f65765e66668de372ee2c8bb2b5 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Fri, 1 Sep 2023 14:09:06 -0400 Subject: [PATCH 024/970] xslt --- .../edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 7739408e408..9076680754e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -2123,13 +2123,7 @@ private static boolean checkParentElement(XMLStreamWriter xmlw, String elementNa public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { try { - //File xsltfile = new File("/home/victoria/fop-2.8/fop/ddi-to-fo.xsl"); - //URL resource = DdiExportUtil.class.getResource("edu/harvard/iq/dataverse/ddi-to-fo.xsl"); - //File xsltfile = new File(resource.toURI()); InputStream styleSheetInput = DdiExportUtil.class.getClassLoader().getResourceAsStream("edu/harvard/iq/dataverse/ddi-to-fo.xsl"); - logger.info("start datasetPdfDDI"); - //InputStream xsltfile = DdiExportUtil.class.getClassLoader().getResourceAsStream( - // "edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl"); final FopFactory fopFactory = FopFactory.newInstance(new File(".").toURI()); FOUserAgent foUserAgent = fopFactory.newFOUserAgent(); @@ -2153,7 +2147,6 @@ public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream transformer.transform(src, res); } catch (Exception e) { - logger.info("First try"); logger.severe(e.getMessage()); } } catch (Exception e) { From 12c463a8fa41c44ee3a94926299ceffaa7d6a114 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Tue, 5 Sep 2023 12:19:17 -0400 Subject: [PATCH 025/970] inc --- .../dataverse/export/PdfCodeBookExporter.java | 36 +- .../dataverse/export/ddi/DdiExportUtil.java | 7 +- .../harvard/iq/dataverse/ddi-pdf/i18n.inc.xsl | 5 + .../ddi-pdf/messages_en.properties.xml | 174 +++++++ .../ddi-pdf/messages_es.properties.xml | 170 +++++++ .../ddi-pdf/messages_fr.properties.xml | 173 +++++++ .../ddi-pdf/messages_ja.properties.xml | 161 +++++++ .../ddi-pdf/messages_nn.properties.xml | 174 +++++++ .../ddi-pdf/messages_no.properties.xml | 174 +++++++ .../ddi-pdf/messages_ru.properties.xml | 169 +++++++ .../edu/harvard/iq/dataverse/ddi-to-fo.xsl | 448 +++++++++--------- 11 files changed, 1446 insertions(+), 245 deletions(-) create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/i18n.inc.xsl create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_en.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_es.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_fr.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ja.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_nn.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_no.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ru.properties.xml diff --git a/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java index e0d5171e30c..4772c09ffd5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java @@ -3,25 +3,23 @@ package edu.harvard.iq.dataverse.export; import com.google.auto.service.AutoService; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetVersion; - import edu.harvard.iq.dataverse.export.ddi.DdiExportUtil; - import io.gdcc.spi.export.ExportDataProvider; - import io.gdcc.spi.export.ExportException; - import io.gdcc.spi.export.Exporter; - import edu.harvard.iq.dataverse.util.BundleUtil; - - import javax.json.JsonObject; - import javax.ws.rs.core.MediaType; - import javax.xml.stream.XMLStreamException; - import java.io.File; - import java.io.IOException; - import java.io.InputStream; - import java.io.OutputStream; - import java.nio.file.Path; - import java.nio.file.Paths; - import java.util.Locale; - import java.util.Optional; + +import edu.harvard.iq.dataverse.export.ddi.DdiExportUtil; +import io.gdcc.spi.export.ExportDataProvider; +import io.gdcc.spi.export.ExportException; +import io.gdcc.spi.export.Exporter; +import edu.harvard.iq.dataverse.util.BundleUtil; +import jakarta.ws.rs.core.MediaType; + +import javax.xml.stream.XMLStreamException; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Locale; +import java.util.Optional; @AutoService(Exporter.class) public class PdfCodeBookExporter implements Exporter { diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 9ed89a6fe94..aab728c7773 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -76,6 +76,7 @@ import org.apache.fop.apps.Fop; import org.apache.fop.apps.FopFactory; import org.apache.fop.apps.MimeConstants; +import java.io.File; public class DdiExportUtil { @@ -2108,6 +2109,7 @@ private static boolean checkParentElement(XMLStreamWriter xmlw, String elementNa public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { try { + String sysId = DdiExportUtil.class.getClassLoader().getResource("edu/harvard/iq/dataverse/ddi-to-fo.xsl").toURI().toString(); InputStream styleSheetInput = DdiExportUtil.class.getClassLoader().getResourceAsStream("edu/harvard/iq/dataverse/ddi-to-fo.xsl"); final FopFactory fopFactory = FopFactory.newInstance(new File(".").toURI()); @@ -2117,7 +2119,9 @@ public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream Fop fop = fopFactory.newFop(MimeConstants.MIME_PDF, foUserAgent, outputStream); // Setup XSLT TransformerFactory factory = TransformerFactory.newInstance(); - Transformer transformer = factory.newTransformer(new StreamSource(styleSheetInput)); + Source mySrc = new StreamSource(styleSheetInput); + mySrc.setSystemId(sysId); + Transformer transformer = factory.newTransformer(mySrc); // Set the value of a in the stylesheet transformer.setParameter("versionParam", "2.0"); @@ -2135,7 +2139,6 @@ public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream logger.severe(e.getMessage()); } } catch (Exception e) { - logger.info("Second try"); logger.severe(e.getMessage()); } } diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/i18n.inc.xsl b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/i18n.inc.xsl new file mode 100644 index 00000000000..edf876f3b04 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/i18n.inc.xsl @@ -0,0 +1,5 @@ + + + + + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_en.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_en.properties.xml new file mode 100644 index 00000000000..d8e98dfd3c6 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_en.properties.xml @@ -0,0 +1,174 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 +Valid +Frequency table not shown +Derivation +discrete +Data Collection Mode +Other Processing +Other Acknowledgment(s) +Untitled +Identification +Click here to access/export data files from Nesstar format +Value +Percentage +Weighting +Primary Investigator(s) +This document was generated using the +Sampling +Cases +Access Conditions +Source +Modalities +Rights & Disclaimer +Definition +Estimates of Sampling Error +Data Files +Geographic Coverage +April +Mean +Metadata Production +Data Collection +Missing Data +Scripts and programs +Variable(s) +Interviewer instructions +Funding Agency/ies +November +Missing +Version +Universe +Contributor(s) +Access Authority +Data Processing & Appraisal +Scope +Administrative documents +StdDev +Contact(s) +Label +Technical documents +Decimal(s) +Type +Literal question +Concepts +Range +Abstract +June +Supervision +Other Forms of Data Appraisal +References +Accessibility +Data Collection Dates +Data Editing +Questionnaires +Valid case(s) +Reports and analytical documents +Copyright +Documentation +Deviations from Sample Design +Publisher(s) +February +Dataset contains +Acknowledgment(s) +Continuous +Standard deviation +Variables Description +Producer +Production Date + +The Explorer allows you to view data files and export them to common statistical formats +Discrete +Group +July +Filename +Cases +Name +Warning: these figures indicate the number of cases found in the data file. They cannot be interpreted as summary statistics of the population of interest. +Statistical tables +December +Subjects +Processing Checks +software +Interviewer's instructions +Table of Contents +Document Information +Subgroup(s) +Keywords +group(s) +W +Weight +Files Description +Notes +Data Collection Notes +file(s) +continuous +Disclaimer +Content +variable(s) +Other Producer(s) +Producers & Sponsors +Data Cleaning Notes +Distributor(s) +Overview +Citation Requirements +September +Category +Confidentiality +Statistics +May +Undetermined +Structure +file +Pre-question +Response Rate +Width +Recoding and Derivation +Series +October +Unit of Analysis +Data Processing Notes +Kind of Data +File +Time Period(s) +File Content +Invalid +Vars +cont. +Key(s) +Question +Source of information +Imputation +Security +To open this file, you will need the free +Other resources +Data Dictionnary +Information +January +Other documents +Minimum +Scope & Coverage +Metadata Producer(s) +Show more info +Data Collector(s) +Post-question +Topics +Sampling Procedure +File Structure +Variables List +Format +Sampling Notes +Variables Group(s) +Description +Categories +Maximum +Depositor(s) +August +NW +Cover Page +Weighted +March + total - showing a subset of +Countries +question details + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_es.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_es.properties.xml new file mode 100644 index 00000000000..9cfcdaf6e7e --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_es.properties.xml @@ -0,0 +1,170 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 +Válido +No se presentan las tablas de frecuencias +Derivación +discreta +Método de Recolección +Otros relacionados al procesamiento +Otros Reconocimientos +Sin título +Identificación +Presione aquí para acceder/exportar al archivo(s) de datos +Valor +Porcentaje +Ponderando +Investigadores Principales +Este documento fue producido utilizando el +Muestreo +Casos +Condiciones de uso +Fuente +Modalidades +Derechos y Notas Legales +Definición +Estimaciones del Error Muestral +Archivo de Datos +Cobertura Geográfica +Abril +Media +Producción de los Metadatos +Recolección de Datos +Datos perdidos +Programas informáticos +Variable(s) +Manual del encuestador +Agencias Auspiciadoras +Noviembre +Valores perdidos +Versión +Universo +Contribuidor(es) +Institución Propietaria +Tratamiento y Validación de Datos +Dominio Temático +Documentos Administrativos +Desviación +Contacto(s) +Etiqueta +Documentos Técnicos +Decimal(es) +Tipo +Pregunta textual +Conceptos +Rango +Resumen +Junio +Supervisión +Otras Formas de Validación de los Datos +Referencias +Accesibilidad +Fechas de Recolección de Datos +Procesamiento de Datos +Cuestionarios +Casos válidos +Reportes y documentos analíticos +Derechos de Autor +Documentación +Modificaciones al Diseño Muestral +Editor(es) +Febrero +Contenido de la Base de Datos +Reconocimiento(s) +Contínua +Desviación estándar +Descripción de la variable +Productor +Fecha de Producción +El Explorador NESSTAR permite visualizar los archivos de datos y exportarlos a diferentes formatos estadísticos +Discreta +Grupo +Julio +Nombre del Archivo +Casos +Nombre +Cuadros estadísticos +Diciembre +Temas +Controles de Tratamiento +software +Manual del encuestador +Indice +Información acerca de la Documentación +Subgrupo(s) +Palabra Clave +grupo(s) +P +Ponderador +Descripción de los Archivos +Notas +Notas sobre la Recolección de Datos +archivo(s) +continua +Nota Legal +Contenido +variable(s) +Otros Productores +Productores y Auspiciadores +Notas acerca de la Depuración de los Datos +Distribuidor(es) +Resumen General +Forma de citar +Septiembre +Categoría +Confidencialidad +Estadística + +Mayo +Indeterminado +Estructura +archivo +Pre-pregunta +Tasa de Respuesta +Ancho +Recodificación y Derivación +Series +Octubre +Unidad de Análisis +Notas sobre el Procesamiento de Datos +Tipo de Datos +Archivo +Periodo de Referencia +Contenido del Archivo +Inválido +Vars. +cont. +Clave(s) +Pregunta +Fuente de información +Imputación +Seguridad +Para abrir este archivo se necesita el software gratuito +Otros recursos +Diccionario de Datos +Información +Enero +Otros documentos +Mínimo +Cobertura y Dominio Temático +Productor de los Metadatos +Mostrar más información +Entrevistador(es) +Pos-pregunta +Temas +Procedimiento de Muestreo +Estructura del Archivo +Lista de variables +Formato +Notas sobre el Muestreo +Grupo(s) de Variables +Descripción +Categorías +Máximo +Depositante(s) +Agosto +NP +Carátula +Ponderado +Marzo + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_fr.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_fr.properties.xml new file mode 100644 index 00000000000..9fa4d2178b1 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_fr.properties.xml @@ -0,0 +1,173 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 +Valide +Tableau de fréquences non-affiché +Mode de calcul +discrète +Méthode de collecte +Autre traitement +Autre(s) remerciement(s) +Sans titre +Identification +Cliquer ici pour accéder/exporter les fichiers de données du format Nesstar +Valeur +Pourcentage +Pondération +Enquêteur(s) principal/aux +Ce document a été généré à l'aide du +Echantillonage +Enregistrements +Conditions d'accès +Source +Modalités +Responsabilité et droits d'auteurs +Définition +Estimation des erreurs d'échantillonage +Fichiers de données +Couverture géographique +Avril +Moyenne +Production des méta-données +Collecte des données +Valeures manquantes +Programmes informatiques +Variable(s) +Instructions aux enquêteurs +Financement +Novembre +Manquant +Version +Univers +Contributeur(s) +Agence propriétaire +Traitement et évaluation des données +Domaines thématiques +Documents administratifs +Ecart type +Contact(s) +Libellé +Documents techniques +Décimale(s) +Type +Formulation de la question +Concepts +Gamme +Résumé +Juin +Supervision +Autres formes d'évaluation des données +Références +Accessibilité +Dates de collecte +Edition des données +Questionnaires +Cas valide(s) +Rapports et documents analytiques +Droits d'auteurs +Documentation +Déviations par rapport à l'échantillon initial +Editeur(s) +Février +Le jeu de données contient +Remerciement(s) +Continue +Ecart type +Description des variables +Producteur +Date de production + +L'Explorer vous permet d'accéder aux données et de les exporter vers les formats statistiques les plus courants +Discrète +Groupe +Juillet +Nom du fichier +Enreg. +Nom +Avertissement: Ces chiffres indiquent le nombre de cas identifiés dans le fichier de données. Ils ne peuvent pas être interpretés comme étant représentatifs de la population concernée. +Tableaux statistiques +Décembre +Sujets +Contrôles de traitement + +Instructions aux enquêteurs +Table des matières +Informations sur le document +Sous-groupe(s) +Mots-clé +groupe(s) +P +Pondération +Description des fichiers +Notes +Notes sur la collecte +fichier(s) +continue +Responsabilité(s) +Contenu +variable(s) +Autre(s) producteur(s) +Producteurs et sponsors +Notes sur l'apurement des données +Distributeur(s) +Aperçu +Citation +Septembre +Catégorie +Confidentialité +Statistiques +Mai +Indéterminé +Structure +fichier +Pré-question +Taux de réponse +Taille +Formulation de la question +Recodage et dérivation +Série +Octobre +Unité d'analyse +Notes sur le traitement des données +Type d'étude +Fichier +Période(s) de référence +Contenu du fichier +Non-valide +Vars +suite +Clé(s) +Question +Source d'information +Imputation +Sécurité +Pour ouvrir ce fichier, vous avez besoin du logiciel gratuit +Autres resources +Dictionnaire des variables +Information +Janvier +Autres documents +Minimum +Domaines thématiques et couverture +Producteur(s) des méta-données +Information complémentaire +Enquêteurs +Post-question +Thèmes +Procédure d'échantillonage +Structure du fichier +Liste des variables +Format +Notes sur l'échantillonage +Groupe(s) de variables +Description +Catégories +Maximum +Dépositaire(s) +Août +NP +Couverture +Pondéré +Mars +question details + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ja.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ja.properties.xml new file mode 100644 index 00000000000..bc5dbb06154 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ja.properties.xml @@ -0,0 +1,161 @@ + + +Generated by Properties2Xml on Tue Feb 13 13:55:43 EST 2007 +有効な +度数表(Frequency table)は表示されません +由来 +不連続な +データ収集モード +その他の確認事項 +識別番号 +データファイルにアクセスするにはここをクリックしてください +無題 +値 +割合 +ウェイティング +第一次調査官 +この文書はToolkitを使用して作られました +サンプリング +ケース +アクセス条件 +情報源 +様相 +権利及び声明文 +定義 +サンプルエラーの見積もり +データファイル +地理的な適用範囲 +４月 +平均 +メタデータ製作 +データ収集 +損失データ +スクリプトおよびプログラム +可変的 +面接者の指示 +出資機関 +１１月 +バージョン +共通の +貢献者 +アクセス権限 +データ処理、評価 +範囲, 領域 +管理用文章 +連絡先 +ラベル +技術的な文書 +小数点 +タイプ +文字の質問 +概念 +範囲 +要約 +６月 +監督 +その他ファーマットのデータ評価 +参照 +アクセス、入手法 +データ収集日 +データ編集 +質問 +レポートおよび分析的な文書 +有効な場合 +コピーライト +書類 +サンプルデザインによる偏差 +発行者 +２月 +データセットに含まれる +確認事項 +連続的な +標準偏差 +変数の記述 +製作者 +製作日 +” Explorer”によってデータファイルを参照することも一般的に使えわれている統計データフォーマットに変換。抽出することも可能です +不連続性 +グループ +７月 +ファイルの名前 +ケース +名前 +統計表 +１２月 +主題, 内容 +工程監査 +ソフト +面接者への指示 +目録 +書類の情報 +サブグループ +キーワード + +グループ +ウェイト +ファイルの詳細 +メモ +データ収集メモ +ファイル +継続的な +声明文 +内容 +変数 +その他の製作者 +製作者とスポンサー +データクリーニングメモ +分配者 +概略 +引用する場合の必要条件 +９月 +カテゴリー +機密性、コンフィデンシャリティー +５月 +未定 +構造 +ファイル +調査前の質問 +回答比率 +幅 +記録と誘導 +シリー +１０月 +分析の単位 +データ処理メモ +データの種類 + +ファイル +期間 +ファイルの内容 +無効 +キー +情報源 +非難 +セキュリティー +このファイルを開けるには、無料で配布されているNesstar Explorer が必要です。 +その他の資料 +データ辞典 +情報 +１月 +その他の書類 +最小値 +規模及び適用範囲 +メタデータ製作者 +さらにインフォメーションを表示 +データ収集者 +調査後の質問 +サンプリングの手順 +ファイルの構造 +変数のリスト +フォーマット +サンプリングメモ +変数のグループ +詳細 +カテゴリー +最大値 +デポジター、提供者、供託者 +８月 +表紙 +ウェイトされた +３月 + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_nn.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_nn.properties.xml new file mode 100644 index 00000000000..fdf14f5dfcd --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_nn.properties.xml @@ -0,0 +1,174 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 +Gyldige +Frekvenstabell ikke vist +Avledning +diskret +Type datainnsamling +Annen prosessering +Andre identifikatorer og krediteringer +Uten tittel +Identifisering +Click here to access/export data files from Nesstar format +Verdi +Prosent +Vekting +Primary Investigator(s) +Dette dokumentet ble generert av +Utvalg +Enheter +Tilgangsbetingelser +Kilde +Modaliteter +Rights & Disclaimer +Definisjon +Estimert utvalgsfeil +Datafiler +Geografisk omfang +April +Mean +Metadata-produksjon +Datainnsamling +Manglende data +Script og programmer +Variable(r) +Instruksjoner til intervjueren +Sponsor/finansierende institusjon(er) +November +Missing +Versjon +Univers +Bidragsyter(e) +Tilgangskontrollør +Dataprosessering og -evaluering +Omfang +Administrative dokumenter +Standardavvik +Kontaktperson(er) +Merkelapp +Tekniske dokumenter +Desimal(er) +Type +Spørsmålstekst +Begrep(er) +Rekkevidde +Sammendrag +Juni +Supervision +Andre former for dataevaluering +Referanser +Tilgjengelighet +Datainnsamlingsdatoer +Dataredigering +Spørreskjema +Gyldige enheter +Rapporter og analysedokumenter +Copyright +Dokumentasjon +Avvik fra utvalgsdesign +Utgiver(e) +Februar +Datasettet inneholder +Krediteringer +Kontinuerlig +Standardavvik +Variabelbeskrivelse +Produsent +Produksjonsdato + +The Explorer allows you to view data files and export them to common statistical formats +Diskret +Gruppe +Juli +Filnavn +Enheter +Navn +Advarsel: disse tallene indikerer antall enheter (cases) i datafilen. De kan ikke tolkes som oppsummert statistikk for populasjonen. +Statistiske tabeller +Desember +Emner +Prosesseringssjekk +programvare +Instruksjoner til intervjueren +Innholdsfortegnelse +Dokumentinformasjon +Undergruppe(r) +Nøkkelord +gruppe(r) +W +Vekt +Filbeskrivelse +Kommentarer +Datainnsamlingskommentarer +file(r) +kontinuerlig +Fraskrivelse +Innhold +variable(r) +Andre produsenter +Produsenter og sponsorer +Kommentarer om datarensing +Distributør(er) +Oversikt +Sitatkrav +September +Kategori +Konfidensialitet +Statistikk +Mai +Uavklart +Struktur +fil +Tekst før spørsmål +Responsrate +Bredde +Omkodinger og utledninger +Serie +Oktober +Analyseenhet +Dataprosesseringskommentarer +Datatype +Fil +Tidsperiode(r) +Filinnhold +Ugyldig +Variabler +kont. +Nøkler +Spørsmål +Kilde for informasjon +Imputasjon +Sikkerhet +For å åpne denne filen trenger du følgende gratisverktøy +Andre ressurser +Dataordbok +Informasjon +Januar +Andre dokumenter +Minimum +Omfang +Metadataprodusenter +Vis mer informasjon +Datainnsamler(e) +Tekst etter spørsmål +Emner +Utvalgsprosedyre +Filstruktur +Variabelliste +Format +Utvalgskommentarer +Variabelgrupper +Beskrivelse +Kategorier +Maksimum +Utgiver(e) +August +NW +Forside +Vektet +Mars + total - viser et utvalg av +Land +spørsmålsdetaljer + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_no.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_no.properties.xml new file mode 100644 index 00000000000..fdf14f5dfcd --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_no.properties.xml @@ -0,0 +1,174 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 +Gyldige +Frekvenstabell ikke vist +Avledning +diskret +Type datainnsamling +Annen prosessering +Andre identifikatorer og krediteringer +Uten tittel +Identifisering +Click here to access/export data files from Nesstar format +Verdi +Prosent +Vekting +Primary Investigator(s) +Dette dokumentet ble generert av +Utvalg +Enheter +Tilgangsbetingelser +Kilde +Modaliteter +Rights & Disclaimer +Definisjon +Estimert utvalgsfeil +Datafiler +Geografisk omfang +April +Mean +Metadata-produksjon +Datainnsamling +Manglende data +Script og programmer +Variable(r) +Instruksjoner til intervjueren +Sponsor/finansierende institusjon(er) +November +Missing +Versjon +Univers +Bidragsyter(e) +Tilgangskontrollør +Dataprosessering og -evaluering +Omfang +Administrative dokumenter +Standardavvik +Kontaktperson(er) +Merkelapp +Tekniske dokumenter +Desimal(er) +Type +Spørsmålstekst +Begrep(er) +Rekkevidde +Sammendrag +Juni +Supervision +Andre former for dataevaluering +Referanser +Tilgjengelighet +Datainnsamlingsdatoer +Dataredigering +Spørreskjema +Gyldige enheter +Rapporter og analysedokumenter +Copyright +Dokumentasjon +Avvik fra utvalgsdesign +Utgiver(e) +Februar +Datasettet inneholder +Krediteringer +Kontinuerlig +Standardavvik +Variabelbeskrivelse +Produsent +Produksjonsdato + +The Explorer allows you to view data files and export them to common statistical formats +Diskret +Gruppe +Juli +Filnavn +Enheter +Navn +Advarsel: disse tallene indikerer antall enheter (cases) i datafilen. De kan ikke tolkes som oppsummert statistikk for populasjonen. +Statistiske tabeller +Desember +Emner +Prosesseringssjekk +programvare +Instruksjoner til intervjueren +Innholdsfortegnelse +Dokumentinformasjon +Undergruppe(r) +Nøkkelord +gruppe(r) +W +Vekt +Filbeskrivelse +Kommentarer +Datainnsamlingskommentarer +file(r) +kontinuerlig +Fraskrivelse +Innhold +variable(r) +Andre produsenter +Produsenter og sponsorer +Kommentarer om datarensing +Distributør(er) +Oversikt +Sitatkrav +September +Kategori +Konfidensialitet +Statistikk +Mai +Uavklart +Struktur +fil +Tekst før spørsmål +Responsrate +Bredde +Omkodinger og utledninger +Serie +Oktober +Analyseenhet +Dataprosesseringskommentarer +Datatype +Fil +Tidsperiode(r) +Filinnhold +Ugyldig +Variabler +kont. +Nøkler +Spørsmål +Kilde for informasjon +Imputasjon +Sikkerhet +For å åpne denne filen trenger du følgende gratisverktøy +Andre ressurser +Dataordbok +Informasjon +Januar +Andre dokumenter +Minimum +Omfang +Metadataprodusenter +Vis mer informasjon +Datainnsamler(e) +Tekst etter spørsmål +Emner +Utvalgsprosedyre +Filstruktur +Variabelliste +Format +Utvalgskommentarer +Variabelgrupper +Beskrivelse +Kategorier +Maksimum +Utgiver(e) +August +NW +Forside +Vektet +Mars + total - viser et utvalg av +Land +spørsmålsdetaljer + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ru.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ru.properties.xml new file mode 100644 index 00000000000..06fde85af5e --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ru.properties.xml @@ -0,0 +1,169 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 +Валидный +Частотная таблица не выводится +Расчет +дискретная +Способ сбора данных +Прочая обработка +Другие участники +Безымянный +Индентификация +Щелкните здесь, чтобы получить доступ к файлам или экспортировать их +Значение +Процент +Взвешивание +Первичный(е) исследователь(и) +Документ был сгенерирован с использованием +Выборка +Наблюдения +Условия доступа +Источник +Модальности +Авторские права и ограничения ответственности +Определение +Оценки ошибок выборки +Файлы данных +Географический охват +Апрель +Среднее +Разработка метаданных +Сбор данных +Пропущенные данные +Скрипты и программы +Переменная(ые) +Инструкции интервьюеру +Кто финансировал +Ноябрь +Пропущенные +Версия +Генеральная совокупность +Участник(и) +Права доступа +Обработка и инспекция данных +Охват +Административные документы +СтдОткл +Контак(ы) +Метка +Технические документы +Десятичные +Тип +Текст вопроса +Концепции +Диапазон +Резюме +Июнь +Контроль +Другие формы инспекции данных +Установки +Доступность +Даты сбора данных +Редактирование данных +Вопросники +Валидное(ые) наблюдение(я) +Отчеты и аналитические документы +Авторские права +Документация +Отклонения от дизайна выборки +Издатель(и) +Февраль +Набор данных содержит +Участник(и) +Непрерывная +Стандартное отклонение +Описание переменных +Разработчик +Дата разработки +Проводник позволяет просматривать файлы данных и экспортировать их в распространенные статистические форматы +Дикретная +Группа +Июль +Имя файла +Наблюдения +Имя +Статистичсекие таблицы +Декабрь +Темы +Контроль обработки +программное обеспечение +Инструкции интервьюеру +Оглавление +Информация о документе +Подгруппа(ы) +Ключевые слова +группа(ы) +B +Вес +Описание файла +Примечания +Примечания по сбору данных +файл(ы) +непрерывная +Ограничения ответственности +Содержание +переменная(ые) +Другие разработчики +Разработчики и спонсоры +Примечания по чистке данных +Дистрибьютор(ы) +Обзор +Требования по цитированию +Сентябрь +Категория +Конфиденциальность +Статистики +Май +Неопределенный +Структура +файл +Текст, предваряющий вопрос +Доля ответов +Ширина +Перекодировка и расчеты +Серия +Октябрь +Единица анализа +Примечания по обработке данных +Тип данных +Файл +Период(ы) времени +Содержание файла +Некорректный +Переменные +непр. +Ключ(и) +Вопрос +Источник информации +Импутация +Безопасность +Чтобы открыть этот файл, необходимо иметь свободным +Прочие источники +Словарь данных +Информация +Январь +Прочие документы +Минимум +Охват и покрытие +Разработчик(и) метаданных +Показать дополнительную информацию +Кто собирал данные +Текст после вопроса +Разделы +Процедура выборки +Структура файла +Список переменных +Формат +Примечания по выборке +Группа(ы) переменных +Описание +Категории +Максимум +Депозитор(ы) +Август +HB +Титульный лист +взвешенные +Март + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl index 1c03d5caf34..c25c76246d6 100644 --- a/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl @@ -66,8 +66,7 @@ Report optional text --> - - + @@ -110,7 +109,7 @@ - + @@ -124,6 +123,7 @@ Times + - 1 + 1 0 @@ -381,21 +381,21 @@ - + - + - + @@ -403,33 +403,33 @@ - + - + - + - + - + @@ -437,21 +437,21 @@ + select="$msg/*/entry[@key='Data_Processing_and_Appraisal']"/> - + - + @@ -461,7 +461,7 @@ - + @@ -486,7 +486,7 @@ - + @@ -503,7 +503,7 @@ - + @@ -528,7 +528,7 @@ - + @@ -569,7 +569,7 @@ - + @@ -642,7 +642,7 @@ - + @@ -657,7 +657,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="$msg/*/entry[@key='Metadata_Producers']"/> + select="$msg/*/entry[@key='Production_Date']"/> + select="$msg/*/entry[@key='Version']"/> + select="$msg/*/entry[@key='Identification']"/> - + @@ -745,7 +745,7 @@ - + @@ -764,14 +764,14 @@ - + - + @@ -782,7 +782,7 @@ + select="$msg/*/entry[@key='Scope_and_Coverage']"/> @@ -793,7 +793,7 @@ + select="$msg/*/entry[@key='Producers_and_Sponsors']"/> @@ -803,7 +803,7 @@ - + @@ -813,7 +813,7 @@ - + @@ -825,7 +825,7 @@ internal-destination="data-processing-and-appraisal" text-decoration="underline" color="blue"> + select="$msg/*/entry[@key='Data_Processing_and_Appraisal']"/> @@ -836,7 +836,7 @@ - + @@ -847,7 +847,7 @@ + select="$msg/*/entry[@key='Rights_and_Disclaimer']"/> @@ -858,7 +858,7 @@ + select="$msg/*/entry[@key='Files_Description']"/> @@ -891,7 +891,7 @@ - + @@ -924,7 +924,7 @@ - + @@ -949,7 +949,7 @@ + select="$msg/*/entry[@key='Variables_Description']"/> @@ -982,7 +982,7 @@ - + @@ -1002,7 +1002,7 @@ - + @@ -1065,7 +1065,7 @@ - + @@ -1078,7 +1078,7 @@ - + + select="$msg/*/entry[@key='Identification']"/> - @@ -1130,7 +1130,7 @@ select="/ddi:codeBook/ddi:stdyDscr/ddi:citation/ddi:verStmt/ddi:version"> : + select="$msg/*/entry[@key='Production_Date']"/>: @@ -1139,7 +1139,7 @@ - @@ -1156,7 +1156,7 @@ - + - + select="$msg/*/entry[@key='Kind_of_Data']"/> + select="$msg/*/entry[@key='Unit_of_Analysis']"/> + select="$msg/*/entry[@key='Scope_and_Coverage']"/> @@ -1257,7 +1257,7 @@ - + - @@ -1299,7 +1299,7 @@ - + - + @@ -1344,7 +1344,7 @@ - + @@ -1361,7 +1361,7 @@ border="{$default-border}" padding="{$cell-padding}"> - @@ -1422,7 +1422,7 @@ padding="{$cell-padding}"> @@ -1444,7 +1444,7 @@ padding="{$cell-padding}"> + select="$msg/*/entry[@key='Other_Producers']"/> + select="$msg/*/entry[@key='Funding_Agencies']"/> @@ -1514,7 +1514,7 @@ - @@ -1534,7 +1534,7 @@ padding="{$cell-padding}"> + select="$msg/*/entry[@key='Sampling']"/> + select="$msg/*/entry[@key='Sampling_Notes']"/> + select="$msg/*/entry[@key='Sampling_Procedure']"/> + select="$msg/*/entry[@key='Response_Rate']"/> + select="$msg/*/entry[@key='Weighting']"/> + select="$msg/*/entry[@key='Data_Collection']"/> @@ -1672,7 +1672,7 @@ padding="{$cell-padding}"> @@ -1694,7 +1694,7 @@ padding="{$cell-padding}"> + select="$msg/*/entry[@key='Time_Periods']"/> @@ -1738,7 +1738,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="$msg/*/entry[@key='Questionnaires']"/> + select="$msg/*/entry[@key='Data_Collectors']"/> + select="$msg/*/entry[@key='Supervision']"/> - + @@ -1885,7 +1885,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="$msg/*/entry[@key='Data_Editing']"/> + select="$msg/*/entry[@key='Other_Processing']"/> + select="$msg/*/entry[@key='Accessibility']"/> @@ -1980,7 +1980,7 @@ padding="{$cell-padding}"> + select="$msg/*/entry[@key='Access_Authority']"/> - @@ -2022,7 +2022,7 @@ padding="{$cell-padding}"> + select="$msg/*/entry[@key='Distributors']"/> + select="$msg/*/entry[@key='Depositors']"/> + select="$msg/*/entry[@key='Confidentiality']"/> + select="$msg/*/entry[@key='Access_Conditions']"/> @@ -2142,7 +2142,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="$msg/*/entry[@key='Disclaimer']"/> + select="$msg/*/entry[@key='Copyright']"/> - + @@ -2194,15 +2194,15 @@ - + - + - + @@ -2218,7 +2218,7 @@ - + @@ -2227,16 +2227,16 @@ - + - + - - + + - + @@ -2262,17 +2262,17 @@ - + - + - + - + @@ -2290,7 +2290,7 @@ - + @@ -2299,17 +2299,17 @@ - + - + - + - + @@ -2327,7 +2327,7 @@ - + @@ -2336,7 +2336,7 @@ - + @@ -2372,7 +2372,7 @@ + select="$msg/*/entry[@key='Reports_and_analytical_documents']"/> @@ -2389,7 +2389,7 @@ space-after="0.03in"> - + @@ -2407,7 +2407,7 @@ + select="$msg/*/entry[@key='Technical_documents']"/> @@ -2425,7 +2425,7 @@ + select="$msg/*/entry[@key='Administrative_documents']"/> @@ -2442,7 +2442,7 @@ space-after="0.03in"> - + @@ -2459,7 +2459,7 @@ space-after="0.03in"> - + @@ -2477,7 +2477,7 @@ + select="$msg/*/entry[@key='Statistical_tables']"/> @@ -2495,7 +2495,7 @@ + select="$msg/*/entry[@key='Scripts_and_programs']"/> @@ -2512,7 +2512,7 @@ space-after="0.03in"> - + @@ -2532,7 +2532,7 @@ + select="$msg/*/entry[@key='Reports_and_analytical_documents']"/> - + - + - - + - + - + @@ -2602,7 +2602,7 @@ - + @@ -2611,7 +2611,7 @@ - + *** - + **** @@ -2749,7 +2749,7 @@ - # # @@ -2762,7 +2762,7 @@ - # # @@ -2776,18 +2776,18 @@ - + - : + : - :  + :  @@ -2811,7 +2811,7 @@ - + @@ -2823,7 +2823,7 @@ - + @@ -2835,7 +2835,7 @@ - + @@ -2847,7 +2847,7 @@ - + @@ -2859,7 +2859,7 @@ - + @@ -2871,7 +2871,7 @@ - + @@ -2905,10 +2905,10 @@ - + - + @@ -2955,7 +2955,7 @@ - + : @@ -3059,7 +3059,7 @@ - + @@ -3080,7 +3080,7 @@ - + @@ -3092,7 +3092,7 @@ - + @@ -3104,7 +3104,7 @@ - + @@ -3116,7 +3116,7 @@ - + @@ -3239,13 +3239,13 @@ - + - + - + @@ -3322,7 +3322,7 @@   - + @@ -3369,7 +3369,7 @@ : - + @@ -3382,7 +3382,7 @@ - + [= + select="$msg/*/entry[@key='Type']"/>= + select="$msg/*/entry[@key='discrete']"/> + select="$msg/*/entry[@key='continuous']"/> ] @@ -3404,7 +3404,7 @@ [== - @@ -3419,13 +3419,13 @@ [= = - ] [ - + =* / @@ -3437,7 +3437,7 @@ @@ -3452,12 +3452,12 @@ - + [ + select="$msg/*/entry[@key='Abbrev_NotWeighted']"/> / - + ] @@ -3470,18 +3470,18 @@ - + select="$msg/*/entry[@key='Invalid']"/> - + - @@ -3517,7 +3517,7 @@ - + - + - + - + - + - + @@ -3638,7 +3638,7 @@ - + + select="$msg/*/entry[@key='Recoding_and_Derivation']"/> - + - + - + - - + + - @@ -3798,7 +3798,7 @@ padding="{$cell-padding}" text-align="center"> + select="$msg/*/entry[@key='Cases_Abbreviation']"/> @@ -3806,7 +3806,7 @@ padding="{$cell-padding}" text-align="center"> + select="$msg/*/entry[@key='Weighted']"/> @@ -3814,9 +3814,9 @@ padding="{$cell-padding}" text-align="center"> + select="$msg/*/entry[@key='Percentage']"/> () + select="$msg/*/entry[@key='Weighted']"/>) @@ -3950,16 +3950,16 @@ + select="$msg/*/entry[@key='SumStat_Warning']"/> - + ( - ) + ) @@ -4003,7 +4003,7 @@ - *** + *** *** @@ -4055,7 +4055,7 @@ - + @@ -4070,7 +4070,7 @@ - + @@ -4085,7 +4085,7 @@ - + @@ -4100,7 +4100,7 @@ - + @@ -4162,38 +4162,38 @@ - + - + - + - + - + - + - + @@ -4257,40 +4257,40 @@ - + - + - + - + - + - + - + - + - + - + - + - + From 4579283f8e5039cbbc6c8df43ac73ee5fd63b967 Mon Sep 17 00:00:00 2001 From: bencomp Date: Mon, 18 Sep 2023 00:07:16 +0200 Subject: [PATCH 026/970] Add basic DatasetFieldTest --- .../iq/dataverse/DatasetFieldTest.java | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 src/test/java/edu/harvard/iq/dataverse/DatasetFieldTest.java diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldTest.java new file mode 100644 index 00000000000..23ba2d69fff --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldTest.java @@ -0,0 +1,41 @@ +package edu.harvard.iq.dataverse; + +import edu.harvard.iq.dataverse.DatasetFieldType.FieldType; +import edu.harvard.iq.dataverse.mocks.MocksFactory; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +class DatasetFieldTest { + @Test + void testCreateNewEmptyDatasetField_withEmptyTemplate() { + Template template = new Template(); + + DatasetField field = DatasetField.createNewEmptyDatasetField(new DatasetFieldType("subject", FieldType.TEXT, false), template); + assertTrue(field.getTemplate() == template); + assertTrue(template.getDatasetFields().isEmpty()); + } + + @Test + void testNotEqualDatasetFields() { + DatasetField field1 = DatasetField.createNewEmptyDatasetField(new DatasetFieldType("subject", FieldType.TEXT, false), new Template()); + field1.setId(MocksFactory.nextId()); + DatasetField field2 = DatasetField.createNewEmptyDatasetField(new DatasetFieldType("subject", FieldType.TEXT, false), new Template()); + field2.setId(MocksFactory.nextId()); + + assertNotEquals(field1, field2); + } + + @Test + void testEqualDatasetFields() { + DatasetField field1 = DatasetField.createNewEmptyDatasetField(new DatasetFieldType("subject", FieldType.TEXT, false), new Template()); + field1.setId(100L); + DatasetField field2 = DatasetField.createNewEmptyDatasetField(new DatasetFieldType("subject", FieldType.TEXT, false), new Template()); + field2.setId(100L); + + assertEquals(field1, field2); + } +} \ No newline at end of file From 87bfe16c6a4390075bd0b481071ebddf2f1c1e5d Mon Sep 17 00:00:00 2001 From: bencomp Date: Mon, 18 Sep 2023 00:25:52 +0200 Subject: [PATCH 027/970] Test DatasetField identities --- .../iq/dataverse/DatasetFieldTest.java | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldTest.java index 23ba2d69fff..97999af3244 100644 --- a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNull; class DatasetFieldTest { @@ -21,12 +22,17 @@ void testCreateNewEmptyDatasetField_withEmptyTemplate() { @Test void testNotEqualDatasetFields() { - DatasetField field1 = DatasetField.createNewEmptyDatasetField(new DatasetFieldType("subject", FieldType.TEXT, false), new Template()); + DatasetFieldType type1 = new DatasetFieldType("subject", FieldType.TEXT, false); + Template template1 = new Template(); + DatasetField field1 = DatasetField.createNewEmptyDatasetField(type1, template1); field1.setId(MocksFactory.nextId()); - DatasetField field2 = DatasetField.createNewEmptyDatasetField(new DatasetFieldType("subject", FieldType.TEXT, false), new Template()); + DatasetFieldType type2 = new DatasetFieldType("subject", FieldType.TEXT, false); + Template template2 = new Template(); + DatasetField field2 = DatasetField.createNewEmptyDatasetField(type2, template2); field2.setId(MocksFactory.nextId()); assertNotEquals(field1, field2); + assertNotEquals(field1, template2); } @Test @@ -34,8 +40,25 @@ void testEqualDatasetFields() { DatasetField field1 = DatasetField.createNewEmptyDatasetField(new DatasetFieldType("subject", FieldType.TEXT, false), new Template()); field1.setId(100L); DatasetField field2 = DatasetField.createNewEmptyDatasetField(new DatasetFieldType("subject", FieldType.TEXT, false), new Template()); + + // Fields are not equal before both have IDs set + assertNotEquals(field1, field2); + field2.setId(100L); assertEquals(field1, field2); } + + @Test + void testCopyDatasetFields() { + DatasetField field1 = DatasetField.createNewEmptyDatasetField(new DatasetFieldType("subject", FieldType.TEXT, false), new Template()); + field1.setId(100L); + DatasetField field2 = field1.copy(field1.getTemplate()); + + assertNull(field2.getId()); + // A copy of a field should not be equal + assertNotEquals(field1, field2); + + assertEquals(field2.getDatasetFieldType(), field1.getDatasetFieldType()); + } } \ No newline at end of file From fbf1f9cbd05e18031170f206b3352c9a424cab16 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Mon, 16 Oct 2023 15:36:27 +0200 Subject: [PATCH 028/970] RO-Crate metadata mime type detection --- .../edu/harvard/iq/dataverse/util/FileUtil.java | 15 ++++++++++++++- .../MimeTypeDetectionByFileName.properties | 2 ++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 1ad389fb0e2..a7c3488e57f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -86,6 +86,7 @@ import java.util.HashMap; import java.util.List; import java.util.Optional; +import java.util.ResourceBundle; import java.util.UUID; import java.util.logging.Level; import java.util.logging.Logger; @@ -182,6 +183,7 @@ public class FileUtil implements java.io.Serializable { public static final String MIME_TYPE_NETCDF = "application/netcdf"; public static final String MIME_TYPE_XNETCDF = "application/x-netcdf"; public static final String MIME_TYPE_HDF5 = "application/x-hdf5"; + public static final String MIME_TYPE_RO_CRATE = "application/ld+json; profile=\"http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate\""; // File type "thumbnail classes" tags: @@ -421,6 +423,11 @@ public static String retestIngestableFileType(File file, String fileType) { } public static String determineFileType(File f, String fileName) throws IOException{ + final ResourceBundle bundle = BundleUtil.getResourceBundle("MimeTypeDetectionByFileName"); + if (bundle.keySet().contains(fileName)) { + return bundle.getString(fileName); + } + String fileType = null; String fileExtension = getFileExtension(fileName); @@ -545,6 +552,11 @@ public static String determineFileType(File f, String fileName) throws IOExcepti } public static String determineFileTypeByNameAndExtension(String fileName) { + final ResourceBundle bundle = BundleUtil.getResourceBundle("MimeTypeDetectionByFileName"); + if (bundle.keySet().contains(fileName)) { + return bundle.getString(fileName); + } + String mimetypesFileTypeMapResult = MIME_TYPE_MAP.getContentType(fileName); logger.fine("MimetypesFileTypeMap type by extension, for " + fileName + ": " + mimetypesFileTypeMapResult); if (mimetypesFileTypeMapResult != null) { @@ -825,7 +837,8 @@ public static boolean useRecognizedType(String suppliedContentType, String recog || canIngestAsTabular(recognizedType) || recognizedType.equals("application/fits-gzipped") || recognizedType.equalsIgnoreCase(ShapefileHandler.SHAPEFILE_FILE_TYPE) || recognizedType.equalsIgnoreCase(BagItFileHandler.FILE_TYPE) - || recognizedType.equals(MIME_TYPE_ZIP)) { + || recognizedType.equals(MIME_TYPE_ZIP) + || recognizedType.equals(MIME_TYPE_RO_CRATE)) { return true; } return false; diff --git a/src/main/java/propertyFiles/MimeTypeDetectionByFileName.properties b/src/main/java/propertyFiles/MimeTypeDetectionByFileName.properties index 70b0c4e371e..5c1a22bfd5f 100644 --- a/src/main/java/propertyFiles/MimeTypeDetectionByFileName.properties +++ b/src/main/java/propertyFiles/MimeTypeDetectionByFileName.properties @@ -2,3 +2,5 @@ Makefile=text/x-makefile Snakemake=text/x-snakemake Dockerfile=application/x-docker-file Vagrantfile=application/x-vagrant-file +ro-crate-metadata.json=application/ld+json; profile="http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate" +ro-crate-metadata.jsonld=application/ld+json; profile="http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate" From fbc6a54ba067e24f135d11cc7f66a950838c45a0 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Mon, 16 Oct 2023 19:34:46 +0200 Subject: [PATCH 029/970] fixed display and facet string for mime types with profile + test + reslease notes --- .../10015-RO-Crate-metadata-file.md | 10 +++++++ .../harvard/iq/dataverse/util/FileUtil.java | 10 +++++++ .../propertyFiles/MimeTypeDisplay.properties | 1 + .../propertyFiles/MimeTypeFacets.properties | 1 + .../iq/dataverse/util/FileUtilTest.java | 29 +++++++++++++++++++ .../resources/fileutil/ro-crate-metadata.json | 1 + 6 files changed, 52 insertions(+) create mode 100644 doc/release-notes/10015-RO-Crate-metadata-file.md create mode 100644 src/test/resources/fileutil/ro-crate-metadata.json diff --git a/doc/release-notes/10015-RO-Crate-metadata-file.md b/doc/release-notes/10015-RO-Crate-metadata-file.md new file mode 100644 index 00000000000..4b018a634f7 --- /dev/null +++ b/doc/release-notes/10015-RO-Crate-metadata-file.md @@ -0,0 +1,10 @@ +Detection of mime-types based on a filename with extension and detection of the RO-Crate metadata files. + +From now on, filenames with extensions can be added into `MimeTypeDetectionByFileName.properties` file. Filenames added there will take precedence over simply recognizing files by extensions. For example, two new filenames are added into that file: +``` +ro-crate-metadata.json=application/ld+json; profile="http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate" +ro-crate-metadata.jsonld=application/ld+json; profile="http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate" +``` + +Therefore, files named `ro-crate-metadata.json` will be then detected as RO-Crated metadata files from now on, instead as generic `JSON` files. +For more information on the RO-Crate specifications, see https://www.researchobject.org/ro-crate diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index a7c3488e57f..baff17af601 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -280,6 +280,11 @@ public static String getUserFriendlyFileType(DataFile dataFile) { if (fileType.equalsIgnoreCase(ShapefileHandler.SHAPEFILE_FILE_TYPE)){ return ShapefileHandler.SHAPEFILE_FILE_TYPE_FRIENDLY_NAME; } + try { + return BundleUtil.getStringFromPropertyFile(fileType,"MimeTypeDisplay" ); + } catch (MissingResourceException e) { + //NOOP: we will try again after trimming ";" + } if (fileType.contains(";")) { fileType = fileType.substring(0, fileType.indexOf(";")); } @@ -294,6 +299,11 @@ public static String getUserFriendlyFileType(DataFile dataFile) { } public static String getIndexableFacetFileType(DataFile dataFile) { + try { + return BundleUtil.getStringFromDefaultPropertyFile(dataFile.getContentType(),"MimeTypeFacets" ); + } catch (MissingResourceException e) { + //NOOP: we will try again after trimming ";" + } String fileType = getFileType(dataFile); try { return BundleUtil.getStringFromDefaultPropertyFile(fileType,"MimeTypeFacets" ); diff --git a/src/main/java/propertyFiles/MimeTypeDisplay.properties b/src/main/java/propertyFiles/MimeTypeDisplay.properties index 295ac226fa1..8486a113116 100644 --- a/src/main/java/propertyFiles/MimeTypeDisplay.properties +++ b/src/main/java/propertyFiles/MimeTypeDisplay.properties @@ -222,5 +222,6 @@ text/xml-graphml=GraphML Network Data application/octet-stream=Unknown application/x-docker-file=Docker Image File application/x-vagrant-file=Vagrant Image File +application/ld+json;\u0020profile\u003d\u0022http\u003a//www.w3.org/ns/json-ld#flattened\u0020http\u003a//www.w3.org/ns/json-ld#compacted\u0020https\u003a//w3id.org/ro/crate\u0022=RO-Crate metadata # Dataverse-specific application/vnd.dataverse.file-package=Dataverse Package diff --git a/src/main/java/propertyFiles/MimeTypeFacets.properties b/src/main/java/propertyFiles/MimeTypeFacets.properties index aaab66f20ae..831c509b860 100644 --- a/src/main/java/propertyFiles/MimeTypeFacets.properties +++ b/src/main/java/propertyFiles/MimeTypeFacets.properties @@ -224,5 +224,6 @@ video/webm=Video text/xml-graphml=Network Data # Other application/octet-stream=Unknown +application/ld+json;\u0020profile\u003d\u0022http\u003a//www.w3.org/ns/json-ld#flattened\u0020http\u003a//www.w3.org/ns/json-ld#compacted\u0020https\u003a//w3id.org/ro/crate\u0022=Metadata # Dataverse-specific application/vnd.dataverse.file-package=Data diff --git a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java index 2cfe9f25d7e..396d613e768 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java @@ -371,4 +371,33 @@ public void testHdf4File() throws IOException { assertEquals("application/octet-stream", contentType); } + @Test + public void testDetermineFileTypeROCrate() { + final String roCrateContentType = "application/ld+json; profile=\"http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate\""; + final DataFile rocrate = new DataFile(roCrateContentType); + + assertEquals(roCrateContentType, rocrate.getContentType()); + assertEquals("RO-Crate metadata", FileUtil.getUserFriendlyFileType(rocrate)); + assertEquals("Metadata", FileUtil.getIndexableFacetFileType(rocrate)); + + final File roCrateFile = new File("src/test/resources/fileutil/ro-crate-metadata.json"); + if (roCrateFile.exists()) { + try { + assertEquals(roCrateContentType, FileUtil.determineFileType(roCrateFile, "ro-crate-metadata.json")); + } catch (IOException ex) { + Logger.getLogger(FileUtilTest.class.getName()).log(Level.SEVERE, null, ex); + } + } else { + fail("File does not exist: " + roCrateFile.toPath().toString()); + } + + // test ";" removal + final String dockerFileWithProfile = "application/x-docker-file; profile=\"http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate\""; + final DataFile dockerDataFile = new DataFile(dockerFileWithProfile); + + assertEquals(dockerFileWithProfile, dockerDataFile.getContentType()); + assertEquals("Docker Image File", FileUtil.getUserFriendlyFileType(dockerDataFile)); + assertEquals("Code", FileUtil.getIndexableFacetFileType(dockerDataFile)); + } + } diff --git a/src/test/resources/fileutil/ro-crate-metadata.json b/src/test/resources/fileutil/ro-crate-metadata.json new file mode 100644 index 00000000000..5ac4b24ad2b --- /dev/null +++ b/src/test/resources/fileutil/ro-crate-metadata.json @@ -0,0 +1 @@ +{"@context":["https://w3id.org/ro/crate/1.1/context",{"metadataContext":"https://language-archives.services/metadata-definitions#metadataContext","orthographicNotes":"https://language-archives.services/metadata-definitions#orthographicNotes","media":"https://language-archives.services/metadata-definitions#media","comments":"https://language-archives.services/metadata-definitions#comments","private":"https://language-archives.services/metadata-definitions#private","depositFormReceived":"https://language-archives.services/metadata-definitions#depositFormReceived","subjectLanguages":"https://language-archives.services/metadata-definitions#subjectLanguages","originatedOn":"https://language-archives.services/metadata-definitions#originatedOn","languageAsGiven":"https://language-archives.services/metadata-definitions#languageAsGiven","contentLanguages":"https://language-archives.services/metadata-definitions#contentLanguages","dialect":"https://language-archives.services/metadata-definitions#dialect","discourseType":"https://language-archives.services/metadata-definitions#discourseType","bornDigital":"https://language-archives.services/metadata-definitions#bornDigital","accessNarrative":"https://language-archives.services/metadata-definitions#accessNarrative","receivedOn":"https://language-archives.services/metadata-definitions#receivedOn","digitisedOn":"https://language-archives.services/metadata-definitions#digitisedOn","ingestNotes":"https://language-archives.services/metadata-definitions#ingestNotes","adminComment":"https://language-archives.services/metadata-definitions#adminComment","external":"https://language-archives.services/metadata-definitions#external","dataType":"https://language-archives.services/metadata-definitions#dataType","citeAs":"https://language-archives.services/metadata-definitions#citeAs","originatedOnNarrative":"https://language-archives.services/metadata-definitions#originatedOnNarrative","countries":"https://language-archives.services/metadata-definitions#countries","tapesReturned":"https://language-archives.services/metadata-definitions#tapesReturned","originalMedia":"https://language-archives.services/metadata-definitions#originalMedia","metadataExportable":"https://language-archives.services/metadata-definitions#metadataExportable","metadataImportedOn":"https://language-archives.services/metadata-definitions#metadataImportedOn","metadataExportedOn":"https://language-archives.services/metadata-definitions#metadataExportedOn","tracking":"https://language-archives.services/metadata-definitions#tracking","fieldsOfResearch":"https://language-archives.services/metadata-definitions#fieldsOfResearch","role":"https://language-archives.services/metadata-definitions#role","doi":"https://language-archives.services/metadata-definitions#doi","sampleRate":"https://language-archives.services/metadata-definitions#sampleRate","channels":"https://language-archives.services/metadata-definitions#channels","fps":"https://language-archives.services/metadata-definitions#fps","essenceId":"https://language-archives.services/metadata-definitions#essenceId"}],"@graph":[{"@id":"#Erakor village","@type":"Place","name":"Erakor village"},{"@id":"#country_Australia","@type":"Country","name":"Australia"},{"@id":"#country_Vanuatu","@type":"Country","code":"VU","name":"Vanuatu"},{"@id":"#country_null","@type":"Country"},{"@id":"#geo_166.427,-22.283,166.467,-22.241","@type":"GeoShape","box":"166.427,-22.283 166.467,-22.241"},{"@id":"#geo_168.159,-17.83,168.594,-17.585","@type":"GeoShape","box":"168.159,-17.83 168.594,-17.585"},{"@id":"#geo_168.217,-17.8235,168.317,-17.7235","@type":"GeoShape","box":"168.217,-17.8235 168.317,-17.7235"},{"@id":"#identifier_collectionId","@type":"PropertyValue","name":"collectionIdentifier","value":"NT1"},{"@id":"#identifier_doi","@type":"PropertyValue","name":"doi","value":"10.4225/72/56F94A61DA9EC"},{"@id":"#identifier_domain","@type":"PropertyValue","name":"domain","value":"paradisec.org.au"},{"@id":"#identifier_hashid","@type":"PropertyValue","name":"hashId","value":"72b3dc1401c8ff06aacba0990a128fc113cf9ad5275f494b05c1142177356561bd7f4c0e8800bade2cbbbed75f6d9d019894735ad7e40762684d243a442d658a"},{"@id":"#identifier_id","@type":"PropertyValue","name":"id","value":"/paradisec.org.au/NT1/98007"},{"@id":"#identifier_itemId","@type":"PropertyValue","name":"itemIdentifier","value":"98007"},{"@id":"#language_bis","@type":"Language","code":"bis","location":{"@id":"#geo_166.427,-22.283,166.467,-22.241"},"name":"Bislama"},{"@id":"#language_erk","@type":"Language","code":"erk","location":{"@id":"#geo_168.159,-17.83,168.594,-17.585"},"name":"Efate, South"},{"@id":"#place_geo_168.217,-17.8235,168.317,-17.7235","@type":"Place","geo":{"@id":"#geo_168.217,-17.8235,168.317,-17.7235"}},{"@id":"./","@type":["Dataset","RepositoryObject"],"additionalType":"item","contentLocation":[{"@id":"#Erakor village"},{"@id":"#place_geo_168.217,-17.8235,168.317,-17.7235"}],"contributor":[{"@id":"http://nla.gov.au/nla.party-479603"},{"@id":"Kalsarap Namaf"},{"@id":"Iokopeth null"},{"@id":"John Maklen"},{"@id":"Waia Tenene"}],"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2018-05-17T04:13:04.000Z","description":"NT1-98007. Text #047 (speaker is John Maklen. Text title: History of villages before Erakor); Text #048 (speaker is John Maklen. Text title: Mantu the flying fox and Erromango); Text #049. Text title: Asaraf (speaker is John Maklen);Text #050. Text title: Mumu and Kotkot (speaker is John Maklen); Text #051. Text title: Natopu ni Erakor—the spirit who lives at Erakor (speaker is John Maklen);Text #038. Text title: The need for respect (speaker is Iokopeth) Stories can be seen at NT8-TEXT. There are time-aligned transcripts of this item and handwritten transcripts by Manuel Wayane scanned as jpg files.","hasPart":[{"@id":"NT1-98007-001.jpg"},{"@id":"NT1-98007-002.jpg"},{"@id":"NT1-98007-003.jpg"},{"@id":"NT1-98007-004.jpg"},{"@id":"NT1-98007-005.jpg"},{"@id":"NT1-98007-006.jpg"},{"@id":"NT1-98007-007.jpg"},{"@id":"NT1-98007-008.jpg"},{"@id":"NT1-98007-009.jpg"},{"@id":"NT1-98007-010.jpg"},{"@id":"NT1-98007-011.jpg"},{"@id":"NT1-98007-012.jpg"},{"@id":"NT1-98007-013.jpg"},{"@id":"NT1-98007-014.jpg"},{"@id":"NT1-98007-015.jpg"},{"@id":"NT1-98007-016.jpg"},{"@id":"NT1-98007-017.jpg"},{"@id":"NT1-98007-018.jpg"},{"@id":"NT1-98007-019.jpg"},{"@id":"NT1-98007-020.jpg"},{"@id":"NT1-98007-021.jpg"},{"@id":"NT1-98007-022.jpg"},{"@id":"NT1-98007-023.jpg"},{"@id":"NT1-98007-024.jpg"},{"@id":"NT1-98007-025.jpg"},{"@id":"NT1-98007-026.jpg"},{"@id":"NT1-98007-027.jpg"},{"@id":"NT1-98007-028.jpg"},{"@id":"NT1-98007-029.jpg"},{"@id":"NT1-98007-030.jpg"},{"@id":"NT1-98007-031.jpg"},{"@id":"NT1-98007-98007A.mp3"},{"@id":"NT1-98007-98007A.wav"},{"@id":"NT1-98007-98007B.mp3"},{"@id":"NT1-98007-98007B.wav"},{"@id":"NT1-98007-98007az.xml"},{"@id":"NT1-98007-A.tab"},{"@id":"NT1-98007-A.xml"},{"@id":"NT1-98007-B.tab"},{"@id":"NT1-98007-B.xml"},{"@id":"NT1-98007-98007A.ixt"},{"@id":"NT1-98007-98007A.trs"},{"@id":"NT1-98007-98007A.flextext"},{"@id":"NT1-98007-98007A.eaf"},{"@id":"NT1-98007-98007B.eaf"}],"identifier":[{"@id":"#identifier_domain"},{"@id":"#identifier_id"},{"@id":"#identifier_hashid"},{"@id":"#identifier_itemId"},{"@id":"#identifier_collectionId"},{"@id":"#identifier_doi"}],"license":{"@id":"_:b0"},"memberOf":{"@id":"/paradisec.org.au/NT1"},"name":"Recordings in South Efate","publisher":{"@id":"http://nla.gov.au/nla.party-593909"},"bornDigital":0,"contentLanguages":[{"@id":"#language_bis"},{"@id":"#language_erk"}],"countries":{"@id":"#country_Vanuatu"},"digitisedOn":"Sun Dec 31 2000 13:00:00 GMT+0000 (Coordinated Universal Time)","external":0,"languageAsGiven":"Nafsan","metadataExportable":1,"originalMedia":"audiocassette","originatedOn":"1998-10-03","private":0,"subjectLanguages":{"@id":"#language_erk"},"tapesReturned":0},{"@id":"Iokopeth null","@type":"Person","givenName":"Iokopeth","homeLocation":{"@id":"#country_null"},"name":"Iokopeth","role":{"@id":"role_speaker"}},{"@id":"John Maklen","@type":"Person","familyName":"Maklen","givenName":"John","homeLocation":{"@id":"#country_null"},"name":"John Maklen","role":{"@id":"role_speaker"}},{"@id":"Kalsarap Namaf","@type":"Person","familyName":"Namaf","givenName":"Kalsarap","homeLocation":{"@id":"#country_null"},"name":"Kalsarap Namaf","role":{"@id":"role_speaker"}},{"@id":"NT1-98007-001.jpg","@type":"File","contentSize":1658368,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:01:31.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-001.jpg","doi":"10.4225/72/575C8A369D680","essenceId":1010709},{"@id":"NT1-98007-002.jpg","@type":"File","contentSize":1816576,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:01:37.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-002.jpg","doi":"10.4225/72/575C8A3C15C98","essenceId":1010710},{"@id":"NT1-98007-003.jpg","@type":"File","contentSize":1811968,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:01:43.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-003.jpg","doi":"10.4225/72/575C8A41DD973","essenceId":1010711},{"@id":"NT1-98007-004.jpg","@type":"File","contentSize":1827840,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:01:48.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-004.jpg","doi":"10.4225/72/575C8A4767685","essenceId":1010712},{"@id":"NT1-98007-005.jpg","@type":"File","contentSize":1853440,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:01:54.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-005.jpg","doi":"10.4225/72/575C8A4CD8971","essenceId":1010713},{"@id":"NT1-98007-006.jpg","@type":"File","contentSize":1796608,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:01:59.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-006.jpg","doi":"10.4225/72/575C8A525C618","essenceId":1010714},{"@id":"NT1-98007-007.jpg","@type":"File","contentSize":1780224,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:02:05.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-007.jpg","doi":"10.4225/72/575C8A5810189","essenceId":1010715},{"@id":"NT1-98007-008.jpg","@type":"File","contentSize":1737728,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:02:11.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-008.jpg","doi":"10.4225/72/575C8A5DB1113","essenceId":1010716},{"@id":"NT1-98007-009.jpg","@type":"File","contentSize":1781760,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:02:16.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-009.jpg","doi":"10.4225/72/575C8A63479C1","essenceId":1010717},{"@id":"NT1-98007-010.jpg","@type":"File","contentSize":1797632,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:02:22.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-010.jpg","doi":"10.4225/72/575C8A68B23D2","essenceId":1010718},{"@id":"NT1-98007-011.jpg","@type":"File","contentSize":1800704,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:02:28.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-011.jpg","doi":"10.4225/72/575C8A6E73D01","essenceId":1010719},{"@id":"NT1-98007-012.jpg","@type":"File","contentSize":1822720,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:02:33.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-012.jpg","doi":"10.4225/72/575C8A742DE00","essenceId":1010720},{"@id":"NT1-98007-013.jpg","@type":"File","contentSize":1809920,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:02:39.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-013.jpg","doi":"10.4225/72/575C8A79B1B0F","essenceId":1010721},{"@id":"NT1-98007-014.jpg","@type":"File","contentSize":1821696,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:02:44.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-014.jpg","doi":"10.4225/72/575C8A7F3F253","essenceId":1010722},{"@id":"NT1-98007-015.jpg","@type":"File","contentSize":1626624,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:02:50.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-015.jpg","doi":"10.4225/72/575C8A84C0022","essenceId":1010723},{"@id":"NT1-98007-016.jpg","@type":"File","contentSize":1633792,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:02:56.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-016.jpg","doi":"10.4225/72/575C8A8A9A944","essenceId":1010724},{"@id":"NT1-98007-017.jpg","@type":"File","contentSize":1870336,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:03:01.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-017.jpg","doi":"10.4225/72/575C8A90506E6","essenceId":1010725},{"@id":"NT1-98007-018.jpg","@type":"File","contentSize":1858560,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:03:07.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-018.jpg","doi":"10.4225/72/575C8A95DF462","essenceId":1010726},{"@id":"NT1-98007-019.jpg","@type":"File","contentSize":1852416,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:03:12.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-019.jpg","doi":"10.4225/72/575C8A9B56F2F","essenceId":1010727},{"@id":"NT1-98007-020.jpg","@type":"File","contentSize":1838080,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:03:18.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-020.jpg","doi":"10.4225/72/575C8AA0F09B8","essenceId":1010728},{"@id":"NT1-98007-021.jpg","@type":"File","contentSize":1861120,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:03:24.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-021.jpg","doi":"10.4225/72/575C8AA6B23AF","essenceId":1010729},{"@id":"NT1-98007-022.jpg","@type":"File","contentSize":1835008,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:03:29.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-022.jpg","doi":"10.4225/72/575C8AAC3A545","essenceId":1010730},{"@id":"NT1-98007-023.jpg","@type":"File","contentSize":1827328,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:03:35.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-023.jpg","doi":"10.4225/72/575C8AB1C991E","essenceId":1010731},{"@id":"NT1-98007-024.jpg","@type":"File","contentSize":1805312,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:03:40.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-024.jpg","doi":"10.4225/72/575C8AB74847D","essenceId":1010732},{"@id":"NT1-98007-025.jpg","@type":"File","contentSize":1912832,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:03:46.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-025.jpg","doi":"10.4225/72/575C8ABCD2B44","essenceId":1010733},{"@id":"NT1-98007-026.jpg","@type":"File","contentSize":1889792,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:03:51.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-026.jpg","doi":"10.4225/72/575C8AC26D69E","essenceId":1010734},{"@id":"NT1-98007-027.jpg","@type":"File","contentSize":1878528,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:03:57.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-027.jpg","doi":"10.4225/72/575C8AC7F3886","essenceId":1010735},{"@id":"NT1-98007-028.jpg","@type":"File","contentSize":1868288,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:04:02.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-028.jpg","doi":"10.4225/72/575C8ACD72196","essenceId":1010736},{"@id":"NT1-98007-029.jpg","@type":"File","contentSize":1859584,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:04:08.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-029.jpg","doi":"10.4225/72/575C8AD2E8E82","essenceId":1010737},{"@id":"NT1-98007-030.jpg","@type":"File","contentSize":1859072,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:04:13.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-030.jpg","doi":"10.4225/72/575C8AD8775E6","essenceId":1010738},{"@id":"NT1-98007-031.jpg","@type":"File","contentSize":1708544,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:04:19.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-031.jpg","doi":"10.4225/72/575C8ADDE64B8","essenceId":1010739},{"@id":"NT1-98007-98007A.eaf","@type":"File","contentSize":165674,"dateCreated":"2016-08-01T05:00:06.000Z","dateModified":"2016-08-01T16:01:41.000Z","encodingFormat":"application/xml","name":"NT1-98007-98007A.eaf","doi":"10.4225/72/579F725FDD059","essenceId":1100487},{"@id":"NT1-98007-98007A.flextext","@type":"File","contentSize":141244,"dateCreated":"2016-05-20T04:00:06.000Z","dateModified":"2016-06-24T12:41:36.000Z","encodingFormat":"application/xml","name":"NT1-98007-98007A.flextext","doi":"10.4225/72/576D2A7B75614","essenceId":1086277},{"@id":"NT1-98007-98007A.ixt","@type":"File","contentSize":40299,"dateCreated":"2016-04-18T07:00:07.000Z","dateModified":"2016-06-24T08:25:06.000Z","encodingFormat":"application/xml","name":"NT1-98007-98007A.ixt","doi":"10.4225/72/576CEE5CED1FB","essenceId":1085095},{"@id":"NT1-98007-98007A.mp3","@type":"File","bitrate":128009,"contentSize":43667584,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2019-11-29T09:17:34.000Z","duration":2729.02,"encodingFormat":"audio/mpeg","name":"NT1-98007-98007A.mp3","channels":2,"doi":"10.4225/72/575C8AE370B93","essenceId":1010740,"sampleRate":44100},{"@id":"NT1-98007-98007A.trs","@type":"File","contentSize":28292,"dateCreated":"2016-04-26T10:00:06.000Z","dateModified":"2016-06-24T08:45:27.000Z","encodingFormat":"application/xml","name":"NT1-98007-98007A.trs","doi":"10.4225/72/576CF32101764","essenceId":1085293},{"@id":"NT1-98007-98007A.wav","@type":"File","bitrate":4608000,"contentSize":1571894006,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2019-11-29T09:16:52.000Z","duration":2728.98,"encodingFormat":"audio/x-wav","name":"NT1-98007-98007A.wav","channels":2,"doi":"10.4225/72/575C8AE8E6E6B","essenceId":1010741,"sampleRate":96000},{"@id":"NT1-98007-98007B.eaf","@type":"File","contentSize":118748,"dateCreated":"2016-08-01T05:00:07.000Z","dateModified":"2016-08-01T16:01:47.000Z","encodingFormat":"application/xml","name":"NT1-98007-98007B.eaf","doi":"10.4225/72/579F7265746C0","essenceId":1100493},{"@id":"NT1-98007-98007B.mp3","@type":"File","bitrate":128007,"contentSize":35305600,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2019-11-29T09:12:58.000Z","duration":2206.47,"encodingFormat":"audio/mpeg","name":"NT1-98007-98007B.mp3","channels":2,"doi":"10.4225/72/575C8AEE64BCA","essenceId":1010742,"sampleRate":44100},{"@id":"NT1-98007-98007B.wav","@type":"File","bitrate":4608000,"contentSize":1270917002,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2019-11-29T09:15:22.000Z","duration":2206.45,"encodingFormat":"audio/x-wav","name":"NT1-98007-98007B.wav","channels":2,"doi":"10.4225/72/575C8AF3D2DA0","essenceId":1010743,"sampleRate":96000},{"@id":"NT1-98007-98007az.xml","@type":"File","contentSize":48755,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:04:46.000Z","encodingFormat":"application/xml","name":"NT1-98007-98007az.xml","doi":"10.4225/72/575C8AF94896E","essenceId":1010744},{"@id":"NT1-98007-A.tab","@type":"File","contentSize":27810,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:04:57.000Z","encodingFormat":"text/plain","name":"NT1-98007-A.tab","doi":"10.4225/72/575C8B043764B","essenceId":1010746},{"@id":"NT1-98007-A.xml","@type":"File","contentSize":48788,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:05:03.000Z","encodingFormat":"application/xml","name":"NT1-98007-A.xml","doi":"10.4225/72/575C8B09BDC88","essenceId":1010747},{"@id":"NT1-98007-B.tab","@type":"File","contentSize":20239,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:05:08.000Z","encodingFormat":"text/plain","name":"NT1-98007-B.tab","doi":"10.4225/72/575C8B0F4F8F1","essenceId":1010748},{"@id":"NT1-98007-B.xml","@type":"File","contentSize":35289,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:05:14.000Z","encodingFormat":"application/xml","name":"NT1-98007-B.xml","doi":"10.4225/72/575C8B14C6702","essenceId":1010749},{"@id":"Waia Tenene","@type":"Person","familyName":"Tenene","givenName":"Waia","homeLocation":{"@id":"#country_null"},"name":"Waia Tenene","role":{"@id":"role_speaker"}},{"@id":"_:b0","@type":"CreativeWork","name":"Open (subject to agreeing to PDSC access conditions)"},{"@id":"http://nla.gov.au/nla.party-479603","@type":"Person","email":"thien@unimelb.edu.au","familyName":"Thieberger","givenName":"Nick","homeLocation":{"@id":"#country_Australia"},"name":"Nick Thieberger","role":[{"@id":"role_collector"},{"@id":"role_depositor"},{"@id":"role_recorder"}]},{"@id":"http://nla.gov.au/nla.party-593909","@type":"Organization","name":"University of Melbourne"},{"@id":"ro-crate-metadata.json","@type":"CreativeWork","conformsTo":{"@id":"https://w3id.org/ro/crate/1.1/context"},"about":{"@id":"./"}},{"@id":"role_collector","@type":"Role","name":"collector"},{"@id":"role_depositor","@type":"Role","name":"depositor"},{"@id":"role_recorder","@type":"Role","name":"recorder"},{"@id":"role_speaker","@type":"Role","name":"speaker"}]} From 963a00e96f71936903809e0aa6eb9deb3d312def Mon Sep 17 00:00:00 2001 From: Ben Companjen Date: Thu, 2 Nov 2023 10:22:36 +0100 Subject: [PATCH 030/970] Return valid JSON from API blocking filters --- .../edu/harvard/iq/dataverse/api/ApiBlockingFilter.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/ApiBlockingFilter.java b/src/main/java/edu/harvard/iq/dataverse/api/ApiBlockingFilter.java index 0e5b8226310..b51b1aa2612 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/ApiBlockingFilter.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/ApiBlockingFilter.java @@ -49,7 +49,7 @@ public void doBlock(ServletRequest sr, ServletResponse sr1, FilterChain fc) thro @Override public void doBlock(ServletRequest sr, ServletResponse sr1, FilterChain fc) throws IOException, ServletException { HttpServletResponse httpResponse = (HttpServletResponse) sr1; - httpResponse.getWriter().println("{ status:\"error\", message:\"Endpoint blocked. Please contact the dataverse administrator\"}" ); + httpResponse.getWriter().println("{ \"status\":\"error\", \"message\":\"Endpoint blocked. Please contact the dataverse administrator\"}" ); httpResponse.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); httpResponse.setContentType("application/json"); } @@ -67,7 +67,7 @@ public void doBlock(ServletRequest sr, ServletResponse sr1, FilterChain fc) thro fc.doFilter(sr, sr1); } else { HttpServletResponse httpResponse = (HttpServletResponse) sr1; - httpResponse.getWriter().println("{ status:\"error\", message:\"Endpoint available from localhost only. Please contact the dataverse administrator\"}" ); + httpResponse.getWriter().println("{ \"status\":\"error\", \"message\":\"Endpoint available from localhost only. Please contact the dataverse administrator\"}" ); httpResponse.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); httpResponse.setContentType("application/json"); } @@ -102,7 +102,7 @@ public void doBlock(ServletRequest sr, ServletResponse sr1, FilterChain fc) thro if ( block ) { HttpServletResponse httpResponse = (HttpServletResponse) sr1; - httpResponse.getWriter().println("{ status:\"error\", message:\"Endpoint available using API key only. Please contact the dataverse administrator\"}" ); + httpResponse.getWriter().println("{ \"status\":\"error\", \"message\":\"Endpoint available using API key only. Please contact the dataverse administrator\"}" ); httpResponse.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); httpResponse.setContentType("application/json"); } else { From cd154813eea9a4f937d6b71645097f3d04867275 Mon Sep 17 00:00:00 2001 From: Ben Companjen Date: Thu, 2 Nov 2023 23:52:49 +0100 Subject: [PATCH 031/970] Add PR release notes --- doc/release-notes/api-blocking-filter-json.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/release-notes/api-blocking-filter-json.md diff --git a/doc/release-notes/api-blocking-filter-json.md b/doc/release-notes/api-blocking-filter-json.md new file mode 100644 index 00000000000..337ff82dd8b --- /dev/null +++ b/doc/release-notes/api-blocking-filter-json.md @@ -0,0 +1,3 @@ +* When any `ApiBlockingFilter` policy applies to a request, the JSON in the body of the error response is now valid JSON. + In case an API client did any special processing to allow it to parse the body, that is no longer necessary. + The status code of such responses has not changed. From 3764bb2358aac756ed7bf97e052441836d5512e8 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 8 Nov 2023 14:34:48 -0500 Subject: [PATCH 032/970] fix guestbook preview stop refresh of datasetForm --- src/main/webapp/dataset-license-terms.xhtml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/webapp/dataset-license-terms.xhtml b/src/main/webapp/dataset-license-terms.xhtml index c54d94442ea..6d5b0a5fe4f 100644 --- a/src/main/webapp/dataset-license-terms.xhtml +++ b/src/main/webapp/dataset-license-terms.xhtml @@ -482,7 +482,7 @@
- #{bundle['file.dataFilesTab.terms.list.guestbook']}   + #{bundle['file.dataFilesTab.terms.list.guestbook']}  
@@ -524,7 +524,7 @@ + update=":datasetForm:previewGuestbook" oncomplete="PF('viewGuestbook').show();"/>
@@ -563,7 +563,7 @@ + update=":datasetForm:previewGuestbook" oncomplete="PF('viewGuestbook').show();"/> From 0ead2c24ff7855396f83f07e724af7ce4b18f7be Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Fri, 1 Dec 2023 17:24:40 +0100 Subject: [PATCH 033/970] incmoplete metadata label visibility setting --- .../edu/harvard/iq/dataverse/DatasetPage.java | 10 ++++----- .../edu/harvard/iq/dataverse/FilePage.java | 18 ++++++++++----- .../iq/dataverse/mydata/DataRetrieverAPI.java | 17 ++++++++++++-- .../search/SearchIncludeFragment.java | 11 +++++++--- .../iq/dataverse/search/SolrSearchResult.java | 22 +++++++++++++++---- .../iq/dataverse/settings/JvmSettings.java | 1 + src/main/webapp/file.xhtml | 2 +- 7 files changed, 61 insertions(+), 20 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index fc18257196d..7ab24e69152 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -2210,13 +2210,13 @@ private void displayPublishMessage(){ public boolean isValid() { if (valid == null) { - DatasetVersion version = dataset.getLatestVersion(); - if (!version.isDraft()) { + if (workingVersion.isDraft() || (canUpdateDataset() && JvmSettings.UI_SHOW_VALIDITY_LABEL.lookupOptional(Boolean.class).orElse(false))) { + final DatasetVersion newVersion = workingVersion.cloneDatasetVersion(); + newVersion.setDatasetFields(newVersion.initDatasetFields()); + valid = newVersion.isValid(); + } else { valid = true; } - DatasetVersion newVersion = version.cloneDatasetVersion(); - newVersion.setDatasetFields(newVersion.initDatasetFields()); - valid = newVersion.isValid(); } return valid; } diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index bfae80ade27..d9a0fe2d9b0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -31,6 +31,7 @@ import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean.MakeDataCountEntry; import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.FileUtil; @@ -283,13 +284,20 @@ private void displayPublishMessage(){ } } + Boolean valid = null; + public boolean isValid() { - if (!fileMetadata.getDatasetVersion().isDraft()) { - return true; + if (valid == null) { + final DatasetVersion workingVersion = fileMetadata.getDatasetVersion(); + if (workingVersion.isDraft() || (canUpdateDataset() && JvmSettings.UI_SHOW_VALIDITY_LABEL.lookupOptional(Boolean.class).orElse(false))) { + final DatasetVersion newVersion = workingVersion.cloneDatasetVersion(); + newVersion.setDatasetFields(newVersion.initDatasetFields()); + valid = newVersion.isValid(); + } else { + valid = true; + } } - DatasetVersion newVersion = fileMetadata.getDatasetVersion().cloneDatasetVersion(); - newVersion.setDatasetFields(newVersion.initDatasetFields()); - return newVersion.isValid(); + return valid; } private boolean canViewUnpublishedDataset() { diff --git a/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java b/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java index e9898031343..be81de9267d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java +++ b/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java @@ -3,11 +3,13 @@ */ package edu.harvard.iq.dataverse.mydata; +import edu.harvard.iq.dataverse.DatasetServiceBean; import edu.harvard.iq.dataverse.DataverseRoleServiceBean; import edu.harvard.iq.dataverse.DataverseServiceBean; import edu.harvard.iq.dataverse.DataverseSession; import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.DvObjectServiceBean; +import edu.harvard.iq.dataverse.PermissionsWrapper; import edu.harvard.iq.dataverse.RoleAssigneeServiceBean; import edu.harvard.iq.dataverse.api.auth.AuthRequired; import edu.harvard.iq.dataverse.search.SearchServiceBean; @@ -25,6 +27,7 @@ import edu.harvard.iq.dataverse.search.SearchException; import edu.harvard.iq.dataverse.search.SearchFields; import edu.harvard.iq.dataverse.search.SortBy; +import edu.harvard.iq.dataverse.settings.JvmSettings; import java.util.List; import java.util.Map; @@ -64,7 +67,9 @@ public class DataRetrieverAPI extends AbstractApiBean { private static final String retrieveDataPartialAPIPath = "retrieve"; @Inject - DataverseSession session; + DataverseSession session; + @Inject + PermissionsWrapper permissionsWrapper; @EJB DataverseRoleServiceBean dataverseRoleService; @@ -82,6 +87,8 @@ public class DataRetrieverAPI extends AbstractApiBean { //MyDataQueryHelperServiceBean myDataQueryHelperServiceBean; @EJB GroupServiceBean groupService; + @EJB + DatasetServiceBean datasetService; private List roleList; private DataverseRolePermissionHelper rolePermissionHelper; @@ -482,12 +489,14 @@ private JsonArrayBuilder formatSolrDocs(SolrQueryResponse solrResponse, RoleTagR JsonObjectBuilder myDataCardInfo; JsonArrayBuilder rolesForCard; + DataverseRequest dataverseRequest = createDataverseRequest(authUser); for (SolrSearchResult doc : solrQueryResponse.getSolrSearchResults()){ // ------------------------------------------- // (a) Get core card data from solr // ------------------------------------------- - myDataCardInfo = doc.getJsonForMyData(); + + myDataCardInfo = doc.getJsonForMyData(isValid(doc, dataverseRequest)); if (!doc.getEntity().isInstanceofDataFile()){ String parentAlias = dataverseService.getParentAliasString(doc); @@ -510,4 +519,8 @@ private JsonArrayBuilder formatSolrDocs(SolrQueryResponse solrResponse, RoleTagR return jsonSolrDocsArrayBuilder; } + + private boolean isValid(SolrSearchResult result, DataverseRequest dataverseRequest) { + return result.isValid(x -> permissionsWrapper.canUpdateDataset(dataverseRequest, datasetService.find(x.getEntityId()))); + } } \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java index 2ce06541afa..d0100bd79a4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java @@ -22,6 +22,7 @@ import edu.harvard.iq.dataverse.ThumbnailServiceWrapper; import edu.harvard.iq.dataverse.WidgetWrapper; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.BundleUtil; import java.time.LocalDate; import java.util.ArrayList; @@ -351,8 +352,7 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused * https://github.com/IQSS/dataverse/issues/84 */ int numRows = 10; - HttpServletRequest httpServletRequest = (HttpServletRequest) FacesContext.getCurrentInstance().getExternalContext().getRequest(); - DataverseRequest dataverseRequest = new DataverseRequest(session.getUser(), httpServletRequest); + DataverseRequest dataverseRequest = getDataverseRequest(); List dataverses = new ArrayList<>(); dataverses.add(dataverse); solrQueryResponse = searchService.search(dataverseRequest, dataverses, queryToPassToSolr, filterQueriesFinal, sortField, sortOrder.toString(), paginationStart, onlyDataRelatedToMe, numRows, false, null, null); @@ -1395,8 +1395,13 @@ public boolean isActivelyEmbargoed(SolrSearchResult result) { } } + private DataverseRequest getDataverseRequest() { + final HttpServletRequest httpServletRequest = (HttpServletRequest) FacesContext.getCurrentInstance().getExternalContext().getRequest(); + return new DataverseRequest(session.getUser(), httpServletRequest); + } + public boolean isValid(SolrSearchResult result) { - return result.isValid(); + return result.isValid(x -> permissionsWrapper.canUpdateDataset(getDataverseRequest(), datasetService.find(x.getEntityId()))); } public enum SortOrder { diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java index 6ad7f9dbbf6..72af8b158dc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java @@ -7,6 +7,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Predicate; import java.util.logging.Logger; import jakarta.json.Json; @@ -26,6 +27,7 @@ import edu.harvard.iq.dataverse.MetadataBlock; import edu.harvard.iq.dataverse.api.Util; import edu.harvard.iq.dataverse.dataset.DatasetThumbnail; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.DateUtil; import edu.harvard.iq.dataverse.util.json.JsonPrinter; import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; @@ -417,7 +419,7 @@ public JsonObject toJsonObject(boolean showRelevance, boolean showEntityIds, boo * * @return */ - public JsonObjectBuilder getJsonForMyData() { + public JsonObjectBuilder getJsonForMyData(boolean isValid) { JsonObjectBuilder myDataJson = json(true, true, true);// boolean showRelevance, boolean showEntityIds, boolean showApiUrls) @@ -425,7 +427,7 @@ public JsonObjectBuilder getJsonForMyData() { .add("is_draft_state", this.isDraftState()).add("is_in_review_state", this.isInReviewState()) .add("is_unpublished_state", this.isUnpublishedState()).add("is_published", this.isPublishedState()) .add("is_deaccesioned", this.isDeaccessionedState()) - .add("is_valid", this.isValid()) + .add("is_valid", isValid) .add("date_to_display_on_card", getDateToDisplayOnCard()); // Add is_deaccessioned attribute, even though MyData currently screens any deaccessioned info out @@ -1265,7 +1267,19 @@ public void setDatasetValid(Boolean datasetValid) { this.datasetValid = datasetValid == null || Boolean.valueOf(datasetValid); } - public boolean isValid() { - return datasetValid; + public boolean isValid(Predicate canUpdateDataset) { + if (this.datasetValid) { + return true; + } + if (!this.getType().equals("datasets")) { + return true; + } + if (this.isDraftState()) { + return false; + } + if (!JvmSettings.UI_SHOW_VALIDITY_LABEL.lookupOptional(Boolean.class).orElse(false)) { + return true; + } + return !canUpdateDataset.test(this); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java index cc3272413c7..9f0371178c0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java @@ -145,6 +145,7 @@ public enum JvmSettings { SCOPE_UI(PREFIX, "ui"), UI_ALLOW_REVIEW_INCOMPLETE(SCOPE_UI, "allow-review-for-incomplete"), UI_SHOW_VALIDITY_FILTER(SCOPE_UI, "show-validity-filter"), + UI_SHOW_VALIDITY_LABEL(SCOPE_UI, "show-label-for-incomplete-when-published"), // NetCDF SETTINGS SCOPE_NETCDF(PREFIX, "netcdf"), diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml index f69b5c35afd..a5218aedaa6 100644 --- a/src/main/webapp/file.xhtml +++ b/src/main/webapp/file.xhtml @@ -76,7 +76,7 @@ - + From 0cd23fbffb0d3c4a75cb3948333d4c71e7e5110e Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Tue, 5 Dec 2023 18:28:26 +0100 Subject: [PATCH 034/970] added documentation --- ...10116-incomplete-metadata-label-setting.md | 1 + .../source/installation/config.rst | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 doc/release-notes/10116-incomplete-metadata-label-setting.md diff --git a/doc/release-notes/10116-incomplete-metadata-label-setting.md b/doc/release-notes/10116-incomplete-metadata-label-setting.md new file mode 100644 index 00000000000..5fd2de2f287 --- /dev/null +++ b/doc/release-notes/10116-incomplete-metadata-label-setting.md @@ -0,0 +1 @@ +Bug fixed for the ``incomplete metadata`` label being shown for published dataset with incomplete metadata in certain scenarios. This label will now only be shown for draft versions of such datasets. This label can also be made visible for published datasets with the new option ``dataverse.api.show-label-for-incomplete-when-published``, and will be only shown on datasets with incomplete metadata that the logged-in user can edit. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index c707daaad07..bc1f01576c9 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2425,6 +2425,25 @@ Defaults to ``false``. Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_API_ALLOW_INCOMPLETE_METADATA``. Will accept ``[tT][rR][uU][eE]|1|[oO][nN]`` as "true" expressions. +.. _dataverse.api.show-label-for-incomplete-when-published: + +dataverse.api.show-label-for-incomplete-when-published +++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +Even when you do not allow incomplete metadata to be saved in dataverse, some metadata may end up being incomplete, e.g., after making a metadata field mandatory. Datasets where that field is +not filled out, become incomplete, and therefore can be labeled with the ``incomplete metadat`` label. By default, this label is only shown for draft datasets. However, in some situations, you +may want to show the label also for published datasets. For example, when an admin needs to have an overview of the datasets that are no longer valid after a metadata fields update, +enabling this option will show the ``incomplete metadata`` labels on published datasets, but only to the users that can edit the specific datasets. Note that you need to reindex the datasets +after changing the metadata definitions. Reindexing will update the labels and other dataset information according to the new situation. + +When enabled, published datasets with incomplete metadata will have an ``incomplete metadata`` label attached to them, but only for the datasets that the user can edit. You can list these datasets, +for example, with the validity of metadata filter shown in "My Data" page that can be turned on by enabling the :ref:`dataverse.ui.show-validity-filter` option. + +Defaults to ``false``. + +Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable +``DATAVERSE_API_SHOW_LABEL_FOR_INCOMPLETE_WHEN_PUBLISHED``. Will accept ``[tT][rR][uU][eE]|1|[oO][nN]`` as "true" expressions. + .. _dataverse.signposting.level1-author-limit: dataverse.signposting.level1-author-limit @@ -2475,6 +2494,8 @@ Defaults to ``false``. Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_UI_ALLOW_REVIEW_FOR_INCOMPLETE``. Will accept ``[tT][rR][uU][eE]|1|[oO][nN]`` as "true" expressions. +.. _dataverse.ui.show-validity-filter: + dataverse.ui.show-validity-filter +++++++++++++++++++++++++++++++++ From 542a19824aaec5a24d4fd000c61c01c9629db08d Mon Sep 17 00:00:00 2001 From: Ludovic DANIEL Date: Wed, 6 Dec 2023 17:43:21 +0100 Subject: [PATCH 035/970] Fixed forgotten code to make localeStrValue of CVV depends on :MetadataLanguages of the dataset --- src/main/webapp/metadataFragment.xhtml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index 200d2917b9a..442ee5bc287 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -292,7 +292,7 @@ @@ -368,12 +368,12 @@ - + @@ -387,7 +387,7 @@ rendered="#{subdsf.datasetFieldType.allowMultiples}" label="#{bundle.select}" multiple="true" filter="#{(subdsf.datasetFieldType.controlledVocabularyValues.size() lt 10) ? 'false':'true'}" filterMatchMode="contains" showHeader="#{(subdsf.datasetFieldType.controlledVocabularyValues.size() lt 10) ? 'false':'true'}"> - +
From 02f2edc2fddd3cf0c39ac6bbdb4381ff0b7ae744 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Thu, 7 Dec 2023 15:48:24 +0100 Subject: [PATCH 036/970] typo fix --- doc/sphinx-guides/source/installation/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index bc1f01576c9..46a83610871 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2431,7 +2431,7 @@ dataverse.api.show-label-for-incomplete-when-published ++++++++++++++++++++++++++++++++++++++++++++++++++++++ Even when you do not allow incomplete metadata to be saved in dataverse, some metadata may end up being incomplete, e.g., after making a metadata field mandatory. Datasets where that field is -not filled out, become incomplete, and therefore can be labeled with the ``incomplete metadat`` label. By default, this label is only shown for draft datasets. However, in some situations, you +not filled out, become incomplete, and therefore can be labeled with the ``incomplete metadata`` label. By default, this label is only shown for draft datasets. However, in some situations, you may want to show the label also for published datasets. For example, when an admin needs to have an overview of the datasets that are no longer valid after a metadata fields update, enabling this option will show the ``incomplete metadata`` labels on published datasets, but only to the users that can edit the specific datasets. Note that you need to reindex the datasets after changing the metadata definitions. Reindexing will update the labels and other dataset information according to the new situation. From a83c5df8e6f768f13f647622dee2d2ed43099ed0 Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Tue, 12 Dec 2023 15:06:00 -0500 Subject: [PATCH 037/970] #10175 docs: correct TRS JSON quotes --- doc/sphinx-guides/source/developers/big-data-support.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index 8d891e63317..ccdaf8940f2 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -116,7 +116,7 @@ The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.Data export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV - export JSON_DATA="{'description':'My description.','directoryLabel':'data/subdir1','categories':['Data'], 'restrict':'false', 'storageIdentifier':'trs://images/dataverse_project_logo.svg', 'fileName':'dataverse_logo.svg', 'mimeType':'image/svg+xml', 'checksum': {'@type': 'SHA-1', '@value': '123456'}}" + export JSON_DATA='{"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"trs://images/dataverse_project_logo.svg", "fileName":"dataverse_logo.svg", "mimeType":"image/svg+xml", "checksum": {"@type": "SHA-1", "@value": "123456"}}' curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/add?persistentId=$PERSISTENT_IDENTIFIER" -F "jsonData=$JSON_DATA" From f6e5db29f0977187128d99f42b32c08c53af5e74 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Tue, 19 Dec 2023 16:42:44 +0100 Subject: [PATCH 038/970] option renamed: show-label-for-incomplete-when-published -> show-validity-label-when-published --- .../10116-incomplete-metadata-label-setting.md | 2 +- doc/sphinx-guides/source/installation/config.rst | 6 +++--- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 2 +- src/main/java/edu/harvard/iq/dataverse/FilePage.java | 2 +- .../edu/harvard/iq/dataverse/search/SolrSearchResult.java | 2 +- .../java/edu/harvard/iq/dataverse/settings/JvmSettings.java | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/release-notes/10116-incomplete-metadata-label-setting.md b/doc/release-notes/10116-incomplete-metadata-label-setting.md index 5fd2de2f287..970f2c922ae 100644 --- a/doc/release-notes/10116-incomplete-metadata-label-setting.md +++ b/doc/release-notes/10116-incomplete-metadata-label-setting.md @@ -1 +1 @@ -Bug fixed for the ``incomplete metadata`` label being shown for published dataset with incomplete metadata in certain scenarios. This label will now only be shown for draft versions of such datasets. This label can also be made visible for published datasets with the new option ``dataverse.api.show-label-for-incomplete-when-published``, and will be only shown on datasets with incomplete metadata that the logged-in user can edit. +Bug fixed for the ``incomplete metadata`` label being shown for published dataset with incomplete metadata in certain scenarios. This label will now only be shown for draft versions of such datasets. This label can also be made visible for published datasets with the new option ``dataverse.ui.show-validity-label-when-published``, and will be only shown on datasets with incomplete metadata that the logged-in user can edit. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 29e53e80d68..70f35cc8797 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2444,10 +2444,10 @@ Defaults to ``false``. Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_API_ALLOW_INCOMPLETE_METADATA``. Will accept ``[tT][rR][uU][eE]|1|[oO][nN]`` as "true" expressions. -.. _dataverse.api.show-label-for-incomplete-when-published: +.. _dataverse.ui.show-validity-label-when-published: -dataverse.api.show-label-for-incomplete-when-published -++++++++++++++++++++++++++++++++++++++++++++++++++++++ +dataverse.ui.show-validity-label-when-published ++++++++++++++++++++++++++++++++++++++++++++++++ Even when you do not allow incomplete metadata to be saved in dataverse, some metadata may end up being incomplete, e.g., after making a metadata field mandatory. Datasets where that field is not filled out, become incomplete, and therefore can be labeled with the ``incomplete metadata`` label. By default, this label is only shown for draft datasets. However, in some situations, you diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 5402ad01cbc..66ade0c621c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -2243,7 +2243,7 @@ private void displayPublishMessage(){ public boolean isValid() { if (valid == null) { - if (workingVersion.isDraft() || (canUpdateDataset() && JvmSettings.UI_SHOW_VALIDITY_LABEL.lookupOptional(Boolean.class).orElse(false))) { + if (workingVersion.isDraft() || (canUpdateDataset() && JvmSettings.UI_SHOW_VALIDITY_LABEL_WHEN_PUBLISHED.lookupOptional(Boolean.class).orElse(false))) { final DatasetVersion newVersion = workingVersion.cloneDatasetVersion(); newVersion.setDatasetFields(newVersion.initDatasetFields()); valid = newVersion.isValid(); diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index b7afa83aea0..34624154b12 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -289,7 +289,7 @@ private void displayPublishMessage(){ public boolean isValid() { if (valid == null) { final DatasetVersion workingVersion = fileMetadata.getDatasetVersion(); - if (workingVersion.isDraft() || (canUpdateDataset() && JvmSettings.UI_SHOW_VALIDITY_LABEL.lookupOptional(Boolean.class).orElse(false))) { + if (workingVersion.isDraft() || (canUpdateDataset() && JvmSettings.UI_SHOW_VALIDITY_LABEL_WHEN_PUBLISHED.lookupOptional(Boolean.class).orElse(false))) { final DatasetVersion newVersion = workingVersion.cloneDatasetVersion(); newVersion.setDatasetFields(newVersion.initDatasetFields()); valid = newVersion.isValid(); diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java index 72af8b158dc..496ccf33ee3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java @@ -1277,7 +1277,7 @@ public boolean isValid(Predicate canUpdateDataset) { if (this.isDraftState()) { return false; } - if (!JvmSettings.UI_SHOW_VALIDITY_LABEL.lookupOptional(Boolean.class).orElse(false)) { + if (!JvmSettings.UI_SHOW_VALIDITY_LABEL_WHEN_PUBLISHED.lookupOptional(Boolean.class).orElse(false)) { return true; } return !canUpdateDataset.test(this); diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java index 3c1ae689263..7ef2030a616 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java @@ -145,7 +145,7 @@ public enum JvmSettings { SCOPE_UI(PREFIX, "ui"), UI_ALLOW_REVIEW_INCOMPLETE(SCOPE_UI, "allow-review-for-incomplete"), UI_SHOW_VALIDITY_FILTER(SCOPE_UI, "show-validity-filter"), - UI_SHOW_VALIDITY_LABEL(SCOPE_UI, "show-label-for-incomplete-when-published"), + UI_SHOW_VALIDITY_LABEL_WHEN_PUBLISHED(SCOPE_UI, "show-validity-label-when-published"), // NetCDF SETTINGS SCOPE_NETCDF(PREFIX, "netcdf"), From 2dc84db8f202010db091f987aacd92113731ce8f Mon Sep 17 00:00:00 2001 From: Ludovic DANIEL Date: Wed, 20 Dec 2023 17:38:57 +0100 Subject: [PATCH 039/970] #9317 - Allowing to delete saved search --- .../iq/dataverse/api/SavedSearches.java | 14 +++-- .../savedsearch/SavedSearchServiceBean.java | 62 ++++++++++++++++--- 2 files changed, 61 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/SavedSearches.java b/src/main/java/edu/harvard/iq/dataverse/api/SavedSearches.java index 5d0365d022e..cc1d7483c29 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/SavedSearches.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/SavedSearches.java @@ -173,16 +173,18 @@ public Response add(JsonObject body) { @DELETE @Path("{id}") - public Response delete(@PathParam("id") long doomedId) { - boolean disabled = true; - if (disabled) { - return error(BAD_REQUEST, "Saved Searches can not safely be deleted because links can not safely be deleted. See https://github.com/IQSS/dataverse/issues/1364 for details."); - } + public Response delete(@PathParam("id") long doomedId, @QueryParam("unlink") boolean unlink) { SavedSearch doomed = savedSearchSvc.find(doomedId); if (doomed == null) { return error(NOT_FOUND, "Could not find saved search id " + doomedId); } - boolean wasDeleted = savedSearchSvc.delete(doomedId); + boolean wasDeleted; + try { + wasDeleted = savedSearchSvc.delete(doomedId, unlink); + } catch (Exception e) { + return error(INTERNAL_SERVER_ERROR, "Problem while trying to unlink links of saved search id " + doomedId); + } + if (wasDeleted) { return ok(Json.createObjectBuilder().add("Deleted", doomedId)); } else { diff --git a/src/main/java/edu/harvard/iq/dataverse/search/savedsearch/SavedSearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/savedsearch/SavedSearchServiceBean.java index 7fc2bdf79a3..eded3a7af8a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/savedsearch/SavedSearchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/savedsearch/SavedSearchServiceBean.java @@ -2,29 +2,28 @@ import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetLinkingDataverse; +import edu.harvard.iq.dataverse.DatasetLinkingServiceBean; import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.DataverseLinkingDataverse; +import edu.harvard.iq.dataverse.DataverseLinkingServiceBean; import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.DvObjectServiceBean; import edu.harvard.iq.dataverse.EjbDataverseEngine; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.GuestUser; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; -import edu.harvard.iq.dataverse.search.SearchServiceBean; -import edu.harvard.iq.dataverse.search.SolrQueryResponse; -import edu.harvard.iq.dataverse.search.SolrSearchResult; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.impl.DeleteDatasetLinkingDataverseCommand; +import edu.harvard.iq.dataverse.engine.command.impl.DeleteDataverseLinkingDataverseCommand; import edu.harvard.iq.dataverse.engine.command.impl.LinkDatasetCommand; import edu.harvard.iq.dataverse.engine.command.impl.LinkDataverseCommand; import edu.harvard.iq.dataverse.search.SearchException; import edu.harvard.iq.dataverse.search.SearchFields; +import edu.harvard.iq.dataverse.search.SearchServiceBean; +import edu.harvard.iq.dataverse.search.SolrQueryResponse; +import edu.harvard.iq.dataverse.search.SolrSearchResult; import edu.harvard.iq.dataverse.search.SortBy; import edu.harvard.iq.dataverse.util.SystemConfig; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; import jakarta.ejb.EJB; import jakarta.ejb.Schedule; import jakarta.ejb.Stateless; @@ -39,6 +38,12 @@ import jakarta.persistence.TypedQuery; import jakarta.servlet.http.HttpServletRequest; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + @Stateless @Named public class SavedSearchServiceBean { @@ -50,6 +55,10 @@ public class SavedSearchServiceBean { @EJB DvObjectServiceBean dvObjectService; @EJB + protected DatasetLinkingServiceBean dsLinkingService; + @EJB + protected DataverseLinkingServiceBean dvLinkingService; + @EJB EjbDataverseEngine commandEngine; @EJB SystemConfig systemConfig; @@ -101,11 +110,15 @@ public SavedSearch add(SavedSearch toPersist) { return persisted; } - public boolean delete(long id) { + public boolean delete(long id, boolean unlink) throws SearchException, CommandException { SavedSearch doomed = find(id); boolean wasDeleted = false; if (doomed != null) { System.out.println("deleting saved search id " + doomed.getId()); + if(unlink) { + DataverseRequest dataverseRequest = new DataverseRequest(doomed.getCreator(), getHttpServletRequest()); + unLinksForSingleSavedSearch(dataverseRequest, doomed); + } em.remove(doomed); em.flush(); wasDeleted = true; @@ -240,6 +253,37 @@ public JsonObjectBuilder makeLinksForSingleSavedSearch(DataverseRequest dvReq, S return response; } + public void unLinksForSingleSavedSearch(DataverseRequest dvReq, SavedSearch savedSearch) throws SearchException, CommandException { + logger.info("UNLINK SAVED SEARCH (" + savedSearch.getId() + ") START search and unlink process"); + Date start = new Date(); + Dataverse linkingDataverse = savedSearch.getDefinitionPoint(); + + SolrQueryResponse queryResponse = findHits(savedSearch); + for (SolrSearchResult solrSearchResult : queryResponse.getSolrSearchResults()) { + + DvObject dvObjectThatDefinitionPointWillLinkTo = dvObjectService.findDvObject(solrSearchResult.getEntityId()); + if (dvObjectThatDefinitionPointWillLinkTo == null) { + continue; + } + + if (dvObjectThatDefinitionPointWillLinkTo.isInstanceofDataverse()) { + Dataverse linkedDataverse = (Dataverse) dvObjectThatDefinitionPointWillLinkTo; + DataverseLinkingDataverse dvld = dvLinkingService.findDataverseLinkingDataverse(linkedDataverse.getId(), linkingDataverse.getId()); + if(dvld != null) { + Dataverse dv = commandEngine.submitInNewTransaction(new DeleteDataverseLinkingDataverseCommand(dvReq, linkingDataverse, dvld, true)); + } + } else if (dvObjectThatDefinitionPointWillLinkTo.isInstanceofDataset()) { + Dataset linkedDataset = (Dataset) dvObjectThatDefinitionPointWillLinkTo; + DatasetLinkingDataverse dsld = dsLinkingService.findDatasetLinkingDataverse(linkedDataset.getId(), linkingDataverse.getId()); + if(dsld != null) { + Dataset ds = commandEngine.submitInNewTransaction(new DeleteDatasetLinkingDataverseCommand(dvReq, linkedDataset, dsld, true)); + } + } + } + + logger.info("UNLINK SAVED SEARCH (" + savedSearch.getId() + ") total time in ms: " + (new Date().getTime() - start.getTime())); + } + private SolrQueryResponse findHits(SavedSearch savedSearch) throws SearchException { String sortField = SearchFields.TYPE; // first return dataverses, then datasets String sortOrder = SortBy.DESCENDING; From ac80e0a3fcb0a2f98839920c86b4f2abd47539b5 Mon Sep 17 00:00:00 2001 From: Ludovic DANIEL Date: Fri, 5 Jan 2024 17:26:44 +0100 Subject: [PATCH 040/970] #9317 - Adding release note --- doc/release-notes/9317-delete-saved-search.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 doc/release-notes/9317-delete-saved-search.md diff --git a/doc/release-notes/9317-delete-saved-search.md b/doc/release-notes/9317-delete-saved-search.md new file mode 100644 index 00000000000..5fb6edd3642 --- /dev/null +++ b/doc/release-notes/9317-delete-saved-search.md @@ -0,0 +1,4 @@ +### Saved search deletion + +Saved search can now be removed using API `/api/admin/savedsearches/$id`. See PR #10198. +This is reflected in the [Saved Search Native API section](https://guides.dataverse.org/en/latest/api/native-api.html#saved-search) of the Guide. \ No newline at end of file From e1bf3c35f2c085d6831d8f3859a123da80745741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konrad=20Per=C5=82owski?= Date: Mon, 15 Jan 2024 12:45:02 +0100 Subject: [PATCH 041/970] Add CrossRef provider logic --- .../source/installation/config.rst | 9 + pom.xml | 5 + .../iq/dataverse/CrossRefRESTfullClient.java | 118 +++++++ .../dataverse/DOICrossRefRegisterService.java | 313 ++++++++++++++++++ .../iq/dataverse/DOICrossRefServiceBean.java | 117 +++++++ .../iq/dataverse/EjbDataverseEngine.java | 8 + .../iq/dataverse/GlobalIdServiceBean.java | 3 +- .../engine/command/CommandContext.java | 24 +- .../iq/dataverse/settings/JvmSettings.java | 11 +- .../dataverse/crossref_metadata_template.xml | 29 ++ .../dataverse/engine/TestCommandContext.java | 5 + 11 files changed, 619 insertions(+), 23 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/CrossRefRESTfullClient.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/DOICrossRefRegisterService.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/DOICrossRefServiceBean.java create mode 100644 src/main/resources/edu/harvard/iq/dataverse/crossref_metadata_template.xml diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index a7d7905ca4a..e7ba624222e 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -245,6 +245,15 @@ this provider. - :ref:`dataverse.pid.ezid.username` - :ref:`dataverse.pid.ezid.password` +**JVM Options for CrossRef:** + +- :ref:`dataverse.pid.crossref.url` +- :ref:`dataverse.pid.crossref.rest-api-url` +- :ref:`dataverse.pid.crossref.username` +- :ref:`dataverse.pid.crossref.password` +- :ref:`dataverse.pid.crossref.depositor` +- :ref:`dataverse.pid.crossref.depositor-email` + **Database Settings:** - :ref:`:DoiProvider <:DoiProvider>` diff --git a/pom.xml b/pom.xml index 7c12a45135c..72ef3391524 100644 --- a/pom.xml +++ b/pom.xml @@ -653,6 +653,11 @@ 3.2.0 test + + org.apache.httpcomponents + fluent-hc + 4.5.14 + diff --git a/src/main/java/edu/harvard/iq/dataverse/CrossRefRESTfullClient.java b/src/main/java/edu/harvard/iq/dataverse/CrossRefRESTfullClient.java new file mode 100644 index 00000000000..4b6728eca57 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/CrossRefRESTfullClient.java @@ -0,0 +1,118 @@ +package edu.harvard.iq.dataverse; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.fluent.Request; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.mime.HttpMultipartMode; +import org.apache.http.entity.mime.MultipartEntityBuilder; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.logging.Level; +import java.util.logging.Logger; + +public class CrossRefRESTfullClient implements Closeable { + + private static final Logger logger = Logger.getLogger(CrossRefRESTfullClient.class.getCanonicalName()); + + private final String url; + private final String apiUrl; + private final String username; + private final String password; + private final CloseableHttpClient httpClient; + private final HttpClientContext context; + private final String encoding = "utf-8"; + + public CrossRefRESTfullClient(String url, String apiUrl, String username, String password) { + this.url = url; + this.apiUrl = apiUrl; + this.username = username; + this.password = password; + try { + context = HttpClientContext.create(); + CredentialsProvider credsProvider = new BasicCredentialsProvider(); + credsProvider.setCredentials(new AuthScope(null, -1), + new UsernamePasswordCredentials(username, password)); + context.setCredentialsProvider(credsProvider); + + httpClient = HttpClients.createDefault(); + } catch (Exception ioe) { + close(); + logger.log(Level.SEVERE,"Fail to init Client",ioe); + throw new RuntimeException("Fail to init Client", ioe); + } + } + + public void close() { + if (this.httpClient != null) { + try { + httpClient.close(); + } catch (IOException io) { + logger.warning("IOException closing hhtpClient: " + io.getMessage()); + } + } + } + + public String getMetadata(String doi) { + HttpGet httpGet = new HttpGet(this.apiUrl + "/works/" + doi); + httpGet.setHeader("Accept", "application/json"); + try { + HttpResponse response = httpClient.execute(httpGet); + String data = EntityUtils.toString(response.getEntity(), encoding); + if (response.getStatusLine().getStatusCode() != 200) { + String errMsg = "Response from getMetadata: " + response.getStatusLine().getStatusCode() + ", " + data; + logger.info(errMsg); + throw new RuntimeException(errMsg); + } + return data; + } catch (IOException ioe) { + logger.info("IOException when get metadata"); + throw new RuntimeException("IOException when get metadata", ioe); + } + } + + public String postMetadata(String xml) throws IOException { + HttpEntity entity = MultipartEntityBuilder.create() + .addTextBody("operation", "doMDUpload") + .addTextBody("login_id", username) + .addTextBody("login_passwd", password) + .addBinaryBody("fname", xml.getBytes(StandardCharsets.UTF_8), ContentType.APPLICATION_XML, "metadata.xml") + .setMode(HttpMultipartMode.BROWSER_COMPATIBLE) + .build(); + HttpResponse response = Request.Post(url + "/servlet/deposit") + .body(entity) + .setHeader("Accept", "*/*") + .execute().returnResponse(); + + String data = EntityUtils.toString(response.getEntity(), encoding); + if (response.getStatusLine().getStatusCode() != 200) { + String errMsg = "Response from postMetadata: " + response.getStatusLine().getStatusCode() + ", " + data; + logger.info(errMsg); + throw new IOException(errMsg); + } + return data; + } + + public boolean testDOIExists(String doi) throws IOException { + HttpGet httpGet = new HttpGet(this.apiUrl + "/works/" + doi); + httpGet.setHeader("Accept", "application/json"); + HttpResponse response = httpClient.execute(httpGet); + if (response.getStatusLine().getStatusCode() != 200) { + EntityUtils.consumeQuietly(response.getEntity()); + return false; + } + EntityUtils.consumeQuietly(response.getEntity()); + return true; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/DOICrossRefRegisterService.java b/src/main/java/edu/harvard/iq/dataverse/DOICrossRefRegisterService.java new file mode 100644 index 00000000000..6e8ed20d570 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/DOICrossRefRegisterService.java @@ -0,0 +1,313 @@ +package edu.harvard.iq.dataverse; + +import com.fasterxml.jackson.databind.ObjectMapper; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import jakarta.ejb.EJB; +import jakarta.ejb.Stateless; +import org.apache.commons.text.StringEscapeUtils; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static edu.harvard.iq.dataverse.util.SystemConfig.getDataverseSiteUrlStatic; + +@Stateless +public class DOICrossRefRegisterService { + private static final Logger logger = Logger.getLogger(DOICrossRefRegisterService.class.getCanonicalName()); + + @EJB + DataverseServiceBean dataverseService; + + private CrossRefRESTfullClient client = null; + + private CrossRefRESTfullClient getClient() { + if (client == null) { + client = new CrossRefRESTfullClient( + JvmSettings.CROSSREF_URL.lookup(), + JvmSettings.CROSSREF_REST_API_URL.lookup(), + JvmSettings.CROSSREF_USERNAME.lookup(), + JvmSettings.CROSSREF_PASSWORD.lookup() + ); + } + return client; + } + + public boolean testDOIExists(String identifier) { + boolean doiExists; + try { + CrossRefRESTfullClient client = getClient(); + doiExists = client.testDOIExists(identifier.substring(identifier.indexOf(":") + 1)); + } catch (Exception e) { + logger.log(Level.INFO, identifier, e); + return false; + } + return doiExists; + } + + public HashMap getMetadata(String identifier) throws IOException { + HashMap metadata = new HashMap<>(); + try { + CrossRefRESTfullClient client = getClient(); + String jsonMetadata = client.getMetadata(identifier.substring(identifier.indexOf(":") + 1)); + Map mappedJson = new ObjectMapper().readValue(jsonMetadata, HashMap.class); + logger.log(Level.FINE, jsonMetadata); + metadata.put("_status", mappedJson.get("status").toString()); + } catch (RuntimeException e) { + logger.log(Level.INFO, identifier, e); + } + return metadata; + } + + public String reserveIdentifier(String identifier, DvObject dvObject) throws IOException { + logger.info("Crossref reserveIdentifier"); + String xmlMetadata = getMetadataFromDvObject(identifier, dvObject); + + CrossRefRESTfullClient client = getClient(); + return client.postMetadata(xmlMetadata); + } + + public void modifyIdentifier(String identifier, DvObject dvObject) throws IOException { + logger.info("Crossref modifyIdentifier"); + String xmlMetadata = getMetadataFromDvObject(identifier, dvObject); + + CrossRefRESTfullClient client = getClient(); + client.postMetadata(xmlMetadata); + } + + public String getMetadataFromDvObject(String identifier, DvObject dvObject) { + Dataset dataset; + + if (dvObject instanceof Dataset) { + dataset = (Dataset) dvObject; + } else { + dataset = (Dataset) dvObject.getOwner(); + } + + CrossRefMetadataTemplate metadataTemplate = new CrossRefMetadataTemplate(); + metadataTemplate.setIdentifier(identifier.substring(identifier.indexOf(':') + 1)); + metadataTemplate.setAuthors(dataset.getLatestVersion().getDatasetAuthors()); + metadataTemplate.setDepositor(JvmSettings.CROSSREF_DEPOSITOR.lookup()); + metadataTemplate.setDepositorEmail(JvmSettings.CROSSREF_DEPOSITOR_EMAIL.lookup()); + metadataTemplate.setInstitution(dataverseService.getRootDataverseName()); + + String title = dvObject.getCurrentName(); + if (dvObject.isInstanceofDataFile()) { + //Note file title is not currently escaped the way the dataset title is, so adding it here. + title = StringEscapeUtils.escapeXml10(title); + } + + if (title.isEmpty() || title.equals(DatasetField.NA_VALUE)) { + title = AbstractGlobalIdServiceBean.UNAVAILABLE; + } + + metadataTemplate.setTitle(title); + + return metadataTemplate.generateXML(); + } +} + +class CrossRefMetadataTemplate { + + private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.edu.harvard.iq.dataverse.CrossRefMetadataTemplate"); + private static String template; + + static { + try (InputStream in = CrossRefMetadataTemplate.class.getResourceAsStream("crossref_metadata_template.xml")) { + template = CrossRefFileUtil.readAndClose(in, "utf-8"); + } catch (Exception e) { + logger.log(Level.SEVERE, "crossref metadata template load error"); + logger.log(Level.SEVERE, "String " + e); + logger.log(Level.SEVERE, "localized message " + e.getLocalizedMessage()); + logger.log(Level.SEVERE, "cause " + e.getCause()); + logger.log(Level.SEVERE, "message " + e.getMessage()); + } + } + + private final String timestamp = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()); + private String institution; + private String depositor; + private String depositorEmail; + private String databaseTitle; + private String identifier; + private String title; + private final String baseUrl = getDataverseSiteUrlStatic(); + private List authors; + + public List getAuthors() { + return authors; + } + + public void setAuthors(List authors) { + this.authors = authors; + } + + public CrossRefMetadataTemplate() { + } + + public String generateXML() { + String xmlMetadata = template.replace("${depositor}", depositor) + .replace("${depositorEmail}", depositorEmail) + .replace("${title}", title) + .replace("${institution}", institution) + .replace("${batchId}", identifier + " " + timestamp) + .replace("${timestamp}", timestamp); + + StringBuilder datasetElement = new StringBuilder(); + datasetElement.append(""); + + StringBuilder contributorsElement = new StringBuilder(); + if (authors != null && !authors.isEmpty()) { + contributorsElement.append(""); + for (DatasetAuthor author : authors) { + contributorsElement.append(""); + contributorsElement.append(author.getName().getDisplayValue()); + contributorsElement.append(""); + contributorsElement.append(author.getName().getDisplayValue()); + contributorsElement.append(""); + + if (author.getAffiliation() != null && !author.getAffiliation().getDisplayValue().isEmpty()) { + contributorsElement.append("") + .append(author.getAffiliation().getDisplayValue()) + .append(""); + } + + if (author.getIdType() != null && + author.getIdValue() != null && + !author.getIdType().isEmpty() && + !author.getIdValue().isEmpty() && + author.getAffiliation() != null && + !author.getAffiliation().getDisplayValue().isEmpty()) { + if (author.getIdType().equals("ORCID")) { + contributorsElement.append("").append("https://orcid.org/").append(author.getIdValue()).append(""); + } + if (author.getIdType().equals("ISNI")) { + contributorsElement.append("").append(author.getIdValue()).append(""); + } + if (author.getIdType().equals("LCNA")) { + contributorsElement.append("").append(author.getIdValue()).append(""); + } + } + + contributorsElement.append(""); + } + contributorsElement.append(""); + + } else { + contributorsElement.append("") + .append(AbstractGlobalIdServiceBean.UNAVAILABLE) + .append(""); + } + + datasetElement.append(contributorsElement); + + datasetElement.append("") + .append(this.title) + .append(""); + + datasetElement.append("") + .append(this.identifier) + .append("") + .append(this.baseUrl).append("/dataset.xhtml?persistentId=doi:").append(this.identifier) + .append(""); + + datasetElement.append(""); + xmlMetadata = xmlMetadata.replace("${datasets}", datasetElement.toString()); + return xmlMetadata; + } + + public static String getTemplate() { + return template; + } + + public static void setTemplate(String template) { + CrossRefMetadataTemplate.template = template; + } + + public String getIdentifier() { + return identifier; + } + + public String getDepositor() { + return depositor; + } + + public void setDepositor(String depositor) { + this.depositor = depositor; + } + + public void setIdentifier(String identifier) { + this.identifier = identifier; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getInstitution() { + return institution; + } + + public void setInstitution(String institution) { + this.institution = institution; + } + + public String getDepositorEmail() { + return depositorEmail; + } + + public void setDepositorEmail(String depositorEmail) { + this.depositorEmail = depositorEmail; + } + + public String getDatabaseTitle() { + return databaseTitle; + } + + public void setDatabaseTitle(String databaseTitle) { + this.databaseTitle = databaseTitle; + } +} + +class CrossRefFileUtil { + + public static void close(InputStream in) { + if (in != null) { + try { + in.close(); + } catch (IOException e) { + throw new RuntimeException("Fail to close InputStream"); + } + } + } + + public static String readAndClose(InputStream inStream, String encoding) { + ByteArrayOutputStream outStream = new ByteArrayOutputStream(); + byte[] buf = new byte[128]; + String data; + try { + int cnt; + while ((cnt = inStream.read(buf)) >= 0) { + outStream.write(buf, 0, cnt); + } + data = outStream.toString(encoding); + } catch (IOException ioe) { + throw new RuntimeException("IOException"); + } finally { + close(inStream); + } + return data; + } +} + diff --git a/src/main/java/edu/harvard/iq/dataverse/DOICrossRefServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DOICrossRefServiceBean.java new file mode 100644 index 00000000000..50fafc5f036 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/DOICrossRefServiceBean.java @@ -0,0 +1,117 @@ +package edu.harvard.iq.dataverse; + +import jakarta.ejb.EJB; +import jakarta.ejb.Stateless; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +@Stateless +public class DOICrossRefServiceBean extends DOIServiceBean { + private static final Logger logger = Logger.getLogger(DOICrossRefServiceBean.class.getCanonicalName()); + + @EJB + DOICrossRefRegisterService doiCrossRefRegisterService; + + + @Override + public boolean alreadyRegistered(GlobalId pid, boolean noProviderDefault) throws Exception { + logger.info("CrossRef alreadyRegistered"); + if (pid == null || pid.asString().isEmpty()) { + logger.fine("No identifier sent."); + return false; + } + boolean alreadyExists; + String identifier = pid.asString(); + try { + alreadyExists = doiCrossRefRegisterService.testDOIExists(identifier); + } catch (Exception e) { + logger.log(Level.WARNING, "alreadyExists failed"); + return false; + } + return alreadyExists; + } + + @Override + public boolean registerWhenPublished() { + return true; + } + + @Override + public List getProviderInformation() { + return List.of("CrossRef", "https://status.crossref.org/"); + } + + @Override + protected String getProviderKeyName() { + return "CrossRef"; + } + + @Override + public String createIdentifier(DvObject dvObject) throws Throwable { + logger.info("CrossRef createIdentifier"); + if (dvObject.getIdentifier() == null || dvObject.getIdentifier().isEmpty()) { + dvObject = generateIdentifier(dvObject); + } + String identifier = getIdentifier(dvObject); + try { + String retString = doiCrossRefRegisterService.reserveIdentifier(identifier, dvObject); + logger.log(Level.FINE, "CrossRef create DOI identifier retString : " + retString); + return retString; + } catch (Exception e) { + logger.log(Level.WARNING, "CrossRef Identifier not created: create failed", e); + throw e; + } + } + + @Override + public Map getIdentifierMetadata(DvObject dvObject) { + logger.info("CrossRef getIdentifierMetadata"); + String identifier = getIdentifier(dvObject); + Map metadata = new HashMap<>(); + try { + metadata = doiCrossRefRegisterService.getMetadata(identifier); + } catch (Exception e) { + logger.log(Level.WARNING, "getIdentifierMetadata failed", e); + } + return metadata; + } + + @Override + public String modifyIdentifierTargetURL(DvObject dvObject) throws Exception { + logger.info("CrossRef modifyIdentifier"); + String identifier = getIdentifier(dvObject); + try { + doiCrossRefRegisterService.modifyIdentifier(identifier, dvObject); + } catch (Exception e) { + logger.log(Level.WARNING, "modifyMetadata failed", e); + throw e; + } + return identifier; + } + + @Override + public void deleteIdentifier(DvObject dvo) throws Exception { + logger.info("CrossRef deleteIdentifier"); + } + + @Override + public boolean publicizeIdentifier(DvObject dvObject) { + logger.info("CrossRef updateIdentifierStatus"); + if (dvObject.getIdentifier() == null || dvObject.getIdentifier().isEmpty()) { + dvObject = generateIdentifier(dvObject); + } + String identifier = getIdentifier(dvObject); + + try { + doiCrossRefRegisterService.reserveIdentifier(identifier, dvObject); + return true; + } catch (Exception e) { + logger.log(Level.WARNING, "modifyMetadata failed: " + e.getMessage(), e); + return false; + } + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java b/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java index 5a689c06019..cfb17caf5a0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java +++ b/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java @@ -119,6 +119,9 @@ public class EjbDataverseEngine { @EJB DOIDataCiteServiceBean doiDataCite; + @EJB + DOICrossRefServiceBean doiCrossRef; + @EJB FakePidProviderServiceBean fakePidProvider; @@ -493,6 +496,11 @@ public DOIDataCiteServiceBean doiDataCite() { return doiDataCite; } + @Override + public DOICrossRefServiceBean doiCrossRef() { + return doiCrossRef; + } + @Override public FakePidProviderServiceBean fakePidProvider() { return fakePidProvider; diff --git a/src/main/java/edu/harvard/iq/dataverse/GlobalIdServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/GlobalIdServiceBean.java index aebf13778c3..ec96bcf603f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GlobalIdServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/GlobalIdServiceBean.java @@ -204,8 +204,9 @@ class BeanDispatcher { switch ( doiProvider ) { case "EZID": return ctxt.doiEZId(); case "DataCite": return ctxt.doiDataCite(); + case "CrossRef": return ctxt.doiCrossRef(); case "FAKE": return ctxt.fakePidProvider(); - default: + default: logger.log(Level.SEVERE, "Unknown doiProvider: {0}", doiProvider); return null; } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java index f74c1222bb0..596a76be172 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java @@ -1,29 +1,9 @@ package edu.harvard.iq.dataverse.engine.command; -import edu.harvard.iq.dataverse.DOIDataCiteServiceBean; -import edu.harvard.iq.dataverse.DOIEZIdServiceBean; -import edu.harvard.iq.dataverse.HandlenetServiceBean; -import edu.harvard.iq.dataverse.DataFileServiceBean; -import edu.harvard.iq.dataverse.DatasetLinkingServiceBean; -import edu.harvard.iq.dataverse.DatasetServiceBean; -import edu.harvard.iq.dataverse.DatasetVersionServiceBean; -import edu.harvard.iq.dataverse.DataverseFacetServiceBean; -import edu.harvard.iq.dataverse.DataverseFieldTypeInputLevelServiceBean; -import edu.harvard.iq.dataverse.DataverseLinkingServiceBean; -import edu.harvard.iq.dataverse.DataverseRoleServiceBean; -import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean; -import edu.harvard.iq.dataverse.DvObjectServiceBean; -import edu.harvard.iq.dataverse.FeaturedDataverseServiceBean; -import edu.harvard.iq.dataverse.FileDownloadServiceBean; -import edu.harvard.iq.dataverse.GuestbookResponseServiceBean; -import edu.harvard.iq.dataverse.GuestbookServiceBean; import edu.harvard.iq.dataverse.search.IndexServiceBean; -import edu.harvard.iq.dataverse.PermissionServiceBean; -import edu.harvard.iq.dataverse.RoleAssigneeServiceBean; import edu.harvard.iq.dataverse.search.SearchServiceBean; -import edu.harvard.iq.dataverse.TemplateServiceBean; -import edu.harvard.iq.dataverse.UserNotificationServiceBean; import edu.harvard.iq.dataverse.actionlogging.ActionLogServiceBean; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.groups.GroupServiceBean; @@ -104,6 +84,8 @@ public interface CommandContext { public DOIDataCiteServiceBean doiDataCite(); + public DOICrossRefServiceBean doiCrossRef(); + public FakePidProviderServiceBean fakePidProvider(); public HandlenetServiceBean handleNet(); diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java index 3bc06738a7e..0600175deb5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java @@ -104,7 +104,16 @@ public enum JvmSettings { DATACITE_REST_API_URL(SCOPE_PID_DATACITE, "rest-api-url", "doi.dataciterestapiurlstring", "doi.mdcbaseurlstring"), DATACITE_USERNAME(SCOPE_PID_DATACITE, "username", "doi.username"), DATACITE_PASSWORD(SCOPE_PID_DATACITE, "password", "doi.password"), - + + // PROVIDER CROSSREF + SCOPE_PID_CROSSREF(SCOPE_PID, "crossref"), + CROSSREF_URL(SCOPE_PID_CROSSREF, "url"), + CROSSREF_REST_API_URL(SCOPE_PID_CROSSREF, "rest-api-url"), + CROSSREF_USERNAME(SCOPE_PID_CROSSREF, "username", "doi.username"), + CROSSREF_PASSWORD(SCOPE_PID_CROSSREF, "password", "doi.password"), + CROSSREF_DEPOSITOR(SCOPE_PID_CROSSREF, "depositor"), + CROSSREF_DEPOSITOR_EMAIL(SCOPE_PID_CROSSREF, "depositor-email"), + // PROVIDER PERMALINK SCOPE_PID_PERMALINK(SCOPE_PID, "permalink"), PERMALINK_BASEURL(SCOPE_PID_PERMALINK, "base-url", "perma.baseurlstring"), diff --git a/src/main/resources/edu/harvard/iq/dataverse/crossref_metadata_template.xml b/src/main/resources/edu/harvard/iq/dataverse/crossref_metadata_template.xml new file mode 100644 index 00000000000..f37ed63cfff --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/crossref_metadata_template.xml @@ -0,0 +1,29 @@ + + + + ${batchId} + ${timestamp} + + ${depositor} + ${depositorEmail} + + Crossref + + + + + + ${title} + + + ${institution} + + + ${datasets} + + + diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java b/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java index a80adb33b8d..15810f85d39 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java @@ -131,6 +131,11 @@ public DOIDataCiteServiceBean doiDataCite() { return null; } + @Override + public DOICrossRefServiceBean doiCrossRef() { + return null; + } + @Override public FakePidProviderServiceBean fakePidProvider() { return null; From a08fbe2b345dd9f10998b758eb962f899bd6653e Mon Sep 17 00:00:00 2001 From: Martin Amouzou <85512093+martinAmouzou@users.noreply.github.com> Date: Wed, 24 Jan 2024 16:13:02 +0100 Subject: [PATCH 042/970] Update version-control.rst --- doc/sphinx-guides/source/developers/version-control.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/sphinx-guides/source/developers/version-control.rst b/doc/sphinx-guides/source/developers/version-control.rst index c36c7d1e963..a97adf4149b 100644 --- a/doc/sphinx-guides/source/developers/version-control.rst +++ b/doc/sphinx-guides/source/developers/version-control.rst @@ -286,5 +286,8 @@ GitHub documents how to make changes to a fork at https://help.github.com/articl git push OdumInstitute 4709-postgresql_96 ---- +Develop branch and pull request +-------------------------------------------- +Please do not use your forked develop branch to push a PR and follow the `1st scenario: preparing the first pull request of Version Control Guide `. Previous: :doc:`troubleshooting` | Next: :doc:`sql-upgrade-scripts` From 8277ac5adac8511efa9bb92d3f51dcf99973e67b Mon Sep 17 00:00:00 2001 From: Martin Amouzou <85512093+martinAmouzou@users.noreply.github.com> Date: Wed, 24 Jan 2024 16:15:10 +0100 Subject: [PATCH 043/970] Update version-control.rst --- doc/sphinx-guides/source/developers/version-control.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/version-control.rst b/doc/sphinx-guides/source/developers/version-control.rst index a97adf4149b..11ce0c0d144 100644 --- a/doc/sphinx-guides/source/developers/version-control.rst +++ b/doc/sphinx-guides/source/developers/version-control.rst @@ -286,8 +286,9 @@ GitHub documents how to make changes to a fork at https://help.github.com/articl git push OdumInstitute 4709-postgresql_96 ---- + Develop branch and pull request -------------------------------------------- -Please do not use your forked develop branch to push a PR and follow the `1st scenario: preparing the first pull request of Version Control Guide `. +Please do not use your forked develop branch to push a PR and follow the `1st scenario: preparing the first pull request of Version Control Guide `_. Previous: :doc:`troubleshooting` | Next: :doc:`sql-upgrade-scripts` From be8c167a6454bc86535ce77e66dedb1d76ca0b08 Mon Sep 17 00:00:00 2001 From: Martin Amouzou <85512093+martinAmouzou@users.noreply.github.com> Date: Wed, 24 Jan 2024 16:53:13 +0100 Subject: [PATCH 044/970] Update version-control.rst --- doc/sphinx-guides/source/developers/version-control.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/version-control.rst b/doc/sphinx-guides/source/developers/version-control.rst index 11ce0c0d144..5b809782f76 100644 --- a/doc/sphinx-guides/source/developers/version-control.rst +++ b/doc/sphinx-guides/source/developers/version-control.rst @@ -289,6 +289,6 @@ GitHub documents how to make changes to a fork at https://help.github.com/articl Develop branch and pull request -------------------------------------------- -Please do not use your forked develop branch to push a PR and follow the `1st scenario: preparing the first pull request of Version Control Guide `_. +Please do not use your forked develop branch to push a PR and follow the `1st scenario: preparing the first pull request of Version Control Guide `_. Previous: :doc:`troubleshooting` | Next: :doc:`sql-upgrade-scripts` From 8c5113b9c3f4e8380df45d91c84fdc3acdaf7f1a Mon Sep 17 00:00:00 2001 From: Martin Amouzou <85512093+martinAmouzou@users.noreply.github.com> Date: Wed, 24 Jan 2024 17:04:48 +0100 Subject: [PATCH 045/970] Update version-control.rst --- doc/sphinx-guides/source/developers/version-control.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/version-control.rst b/doc/sphinx-guides/source/developers/version-control.rst index 5b809782f76..35e37ac0471 100644 --- a/doc/sphinx-guides/source/developers/version-control.rst +++ b/doc/sphinx-guides/source/developers/version-control.rst @@ -289,6 +289,6 @@ GitHub documents how to make changes to a fork at https://help.github.com/articl Develop branch and pull request -------------------------------------------- -Please do not use your forked develop branch to push a PR and follow the `1st scenario: preparing the first pull request of Version Control Guide `_. +Please do not use your forked develop branch to push a PR and follow the `1st scenario: preparing the first pull request of Version Control Guide `_. Previous: :doc:`troubleshooting` | Next: :doc:`sql-upgrade-scripts` From 1edb8bac22046dffd5c7c21197e013bb3be80a60 Mon Sep 17 00:00:00 2001 From: Martin Amouzou <85512093+martinAmouzou@users.noreply.github.com> Date: Wed, 24 Jan 2024 17:07:58 +0100 Subject: [PATCH 046/970] Update version-control.rst --- doc/sphinx-guides/source/developers/version-control.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/version-control.rst b/doc/sphinx-guides/source/developers/version-control.rst index 35e37ac0471..99e33164417 100644 --- a/doc/sphinx-guides/source/developers/version-control.rst +++ b/doc/sphinx-guides/source/developers/version-control.rst @@ -289,6 +289,6 @@ GitHub documents how to make changes to a fork at https://help.github.com/articl Develop branch and pull request -------------------------------------------- -Please do not use your forked develop branch to push a PR and follow the `1st scenario: preparing the first pull request of Version Control Guide `_. +Please do not use your forked develop branch to push a PR and follow the `1st scenario: preparing the first pull request of Version Control Guide <#summary-of-git-commands>`_. Previous: :doc:`troubleshooting` | Next: :doc:`sql-upgrade-scripts` From 6df751346325948935582836a69b6e5378fb1b06 Mon Sep 17 00:00:00 2001 From: konradperlowski Date: Wed, 31 Jan 2024 10:31:03 +0100 Subject: [PATCH 047/970] Add CrossRef config properties description --- .../source/installation/config.rst | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index e7ba624222e..24f1d0cce12 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2370,6 +2370,50 @@ should delete the old JVM option and the wrapped password alias, then recreate as shown for :ref:`dataverse.pid.datacite.password` but with the EZID alias name. +.. _dataverse.pid.crossref.url: + +dataverse.pid.crossref.url +++++++++++++++++++++++++++ + +CrossRef url used to post metadata. + +.. _dataverse.pid.crossref.rest-api-url: + +dataverse.pid.crossref.rest-api-url ++++++++++++++++++++++++++++++++++++ + +CrossRef API url to retrieve metadata information + +.. _dataverse.pid.crossref.username: + +dataverse.pid.crossref.username ++++++++++++++++++++++++++++++++ + +CrossRef uses `HTTP Basic authentication `_ +for their APIs. +- Used in conjunction with :ref:`dataverse.pid.crossref.url` and :ref:`dataverse.pid.crossref.password`. + +.. _dataverse.pid.crossref.password: + +dataverse.pid.crossref.password ++++++++++++++++++++++++++++++++ + +- Used in conjunction with :ref:`dataverse.pid.crossref.url` and :ref:`dataverse.pid.crossref.username`. + +.. _dataverse.pid.crossref.depositor: + +dataverse.pid.crossref.depositor +++++++++++++++++++++++++++++++++ + +The entity, such as a person or organization, that deposited the Dataset in the repository + +.. _dataverse.pid.crossref.depositor-email: + +dataverse.pid.crossref.depositor-email +++++++++++++++++++++++++++++++++++++++ + +Contact email to the indicated Depositor + .. _dataverse.timerServer: dataverse.timerServer From a50f963931a671404ff5160b353aefe7d312afcb Mon Sep 17 00:00:00 2001 From: Ludovic DANIEL Date: Wed, 31 Jan 2024 16:32:05 +0100 Subject: [PATCH 048/970] #9317 - Updated API Guide --- doc/sphinx-guides/source/api/native-api.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 6591c983824..4f84c123109 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -5131,8 +5131,7 @@ The ``$identifier`` should start with an ``@`` if it's a user. Groups start with Saved Search ~~~~~~~~~~~~ -The Saved Search, Linked Dataverses, and Linked Datasets features shipped with Dataverse 4.0, but as a "`superuser-only `_" because they are **experimental** (see `#1364 `_, `#1813 `_, `#1840 `_, `#1890 `_, `#1939 `_, `#2167 `_, `#2186 `_, `#2053 `_, and `#2543 `_). The following API endpoints were added to help people with access to the "admin" API make use of these features in their current form. There is a known issue (`#1364 `_) that once a link to a Dataverse collection or dataset is created, it cannot be removed (apart from database manipulation and reindexing) which is why a ``DELETE`` endpoint for saved searches is neither documented nor functional. The Linked Dataverse collections feature is `powered by Saved Search `_ and therefore requires that the "makelinks" endpoint be executed on a periodic basis as well. - +The Saved Search, Linked Dataverses, and Linked Datasets features are only accessible to superusers except for Linking a dataset. The following API endpoints were added to help people with access to the “admin” API make use of these features in their current form, keep in mind that they are partially experimental. List all saved searches. :: GET http://$SERVER/api/admin/savedsearches/list @@ -5141,6 +5140,10 @@ List a saved search by database id. :: GET http://$SERVER/api/admin/savedsearches/$id +Delete a saved search by database id. The ``unlink=true`` query parameter unlink links (Linked dataset or Dataverse collection) related to the deleted saved search. This parameter should be well considered as you cannot know if the saved search created the links or if someone else did via other API. Also, it may be followed ``/makelinks/all`` depending on the need if other saved searches could recreate some deleted links or by reindexing some Dataverse or Dataset. :: + + DELETE http://$SERVER/api/admin/savedsearches/$id?unlink=true + Execute a saved search by database id and make links to Dataverse collections and datasets that are found. The JSON response indicates which Dataverse collections and datasets were newly linked versus already linked. The ``debug=true`` query parameter adds to the JSON response extra information about the saved search being executed (which you could also get by listing the saved search). :: PUT http://$SERVER/api/admin/savedsearches/makelinks/$id?debug=true From 9110adee7c739b21803da912913b52442a7e1e38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20ROUCOU?= Date: Wed, 31 Jan 2024 18:22:03 +0100 Subject: [PATCH 049/970] Sitemap more than 50000 entries IQSS#8936 --- .../iq/dataverse/sitemap/SiteMapUtil.java | 234 +++++++----------- .../iq/dataverse/sitemap/SiteMapUtilTest.java | 58 ++++- 2 files changed, 143 insertions(+), 149 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtil.java b/src/main/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtil.java index 86ae697f771..3077c41fa14 100644 --- a/src/main/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtil.java @@ -1,194 +1,135 @@ package edu.harvard.iq.dataverse.sitemap; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.Dataverse; -import edu.harvard.iq.dataverse.DvObjectContainer; -import edu.harvard.iq.dataverse.settings.ConfigCheckService; -import edu.harvard.iq.dataverse.settings.JvmSettings; -import edu.harvard.iq.dataverse.util.SystemConfig; -import edu.harvard.iq.dataverse.util.xml.XmlValidator; import java.io.File; import java.io.IOException; import java.net.MalformedURLException; -import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.text.SimpleDateFormat; +import java.text.ParseException; +import java.time.format.DateTimeFormatter; import java.util.List; import java.util.logging.Logger; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.OutputKeys; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerConfigurationException; -import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.xml.sax.SAXException; + +import com.redfin.sitemapgenerator.W3CDateFormat; +import com.redfin.sitemapgenerator.W3CDateFormat.Pattern; +import com.redfin.sitemapgenerator.WebSitemapGenerator; +import com.redfin.sitemapgenerator.WebSitemapUrl; + +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DvObjectContainer; +import edu.harvard.iq.dataverse.settings.ConfigCheckService; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.SystemConfig; public class SiteMapUtil { private static final Logger logger = Logger.getLogger(SiteMapUtil.class.getCanonicalName()); + private static DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - static final String SITEMAP_FILENAME_FINAL = "sitemap.xml"; static final String SITEMAP_FILENAME_STAGED = "sitemap.xml.staged"; + /** @see https://www.sitemaps.org/protocol.html#index */ + static final int SITEMAP_LIMIT = 50000; + - /** - * TODO: Handle more than 50,000 entries in the sitemap. - * - * (As of this writing Harvard Dataverse only has ~3000 dataverses and - * ~30,000 datasets.) - * - * "each Sitemap file that you provide must have no more than 50,000 URLs" - * https://www.sitemaps.org/protocol.html - * - * Consider using a third party library: "One sitemap can contain a maximum - * of 50,000 URLs. (Some sitemaps, like Google News sitemaps, can contain - * only 1,000 URLs.) If you need to put more URLs than that in a sitemap, - * you'll have to use a sitemap index file. Fortunately, WebSitemapGenerator - * can manage the whole thing for you." - * https://github.com/dfabulich/sitemapgen4j - */ public static void updateSiteMap(List dataverses, List datasets) { logger.info("BEGIN updateSiteMap"); - String sitemapPathString = getSitemapPathString(); - String stagedSitemapPathAndFileString = sitemapPathString + File.separator + SITEMAP_FILENAME_STAGED; - String finalSitemapPathAndFileString = sitemapPathString + File.separator + SITEMAP_FILENAME_FINAL; - - Path stagedPath = Paths.get(stagedSitemapPathAndFileString); - if (Files.exists(stagedPath)) { - logger.warning("Unable to update sitemap! The staged file from a previous run already existed. Delete " + stagedSitemapPathAndFileString + " and try again."); + final String dataverseSiteUrl = SystemConfig.getDataverseSiteUrlStatic(); + final String msgErrorFormat = "Problem with %s : %s. The exception is %s"; + final String msgErrorW3CFormat = "%s isn't a valid W3C date time for %s. The exception is %s"; + final String sitemapPathString = getSitemapPathString(); + final String stagedSitemapPathAndFileString = sitemapPathString + File.separator + SITEMAP_FILENAME_STAGED; + final Path stagedSitemapPath = Paths.get(stagedSitemapPathAndFileString); + + if (Files.exists(stagedSitemapPath)) { + logger.warning(String.format( + "Unable to update sitemap! The staged file from a previous run already existed. Delete %s and try again.", + stagedSitemapPathAndFileString)); return; } - DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder documentBuilder = null; + final File directory = new File(sitemapPathString); + if (!directory.exists()) { + directory.mkdir(); + } + + // Use DAY pattern (2024-01-24), local machine timezone + final W3CDateFormat dateFormat = new W3CDateFormat(Pattern.DAY); + WebSitemapGenerator wsg = null; try { - documentBuilder = documentBuilderFactory.newDocumentBuilder(); - } catch (ParserConfigurationException ex) { - logger.warning("Unable to update sitemap! ParserConfigurationException: " + ex.getLocalizedMessage()); + wsg = WebSitemapGenerator.builder(dataverseSiteUrl, directory).autoValidate(true).dateFormat(dateFormat) + .build(); + } catch (MalformedURLException e) { + logger.warning(String.format(msgErrorFormat, "Dataverse site URL", dataverseSiteUrl, e.getLocalizedMessage())); return; } - Document document = documentBuilder.newDocument(); - - Element urlSet = document.createElement("urlset"); - urlSet.setAttribute("xmlns", "http://www.sitemaps.org/schemas/sitemap/0.9"); - urlSet.setAttribute("xmlns:xhtml", "http://www.w3.org/1999/xhtml"); - document.appendChild(urlSet); for (Dataverse dataverse : dataverses) { if (!dataverse.isReleased()) { continue; } - Element url = document.createElement("url"); - urlSet.appendChild(url); - - Element loc = document.createElement("loc"); - String dataverseAlias = dataverse.getAlias(); - loc.appendChild(document.createTextNode(SystemConfig.getDataverseSiteUrlStatic() + "/dataverse/" + dataverseAlias)); - url.appendChild(loc); - - Element lastmod = document.createElement("lastmod"); - lastmod.appendChild(document.createTextNode(getLastModDate(dataverse))); - url.appendChild(lastmod); + final String dvAlias = dataverse.getAlias(); + final String dataverseUrl = dataverseSiteUrl + "/dataverse/" + dvAlias; + final String lastModDate = getLastModDate(dataverse); + try { + final WebSitemapUrl url = new WebSitemapUrl.Options(dataverseUrl).lastMod(lastModDate).build(); + wsg.addUrl(url); + } catch (MalformedURLException e) { + logger.fine(String.format(msgErrorFormat, "dataverse URL", dataverseUrl, e.getLocalizedMessage())); + } catch (ParseException e) { + logger.fine(String.format(msgErrorW3CFormat, lastModDate, "dataverse alias " + dvAlias, e.getLocalizedMessage())); + } } for (Dataset dataset : datasets) { - if (!dataset.isReleased()) { - continue; - } - if (dataset.isHarvested()) { - continue; - } // The deaccessioned check is last because it has to iterate through dataset versions. - if (dataset.isDeaccessioned()) { + if (!dataset.isReleased() || dataset.isHarvested() || dataset.isDeaccessioned()) { continue; } - Element url = document.createElement("url"); - urlSet.appendChild(url); - - Element loc = document.createElement("loc"); - String datasetPid = dataset.getGlobalId().asString(); - loc.appendChild(document.createTextNode(SystemConfig.getDataverseSiteUrlStatic() + "/dataset.xhtml?persistentId=" + datasetPid)); - url.appendChild(loc); - - Element lastmod = document.createElement("lastmod"); - lastmod.appendChild(document.createTextNode(getLastModDate(dataset))); - url.appendChild(lastmod); - } - - TransformerFactory transformerFactory = TransformerFactory.newInstance(); - Transformer transformer = null; - try { - transformer = transformerFactory.newTransformer(); - } catch (TransformerConfigurationException ex) { - logger.warning("Unable to update sitemap! TransformerConfigurationException: " + ex.getLocalizedMessage()); - return; - } - transformer.setOutputProperty(OutputKeys.INDENT, "yes"); - transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2"); - DOMSource source = new DOMSource(document); - File directory = new File(sitemapPathString); - if (!directory.exists()) { - directory.mkdir(); - } - - boolean debug = false; - if (debug) { - logger.info("Writing sitemap to console/logs"); - StreamResult consoleResult = new StreamResult(System.out); + final String datasetPid = dataset.getGlobalId().asString(); + final String datasetUrl = dataverseSiteUrl + "/dataset.xhtml?persistentId=" + datasetPid; + final String lastModDate = getLastModDate(dataset); try { - transformer.transform(source, consoleResult); - } catch (TransformerException ex) { - logger.warning("Unable to print sitemap to the console: " + ex.getLocalizedMessage()); + final WebSitemapUrl url = new WebSitemapUrl.Options(datasetUrl).lastMod(lastModDate).build(); + wsg.addUrl(url); + } catch (MalformedURLException e) { + logger.fine(String.format(msgErrorFormat, "dataset URL", datasetUrl, e.getLocalizedMessage())); + } catch (ParseException e) { + logger.fine(String.format(msgErrorW3CFormat, lastModDate, "dataset " + datasetPid, e.getLocalizedMessage())); } } - logger.info("Writing staged sitemap to " + stagedSitemapPathAndFileString); - StreamResult result = new StreamResult(new File(stagedSitemapPathAndFileString)); + logger.info(String.format("Writing and checking sitemap file into %s", sitemapPathString)); try { - transformer.transform(source, result); - } catch (TransformerException ex) { - logger.warning("Unable to update sitemap! Unable to write staged sitemap to " + stagedSitemapPathAndFileString + ". TransformerException: " + ex.getLocalizedMessage()); - return; - } - - logger.info("Checking staged sitemap for well-formedness. The staged file is " + stagedSitemapPathAndFileString); - try { - XmlValidator.validateXmlWellFormed(stagedSitemapPathAndFileString); + wsg.write(); + if (dataverses.size() + datasets.size() > SITEMAP_LIMIT) { + wsg.writeSitemapsWithIndex(); + } } catch (Exception ex) { - logger.warning("Unable to update sitemap! Staged sitemap file is not well-formed XML! The exception for " + stagedSitemapPathAndFileString + " is " + ex.getLocalizedMessage()); - return; - } - - logger.info("Checking staged sitemap against XML schema. The staged file is " + stagedSitemapPathAndFileString); - URL schemaUrl = null; - try { - schemaUrl = new URL("https://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"); - } catch (MalformedURLException ex) { - // This URL is hard coded and it's fine. We should never get MalformedURLException so we just swallow the exception and carry on. - } - try { - XmlValidator.validateXmlSchema(stagedSitemapPathAndFileString, schemaUrl); - } catch (SAXException | IOException ex) { - logger.warning("Unable to update sitemap! Exception caught while checking XML staged file (" + stagedSitemapPathAndFileString + " ) against XML schema: " + ex.getLocalizedMessage()); + final StringBuffer errorMsg = new StringBuffer("Unable to write or validate sitemap ! The exception is "); + errorMsg.append(ex.getLocalizedMessage()); + // Add causes messages exception + Throwable cause = ex.getCause(); + // Fix limit to 5 causes + final int causeLimit = 5; + int cpt = 0; + while (cause != null && cpt < causeLimit) { + errorMsg.append(" with cause ").append(cause.getLocalizedMessage()); + cause = ex.getCause(); + cpt = cpt + 1; + } + logger.warning(errorMsg.toString()); return; } - Path finalPath = Paths.get(finalSitemapPathAndFileString); - logger.info("Copying staged sitemap from " + stagedSitemapPathAndFileString + " to " + finalSitemapPathAndFileString); + logger.info(String.format("Remove staged sitemap %s", stagedSitemapPathAndFileString)); try { - Files.move(stagedPath, finalPath, StandardCopyOption.REPLACE_EXISTING); + Files.deleteIfExists(stagedSitemapPath); } catch (IOException ex) { - logger.warning("Unable to update sitemap! Unable to copy staged sitemap from " + stagedSitemapPathAndFileString + " to " + finalSitemapPathAndFileString + ". IOException: " + ex.getLocalizedMessage()); + logger.warning("Unable to delete sitemap staged file! IOException: " + ex.getLocalizedMessage()); return; } @@ -199,12 +140,11 @@ private static String getLastModDate(DvObjectContainer dvObjectContainer) { // TODO: Decide if YYYY-MM-DD is enough. https://www.sitemaps.org/protocol.html // says "The date of last modification of the file. This date should be in W3C Datetime format. // This format allows you to omit the time portion, if desired, and use YYYY-MM-DD." - return new SimpleDateFormat("yyyy-MM-dd").format(dvObjectContainer.getModificationTime()); + return dvObjectContainer.getModificationTime().toLocalDateTime().format(formatter); } public static boolean stageFileExists() { - String sitemapPathString = getSitemapPathString(); - String stagedSitemapPathAndFileString = sitemapPathString + File.separator + SITEMAP_FILENAME_STAGED; + String stagedSitemapPathAndFileString = getSitemapPathString() + File.separator + SITEMAP_FILENAME_STAGED; Path stagedPath = Paths.get(stagedSitemapPathAndFileString); if (Files.exists(stagedPath)) { logger.warning("Unable to update sitemap! The staged file from a previous run already existed. Delete " + stagedSitemapPathAndFileString + " and try again."); @@ -212,7 +152,7 @@ public static boolean stageFileExists() { } return false; } - + /** * Lookup the location where to generate the sitemap. * @@ -223,6 +163,6 @@ public static boolean stageFileExists() { */ private static String getSitemapPathString() { return JvmSettings.DOCROOT_DIRECTORY.lookup() + File.separator + "sitemap"; - } + } diff --git a/src/test/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtilTest.java index 41032ffa811..704859be86f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtilTest.java @@ -11,18 +11,19 @@ import java.io.File; import java.io.IOException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Date; import java.util.List; import static org.junit.jupiter.api.Assertions.*; -import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -32,6 +33,10 @@ class SiteMapUtilTest { + // see https://www.sitemaps.org/protocol.html#validating + final String xsdSitemap = "https://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"; + final String xsdSitemapIndex = "https://www.sitemaps.org/schemas/sitemap/0.9/siteindex.xsd"; + @TempDir Path tempDir; Path tempDocroot; @@ -105,7 +110,7 @@ void testUpdateSiteMap() throws IOException, ParseException, SAXException { // then String pathToSiteMap = tempDocroot.resolve("sitemap").resolve("sitemap.xml").toString(); assertDoesNotThrow(() -> XmlValidator.validateXmlWellFormed(pathToSiteMap)); - assertTrue(XmlValidator.validateXmlSchema(pathToSiteMap, new URL("https://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"))); + assertTrue(XmlValidator.validateXmlSchema(pathToSiteMap, new URL(xsdSitemap))); File sitemapFile = new File(pathToSiteMap); String sitemapString = XmlPrinter.prettyPrintXml(new String(Files.readAllBytes(Paths.get(sitemapFile.getAbsolutePath())))); @@ -116,7 +121,56 @@ void testUpdateSiteMap() throws IOException, ParseException, SAXException { assertFalse(sitemapString.contains(unpublishedPid)); assertFalse(sitemapString.contains(harvestedPid)); assertFalse(sitemapString.contains(deaccessionedPid)); + } + + @Test + void testHugeSiteMap() throws IOException, ParseException, SAXException { + // given + final int nbDataverse = 50; + final int nbDataset = 50000; + + final List dataverses = new ArrayList<>(nbDataverse); + for (int i = 1; i <= nbDataverse; i++) { + final Dataverse publishedDataverse = new Dataverse(); + publishedDataverse.setAlias(String.format("publishedDv%s", i)); + publishedDataverse.setModificationTime(new Timestamp(new Date().getTime())); + publishedDataverse.setPublicationDate(new Timestamp(new Date().getTime())); + dataverses.add(publishedDataverse); + } + + final List datasets = new ArrayList<>(nbDataset); + for (int i = 1; i <= nbDataset; i++) { + final Dataset published = new Dataset(); + published.setGlobalId(new GlobalId(DOIServiceBean.DOI_PROTOCOL, "10.666", String.format("FAKE/published%s", i), null, DOIServiceBean.DOI_RESOLVER_URL, null)); + published.setPublicationDate(new Timestamp(new Date().getTime())); + published.setModificationTime(new Timestamp(new Date().getTime())); + datasets.add(published); + } + // when + SiteMapUtil.updateSiteMap(dataverses, datasets); + + // then + final Path siteMapDir = tempDocroot.resolve("sitemap"); + final String pathToSiteMapIndexFile = siteMapDir.resolve("sitemap_index.xml").toString(); + assertDoesNotThrow(() -> XmlValidator.validateXmlWellFormed(pathToSiteMapIndexFile)); + assertTrue(XmlValidator.validateXmlSchema(pathToSiteMapIndexFile, new URL(xsdSitemapIndex))); + + final File sitemapFile = new File(pathToSiteMapIndexFile); + String sitemapString = XmlPrinter.prettyPrintXml(new String(Files.readAllBytes(Paths.get(sitemapFile.getAbsolutePath())), StandardCharsets.UTF_8)); + // System.out.println("sitemap: " + sitemapString); + + assertTrue(sitemapString.contains("sitemap1.xml")); + assertTrue(sitemapString.contains("sitemap2.xml")); + assertTrue(sitemapString.contains("")); + + final String pathToSiteMap1File = siteMapDir.resolve("sitemap1.xml").toString(); + assertDoesNotThrow(() -> XmlValidator.validateXmlWellFormed(pathToSiteMap1File)); + assertTrue(XmlValidator.validateXmlSchema(pathToSiteMap1File, new URL(xsdSitemap))); + + final String pathToSiteMap2File = siteMapDir.resolve("sitemap2.xml").toString(); + assertDoesNotThrow(() -> XmlValidator.validateXmlWellFormed(pathToSiteMap2File)); + assertTrue(XmlValidator.validateXmlSchema(pathToSiteMap2File, new URL(xsdSitemap))); } } From 528e25a50c74976449ad306c7e61b39906671ea1 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Fri, 2 Feb 2024 14:22:58 -0500 Subject: [PATCH 050/970] xsl --- .../edu/harvard/iq/dataverse/ddi-to-fo.xsl | 52 +++---------------- 1 file changed, 7 insertions(+), 45 deletions(-) diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl index c25c76246d6..cb622912960 100644 --- a/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl @@ -109,7 +109,7 @@ - + @@ -123,7 +123,7 @@ Times - + - 1 + 1 0 @@ -3884,50 +3884,17 @@ select="$catgry-freq div $catgry-sum-freq"/> - - - - - - - - - - - - - - - - - - - - 0 - - - - - + + + test="not(@missing='Y')"> - - - - - - + @@ -3947,11 +3914,6 @@ - - - From 60a2795951aa665819cfecea92c4b9e4e0a63b07 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Fri, 2 Feb 2024 17:09:24 -0500 Subject: [PATCH 051/970] xsl instead xslt --- src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl index cb622912960..26fd7c23479 100644 --- a/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl @@ -123,7 +123,7 @@ Times - + + diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index 72fe279fbf8..be502d211d2 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -64,6 +64,7 @@
+
diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml index f69b5c35afd..3a49af29329 100644 --- a/src/main/webapp/file.xhtml +++ b/src/main/webapp/file.xhtml @@ -43,7 +43,7 @@
#{FilePage.fileMetadata.label} - +
@@ -64,22 +64,23 @@

- + - + + - + + value="#{bundle['file.DatasetVersion']} #{FilePage.fileMetadata.datasetVersion.versionNumber}.#{FilePage.fileMetadata.datasetVersion.minorVersionNumber}"/>
@@ -98,9 +99,9 @@ - -
@@ -145,11 +146,11 @@ - - -
@@ -192,7 +193,7 @@
-
    - @@ -227,12 +228,12 @@
    #{bundle['file.editBtn']} - +
    #{bundle['metrics.file.title']} - -
    @@ -317,7 +318,7 @@
    -
    @@ -351,7 +352,7 @@ - @@ -376,7 +377,7 @@ - +
      @@ -393,8 +394,8 @@
      - -
      @@ -443,7 +444,7 @@ #{bundle['file.dataFilesTab.metadata.addBtn']}
    @@ -483,7 +484,7 @@
    -
    @@ -495,7 +496,7 @@ #{bundle['file.metadata.filetags']} -
    +
    @@ -504,7 +505,7 @@
    - + @@ -566,6 +567,13 @@ #{FilePage.file.embargo.reason} + + + #{bundle['file.metadataTab.fileMetadata.retentionReason.label']} + + #{FilePage.file.retention.reason} + + #{bundle['file.metadataTab.fileMetadata.size.label']} @@ -624,8 +632,8 @@ - - + +

    #{bundle['file.share.tip']}

    @@ -634,7 +642,7 @@ #{bundle.close}
    - + @@ -645,7 +653,7 @@ - + @@ -657,7 +665,7 @@ - + @@ -678,7 +686,7 @@ #{bundle.close}
- +

#{bundle['file.compute.fileAccessDenied']}

@@ -690,7 +698,7 @@ + + \ No newline at end of file From 280ed49c48a97781ef8df69ea5d90d3456c81bd4 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 12 Sep 2024 15:31:23 -0400 Subject: [PATCH 848/970] adopt using CDI, fix funderIdentifier element per schema --- .../pidproviders/PidProviderFactoryBean.java | 5 +- .../pidproviders/doi/XmlMetadataTemplate.java | 68 ++++++++++++------- .../doi/datacite/DataCiteDOIProvider.java | 4 -- 3 files changed, 43 insertions(+), 34 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java index 2e3b96b2dd7..d7a19355a78 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java @@ -252,8 +252,5 @@ public PidProvider getDefaultPidGenerator() { return PidUtil.getPidProvider(protocol, authority, shoulder); } } - - public JsonObject getExternalVocabularyValue(String name) { - return datasetFieldServiceBean.getExternalVocabularyValue(name); - } + } \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 93631ad8c0a..fc9fbd79acf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -41,6 +41,7 @@ import edu.harvard.iq.dataverse.DatasetField; import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; import edu.harvard.iq.dataverse.DatasetFieldConstant; +import edu.harvard.iq.dataverse.DatasetFieldServiceBean; import edu.harvard.iq.dataverse.DatasetFieldType; import edu.harvard.iq.dataverse.DatasetFieldValue; import edu.harvard.iq.dataverse.DatasetRelPublication; @@ -70,6 +71,7 @@ import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.xml.XmlPrinter; import edu.harvard.iq.dataverse.util.xml.XmlWriterUtil; +import jakarta.enterprise.inject.spi.CDI; import jakarta.json.JsonObject; public class XmlMetadataTemplate { @@ -623,34 +625,32 @@ private void writeEntityElements(XMLStreamWriter xmlw, String elementName, Strin } if (StringUtils.isNotBlank(affiliation)) { - String orgName = getOrganizationFromRORField(affiliation, attributeMap); - XmlWriterUtil.writeFullElementWithAttributes(xmlw, "affiliation", attributeMap, StringEscapeUtils.escapeXml10(orgName)); - } - xmlw.writeEndElement(); - } - - private String getOrganizationFromRORField(String nameOrRor, Map attributeMap) { - boolean isROR=false; - String orgName = nameOrRor; - ExternalIdentifier externalIdentifier = ExternalIdentifier.ROR; - if (externalIdentifier.isValidIdentifier(nameOrRor)) { - isROR=true; - if(pidProvider instanceof DataCiteDOIProvider dcProvider) { - JsonObject jo = dcProvider.getExternalVocabularyValue(nameOrRor); - if(jo!=null) { + attributeMap.clear(); + boolean isROR=false; + String orgName = affiliation; + ExternalIdentifier externalIdentifier = ExternalIdentifier.ROR; + if (externalIdentifier.isValidIdentifier(orgName)) { + isROR = true; + JsonObject jo = getExternalVocabularyValue(orgName); + if (jo != null) { orgName = jo.getString("termName"); } } - } - - attributeMap.clear(); - if (isROR) { + + if (isROR) { + + attributeMap.put("schemeURI", "https://ror.org"); + attributeMap.put("affiliationIdentifierScheme", "ROR"); + attributeMap.put("affiliationIdentifier", orgName); + } - attributeMap.put("schemeURI", "https://ror.org"); - attributeMap.put("affiliationIdentifierScheme", "ROR"); - attributeMap.put("affiliationIdentifier", nameOrRor); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "affiliation", attributeMap, StringEscapeUtils.escapeXml10(orgName)); } - return orgName; + xmlw.writeEndElement(); + } + + private JsonObject getExternalVocabularyValue(String id) { + return CDI.current().select(DatasetFieldServiceBean.class).get().getExternalVocabularyValue(id); } /** @@ -1498,10 +1498,26 @@ private void writeFundingReferences(XMLStreamWriter xmlw, DvObject dvObject) thr } if (!StringUtils.isBlank(funder)) { fundingReferenceWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "fundingReferences", fundingReferenceWritten); - Map attributeMap = new HashMap<>(); - funder = getOrganizationFromRORField(funder, attributeMap); + boolean isROR=false; + String funderIdentifier = null; + ExternalIdentifier externalIdentifier = ExternalIdentifier.ROR; + if (externalIdentifier.isValidIdentifier(funder)) { + isROR = true; + JsonObject jo = getExternalVocabularyValue(funder); + if (jo != null) { + funderIdentifier = funder; + funder = jo.getString("termName"); + } + } + xmlw.writeStartElement("fundingReference"); // - XmlWriterUtil.writeFullElementWithAttributes(xmlw, "funderName", attributeMap, StringEscapeUtils.escapeXml10(funder)); + XmlWriterUtil.writeFullElement(xmlw, "funderName", StringEscapeUtils.escapeXml10(funder)); + if (isROR) { + Map attributeMap = new HashMap<>(); + attributeMap.put("schemeURI", "https://ror.org"); + attributeMap.put("funderIdentifierType", "ROR"); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "funderIdentifier", attributeMap, StringEscapeUtils.escapeXml10(funderIdentifier)); + } if (StringUtils.isNotBlank(awardNumber)) { XmlWriterUtil.writeFullElement(xmlw, "awardNumber", StringEscapeUtils.escapeXml10(awardNumber)); } diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java index d84b7760a96..5630844fb32 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java @@ -342,9 +342,5 @@ public boolean updateIdentifier(DvObject dvObject) { return false; } } - - public JsonObject getExternalVocabularyValue(String name) { - return pidProviderService.getExternalVocabularyValue(name); - } } From aee157dbe5628c0a89827627ecca74fb4d8d4af3 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 12 Sep 2024 15:35:38 -0400 Subject: [PATCH 849/970] datasetTypes test fix --- src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java index 1dec51cc3ef..e8426b638d7 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java @@ -240,7 +240,8 @@ public void testAddAndDeleteDatasetType() { numbersOnly.prettyPrint(); numbersOnly.then().assertThat().statusCode(BAD_REQUEST.getStatusCode()); - String randomName = UUID.randomUUID().toString().substring(0, 8); + //Avoid all-numeric names (which are not allowed) + String randomName = "A" + UUID.randomUUID().toString().substring(0, 8); String jsonIn = Json.createObjectBuilder().add("name", randomName).build().toString(); System.out.println("adding type with name " + randomName); From f255f1996eb93917170aa7744a28ae29214db02b Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 12 Sep 2024 16:43:58 -0400 Subject: [PATCH 850/970] release note/changelog changes --- doc/release-notes/10632-DataCiteXMLandRelationType.md | 6 +++--- doc/sphinx-guides/source/api/changelog.rst | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/doc/release-notes/10632-DataCiteXMLandRelationType.md b/doc/release-notes/10632-DataCiteXMLandRelationType.md index d90edcad0e2..10e9a830714 100644 --- a/doc/release-notes/10632-DataCiteXMLandRelationType.md +++ b/doc/release-notes/10632-DataCiteXMLandRelationType.md @@ -8,9 +8,9 @@ For details see https://github.com/IQSS/dataverse/pull/10632 and https://github. Multiple backward incompatible changes and bug fixes have been made to API calls (3 of the four of which were not documented) related to updating PID target urls and metadata at the provider service: [Update Target URL for a Published Dataset at the PID provider](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-target-url-for-a-published-dataset-at-the-pid-provider) -[Update Target URL for all Published Datasets at the PID provider](https://dataverse-guide--10632.org.readthedocs.build/en/10632/admin/dataverses-datasets.html#update-target-url-for-all-published-datasets-at-the-pid-provider) -[Update Metadata for a Published Dataset at the PID provider](https://dataverse-guide--10632.org.readthedocs.build/en/10632/admin/dataverses-datasets.html#update-metadata-for-a-published-dataset-at-the-pid-provider) -[Update Metadata for all Published Datasets at the PID provider](https://dataverse-guide--10632.org.readthedocs.build/en/10632/admin/dataverses-datasets.html#update-metadata-for-all-published-datasets-at-the-pid-provider) +[Update Target URL for all Published Datasets at the PID provider](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-target-url-for-all-published-datasets-at-the-pid-provider) +[Update Metadata for a Published Dataset at the PID provider](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-metadata-for-a-published-dataset-at-the-pid-provider) +[Update Metadata for all Published Datasets at the PID provider](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-metadata-for-all-published-datasets-at-the-pid-provider) Upgrade instructions -------------------- diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index a7af3e84b28..b40d32f37d8 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -6,6 +6,12 @@ This API changelog is experimental and we would love feedback on its usefulness. .. contents:: |toctitle| :local: :depth: 1 +v6.4 +---- + +[**/api/datasets/$dataset-id/modifyRegistration**](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-target-url-for-a-published-dataset-at-the-pid-provider) - changes from GET to POST +[**/api/datasets/modifyRegistrationPIDMetadataAll**](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-metadata-for-all-published-datasets-at-the-pid-provider) - changes from GET to POST + v6.3 ---- From 2ecbd692bd5a4a6fbb2702af5c859a2f09e0f577 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 13 Sep 2024 12:26:04 +0200 Subject: [PATCH 851/970] chore(ct): update base image wait4x to 2.14.2 for stdlib update --- modules/container-base/src/main/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/container-base/src/main/docker/Dockerfile b/modules/container-base/src/main/docker/Dockerfile index e3717928285..802db62e5e4 100644 --- a/modules/container-base/src/main/docker/Dockerfile +++ b/modules/container-base/src/main/docker/Dockerfile @@ -106,7 +106,7 @@ EOF ARG JATTACH_VERSION="v2.2" ARG JATTACH_TGZ_CHECKSUM_AMD64="acd9e17f15749306be843df392063893e97bfecc5260eef73ee98f06e5cfe02f" ARG JATTACH_TGZ_CHECKSUM_ARM64="288ae5ed87ee7fe0e608c06db5a23a096a6217c9878ede53c4e33710bdcaab51" -ARG WAIT4X_VERSION="v2.14.1" +ARG WAIT4X_VERSION="v2.14.2" ARG PKGS="jq imagemagick curl unzip wget acl lsof procps netcat-openbsd dumb-init" # Installing the packages in an extra container layer for better caching From f20117406c72f2eef91371ed3276200d58bcbceb Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 13 Sep 2024 14:12:14 +0200 Subject: [PATCH 852/970] chore(ct): update base image wait4x to 2.14.2 for stdlib update #10844 --- modules/container-base/src/main/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/container-base/src/main/docker/Dockerfile b/modules/container-base/src/main/docker/Dockerfile index 29078e6896c..57ab5ac5787 100644 --- a/modules/container-base/src/main/docker/Dockerfile +++ b/modules/container-base/src/main/docker/Dockerfile @@ -106,7 +106,7 @@ EOF ARG JATTACH_VERSION="v2.2" ARG JATTACH_TGZ_CHECKSUM_AMD64="acd9e17f15749306be843df392063893e97bfecc5260eef73ee98f06e5cfe02f" ARG JATTACH_TGZ_CHECKSUM_ARM64="288ae5ed87ee7fe0e608c06db5a23a096a6217c9878ede53c4e33710bdcaab51" -ARG WAIT4X_VERSION="v2.14.0" +ARG WAIT4X_VERSION="v2.14.2" ARG PKGS="jq imagemagick curl unzip wget acl lsof procps netcat-openbsd dumb-init" # Installing the packages in an extra container layer for better caching From 194dae8172cd148233a6f2a0e1fbe9bcad6a0b51 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 13 Sep 2024 10:36:45 -0400 Subject: [PATCH 853/970] don't send contributors w/o contributorType --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index fc9fbd79acf..6259a41017f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -574,7 +574,7 @@ private void writeContributors(XMLStreamWriter xmlw, DvObject dvObject) throws X } // QDR - doesn't have Funder in the contributor type list. // Using a string isn't i18n - if (StringUtils.isNotBlank(contributor) && !StringUtils.equalsIgnoreCase("Funder", contributorType)) { + if (StringUtils.isNotBlank(contributor) && StringUtils.isNotBlank(contributorType) && !StringUtils.equalsIgnoreCase("Funder", contributorType)) { contributorsCreated = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "contributors", contributorsCreated); JsonObject entityObject = PersonOrOrgUtil.getPersonOrOrganization(contributor, false, false); writeEntityElements(xmlw, "contributor", contributorType, entityObject, null, null, null); From 35ff432480800c1b99400bf00db9a64c3456981a Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 13 Sep 2024 15:25:01 -0400 Subject: [PATCH 854/970] relatedIdentifierType is required --- .../pidproviders/doi/XmlMetadataTemplate.java | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 6259a41017f..6ed13a05301 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -1031,6 +1031,13 @@ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) th break; default: if (relatedIdentifier != null) { + //See if it is a GlobalID we know + try { + GlobalId pid = PidUtil.parseAsGlobalID(relatedIdentifier); + relatedIdentifier = pid.asRawIdentifier(); + pubIdType = getCanonicalPublicationType(pid.getProtocol()); + } catch (IllegalArgumentException e) { + } // For non-URL types, if a URL is given, split the string to get a schemeUri try { URL relatedUrl = new URI(relatedIdentifier).toURL(); @@ -1039,16 +1046,16 @@ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) th String site = String.format("%s://%s", protocol, authority); relatedIdentifier = relatedIdentifier.substring(site.length()); attributes.put("schemeURI", site); + pubIdType = "URL"; } catch (URISyntaxException | MalformedURLException | IllegalArgumentException e) { - // Just an identifier + // Just an identifier but without a pubIdType we won't include it + logger.warning("Related Identifier found without type: " + relatedIdentifier); } } } - if (StringUtils.isNotBlank(relatedIdentifier)) { + if (StringUtils.isNotBlank(relatedIdentifier) && StringUtils.isNotBlank(pubIdType)) { // Still have a valid entry - if (pubIdType != null) { - attributes.put("relatedIdentifierType", pubIdType); - } + attributes.put("relatedIdentifierType", pubIdType); attributes.put("relationType", relationType); relatedIdentifiersWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "relatedIdentifiers", relatedIdentifiersWritten); XmlWriterUtil.writeFullElementWithAttributes(xmlw, "relatedIdentifier", attributes, relatedIdentifier); @@ -1115,8 +1122,10 @@ private static String getCanonicalPublicationType(String pubIdType) { relatedIdentifierTypeMap.put("URL".toLowerCase(), "URL"); relatedIdentifierTypeMap.put("URN".toLowerCase(), "URN"); relatedIdentifierTypeMap.put("WOS".toLowerCase(), "WOS"); - // Add entry for Handle protocol so this can be used with GlobalId/getProtocol() + // Add entry for Handle,Perma protocols so this can be used with GlobalId/getProtocol() relatedIdentifierTypeMap.put("hdl".toLowerCase(), "Handle"); + relatedIdentifierTypeMap.put("perma".toLowerCase(), "URL"); + } return relatedIdentifierTypeMap.get(pubIdType); } From 6c8f73e47b5e77467a47d753f49d855c242727f8 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Sat, 14 Sep 2024 10:41:24 -0400 Subject: [PATCH 855/970] flip to prefer identifier over url seeing cases at QDR where the type is DOI, the identifier is the doi and the URL is a non-DOI reference (e.g. at pubmed). --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 6ed13a05301..6b9e6c31660 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -993,10 +993,10 @@ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) th pubIdType = getCanonicalPublicationType(pubIdType); logger.fine("Canonical type: " + pubIdType); - // Prefer url if set, otherwise check identifier - String relatedIdentifier = url; + // Prefer identifier if set, otherwise check url + String relatedIdentifier = identifier; if (StringUtils.isBlank(relatedIdentifier)) { - relatedIdentifier = identifier; + relatedIdentifier = url; } logger.fine("Related identifier: " + relatedIdentifier); // For types where we understand the protocol, get the canonical form From 86aec687f6451713377b9753bef83fbe5eb609e6 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Sat, 14 Sep 2024 10:56:49 -0400 Subject: [PATCH 856/970] Handle case where type is set but there's no identifier --- .../pidproviders/doi/XmlMetadataTemplate.java | 89 ++++++++++--------- 1 file changed, 45 insertions(+), 44 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 6b9e6c31660..77b9f9c63bb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -1000,56 +1000,57 @@ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) th } logger.fine("Related identifier: " + relatedIdentifier); // For types where we understand the protocol, get the canonical form - - switch (pubIdType != null ? pubIdType : "none") { - case "DOI": - if (!(relatedIdentifier.startsWith("doi:") || relatedIdentifier.startsWith("http"))) { - relatedIdentifier = "doi:" + relatedIdentifier; - } - logger.fine("Intermediate Related identifier: " + relatedIdentifier); - try { - GlobalId pid = PidUtil.parseAsGlobalID(relatedIdentifier); - relatedIdentifier = pid.asRawIdentifier(); - } catch (IllegalArgumentException e) { - logger.warning("Invalid DOI: " + e.getLocalizedMessage()); - relatedIdentifier = null; - } - logger.fine("Final Related identifier: " + relatedIdentifier); - break; - case "Handle": - if (!relatedIdentifier.startsWith("hdl:") || !relatedIdentifier.startsWith("http")) { - relatedIdentifier = "hdl:" + relatedIdentifier; - } - try { - GlobalId pid = PidUtil.parseAsGlobalID(relatedIdentifier); - relatedIdentifier = pid.asRawIdentifier(); - } catch (IllegalArgumentException e) { - relatedIdentifier = null; - } - break; - case "URL": - break; - default: - if (relatedIdentifier != null) { - //See if it is a GlobalID we know + if (StringUtils.isNotBlank(relatedIdentifier)) { + switch (pubIdType != null ? pubIdType : "none") { + case "DOI": + if (!(relatedIdentifier.startsWith("doi:") || relatedIdentifier.startsWith("http"))) { + relatedIdentifier = "doi:" + relatedIdentifier; + } + logger.fine("Intermediate Related identifier: " + relatedIdentifier); try { GlobalId pid = PidUtil.parseAsGlobalID(relatedIdentifier); relatedIdentifier = pid.asRawIdentifier(); - pubIdType = getCanonicalPublicationType(pid.getProtocol()); } catch (IllegalArgumentException e) { + logger.warning("Invalid DOI: " + e.getLocalizedMessage()); + relatedIdentifier = null; + } + logger.fine("Final Related identifier: " + relatedIdentifier); + break; + case "Handle": + if (!relatedIdentifier.startsWith("hdl:") || !relatedIdentifier.startsWith("http")) { + relatedIdentifier = "hdl:" + relatedIdentifier; } - // For non-URL types, if a URL is given, split the string to get a schemeUri try { - URL relatedUrl = new URI(relatedIdentifier).toURL(); - String protocol = relatedUrl.getProtocol(); - String authority = relatedUrl.getAuthority(); - String site = String.format("%s://%s", protocol, authority); - relatedIdentifier = relatedIdentifier.substring(site.length()); - attributes.put("schemeURI", site); - pubIdType = "URL"; - } catch (URISyntaxException | MalformedURLException | IllegalArgumentException e) { - // Just an identifier but without a pubIdType we won't include it - logger.warning("Related Identifier found without type: " + relatedIdentifier); + GlobalId pid = PidUtil.parseAsGlobalID(relatedIdentifier); + relatedIdentifier = pid.asRawIdentifier(); + } catch (IllegalArgumentException e) { + relatedIdentifier = null; + } + break; + case "URL": + break; + default: + if (relatedIdentifier != null) { + // See if it is a GlobalID we know + try { + GlobalId pid = PidUtil.parseAsGlobalID(relatedIdentifier); + relatedIdentifier = pid.asRawIdentifier(); + pubIdType = getCanonicalPublicationType(pid.getProtocol()); + } catch (IllegalArgumentException e) { + } + // For non-URL types, if a URL is given, split the string to get a schemeUri + try { + URL relatedUrl = new URI(relatedIdentifier).toURL(); + String protocol = relatedUrl.getProtocol(); + String authority = relatedUrl.getAuthority(); + String site = String.format("%s://%s", protocol, authority); + relatedIdentifier = relatedIdentifier.substring(site.length()); + attributes.put("schemeURI", site); + pubIdType = "URL"; + } catch (URISyntaxException | MalformedURLException | IllegalArgumentException e) { + // Just an identifier but without a pubIdType we won't include it + logger.warning("Related Identifier found without type: " + relatedIdentifier); + } } } } From 203add1bb99bbdc2ffdec1c45c07f51d3b5db79a Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Sat, 14 Sep 2024 11:20:11 -0400 Subject: [PATCH 857/970] map non-standard contributors to Other, remove unused imports --- .../pidproviders/doi/XmlMetadataTemplate.java | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 77b9f9c63bb..c86baf17f90 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -10,16 +10,15 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.logging.Logger; -import java.util.stream.Collectors; import javax.xml.stream.XMLOutputFactory; import javax.xml.stream.XMLStreamException; @@ -27,14 +26,9 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.StringEscapeUtils; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import org.ocpsoft.common.util.Strings; import edu.harvard.iq.dataverse.AlternativePersistentIdentifier; -import edu.harvard.iq.dataverse.ControlledVocabularyValue; import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetAuthor; @@ -42,8 +36,6 @@ import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; import edu.harvard.iq.dataverse.DatasetFieldConstant; import edu.harvard.iq.dataverse.DatasetFieldServiceBean; -import edu.harvard.iq.dataverse.DatasetFieldType; -import edu.harvard.iq.dataverse.DatasetFieldValue; import edu.harvard.iq.dataverse.DatasetRelPublication; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DvObject; @@ -52,23 +44,16 @@ import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.api.Util; -import edu.harvard.iq.dataverse.api.dto.DatasetDTO; -import edu.harvard.iq.dataverse.api.dto.FieldDTO; -import edu.harvard.iq.dataverse.api.dto.MetadataBlockDTO; import edu.harvard.iq.dataverse.dataset.DatasetType; import edu.harvard.iq.dataverse.dataset.DatasetUtil; -import edu.harvard.iq.dataverse.export.DDIExporter; import edu.harvard.iq.dataverse.license.License; import edu.harvard.iq.dataverse.pidproviders.AbstractPidProvider; import edu.harvard.iq.dataverse.pidproviders.PidProvider; import edu.harvard.iq.dataverse.pidproviders.PidUtil; -import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; -import edu.harvard.iq.dataverse.pidproviders.doi.datacite.DataCiteDOIProvider; import edu.harvard.iq.dataverse.pidproviders.handle.HandlePidProvider; import edu.harvard.iq.dataverse.pidproviders.perma.PermaLinkPidProvider; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.PersonOrOrgUtil; -import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.xml.XmlPrinter; import edu.harvard.iq.dataverse.util.xml.XmlWriterUtil; import jakarta.enterprise.inject.spi.CDI; @@ -575,6 +560,7 @@ private void writeContributors(XMLStreamWriter xmlw, DvObject dvObject) throws X // QDR - doesn't have Funder in the contributor type list. // Using a string isn't i18n if (StringUtils.isNotBlank(contributor) && StringUtils.isNotBlank(contributorType) && !StringUtils.equalsIgnoreCase("Funder", contributorType)) { + contributorType = getCanonicalContributorType(contributorType); contributorsCreated = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "contributors", contributorsCreated); JsonObject entityObject = PersonOrOrgUtil.getPersonOrOrganization(contributor, false, false); writeEntityElements(xmlw, "contributor", contributorType, entityObject, null, null, null); @@ -587,6 +573,18 @@ private void writeContributors(XMLStreamWriter xmlw, DvObject dvObject) throws X } } + //List from https://schema.datacite.org/meta/kernel-4/include/datacite-contributorType-v4.xsd + private Set contributorTypes = new HashSet<>(Arrays.asList("ContactPerson", "DataCollector", "DataCurator", "DataManager", "Distributor", "Editor", + "HostingInstitution", "Other", "Producer", "ProjectLeader", "ProjectManager", "ProjectMember", "RegistrationAgency", "RegistrationAuthority", + "RelatedPerson", "ResearchGroup", "RightsHolder", "Researcher", "Sponsor", "Supervisor", "WorkPackageLeader")); + + private String getCanonicalContributorType(String contributorType) { + if(!contributorTypes.contains(contributorType)) { + return "Other"; + } + return contributorType; + } + private void writeEntityElements(XMLStreamWriter xmlw, String elementName, String type, JsonObject entityObject, String affiliation, String nameIdentifier, String nameIdentifierScheme) throws XMLStreamException { xmlw.writeStartElement(elementName); Map attributeMap = new HashMap(); From b06e620335bad37e07622c3e28a186d4505177f4 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Sat, 14 Sep 2024 11:31:50 -0400 Subject: [PATCH 858/970] Treat missing contrib type as Other --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index c86baf17f90..5ca62932e15 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -559,7 +559,7 @@ private void writeContributors(XMLStreamWriter xmlw, DvObject dvObject) throws X } // QDR - doesn't have Funder in the contributor type list. // Using a string isn't i18n - if (StringUtils.isNotBlank(contributor) && StringUtils.isNotBlank(contributorType) && !StringUtils.equalsIgnoreCase("Funder", contributorType)) { + if (StringUtils.isNotBlank(contributor) && !StringUtils.equalsIgnoreCase("Funder", contributorType)) { contributorType = getCanonicalContributorType(contributorType); contributorsCreated = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "contributors", contributorsCreated); JsonObject entityObject = PersonOrOrgUtil.getPersonOrOrganization(contributor, false, false); @@ -579,7 +579,7 @@ private void writeContributors(XMLStreamWriter xmlw, DvObject dvObject) throws X "RelatedPerson", "ResearchGroup", "RightsHolder", "Researcher", "Sponsor", "Supervisor", "WorkPackageLeader")); private String getCanonicalContributorType(String contributorType) { - if(!contributorTypes.contains(contributorType)) { + if(StringUtils.isBlank(contributorType) || !contributorTypes.contains(contributorType)) { return "Other"; } return contributorType; From 66bcbbb12976e03b6f41feae39255b214580f481 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 16 Sep 2024 15:03:39 +0200 Subject: [PATCH 859/970] chore(ci): delete duplicate action after renaming --- .../actions/check-newer-base-image/action.yml | 53 ------------------- 1 file changed, 53 deletions(-) delete mode 100644 .github/actions/check-newer-base-image/action.yml diff --git a/.github/actions/check-newer-base-image/action.yml b/.github/actions/check-newer-base-image/action.yml deleted file mode 100644 index d0c2e92d60a..00000000000 --- a/.github/actions/check-newer-base-image/action.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -name: "Check Newer Base Image" -description: "Determine if a base image is more recent than the derived." -inputs: - base: - description: 'The name and (rolling) tag of the base image to check - or - a shell command to get it' - required: true - derived: - description: 'The name and (rolling) tag of the derived image - or - a shell command to get it' - required: true -outputs: - is-more-recent: - description: "True if base image has a more recent update, false if not." - value: "${{ steps.determine.outputs.is_more_recent }}" - -runs: - using: composite - steps: - - shell: bash - id: determine - run: | - BASE_IMAGE="${{ inputs.base }}" - # Get namespace, default to "library" if not found - BASE_IMAGE_NS="${BASE_IMAGE%/*}" - if [[ "$BASE_IMAGE_NS" = "${BASE_IMAGE}" ]]; then - BASE_IMAGE_NS="library" - fi - BASE_IMAGE_REPO="${BASE_IMAGE%:*}" - BASE_IMAGE_TAG="${BASE_IMAGE#*:}" - - BASE_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${BASE_IMAGE_NS}/repositories/${BASE_IMAGE_REPO}/tags/${BASE_IMAGE_TAG}" | jq -r .last_updated )" - if [[ "$BASE_IMAGE_LAST_UPDATE" = "null" ]]; then - echo "::error title='Invalid Base Image'::Could not find ${BASE_IMAGE} in the registry" - exit 1 - fi - - DERIVED_IMAGE="${{ inputs.derived }}" - # Get namespace, default to "library" if not found - DERIVED_IMAGE_NS="${DERIVED_IMAGE%/*}" - if [[ "${DERIVED_IMAGE_NS}" = "${DERIVED_IMAGE}" ]]; then - DERIVED_IMAGE_NS="library" - fi - DERIVED_IMAGE_REPO="$( echo "${DERIVED_IMAGE%:*}" | cut -f2 -d/ )" - DERIVED_IMAGE_TAG="${DERIVED_IMAGE#*:}" - - DERIVED_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${DERIVED_IMAGE_NS}/repositories/${DERIVED_IMAGE_REPO}/tags/${DERIVED_IMAGE_TAG}" | jq -r .last_updated )" - if [[ "$DERIVED_IMAGE_LAST_UPDATE" = "null" || "$DERIVED_IMAGE_LAST_UPDATE" < "$BASE_IMAGE_LAST_UPDATE" ]]; then - echo "Base image $BASE_IMAGE has a newer release ($BASE_IMAGE_LAST_UPDATE), which is more recent than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" - echo "is_more_recent=true" >> $GITHUB_OUTPUT - else - echo "Base image $BASE_IMAGE ($BASE_IMAGE_LAST_UPDATE) is older than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" - echo "is_more_recent=false" >> $GITHUB_OUTPUT - fi From d3dc5c65b957aa8489168560b4fe398d7e8d8439 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 16 Sep 2024 16:14:21 +0200 Subject: [PATCH 860/970] docs(ct): update base image tag policy from latest discussion #10827 As discussed during the 2024-09-12 containerization working group meeting (see ct.gdcc.io) and on https://dataverse.zulipchat.com/#narrow/stream/375812-containers/topic/change.20version.20scheme.20base.20image.3F/near/469884104 --- .../source/container/base-image.rst | 53 ++++++++++++------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst index 83799397860..8d08f042e5d 100644 --- a/doc/sphinx-guides/source/container/base-image.rst +++ b/doc/sphinx-guides/source/container/base-image.rst @@ -37,39 +37,54 @@ Our tagging is inspired by `Bitnami .`` tag). +Expect disruptive changes in case of high risk security threats. + +- | **Latest** + | Definition: ``latest`` + | Summary: Rolling tag, always pointing to the latest revision of the most current Dataverse release. - | **Rolling Production** | Definition: ``.-`` | Example: ``6.4-noble`` - | Summary: A **rolling tag** for released versions of Dataverse that will always point to latest revision of an immutable production image (see below). - | The **three latest releases** of the Dataverse project will receive regular, scheduled updates to the image such as security patches for the underlying operating system. Content will be fairly stable as disruptive changes like Payara or Java upgrades will be handled in a new major or minor upgrade to Dataverse (a new ``.`` tag). Expect disruptive changes in case of high risk security threats. + | Summary: Rolling tag, pointing to the latest revision of an immutable production image for released versions of Dataverse. - | **Immutable Production** | Definition: ``.--r`` | Example: ``6.4-noble-r1`` | Summary: An **immutable tag** where the revision is incremented for rebuilds of the image. | This image should be especially attractive if you want explict control over when your images are updated. -We offer two rolling tags for developers: +Tags for Development Use +^^^^^^^^^^^^^^^^^^^^^^^^ -- | **Latest** - | Definition: ``latest`` - | Example: ``latest`` - | Summary: A **rolling tag** that tracks the ``develop`` branch (see also :ref:`develop-branch`). (`Dockerfile `__) +All of the tags below are strongly recommended for development purposes only due to their fast changing nature. +In addition to updates due to PR merges, the most recent are undergoing scheduled maintenance to ensure timely security fixes. +When a development cycle of the Dataverse project finishes, maintenance ceases for any tags carrying version numbers. +For now, stale images will be kept on Docker Hub indefinitely. + +- | **Unstable** + | Definition: ``unstable`` + | Summary: Rolling tag, tracking the ``develop`` branch (see also :ref:`develop-branch`). (`Dockerfile `__) | Please expect abrupt changes like new Payara or Java versions as well as OS updates or flavor switches when using this tag. - Usage is recommended for **development purposes only**. +- | **Upcoming** + | Definition: ``.-`` + | Example: ``6.5-noble`` + | Summary: Rolling tag, equivalent to ``unstable`` for current development cycle. + Will roll over to the rolling production tag after a Dataverse release. - | **Flexible Stack** | Definition: ``.--p-j``: - | Example: ``6.4-noble-p6.2024.6-j17`` - | Summary: A **rolling tag** during a development cycle of the Dataverse software (`Dockerfile `__). - | *Note 1:* No updates for tags after a development cycle ends. - Usage is recommended for **development purposes only**. - | *Note 2:* The current tag will always be 1 minor version ahead of Datavere releases. - Example: Assume Dataverse ``6.x`` is released, ``6.(x+1)`` is underway. - The rolling tag in use during the cycle will be ``6.(x+1)-ffffff-p6.202P.P-jJJ``. - See also: :doc:`/developers/making-releases`. - -You can find these tags at https://hub.docker.com/r/gdcc/base/tags + | Example: ``6.5-noble-p6.2024.6-j17`` + | Summary: Rolling tag during a development cycle of the Dataverse software (`Dockerfile `__). + +**NOTE**: The current tag will always be 1 minor version ahead of Datavere releases. +Example: Assume Dataverse ``6.x`` is released, ``6.(x+1)`` is underway. +The rolling tag in use during the cycle will be ``6.(x+1)-FFF`` and ``6.(x+1)-FFF-p6.202P.P-jJJ``. +See also: :doc:`/developers/making-releases`. Image Contents ++++++++++++++ From 08a94bb84542b7980330c4d428a03debe6a95316 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 16 Sep 2024 16:40:09 +0200 Subject: [PATCH 861/970] style,docs(ct): minor tweaks to base image policy as per @pdurbin --- doc/sphinx-guides/source/container/base-image.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst index 8d08f042e5d..84f6c5ece02 100644 --- a/doc/sphinx-guides/source/container/base-image.rst +++ b/doc/sphinx-guides/source/container/base-image.rst @@ -77,11 +77,11 @@ For now, stale images will be kept on Docker Hub indefinitely. | Summary: Rolling tag, equivalent to ``unstable`` for current development cycle. Will roll over to the rolling production tag after a Dataverse release. - | **Flexible Stack** - | Definition: ``.--p-j``: + | Definition: ``.--p-j`` | Example: ``6.5-noble-p6.2024.6-j17`` | Summary: Rolling tag during a development cycle of the Dataverse software (`Dockerfile `__). -**NOTE**: The current tag will always be 1 minor version ahead of Datavere releases. +**NOTE**: In these tags for development usage, the version number will always be 1 minor version ahead of existing Dataverse releases. Example: Assume Dataverse ``6.x`` is released, ``6.(x+1)`` is underway. The rolling tag in use during the cycle will be ``6.(x+1)-FFF`` and ``6.(x+1)-FFF-p6.202P.P-jJJ``. See also: :doc:`/developers/making-releases`. From 8cc9d20326509af6236d51362ab5729087fb1f09 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 16 Sep 2024 16:49:03 +0200 Subject: [PATCH 862/970] feat(ct): switch latest to unstable in base image flow As per latest discussion, we want to keep the unstable tag around. It shall still point to the latest from develop as it has been done before our revised tagging policy. Latest will be used for production images, much more aligned with the Bitnami policy --- .github/workflows/container_maintenance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index c09aad22916..80e063bc222 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -133,7 +133,7 @@ jobs: - name: Configure update of "latest" tag for development branch if: ${{ matrix.branch == env.DEVELOP_BRANCH }} run: | - echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" + echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=unstable" | tee -a "${GITHUB_ENV}" - name: Deploy multi-arch base container image to Docker Hub if: ${{ steps.temurin-check.outputs.is-more-recent == 'true' || steps.package-check.outputs.newer_packages == 'true' || inputs.force_build }} id: build From 466d2ef4ec0c5a33253acca82b861fd05564a571 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 16 Sep 2024 17:03:46 +0200 Subject: [PATCH 863/970] style,docs(ct): minor tweaks to base image policy as per @pdurbin --- doc/sphinx-guides/source/container/base-image.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst index 84f6c5ece02..a0852a5465f 100644 --- a/doc/sphinx-guides/source/container/base-image.rst +++ b/doc/sphinx-guides/source/container/base-image.rst @@ -72,12 +72,12 @@ For now, stale images will be kept on Docker Hub indefinitely. | Summary: Rolling tag, tracking the ``develop`` branch (see also :ref:`develop-branch`). (`Dockerfile `__) | Please expect abrupt changes like new Payara or Java versions as well as OS updates or flavor switches when using this tag. - | **Upcoming** - | Definition: ``.-`` + | Definition: ``.-`` | Example: ``6.5-noble`` | Summary: Rolling tag, equivalent to ``unstable`` for current development cycle. Will roll over to the rolling production tag after a Dataverse release. - | **Flexible Stack** - | Definition: ``.--p-j`` + | Definition: ``.--p-j`` | Example: ``6.5-noble-p6.2024.6-j17`` | Summary: Rolling tag during a development cycle of the Dataverse software (`Dockerfile `__). From 4c9e80fe09e0f607a039d61fd2dfbf190235660e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 16 Sep 2024 17:31:57 +0200 Subject: [PATCH 864/970] fix(ct): remove auth for revision action Maybe we won't run into a rate limit for now, as the limit of 180/s * 100 = 18000 tags/s seems to leave us some headroom for now. --- .github/actions/get-image-revision/action.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/actions/get-image-revision/action.yml b/.github/actions/get-image-revision/action.yml index ca9eedeabe1..6eaf89abba2 100644 --- a/.github/actions/get-image-revision/action.yml +++ b/.github/actions/get-image-revision/action.yml @@ -39,14 +39,17 @@ runs: *) IMAGE_NS_REPO="library/$IMAGE_NS_REPO";; # bare repository name (docker official image); must convert to namespace/repository syntax esac - # Without such a token we run into rate limits + # Without such a token we may run into rate limits + # OB 2024-09-16: for some reason using this token stopped working. Let's go without and see if we really fall into rate limits. token=$( curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE_NS_REPO:pull" ) ALL_TAGS="$( i=0 while [ $? == 0 ]; do i=$((i+1)) - RESULT=$( curl -s -H "Authorization: Bearer $token" "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) + # OB 2024-09-16: for some reason using this token stopped working. Let's go without and see if we really fall into rate limits. + # RESULT=$( curl -s -H "Authorization: Bearer $token" "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) + RESULT=$( curl -s "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) if [[ $( echo "$RESULT" | jq '.message' ) != "null" ]]; then # If we run into an error on the first attempt, that means we have a problem. if [[ "$i" == "1" ]]; then From 86f2c84d3ded51597f2e15f9bc86817e502afcec Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 16 Sep 2024 17:44:39 +0200 Subject: [PATCH 865/970] fix(ct): try to avoid shell substitution in base image flow We might need more backslashes to avoid telling the shell to replace what we want to be a maven property --- .github/workflows/container_maintenance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 80e063bc222..c8fe2fa2937 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -129,7 +129,7 @@ jobs: uses: gdcc/wip-dataverse-base-image/.github/actions/get-image-revision@10478-version-base-img with: image-ref: ${{ env.BASE_IMAGE }} - tag-options-prefix: "-Dbase.image.tag=\\${base.image.tag.release} -Ddocker.imagePropertyConfiguration=override -Ddocker.tags.revision=" + tag-options-prefix: "-Dbase.image.tag=\\\\${base.image.tag.release} -Ddocker.imagePropertyConfiguration=override -Ddocker.tags.revision=" - name: Configure update of "latest" tag for development branch if: ${{ matrix.branch == env.DEVELOP_BRANCH }} run: | From 997f7ebab9c61f578ddeb31197edfed847e3e64b Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 16 Sep 2024 17:54:58 +0200 Subject: [PATCH 866/970] fix(ct): avoid shell substitutions going awry for base image release tag Instead of using a Maven reference and dealing with escaping of $ chars, override the suffix with an empty string --- .github/workflows/container_maintenance.yml | 4 ++-- modules/container-base/pom.xml | 4 ++-- modules/container-base/src/backports/v6.1/001-pom.xml.patch | 4 ++-- modules/container-base/src/backports/v6.2/001-pom.xml.patch | 4 ++-- modules/container-base/src/backports/v6.3/001-pom.xml.patch | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index c8fe2fa2937..b7f8fcf45fa 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -96,7 +96,7 @@ jobs: if [[ "${{ matrix.branch }}" = "${{ env.DEVELOP_BRANCH }}" ]]; then NAME=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout ) else - NAME=$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag="\${base.image.tag.release}" -q -DforceStdout ) + NAME=$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout ) fi echo "BASE_IMAGE=${NAME}" | tee -a "${GITHUB_ENV}" @@ -129,7 +129,7 @@ jobs: uses: gdcc/wip-dataverse-base-image/.github/actions/get-image-revision@10478-version-base-img with: image-ref: ${{ env.BASE_IMAGE }} - tag-options-prefix: "-Dbase.image.tag=\\\\${base.image.tag.release} -Ddocker.imagePropertyConfiguration=override -Ddocker.tags.revision=" + tag-options-prefix: "-Dbase.image.tag.suffix='' -Ddocker.imagePropertyConfiguration=override -Ddocker.tags.revision=" - name: Configure update of "latest" tag for development branch if: ${{ matrix.branch == env.DEVELOP_BRANCH }} run: | diff --git a/modules/container-base/pom.xml b/modules/container-base/pom.xml index e707c6a69c8..b6667077d98 100644 --- a/modules/container-base/pom.xml +++ b/modules/container-base/pom.xml @@ -43,8 +43,8 @@ gdcc/base:${base.image.tag} - ${base.image.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} - ${base.image.version}-${java.image.flavor} + ${base.image.version}-${java.image.flavor}${base.image.tag.suffix} + -p${payara.version}-j${target.java.version} eclipse-temurin:${target.java.version}-jre-${java.image.flavor} noble 1000 diff --git a/modules/container-base/src/backports/v6.1/001-pom.xml.patch b/modules/container-base/src/backports/v6.1/001-pom.xml.patch index ef0fbdda9fc..6498f972889 100644 --- a/modules/container-base/src/backports/v6.1/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.1/001-pom.xml.patch @@ -7,8 +7,8 @@ - unstable - eclipse-temurin:${target.java.version}-jre + gdcc/base:${base.image.tag} -+ ${base.image.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} -+ ${base.image.version}-${java.image.flavor} ++ ${base.image.version}-${java.image.flavor}${base.image.tag.suffix} ++ -p${payara.version}-j${target.java.version} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 diff --git a/modules/container-base/src/backports/v6.2/001-pom.xml.patch b/modules/container-base/src/backports/v6.2/001-pom.xml.patch index ef0fbdda9fc..6498f972889 100644 --- a/modules/container-base/src/backports/v6.2/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.2/001-pom.xml.patch @@ -7,8 +7,8 @@ - unstable - eclipse-temurin:${target.java.version}-jre + gdcc/base:${base.image.tag} -+ ${base.image.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} -+ ${base.image.version}-${java.image.flavor} ++ ${base.image.version}-${java.image.flavor}${base.image.tag.suffix} ++ -p${payara.version}-j${target.java.version} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 diff --git a/modules/container-base/src/backports/v6.3/001-pom.xml.patch b/modules/container-base/src/backports/v6.3/001-pom.xml.patch index ef0fbdda9fc..6498f972889 100644 --- a/modules/container-base/src/backports/v6.3/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.3/001-pom.xml.patch @@ -7,8 +7,8 @@ - unstable - eclipse-temurin:${target.java.version}-jre + gdcc/base:${base.image.tag} -+ ${base.image.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} -+ ${base.image.version}-${java.image.flavor} ++ ${base.image.version}-${java.image.flavor}${base.image.tag.suffix} ++ -p${payara.version}-j${target.java.version} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 From d76556ab422d48ca0044f7be1a67722f4f420a0d Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 16 Sep 2024 16:35:20 -0400 Subject: [PATCH 867/970] add trailing / after hostname for perma base-url #10775 --- doc/sphinx-guides/source/installation/config.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 1a09e575cdc..2f4a605260c 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -536,6 +536,7 @@ be configured with the ``base-url`` of a resolution service. PermaLinks also all Note: +- If you configure ``base-url``, it should include a "/" after the hostname like this: ``https://demo.dataverse.org/``. - When using multiple PermaLink providers, you should avoid ambiguous authority/separator/shoulder combinations that would result in the same overall prefix. - In general, PermaLink authority/shoulder values should be alphanumeric. For other cases, admins may need to consider the potential impact of special characters in S3 storage identifiers, resolver URLs, exports, etc. From 48ccdf2bbcbdf8b0cd739fcd504b2465d4cc93c4 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 09:01:14 +0200 Subject: [PATCH 868/970] ci(ct): remove out of scope actions for #10478 We will deal with shipping the updated application container images separately as part of issue #10618. Adding some comments about why some stuff is still around. --- .github/workflows/container_maintenance.yml | 22 +++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index b7f8fcf45fa..c10215a7ccb 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -141,23 +141,25 @@ jobs: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} ${{ steps.revision-tag.outputs.tag-options }} -Ddocker.platforms=${{ env.PLATFORMS }} echo "rebuild=true" | tee -a "${GITHUB_OUTPUT}" + # TODO: this is here to not drop the knowledge about matrix output workarounds (for now) # - if: always() # name: Save status (workaround for matrix outputs) # run: | # # steps.build.outcome is the status BEFORE continue-on-error # echo "STATUS_$( echo "${{ matrix.branch }}" | tr ".:;,-/ " "_" )=${{ steps.build.outcome }}" | tee -a "${GITHUB_ENV}" - - name: Rebuild application container - if: ${{ steps.build.outputs.rebuild }} - uses: ./.github/actions/deploy-app-container - with: - registry: "" - registry_token: "" - ref: "" - base_image: "" - base_image_tag: "" - + # TODO: As part of issue #10618 we will need to create this action, shipping updated app images + #- name: Rebuild application container + # if: ${{ steps.build.outputs.rebuild }} + # uses: ./.github/actions/deploy-app-container + # with: + # registry: "" + # registry_token: "" + # ref: "" + # base_image: "" + # base_image_tag: "" + # TODO: This job should become part of the matrix as an action, so we don't need to fiddle with matrix outputs hacks #push-app-img: # name: "Rebase & Publish App Image" # permissions: From cc39131d321eab777d0fe4be158e900692870a62 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 09:15:20 +0200 Subject: [PATCH 869/970] ci(ct): reorganise tags for develop branch #10478 Now adding the "upcoming" tag during the develop branch run. Also some reorganizing to ship the tag options using outputs, not env vars. Moving the common Maven option to enable tag overrides to the command instead of the options building. --- .github/workflows/container_maintenance.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index c10215a7ccb..fdef2aac6d4 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -94,11 +94,11 @@ jobs: - name: Determine base image name run: | if [[ "${{ matrix.branch }}" = "${{ env.DEVELOP_BRANCH }}" ]]; then - NAME=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout ) + echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" | tee -a "${GITHUB_ENV}" + echo "BASE_IMAGE_UPCOMING=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}" else - NAME=$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout ) + echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}" fi - echo "BASE_IMAGE=${NAME}" | tee -a "${GITHUB_ENV}" # Figure out if a rebuild is necessary because either there is an updated Java image or our installed packages need updates - name: Check for recent Temurin image updates @@ -129,16 +129,18 @@ jobs: uses: gdcc/wip-dataverse-base-image/.github/actions/get-image-revision@10478-version-base-img with: image-ref: ${{ env.BASE_IMAGE }} - tag-options-prefix: "-Dbase.image.tag.suffix='' -Ddocker.imagePropertyConfiguration=override -Ddocker.tags.revision=" + tag-options-prefix: "-Dbase.image.tag.suffix='' -Ddocker.tags.revision=" - name: Configure update of "latest" tag for development branch + id: develop-tag if: ${{ matrix.branch == env.DEVELOP_BRANCH }} run: | - echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=unstable" | tee -a "${GITHUB_ENV}" + echo "tag-options=-Ddocker.tags.develop=unstable -Ddocker.tags.upcoming=${BASE_IMAGE_UPCOMING#*:}" | tee -a "${GITHUB_OUTPUT}" - name: Deploy multi-arch base container image to Docker Hub if: ${{ steps.temurin-check.outputs.is-more-recent == 'true' || steps.package-check.outputs.newer_packages == 'true' || inputs.force_build }} id: build run: | - mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} ${{ steps.revision-tag.outputs.tag-options }} -Ddocker.platforms=${{ env.PLATFORMS }} + mvn -f modules/container-base -Pct deploy -Ddocker.noCache -Ddocker.platforms=${{ env.PLATFORMS }} \ + -Ddocker.imagePropertyConfiguration=override ${{ steps.develop-tag.outputs.tag-options }} ${{ steps.revision-tag.outputs.tag-options }} echo "rebuild=true" | tee -a "${GITHUB_OUTPUT}" # TODO: this is here to not drop the knowledge about matrix output workarounds (for now) From c6b992a1ac11e514b01be81e12cf03039295a6cb Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 10:30:13 +0200 Subject: [PATCH 870/970] chore(parent): remove OSS plugin snapshot repo after upgrade to DMP 0.45 --- modules/dataverse-parent/pom.xml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 3df06281547..6bea02569ec 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -214,14 +214,6 @@ never - - oss.sonatype.org - https://oss.sonatype.org/content/repositories/snapshots - - true - always - - From 8955ef087717b337a9ba7d777505dbd02cce303f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 10:31:22 +0200 Subject: [PATCH 871/970] build(ct): make application image use new base image flexi stack #10478 --- pom.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 9b4914a1640..d8fe55ba22a 100644 --- a/pom.xml +++ b/pom.xml @@ -999,8 +999,9 @@ unstable false gdcc/base:${base.image.tag} - - rev1 + noble + + ${base.image.version}-${base.image.flavor}-p${payara.version}-j${target.java.version} gdcc/configbaker:${conf.image.tag} ${app.image.tag} From 710cf3cab8b6734d12d5ec2be1134669de5af740 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 10:53:09 +0200 Subject: [PATCH 872/970] style(ct): remove stale comment from base module pom --- modules/container-base/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/container-base/pom.xml b/modules/container-base/pom.xml index b6667077d98..6417b5b91fa 100644 --- a/modules/container-base/pom.xml +++ b/modules/container-base/pom.xml @@ -12,7 +12,6 @@ io.gdcc container-base - ${packaging.type} Container Base Image This module provides an application server base image to be decorated with the Dataverse app. From b043edff6d1846f223ff89d0f0184abc5d726580 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 11:02:13 +0200 Subject: [PATCH 873/970] chore(ct): add comment about apt sec updates detection via list hack For now, we stick to not alter the image more than necessary. Only packages we installed will be upgraded, as these are not part of the normal Java base images. The Java base images receive regular updates and undergo testing. It might be unwise to just install all the security updates we could get. Leaving the option here for later saves the trouble to dig up the solution again. --- .github/workflows/container_maintenance.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index fdef2aac6d4..3a28f355e94 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -120,7 +120,13 @@ jobs: else echo "Base image $BASE_IMAGE has no updates for our custom installed packages" echo "newer_packages=false" >> "${GITHUB_OUTPUT}" - fi + fi + + # TODO: In a future version of this script, we might want to include checking for other security updates, + # not just updates to the packages we installed. + # grep security /etc/apt/sources.list > /tmp/security.list + # apt-get update -oDir::Etc::Sourcelist=/tmp/security.list + # apt-get dist-upgrade -y -oDir::Etc::Sourcelist=/tmp/security.list -oDir::Etc::SourceParts=/bin/false -s - name: Calculate revision number for immutable tag (on release branches only) if: ${{ matrix.branch != env.DEVELOP_BRANCH }} From 9b12bc234f4fe9e455bfce3e66f3df7120f413cc Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 14:43:25 +0200 Subject: [PATCH 874/970] ci(ct): use new setup-maven action in base image push workflow Simplify setup following DRY principle --- .github/workflows/container_base_push.yml | 25 +++++------------------ 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 532c64bd781..ff5c6117f2b 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -31,27 +31,12 @@ jobs: #if: ${{ github.repository_owner == 'IQSS' }} steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Determine Java version from Parent POM - run: | - echo "JAVA_VERSION=$(grep '' modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" >> ${GITHUB_ENV} - - - name: Set up JDK ${{ env.JAVA_VERSION }} - id: setup-java - uses: actions/setup-java@v4 - with: - java-version: ${{ env.JAVA_VERSION }} - distribution: 'temurin' - cache: 'maven' - cache-dependency-path: | - modules/container-base/pom.xml - - name: Download common cache on branch cache miss - if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} - uses: actions/cache/restore@v4 + - name: Checkout and Setup Maven + # TODO: change to upstream location in final PR + uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img with: - key: dataverse-maven-cache - path: ~/.m2/repository + git-reference: ${{ github.ref }} + pom-paths: modules/container-base/pom.xml # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and # on events in context of upstream because secrets. PRs run in context of forks by default! From 4f8499f79b09b8dfb619ae4ece0684c97b5f2999 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 14:44:38 +0200 Subject: [PATCH 875/970] ci(ct): replace logic in base image push workflow Using the same actions and steps as done in the maintenance action should work for this workflow in case of a push event, too. --- .github/workflows/container_base_push.yml | 26 +++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index ff5c6117f2b..29493fcbb5c 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -77,18 +77,32 @@ jobs: # "Files": null}]} # EOF - - name: Calculate revision number for immutable tag + # Determine the base image name we are going to use from here on + - name: Determine base image name run: | - BASE_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" - echo "REVISION_OPTION=-Dbase.image.revision=$( .github/workflows/scripts/get_next_revision.sh "${BASE_IMAGE}" )" | tee -a "${GITHUB_ENV}" + if [[ "${{ github.ref_name }}" = "${{ env.DEVELOPMENT_BRANCH }}" ]]; then + echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" | tee -a "${GITHUB_ENV}" + echo "BASE_IMAGE_UPCOMING=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}" + else + echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}" + fi + - name: Calculate revision number for immutable tag (on release branches only) + if: ${{ github.ref_name != env.DEVELOPMENT_BRANCH }} + id: revision-tag + uses: ./.github/actions/get-image-revision + with: + image-ref: ${{ env.BASE_IMAGE }} + tag-options-prefix: "-Dbase.image.tag.suffix='' -Ddocker.tags.revision=" - name: Configure update of "latest" tag for development branch + id: develop-tag if: ${{ github.ref_name == env.DEVELOPMENT_BRANCH }} run: | - echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" - + echo "tag-options=-Ddocker.tags.develop=unstable -Ddocker.tags.upcoming=${BASE_IMAGE_UPCOMING#*:}" | tee -a "${GITHUB_OUTPUT}" - name: Deploy multi-arch base container image to Docker Hub id: build - run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} ${REVISION_OPTION} -Ddocker.platforms=${{ env.PLATFORMS }} + run: | + mvn -f modules/container-base -Pct deploy -Ddocker.noCache -Ddocker.platforms=${{ env.PLATFORMS }} \ + -Ddocker.imagePropertyConfiguration=override ${{ steps.develop-tag.outputs.tag-options }} ${{ steps.revision-tag.outputs.tag-options }} #push-app-img: # name: "Rebase & Publish App Image" From b43af732f84f7d7d9899ec18ac5b7095b9fac907 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 14:45:08 +0200 Subject: [PATCH 876/970] test(ct): temporarily set dev branch in base image push flow to feature branch --- .github/workflows/container_base_push.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 29493fcbb5c..25dfac32ba8 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -17,7 +17,8 @@ on: env: PLATFORMS: linux/amd64,linux/arm64 - DEVELOPMENT_BRANCH: develop + # TODO: set back to develop for final PR + DEVELOPMENT_BRANCH: 10478-version-base-img jobs: build: From 415bae3a81ca91b0007ae0b769c07ad31bf13839 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 14:47:49 +0200 Subject: [PATCH 877/970] ci(ct): use setup-maven action in app image push workflow to simplify setup Following DRY principle, reuse the steps defined --- .github/workflows/container_app_push.yml | 45 +++++++++--------------- 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index 3344a8e2d0d..7f8c880a6bc 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -38,23 +38,15 @@ jobs: if: ${{ github.repository_owner == 'IQSS' && inputs.branch == '10478-version-base-img' }} steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Set up JDK - id: setup-java - uses: actions/setup-java@v3 - with: - java-version: "17" - distribution: temurin - cache: maven - - name: Download common cache on branch cache miss - if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} - uses: actions/cache/restore@v4 + - name: Checkout and Setup Maven + # TODO: change to upstream location in final PR + uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img with: - path: ~/.m2/repository - key: dataverse-maven-cache - + git-reference: ${{ github.ref_name }} + pom-paths: | + pom.xml + modules/container-configbaker/pom.xml + modules/dataverse-parent/pom.xml - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests) run: > @@ -119,20 +111,15 @@ jobs: if: needs.check-secrets.outputs.available == 'true' && ( github.event_name != 'push' || ( github.event_name == 'push' && contains(fromJSON('["develop", "master"]'), github.ref_name))) steps: - - uses: actions/checkout@v4 - - name: Set up JDK - id: setup-java - uses: actions/setup-java@v3 - with: - java-version: "17" - distribution: temurin - cache: maven - - name: Download common cache on branch cache miss - if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} - uses: actions/cache/restore@v4 + - name: Checkout and Setup Maven + # TODO: change to upstream location in final PR + uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img with: - path: ~/.m2/repository - key: dataverse-maven-cache + git-reference: ${{ github.ref_name }} + pom-paths: | + pom.xml + modules/container-configbaker/pom.xml + modules/dataverse-parent/pom.xml # Depending on context, we push to different targets. Login accordingly. - if: github.event_name != 'pull_request' From 5eb6d1a2cabaff69acbb30b90bc7fce0acc0146e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 14:48:28 +0200 Subject: [PATCH 878/970] test(ct): temporarily enable app image push flow execution in all forks --- .github/workflows/container_app_push.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index 7f8c880a6bc..15ef7b892e8 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -35,7 +35,8 @@ jobs: packages: write pull-requests: write # Only run in upstream repo - avoid unnecessary runs in forks - if: ${{ github.repository_owner == 'IQSS' && inputs.branch == '10478-version-base-img' }} + # TODO: re-enable for final PR + #if: ${{ github.repository_owner == 'IQSS' }} steps: - name: Checkout and Setup Maven From 41c9b2915e79211ee7a48a0b32aed7290b07d69f Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 17 Sep 2024 10:12:41 -0400 Subject: [PATCH 879/970] avoid spurious log warning for others e.g. isbn these were going through the default check for URLs and failing (not a url) leading to a warning. The new code should try URL parsing for URLs, try PID and URL parsing for ones with no type specified, and send the rest of the identifiers w/o any additional (optional) attributes. --- .../pidproviders/doi/XmlMetadataTemplate.java | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 5ca62932e15..fe05b0d4537 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -1026,8 +1026,22 @@ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) th } break; case "URL": + // If a URL is given, split the string to get a schemeUri + try { + URL relatedUrl = new URI(relatedIdentifier).toURL(); + String protocol = relatedUrl.getProtocol(); + String authority = relatedUrl.getAuthority(); + String site = String.format("%s://%s", protocol, authority); + relatedIdentifier = relatedIdentifier.substring(site.length()); + attributes.put("schemeURI", site); + } catch (URISyntaxException | MalformedURLException | IllegalArgumentException e) { + // Just an identifier but without a pubIdType we won't include it + logger.warning("Invalid Identifier of type URL: " + relatedIdentifier); + relatedIdentifier = null; + } break; - default: + case "none": + //Try to identify PIDs and URLs and send them as related identifiers if (relatedIdentifier != null) { // See if it is a GlobalID we know try { @@ -1048,8 +1062,14 @@ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) th } catch (URISyntaxException | MalformedURLException | IllegalArgumentException e) { // Just an identifier but without a pubIdType we won't include it logger.warning("Related Identifier found without type: " + relatedIdentifier); + //Won't be sent since pubIdType is null - could also set relatedIdentifier to null } } + break; + default: + //Some other valid type - we just send the identifier w/o optional attributes + //To Do - validation for other types? + break; } } if (StringUtils.isNotBlank(relatedIdentifier) && StringUtils.isNotBlank(pubIdType)) { From 8dfd5f32f106e17815ddab0c39f781543f430b07 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Tue, 17 Sep 2024 11:20:36 -0400 Subject: [PATCH 880/970] update doc --- doc/sphinx-guides/source/installation/config.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 9bfd2488062..cb3e6dfc187 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -519,6 +519,7 @@ dataverse.pid.*.crossref.depositor dataverse.pid.*.crossref.depositor-email ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +CrossRef is an experimental provider. PID Providers of type ``crossref`` require six additional parameters that define how the provider connects to CrossRef. CrossRef has two APIs that are used in Dataverse: From 71f43992d4587660a298850d58211657bb8ed28f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 14:50:03 +0200 Subject: [PATCH 881/970] ci(ct): use an optional base image ref for app image push flow We need to transfer the determined base image name we might have just built from the calling workflow into this flow. As we provide a default value, this is picked up for pull_requests. --- .github/workflows/container_app_push.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index 15ef7b892e8..68c079b9ee5 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -6,10 +6,11 @@ on: # Instead, push events will trigger from the base image and maven unit tests via workflow_call. workflow_call: inputs: - branch: + base-image-ref: type: string - description: "A tag or branch to checkout for building the image" - required: true + description: "Reference of the base image to build on in full qualified form [/]/:" + required: false + default: "gdcc/base:unstable" pull_request: branches: - develop @@ -21,7 +22,6 @@ on: env: IMAGE_TAG: unstable - BASE_IMAGE_TAG: unstable REGISTRY: "" # Empty means default to Docker Hub PLATFORMS: "linux/amd64,linux/arm64" MASTER_BRANCH_TAG: alpha @@ -53,6 +53,7 @@ jobs: run: > mvn -B -f modules/dataverse-parent -P ct -pl edu.harvard.iq:dataverse -am + -Dbase.image=${{ inputs.base-image-ref }} install # TODO: add smoke / integration testing here (add "-Pct -DskipIntegrationTests=false") @@ -156,11 +157,12 @@ jobs: run: > mvn -B -f modules/dataverse-parent -P ct -pl edu.harvard.iq:dataverse -am + -Dbase.image=${{ inputs.base-image-ref }} install - name: Deploy multi-arch application and configbaker container image run: > mvn - -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }} + -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image=${{ inputs.base-image-ref }} ${{ env.REGISTRY }} -Ddocker.platforms=${{ env.PLATFORMS }} -P ct deploy From 5a0cfa0ab3169d87cd878f40b59e5af33d7cd8ae Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 15:46:15 +0200 Subject: [PATCH 882/970] ci(setup-maven): try to auto-detect git ref It's not so easy to determine the right git ref for different scenarios like PR, etc. Unless explicitly given a ref, try to autodetect the right one to go with. --- .github/actions/setup-maven/action.yml | 4 ++-- .github/workflows/container_app_push.yml | 2 -- .github/workflows/container_base_push.yml | 1 - 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/actions/setup-maven/action.yml b/.github/actions/setup-maven/action.yml index 6edbba1b1a9..4cf09f34231 100644 --- a/.github/actions/setup-maven/action.yml +++ b/.github/actions/setup-maven/action.yml @@ -4,8 +4,8 @@ description: "Determine Java version and setup Maven, including necessary caches inputs: git-reference: description: 'The git reference (branch/tag) to check out' - required: true - default: 'develop' + required: false + default: '${{ github.ref }}' pom-paths: description: "List of paths to Maven POM(s) for cache dependency setup" required: false diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index 68c079b9ee5..555ea97cf52 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -43,7 +43,6 @@ jobs: # TODO: change to upstream location in final PR uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img with: - git-reference: ${{ github.ref_name }} pom-paths: | pom.xml modules/container-configbaker/pom.xml @@ -117,7 +116,6 @@ jobs: # TODO: change to upstream location in final PR uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img with: - git-reference: ${{ github.ref_name }} pom-paths: | pom.xml modules/container-configbaker/pom.xml diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 25dfac32ba8..1365dd69eaf 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -36,7 +36,6 @@ jobs: # TODO: change to upstream location in final PR uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img with: - git-reference: ${{ github.ref }} pom-paths: modules/container-base/pom.xml # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and From 34db758a32044ab0281e8d627c59c0b3d4553c5f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 17:16:51 +0200 Subject: [PATCH 883/970] chore(ct): add notes in flows about adding a path filter We want to avoid duplicate runs which might trigger race conditions for image shipments. --- .github/workflows/container_app_push.yml | 4 ++++ .github/workflows/maven_unit_test.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index 555ea97cf52..e3dbcd4d2c6 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -48,6 +48,10 @@ jobs: modules/container-configbaker/pom.xml modules/dataverse-parent/pom.xml + # TODO: Add a filter step here, that avoids building the image if this is a PR and there are other files touched than declared above. + # Use https://github.com/dorny/paths-filter to solve this. This will ensure we do not run this twice if this workflow + # will be triggered by the other workflows already (base image or java changes) + - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests) run: > mvn -B -f modules/dataverse-parent diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml index 4ad4798bc64..1c9c6c7d5e3 100644 --- a/.github/workflows/maven_unit_test.yml +++ b/.github/workflows/maven_unit_test.yml @@ -156,6 +156,10 @@ jobs: # NOTE: this may be extended with adding a report to the build output, leave a comment, send to Sonarcloud, ... + # TODO: Add a filter step here, that avoids calling the app image release workflow if there are changes to the base image. + # Use https://github.com/dorny/paths-filter to solve this. Will require and additional job or adding to integration-test job. + # This way we ensure that we're not running the app image flow with a non-matching base image. + push-app-img: name: Publish App Image permissions: From 314fa339a6ea162c2bb4dc069a2c71c01a24df89 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 17:24:17 +0200 Subject: [PATCH 884/970] chore(ct): add note about missing triggers for base push flow This is out of scope for #10827, but should be addressed at a later point to avoid duplicated runs with potential race conditions. Also it enables proper rebuilds for preview images when someone is just trying to create a base image change, but which should obviously be tested full chain. --- .github/workflows/container_base_push.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 1365dd69eaf..d52209c5c30 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -15,6 +15,9 @@ on: - 'modules/dataverse-parent/pom.xml' - '.github/workflows/container_base_push.yml' + # TODO: we are missing a workflow_call option here, so we can trigger this flow from pr comments and maven tests (keep the secrets availability in mind!) + # TODO: we are missing a pull_request option here (filter for stuff that would trigger the maven runs!) so we can trigger preview builds for them when coming from the main repo (keep the secrets availability in mind!) + env: PLATFORMS: linux/amd64,linux/arm64 # TODO: set back to develop for final PR From 09f7264f98fa49e303d3fd068306b60f2a7a54a8 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 17:25:05 +0200 Subject: [PATCH 885/970] fix,ci(ct): don't trigger the base push flow for backports Obviously these are meant for the maintenance workflow, not the push flow! --- .github/workflows/container_base_push.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index d52209c5c30..c6283856ac8 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -12,6 +12,7 @@ on: # "Path filters are not evaluated for pushes of tags" https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#onpushpull_requestpull_request_targetpathspaths-ignore paths: - 'modules/container-base/**' + - '!modules/container-base/src/backports/**' - 'modules/dataverse-parent/pom.xml' - '.github/workflows/container_base_push.yml' From 44ec28d61f492804ca491fa6402bf3f5148bff6d Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 17:31:39 +0200 Subject: [PATCH 886/970] ci(ct): trigger app flow from base push flow We detect the tag we have been using in a finalizing step to hand a proper base image ref to the app image workflow to make it work on the images we just pushed to the registry. --- .github/workflows/container_base_push.yml | 39 +++++++++++++++-------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index c6283856ac8..8a9a3212fe9 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -26,7 +26,7 @@ env: jobs: build: - name: Build image + name: Base Image runs-on: ubuntu-latest permissions: contents: read @@ -34,6 +34,8 @@ jobs: # TODO: re-enable for final PR # Only run in upstream repo - avoid unnecessary runs in forks #if: ${{ github.repository_owner == 'IQSS' }} + outputs: + base-image-ref: ${{ steps.finalize.outputs.base-image-ref }} steps: - name: Checkout and Setup Maven @@ -102,25 +104,34 @@ jobs: if: ${{ github.ref_name == env.DEVELOPMENT_BRANCH }} run: | echo "tag-options=-Ddocker.tags.develop=unstable -Ddocker.tags.upcoming=${BASE_IMAGE_UPCOMING#*:}" | tee -a "${GITHUB_OUTPUT}" + - name: Deploy multi-arch base container image to Docker Hub id: build run: | mvn -f modules/container-base -Pct deploy -Ddocker.noCache -Ddocker.platforms=${{ env.PLATFORMS }} \ -Ddocker.imagePropertyConfiguration=override ${{ steps.develop-tag.outputs.tag-options }} ${{ steps.revision-tag.outputs.tag-options }} - #push-app-img: - # name: "Rebase & Publish App Image" - # permissions: - # contents: read - # packages: write - # pull-requests: write - # secrets: inherit - # needs: - # - discover - # - build - # uses: ./.github/workflows/container_app_push.yml - # with: - # branch: ${{ github.ref_name }} + - name: Determine appropriate base image ref for app image + id: finalize + run: | + if [[ "${{ github.ref_name }}" = "${{ env.DEVELOPMENT_BRANCH }}" ]]; then + echo "base-image-ref=${BASE_IMAGE_UPCOMING}" | tee -a "$GITHUB_OUTPUT" + else + echo "base-image-ref=gdcc/base:${{ steps.revision-tag.outputs.revision-tag }}" | tee -a "$GITHUB_OUTPUT" + fi + + push-app-img: + name: "Rebase & Publish App Image" + permissions: + contents: read + packages: write + pull-requests: write + secrets: inherit + needs: + - build + uses: ./.github/workflows/container_app_push.yml + with: + base-image-ref: ${{ needs.build.outputs.base-image-ref }} # TODO: job to update the docker hub description with supported tags and all From 6a05bef4b8e34132b444f04ecf0bc6484da3ac2d Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 17 Sep 2024 13:54:34 -0400 Subject: [PATCH 887/970] changes per review --- doc/sphinx-guides/source/api/changelog.rst | 5 +- .../source/installation/config.rst | 4 + ...dataset-create-new-all-default-fields.json | 2816 ++++++++--------- .../iq/dataverse/DatasetFieldConstant.java | 2 + .../harvard/iq/dataverse/DatasetVersion.java | 65 +- .../iq/dataverse/DatasetVersionUI.java | 30 +- .../dublincore/DublinCoreExportUtil.java | 30 +- .../pidproviders/doi/XmlMetadataTemplate.java | 134 +- 8 files changed, 1549 insertions(+), 1537 deletions(-) diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index b40d32f37d8..c5890fd9996 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -6,11 +6,12 @@ This API changelog is experimental and we would love feedback on its usefulness. .. contents:: |toctitle| :local: :depth: 1 + v6.4 ---- -[**/api/datasets/$dataset-id/modifyRegistration**](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-target-url-for-a-published-dataset-at-the-pid-provider) - changes from GET to POST -[**/api/datasets/modifyRegistrationPIDMetadataAll**](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-metadata-for-all-published-datasets-at-the-pid-provider) - changes from GET to POST +- **/api/datasets/$dataset-id/modifyRegistration**: Changed from GET to POST +- **/api/datasets/modifyRegistrationPIDMetadataAll**: Changed from GET to POST v6.3 diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index b2d9bd3d342..30a333c1614 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -232,6 +232,10 @@ Dataverse can be configured with one or more PID providers, each of which can mi to manage an authority/shoulder combination, aka a "prefix" (PermaLinks also support custom separator characters as part of the prefix), along with an optional list of individual PIDs (with different authority/shoulders) than can be managed with that account. +Dataverse automatically manages assigning PIDs and making them findable when datasets are published. There are also :ref:`API calls that +allow updating the PID target URLs and metadata of already-published datasets manually if needed ` , e.g. if a Dataverse instance is +moved to a new URL or when the software is updated to generate additional metadata or address schema changes at the PID service. + Testing PID Providers +++++++++++++++++++++ diff --git a/scripts/api/data/dataset-create-new-all-default-fields.json b/scripts/api/data/dataset-create-new-all-default-fields.json index e522ab32b1d..84b152690f9 100644 --- a/scripts/api/data/dataset-create-new-all-default-fields.json +++ b/scripts/api/data/dataset-create-new-all-default-fields.json @@ -1,1410 +1,1410 @@ { - "datasetVersion" : { - "license" : { - "name" : "CC0 1.0", - "uri" : "http://creativecommons.org/publicdomain/zero/1.0" - }, - "metadataBlocks" : { - "citation" : { - "displayName" : "Citation Metadata", - "fields" : [{ - "typeName" : "title", - "multiple" : false, - "typeClass" : "primitive", - "value" : "Replication Data for: Title" - }, { - "typeName" : "subtitle", - "multiple" : false, - "typeClass" : "primitive", - "value" : "Subtitle" - }, { - "typeName" : "alternativeTitle", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ - "Alternative Title" - ] - }, { - "typeName" : "alternativeURL", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://AlternativeURL.org" - }, { - "typeName" : "otherId", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "otherIdAgency" : { - "typeName" : "otherIdAgency", - "multiple" : false, - "typeClass" : "primitive", - "value" : "OtherIDAgency1" - }, - "otherIdValue" : { - "typeName" : "otherIdValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "OtherIDIdentifier1" - } - }, { - "otherIdAgency" : { - "typeName" : "otherIdAgency", - "multiple" : false, - "typeClass" : "primitive", - "value" : "OtherIDAgency2" - }, - "otherIdValue" : { - "typeName" : "otherIdValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "OtherIDIdentifier2" - } - } - ] - }, { - "typeName" : "author", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "authorName" : { - "typeName" : "authorName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastAuthor1, FirstAuthor1" - }, - "authorAffiliation" : { - "typeName" : "authorAffiliation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "AuthorAffiliation1" - }, - "authorIdentifierScheme" : { - "typeName" : "authorIdentifierScheme", - "multiple" : false, - "typeClass" : "controlledVocabulary", - "value" : "ORCID" - }, - "authorIdentifier" : { - "typeName" : "authorIdentifier", - "multiple" : false, - "typeClass" : "primitive", - "value" : "AuthorIdentifier1" - } - }, { - "authorName" : { - "typeName" : "authorName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastAuthor2, FirstAuthor2" - }, - "authorAffiliation" : { - "typeName" : "authorAffiliation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "AuthorAffiliation2" - }, - "authorIdentifierScheme" : { - "typeName" : "authorIdentifierScheme", - "multiple" : false, - "typeClass" : "controlledVocabulary", - "value" : "ISNI" - }, - "authorIdentifier" : { - "typeName" : "authorIdentifier", - "multiple" : false, - "typeClass" : "primitive", - "value" : "AuthorIdentifier2" - } - } - ] - }, { - "typeName" : "datasetContact", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "datasetContactName" : { - "typeName" : "datasetContactName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastContact1, FirstContact1" - }, - "datasetContactAffiliation" : { - "typeName" : "datasetContactAffiliation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "ContactAffiliation1" - }, - "datasetContactEmail" : { - "typeName" : "datasetContactEmail", - "multiple" : false, - "typeClass" : "primitive", - "value" : "ContactEmail1@mailinator.com" - } - }, { - "datasetContactName" : { - "typeName" : "datasetContactName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastContact2, FirstContact2" - }, - "datasetContactAffiliation" : { - "typeName" : "datasetContactAffiliation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "ContactAffiliation2" - }, - "datasetContactEmail" : { - "typeName" : "datasetContactEmail", - "multiple" : false, - "typeClass" : "primitive", - "value" : "ContactEmail2@mailinator.com" - } - } - ] - }, { - "typeName" : "dsDescription", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "dsDescriptionValue" : { - "typeName" : "dsDescriptionValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "DescriptionText1" - }, - "dsDescriptionDate" : { - "typeName" : "dsDescriptionDate", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1000-01-01" - } - }, { - "dsDescriptionValue" : { - "typeName" : "dsDescriptionValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "DescriptionText2" - }, - "dsDescriptionDate" : { - "typeName" : "dsDescriptionDate", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1000-02-02" - } - } - ] - }, { - "typeName" : "subject", - "multiple" : true, - "typeClass" : "controlledVocabulary", - "value" : [ - "Agricultural Sciences", - "Business and Management", - "Engineering", - "Law" - ] - }, { - "typeName" : "keyword", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "keywordValue" : { - "typeName" : "keywordValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "KeywordTerm1" - }, - "keywordTermURI" : { - "typeName" : "keywordTermURI", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://keywordTermURI1.org" - }, - "keywordVocabulary" : { - "typeName" : "keywordVocabulary", - "multiple" : false, - "typeClass" : "primitive", - "value" : "KeywordVocabulary1" - }, - "keywordVocabularyURI" : { - "typeName" : "keywordVocabularyURI", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://KeywordVocabularyURL1.org" - } - }, { - "keywordValue" : { - "typeName" : "keywordValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "KeywordTerm2" - }, - "keywordTermURI" : { - "typeName" : "keywordTermURI", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://keywordTermURI2.org" - }, - "keywordVocabulary" : { - "typeName" : "keywordVocabulary", - "multiple" : false, - "typeClass" : "primitive", - "value" : "KeywordVocabulary2" - }, - "keywordVocabularyURI" : { - "typeName" : "keywordVocabularyURI", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://KeywordVocabularyURL2.org" - } - } - ] - }, { - "typeName" : "topicClassification", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "topicClassValue" : { - "typeName" : "topicClassValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "Topic Classification Term1" - }, - "topicClassVocab" : { - "typeName" : "topicClassVocab", - "multiple" : false, - "typeClass" : "primitive", - "value" : "Topic Classification Vocab1" - }, - "topicClassVocabURI" : { - "typeName" : "topicClassVocabURI", - "multiple" : false, - "typeClass" : "primitive", - "value" : "https://TopicClassificationURL1.com" - } - }, { - "topicClassValue" : { - "typeName" : "topicClassValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "Topic Classification Term2" - }, - "topicClassVocab" : { - "typeName" : "topicClassVocab", - "multiple" : false, - "typeClass" : "primitive", - "value" : "Topic Classification Vocab2" - }, - "topicClassVocabURI" : { - "typeName" : "topicClassVocabURI", - "multiple" : false, - "typeClass" : "primitive", - "value" : "https://TopicClassificationURL2.com" - } - } - ] - }, { - "typeName" : "publication", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "publicationRelationType" : { - "typeName" : "publicationRelationType", - "multiple" : false, - "typeClass" : "controlledVocabulary", - "value" : "IsSupplementTo" - }, - "publicationCitation" : { - "typeName" : "publicationCitation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "RelatedPublicationCitation1" - }, - "publicationIDType" : { - "typeName" : "publicationIDType", - "multiple" : false, - "typeClass" : "controlledVocabulary", - "value" : "ark" - }, - "publicationIDNumber" : { - "typeName" : "publicationIDNumber", - "multiple" : false, - "typeClass" : "primitive", - "value" : "RelatedPublicationIDNumber1" - }, - "publicationURL" : { - "typeName" : "publicationURL", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://RelatedPublicationURL1.org" - } - }, { - "publicationCitation" : { - "typeName" : "publicationCitation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "RelatedPublicationCitation2" - }, - "publicationIDType" : { - "typeName" : "publicationIDType", - "multiple" : false, - "typeClass" : "controlledVocabulary", - "value" : "arXiv" - }, - "publicationIDNumber" : { - "typeName" : "publicationIDNumber", - "multiple" : false, - "typeClass" : "primitive", - "value" : "RelatedPublicationIDNumber2" - }, - "publicationURL" : { - "typeName" : "publicationURL", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://RelatedPublicationURL2.org" - } - } - ] - }, { - "typeName" : "notesText", - "multiple" : false, - "typeClass" : "primitive", - "value" : "Notes1" - }, { - "typeName" : "language", - "multiple" : true, - "typeClass" : "controlledVocabulary", - "value" : [ - "Abkhaz", - "Afar" - ] - }, { - "typeName" : "producer", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "producerName" : { - "typeName" : "producerName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastProducer1, FirstProducer1" - }, - "producerAffiliation" : { - "typeName" : "producerAffiliation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "ProducerAffiliation1" - }, - "producerAbbreviation" : { - "typeName" : "producerAbbreviation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "ProducerAbbreviation1" - }, - "producerURL" : { - "typeName" : "producerURL", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://ProducerURL1.org" - }, - "producerLogoURL" : { - "typeName" : "producerLogoURL", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://ProducerLogoURL1.org" - } - }, { - "producerName" : { - "typeName" : "producerName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastProducer2, FirstProducer2" - }, - "producerAffiliation" : { - "typeName" : "producerAffiliation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "ProducerAffiliation2" - }, - "producerAbbreviation" : { - "typeName" : "producerAbbreviation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "ProducerAbbreviation2" - }, - "producerURL" : { - "typeName" : "producerURL", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://ProducerURL2.org" - }, - "producerLogoURL" : { - "typeName" : "producerLogoURL", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://ProducerLogoURL2.org" - } - } - ] - }, { - "typeName" : "productionDate", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1003-01-01" - }, { - "typeName" : "productionPlace", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ - "ProductionPlace" - ] - }, { - "typeName" : "contributor", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "contributorType" : { - "typeName" : "contributorType", - "multiple" : false, - "typeClass" : "controlledVocabulary", - "value" : "Data Collector" - }, - "contributorName" : { - "typeName" : "contributorName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastContributor1, FirstContributor1" - } - }, { - "contributorType" : { - "typeName" : "contributorType", - "multiple" : false, - "typeClass" : "controlledVocabulary", - "value" : "Data Curator" - }, - "contributorName" : { - "typeName" : "contributorName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastContributor2, FirstContributor2" - } - } - ] - }, { - "typeName" : "grantNumber", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "grantNumberAgency" : { - "typeName" : "grantNumberAgency", - "multiple" : false, - "typeClass" : "primitive", - "value" : "GrantInformationGrantAgency1" - }, - "grantNumberValue" : { - "typeName" : "grantNumberValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "GrantInformationGrantNumber1" - } - }, { - "grantNumberAgency" : { - "typeName" : "grantNumberAgency", - "multiple" : false, - "typeClass" : "primitive", - "value" : "GrantInformationGrantAgency2" - }, - "grantNumberValue" : { - "typeName" : "grantNumberValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "GrantInformationGrantNumber2" - } - } - ] - }, { - "typeName" : "distributor", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "distributorName" : { - "typeName" : "distributorName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastDistributor1, FirstDistributor1" - }, - "distributorAffiliation" : { - "typeName" : "distributorAffiliation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "DistributorAffiliation1" - }, - "distributorAbbreviation" : { - "typeName" : "distributorAbbreviation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "DistributorAbbreviation1" - }, - "distributorURL" : { - "typeName" : "distributorURL", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://DistributorURL1.org" - }, - "distributorLogoURL" : { - "typeName" : "distributorLogoURL", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://DistributorLogoURL1.org" - } - }, { - "distributorName" : { - "typeName" : "distributorName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastDistributor2, FirstDistributor2" - }, - "distributorAffiliation" : { - "typeName" : "distributorAffiliation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "DistributorAffiliation2" - }, - "distributorAbbreviation" : { - "typeName" : "distributorAbbreviation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "DistributorAbbreviation2" - }, - "distributorURL" : { - "typeName" : "distributorURL", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://DistributorURL2.org" - }, - "distributorLogoURL" : { - "typeName" : "distributorLogoURL", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://DistributorLogoURL2.org" - } - } - ] - }, { - "typeName" : "distributionDate", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1004-01-01" - }, { - "typeName" : "depositor", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastDepositor, FirstDepositor" - }, { - "typeName" : "dateOfDeposit", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1002-01-01" - }, { - "typeName" : "timePeriodCovered", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "timePeriodCoveredStart" : { - "typeName" : "timePeriodCoveredStart", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1005-01-01" - }, - "timePeriodCoveredEnd" : { - "typeName" : "timePeriodCoveredEnd", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1005-01-02" - } - }, { - "timePeriodCoveredStart" : { - "typeName" : "timePeriodCoveredStart", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1005-02-01" - }, - "timePeriodCoveredEnd" : { - "typeName" : "timePeriodCoveredEnd", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1005-02-02" - } - } - ] - }, { - "typeName" : "dateOfCollection", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "dateOfCollectionStart" : { - "typeName" : "dateOfCollectionStart", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1006-01-01" - }, - "dateOfCollectionEnd" : { - "typeName" : "dateOfCollectionEnd", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1006-01-01" - } - }, { - "dateOfCollectionStart" : { - "typeName" : "dateOfCollectionStart", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1006-02-01" - }, - "dateOfCollectionEnd" : { - "typeName" : "dateOfCollectionEnd", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1006-02-02" - } - } - ] - }, { - "typeName" : "kindOfData", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ - "KindOfData1", - "KindOfData2" - ] - }, { - "typeName" : "series", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "seriesName" : { - "typeName" : "seriesName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "SeriesName" - }, - "seriesInformation" : { - "typeName" : "seriesInformation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "SeriesInformation" - } - } - ] - }, { - "typeName" : "software", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "softwareName" : { - "typeName" : "softwareName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "SoftwareName1" - }, - "softwareVersion" : { - "typeName" : "softwareVersion", - "multiple" : false, - "typeClass" : "primitive", - "value" : "SoftwareVersion1" - } - }, { - "softwareName" : { - "typeName" : "softwareName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "SoftwareName2" - }, - "softwareVersion" : { - "typeName" : "softwareVersion", - "multiple" : false, - "typeClass" : "primitive", - "value" : "SoftwareVersion2" - } - } - ] - }, { - "typeName" : "relatedMaterial", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ - "RelatedMaterial1", - "RelatedMaterial2" - ] - }, { - "typeName" : "relatedDatasets", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ - "RelatedDatasets1", - "RelatedDatasets2" - ] - }, { - "typeName" : "otherReferences", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ - "OtherReferences1", - "OtherReferences2" - ] - }, { - "typeName" : "dataSources", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ - "DataSources1", - "DataSources2" - ] - }, { - "typeName" : "originOfSources", - "multiple" : false, - "typeClass" : "primitive", - "value" : "OriginOfSources" - }, { - "typeName" : "characteristicOfSources", - "multiple" : false, - "typeClass" : "primitive", - "value" : "CharacteristicOfSourcesNoted" - }, { - "typeName" : "accessToSources", - "multiple" : false, - "typeClass" : "primitive", - "value" : "DocumentationAndAccessToSources" - } - ] - }, - "geospatial" : { - "displayName" : "Geospatial Metadata", - "fields" : [{ - "typeName" : "geographicCoverage", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "country" : { - "typeName" : "country", - "multiple" : false, - "typeClass" : "controlledVocabulary", - "value" : "Afghanistan" - }, - "state" : { - "typeName" : "state", - "multiple" : false, - "typeClass" : "primitive", - "value" : "GeographicCoverageStateProvince1" - }, - "city" : { - "typeName" : "city", - "multiple" : false, - "typeClass" : "primitive", - "value" : "GeographicCoverageCity1" - }, - "otherGeographicCoverage" : { - "typeName" : "otherGeographicCoverage", - "multiple" : false, - "typeClass" : "primitive", - "value" : "GeographicCoverageOther1" - } - }, { - "country" : { - "typeName" : "country", - "multiple" : false, - "typeClass" : "controlledVocabulary", - "value" : "Albania" - }, - "state" : { - "typeName" : "state", - "multiple" : false, - "typeClass" : "primitive", - "value" : "GeographicCoverageStateProvince2" - }, - "city" : { - "typeName" : "city", - "multiple" : false, - "typeClass" : "primitive", - "value" : "GeographicCoverageCity2" - }, - "otherGeographicCoverage" : { - "typeName" : "otherGeographicCoverage", - "multiple" : false, - "typeClass" : "primitive", - "value" : "GeographicCoverageOther2" - } - } - ] - }, { - "typeName" : "geographicUnit", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ - "GeographicUnit1", - "GeographicUnit2" - ] - }, { - "typeName" : "geographicBoundingBox", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "westLongitude" : { - "typeName" : "westLongitude", - "multiple" : false, - "typeClass" : "primitive", - "value" : "-72" - }, - "eastLongitude" : { - "typeName" : "eastLongitude", - "multiple" : false, - "typeClass" : "primitive", - "value" : "-70" - }, - "northLatitude" : { - "typeName" : "northLatitude", - "multiple" : false, - "typeClass" : "primitive", - "value" : "43" - }, - "southLatitude" : { - "typeName" : "southLatitude", - "multiple" : false, - "typeClass" : "primitive", - "value" : "42" - } - }, { - "westLongitude" : { - "typeName" : "westLongitude", - "multiple" : false, - "typeClass" : "primitive", - "value" : "-18" - }, - "eastLongitude" : { - "typeName" : "eastLongitude", - "multiple" : false, - "typeClass" : "primitive", - "value" : "-13" - }, - "northLatitude" : { - "typeName" : "northLatitude", - "multiple" : false, - "typeClass" : "primitive", - "value" : "29" - }, - "southLatitude" : { - "typeName" : "southLatitude", - "multiple" : false, - "typeClass" : "primitive", - "value" : "28" - } - } - ] - } - ] - }, - "socialscience" : { - "displayName" : "Social Science and Humanities Metadata", - "fields" : [{ - "typeName" : "unitOfAnalysis", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ - "UnitOfAnalysis1", - "UnitOfAnalysis2" - ] - }, { - "typeName" : "universe", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ - "Universe1", - "Universe2" - ] - }, { - "typeName" : "timeMethod", - "multiple" : false, - "typeClass" : "primitive", - "value" : "TimeMethod" - }, { - "typeName" : "dataCollector", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastDataCollector1, FirstDataCollector1" - }, { - "typeName" : "collectorTraining", - "multiple" : false, - "typeClass" : "primitive", - "value" : "CollectorTraining" - }, { - "typeName" : "frequencyOfDataCollection", - "multiple" : false, - "typeClass" : "primitive", - "value" : "Frequency" - }, { - "typeName" : "samplingProcedure", - "multiple" : false, - "typeClass" : "primitive", - "value" : "SamplingProcedure" - }, { - "typeName" : "targetSampleSize", - "multiple" : false, - "typeClass" : "compound", - "value" : { - "targetSampleActualSize" : { - "typeName" : "targetSampleActualSize", - "multiple" : false, - "typeClass" : "primitive", - "value" : "100" - }, - "targetSampleSizeFormula" : { - "typeName" : "targetSampleSizeFormula", - "multiple" : false, - "typeClass" : "primitive", - "value" : "TargetSampleSizeFormula" - } - } - }, { - "typeName" : "deviationsFromSampleDesign", - "multiple" : false, - "typeClass" : "primitive", - "value" : "MajorDeviationsForSampleDesign" - }, { - "typeName" : "collectionMode", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ - "CollectionMode" - ] - }, { - "typeName" : "researchInstrument", - "multiple" : false, - "typeClass" : "primitive", - "value" : "TypeOfResearchInstrument" - }, { - "typeName" : "dataCollectionSituation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "CharacteristicsOfDataCollectionSituation" - }, { - "typeName" : "actionsToMinimizeLoss", - "multiple" : false, - "typeClass" : "primitive", - "value" : "ActionsToMinimizeLosses" - }, { - "typeName" : "controlOperations", - "multiple" : false, - "typeClass" : "primitive", - "value" : "ControlOperations" - }, { - "typeName" : "weighting", - "multiple" : false, - "typeClass" : "primitive", - "value" : "Weighting" - }, { - "typeName" : "cleaningOperations", - "multiple" : false, - "typeClass" : "primitive", - "value" : "CleaningOperations" - }, { - "typeName" : "datasetLevelErrorNotes", - "multiple" : false, - "typeClass" : "primitive", - "value" : "StudyLevelErrorNotes" - }, { - "typeName" : "responseRate", - "multiple" : false, - "typeClass" : "primitive", - "value" : "ResponseRate" - }, { - "typeName" : "samplingErrorEstimates", - "multiple" : false, - "typeClass" : "primitive", - "value" : "EstimatesOfSamplingError" - }, { - "typeName" : "otherDataAppraisal", - "multiple" : false, - "typeClass" : "primitive", - "value" : "OtherFormsOfDataAppraisal" - }, { - "typeName" : "socialScienceNotes", - "multiple" : false, - "typeClass" : "compound", - "value" : { - "socialScienceNotesType" : { - "typeName" : "socialScienceNotesType", - "multiple" : false, - "typeClass" : "primitive", - "value" : "NotesType" - }, - "socialScienceNotesSubject" : { - "typeName" : "socialScienceNotesSubject", - "multiple" : false, - "typeClass" : "primitive", - "value" : "NotesSubject" - }, - "socialScienceNotesText" : { - "typeName" : "socialScienceNotesText", - "multiple" : false, - "typeClass" : "primitive", - "value" : "NotesText" - } - } - } - ] - }, - "astrophysics" : { - "displayName" : "Astronomy and Astrophysics Metadata", - "fields" : [{ - "typeName" : "astroType", - "multiple" : true, - "typeClass" : "controlledVocabulary", - "value" : [ - "Image", - "Mosaic", - "EventList", - "Cube" - ] - }, { - "typeName" : "astroFacility", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ - "Facility1", - "Facility2" - ] - }, { - "typeName" : "astroInstrument", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ - "Instrument1", - "Instrument2" - ] - }, { - "typeName" : "astroObject", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ - "Object1", - "Object2" - ] - }, { - "typeName" : "resolution.Spatial", - "multiple" : false, - "typeClass" : "primitive", - "value" : "SpatialResolution" - }, { - "typeName" : "resolution.Spectral", - "multiple" : false, - "typeClass" : "primitive", - "value" : "SpectralResolution" - }, { - "typeName" : "resolution.Temporal", - "multiple" : false, - "typeClass" : "primitive", - "value" : "TimeResolution" - }, { - "typeName" : "coverage.Spectral.Bandpass", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ - "Bandpass1", - "Bandpass2" - ] - }, { - "typeName" : "coverage.Spectral.CentralWavelength", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ - "3001", - "3002" - ] - }, { - "typeName" : "coverage.Spectral.Wavelength", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "coverage.Spectral.MinimumWavelength" : { - "typeName" : "coverage.Spectral.MinimumWavelength", - "multiple" : false, - "typeClass" : "primitive", - "value" : "4001" - }, - "coverage.Spectral.MaximumWavelength" : { - "typeName" : "coverage.Spectral.MaximumWavelength", - "multiple" : false, - "typeClass" : "primitive", - "value" : "4002" - } - }, { - "coverage.Spectral.MinimumWavelength" : { - "typeName" : "coverage.Spectral.MinimumWavelength", - "multiple" : false, - "typeClass" : "primitive", - "value" : "4003" - }, - "coverage.Spectral.MaximumWavelength" : { - "typeName" : "coverage.Spectral.MaximumWavelength", - "multiple" : false, - "typeClass" : "primitive", - "value" : "4004" - } - } - ] - }, { - "typeName" : "coverage.Temporal", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "coverage.Temporal.StartTime" : { - "typeName" : "coverage.Temporal.StartTime", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1007-01-01" - }, - "coverage.Temporal.StopTime" : { - "typeName" : "coverage.Temporal.StopTime", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1007-01-02" - } - }, { - "coverage.Temporal.StartTime" : { - "typeName" : "coverage.Temporal.StartTime", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1007-02-01" - }, - "coverage.Temporal.StopTime" : { - "typeName" : "coverage.Temporal.StopTime", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1007-02-02" - } - } - ] - }, { - "typeName" : "coverage.Spatial", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ - "SkyCoverage1", - "SkyCoverage2" - ] - }, { - "typeName" : "coverage.Depth", - "multiple" : false, - "typeClass" : "primitive", - "value" : "200" - }, { - "typeName" : "coverage.ObjectDensity", - "multiple" : false, - "typeClass" : "primitive", - "value" : "300" - }, { - "typeName" : "coverage.ObjectCount", - "multiple" : false, - "typeClass" : "primitive", - "value" : "400" - }, { - "typeName" : "coverage.SkyFraction", - "multiple" : false, - "typeClass" : "primitive", - "value" : "500" - }, { - "typeName" : "coverage.Polarization", - "multiple" : false, - "typeClass" : "primitive", - "value" : "Polarization" - }, { - "typeName" : "redshiftType", - "multiple" : false, - "typeClass" : "primitive", - "value" : "RedshiftType" - }, { - "typeName" : "resolution.Redshift", - "multiple" : false, - "typeClass" : "primitive", - "value" : "600" - }, { - "typeName" : "coverage.RedshiftValue", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "coverage.Redshift.MinimumValue" : { - "typeName" : "coverage.Redshift.MinimumValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "701" - }, - "coverage.Redshift.MaximumValue" : { - "typeName" : "coverage.Redshift.MaximumValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "702" - } - }, { - "coverage.Redshift.MinimumValue" : { - "typeName" : "coverage.Redshift.MinimumValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "703" - }, - "coverage.Redshift.MaximumValue" : { - "typeName" : "coverage.Redshift.MaximumValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "704" - } - } - ] - } - ] - }, - "biomedical" : { - "displayName" : "Life Sciences Metadata", - "fields" : [{ - "typeName" : "studyDesignType", - "multiple" : true, - "typeClass" : "controlledVocabulary", - "value" : [ - "Case Control", - "Cross Sectional", - "Cohort Study", - "Not Specified" - ] - }, { - "typeName" : "studyFactorType", - "multiple" : true, - "typeClass" : "controlledVocabulary", - "value" : [ - "Age", - "Biomarkers", - "Cell Surface Markers", - "Developmental Stage" - ] - }, { - "typeName" : "studyAssayOrganism", - "multiple" : true, - "typeClass" : "controlledVocabulary", - "value" : [ - "Arabidopsis thaliana", - "Bos taurus", - "Caenorhabditis elegans", - "Danio rerio (zebrafish)" - ] - }, { - "typeName" : "studyAssayOtherOrganism", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ - "OtherOrganism1", - "OtherOrganism2" - ] - }, { - "typeName" : "studyAssayMeasurementType", - "multiple" : true, - "typeClass" : "controlledVocabulary", - "value" : [ - "genome sequencing", - "cell sorting", - "clinical chemistry analysis", - "DNA methylation profiling" - ] - }, { - "typeName" : "studyAssayOtherMeasurmentType", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ - "OtherMeasurementType1", - "OtherMeasurementType2" - ] - }, { - "typeName" : "studyAssayTechnologyType", - "multiple" : true, - "typeClass" : "controlledVocabulary", - "value" : [ - "culture based drug susceptibility testing, single concentration", - "culture based drug susceptibility testing, two concentrations", - "culture based drug susceptibility testing, three or more concentrations (minimium inhibitory concentration measurement)", - "flow cytometry" - ] - }, { - "typeName" : "studyAssayPlatform", - "multiple" : true, - "typeClass" : "controlledVocabulary", - "value" : [ - "210-MS GC Ion Trap (Varian)", - "220-MS GC Ion Trap (Varian)", - "225-MS GC Ion Trap (Varian)", - "300-MS quadrupole GC/MS (Varian)" - ] - }, { - "typeName" : "studyAssayCellType", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ - "CellType1", - "CellType2" - ] - } - ] - }, - "journal" : { - "displayName" : "Journal Metadata", - "fields" : [{ - "typeName" : "journalVolumeIssue", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "journalVolume" : { - "typeName" : "journalVolume", - "multiple" : false, - "typeClass" : "primitive", - "value" : "JournalVolume1" - }, - "journalIssue" : { - "typeName" : "journalIssue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "JournalIssue1" - }, - "journalPubDate" : { - "typeName" : "journalPubDate", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1008-01-01" - } - }, { - "journalVolume" : { - "typeName" : "journalVolume", - "multiple" : false, - "typeClass" : "primitive", - "value" : "JournalVolume2" - }, - "journalIssue" : { - "typeName" : "journalIssue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "JournalIssue2" - }, - "journalPubDate" : { - "typeName" : "journalPubDate", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1008-02-01" - } - } - ] - }, { - "typeName" : "journalArticleType", - "multiple" : false, - "typeClass" : "controlledVocabulary", - "value" : "abstract" - } - ] - } - } - } + "datasetVersion" : { + "license" : { + "name" : "CC0 1.0", + "uri" : "http://creativecommons.org/publicdomain/zero/1.0" + }, + "metadataBlocks" : { + "citation" : { + "displayName" : "Citation Metadata", + "fields" : [{ + "typeName" : "title", + "multiple" : false, + "typeClass" : "primitive", + "value" : "Replication Data for: Title" + }, { + "typeName" : "subtitle", + "multiple" : false, + "typeClass" : "primitive", + "value" : "Subtitle" + }, { + "typeName" : "alternativeTitle", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "Alternative Title" + ] + }, { + "typeName" : "alternativeURL", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://AlternativeURL.org" + }, { + "typeName" : "otherId", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "otherIdAgency" : { + "typeName" : "otherIdAgency", + "multiple" : false, + "typeClass" : "primitive", + "value" : "OtherIDAgency1" + }, + "otherIdValue" : { + "typeName" : "otherIdValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "OtherIDIdentifier1" + } + }, { + "otherIdAgency" : { + "typeName" : "otherIdAgency", + "multiple" : false, + "typeClass" : "primitive", + "value" : "OtherIDAgency2" + }, + "otherIdValue" : { + "typeName" : "otherIdValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "OtherIDIdentifier2" + } + } + ] + }, { + "typeName" : "author", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "authorName" : { + "typeName" : "authorName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastAuthor1, FirstAuthor1" + }, + "authorAffiliation" : { + "typeName" : "authorAffiliation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "AuthorAffiliation1" + }, + "authorIdentifierScheme" : { + "typeName" : "authorIdentifierScheme", + "multiple" : false, + "typeClass" : "controlledVocabulary", + "value" : "ORCID" + }, + "authorIdentifier" : { + "typeName" : "authorIdentifier", + "multiple" : false, + "typeClass" : "primitive", + "value" : "AuthorIdentifier1" + } + }, { + "authorName" : { + "typeName" : "authorName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastAuthor2, FirstAuthor2" + }, + "authorAffiliation" : { + "typeName" : "authorAffiliation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "AuthorAffiliation2" + }, + "authorIdentifierScheme" : { + "typeName" : "authorIdentifierScheme", + "multiple" : false, + "typeClass" : "controlledVocabulary", + "value" : "ISNI" + }, + "authorIdentifier" : { + "typeName" : "authorIdentifier", + "multiple" : false, + "typeClass" : "primitive", + "value" : "AuthorIdentifier2" + } + } + ] + }, { + "typeName" : "datasetContact", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "datasetContactName" : { + "typeName" : "datasetContactName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastContact1, FirstContact1" + }, + "datasetContactAffiliation" : { + "typeName" : "datasetContactAffiliation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "ContactAffiliation1" + }, + "datasetContactEmail" : { + "typeName" : "datasetContactEmail", + "multiple" : false, + "typeClass" : "primitive", + "value" : "ContactEmail1@mailinator.com" + } + }, { + "datasetContactName" : { + "typeName" : "datasetContactName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastContact2, FirstContact2" + }, + "datasetContactAffiliation" : { + "typeName" : "datasetContactAffiliation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "ContactAffiliation2" + }, + "datasetContactEmail" : { + "typeName" : "datasetContactEmail", + "multiple" : false, + "typeClass" : "primitive", + "value" : "ContactEmail2@mailinator.com" + } + } + ] + }, { + "typeName" : "dsDescription", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "dsDescriptionValue" : { + "typeName" : "dsDescriptionValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "DescriptionText1" + }, + "dsDescriptionDate" : { + "typeName" : "dsDescriptionDate", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1000-01-01" + } + }, { + "dsDescriptionValue" : { + "typeName" : "dsDescriptionValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "DescriptionText2" + }, + "dsDescriptionDate" : { + "typeName" : "dsDescriptionDate", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1000-02-02" + } + } + ] + }, { + "typeName" : "subject", + "multiple" : true, + "typeClass" : "controlledVocabulary", + "value" : [ + "Agricultural Sciences", + "Business and Management", + "Engineering", + "Law" + ] + }, { + "typeName" : "keyword", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "keywordValue" : { + "typeName" : "keywordValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "KeywordTerm1" + }, + "keywordTermURI" : { + "typeName" : "keywordTermURI", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://keywordTermURI1.org" + }, + "keywordVocabulary" : { + "typeName" : "keywordVocabulary", + "multiple" : false, + "typeClass" : "primitive", + "value" : "KeywordVocabulary1" + }, + "keywordVocabularyURI" : { + "typeName" : "keywordVocabularyURI", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://KeywordVocabularyURL1.org" + } + }, { + "keywordValue" : { + "typeName" : "keywordValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "KeywordTerm2" + }, + "keywordTermURI" : { + "typeName" : "keywordTermURI", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://keywordTermURI2.org" + }, + "keywordVocabulary" : { + "typeName" : "keywordVocabulary", + "multiple" : false, + "typeClass" : "primitive", + "value" : "KeywordVocabulary2" + }, + "keywordVocabularyURI" : { + "typeName" : "keywordVocabularyURI", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://KeywordVocabularyURL2.org" + } + } + ] + }, { + "typeName" : "topicClassification", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "topicClassValue" : { + "typeName" : "topicClassValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "Topic Classification Term1" + }, + "topicClassVocab" : { + "typeName" : "topicClassVocab", + "multiple" : false, + "typeClass" : "primitive", + "value" : "Topic Classification Vocab1" + }, + "topicClassVocabURI" : { + "typeName" : "topicClassVocabURI", + "multiple" : false, + "typeClass" : "primitive", + "value" : "https://TopicClassificationURL1.com" + } + }, { + "topicClassValue" : { + "typeName" : "topicClassValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "Topic Classification Term2" + }, + "topicClassVocab" : { + "typeName" : "topicClassVocab", + "multiple" : false, + "typeClass" : "primitive", + "value" : "Topic Classification Vocab2" + }, + "topicClassVocabURI" : { + "typeName" : "topicClassVocabURI", + "multiple" : false, + "typeClass" : "primitive", + "value" : "https://TopicClassificationURL2.com" + } + } + ] + }, { + "typeName" : "publication", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "publicationRelationType" : { + "typeName" : "publicationRelationType", + "multiple" : false, + "typeClass" : "controlledVocabulary", + "value" : "IsSupplementTo" + }, + "publicationCitation" : { + "typeName" : "publicationCitation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "RelatedPublicationCitation1" + }, + "publicationIDType" : { + "typeName" : "publicationIDType", + "multiple" : false, + "typeClass" : "controlledVocabulary", + "value" : "ark" + }, + "publicationIDNumber" : { + "typeName" : "publicationIDNumber", + "multiple" : false, + "typeClass" : "primitive", + "value" : "RelatedPublicationIDNumber1" + }, + "publicationURL" : { + "typeName" : "publicationURL", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://RelatedPublicationURL1.org" + } + }, { + "publicationCitation" : { + "typeName" : "publicationCitation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "RelatedPublicationCitation2" + }, + "publicationIDType" : { + "typeName" : "publicationIDType", + "multiple" : false, + "typeClass" : "controlledVocabulary", + "value" : "arXiv" + }, + "publicationIDNumber" : { + "typeName" : "publicationIDNumber", + "multiple" : false, + "typeClass" : "primitive", + "value" : "RelatedPublicationIDNumber2" + }, + "publicationURL" : { + "typeName" : "publicationURL", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://RelatedPublicationURL2.org" + } + } + ] + }, { + "typeName" : "notesText", + "multiple" : false, + "typeClass" : "primitive", + "value" : "Notes1" + }, { + "typeName" : "language", + "multiple" : true, + "typeClass" : "controlledVocabulary", + "value" : [ + "Abkhaz", + "Afar" + ] + }, { + "typeName" : "producer", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "producerName" : { + "typeName" : "producerName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastProducer1, FirstProducer1" + }, + "producerAffiliation" : { + "typeName" : "producerAffiliation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "ProducerAffiliation1" + }, + "producerAbbreviation" : { + "typeName" : "producerAbbreviation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "ProducerAbbreviation1" + }, + "producerURL" : { + "typeName" : "producerURL", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://ProducerURL1.org" + }, + "producerLogoURL" : { + "typeName" : "producerLogoURL", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://ProducerLogoURL1.org" + } + }, { + "producerName" : { + "typeName" : "producerName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastProducer2, FirstProducer2" + }, + "producerAffiliation" : { + "typeName" : "producerAffiliation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "ProducerAffiliation2" + }, + "producerAbbreviation" : { + "typeName" : "producerAbbreviation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "ProducerAbbreviation2" + }, + "producerURL" : { + "typeName" : "producerURL", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://ProducerURL2.org" + }, + "producerLogoURL" : { + "typeName" : "producerLogoURL", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://ProducerLogoURL2.org" + } + } + ] + }, { + "typeName" : "productionDate", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1003-01-01" + }, { + "typeName" : "productionPlace", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "ProductionPlace" + ] + }, { + "typeName" : "contributor", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "contributorType" : { + "typeName" : "contributorType", + "multiple" : false, + "typeClass" : "controlledVocabulary", + "value" : "Data Collector" + }, + "contributorName" : { + "typeName" : "contributorName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastContributor1, FirstContributor1" + } + }, { + "contributorType" : { + "typeName" : "contributorType", + "multiple" : false, + "typeClass" : "controlledVocabulary", + "value" : "Data Curator" + }, + "contributorName" : { + "typeName" : "contributorName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastContributor2, FirstContributor2" + } + } + ] + }, { + "typeName" : "grantNumber", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "grantNumberAgency" : { + "typeName" : "grantNumberAgency", + "multiple" : false, + "typeClass" : "primitive", + "value" : "GrantInformationGrantAgency1" + }, + "grantNumberValue" : { + "typeName" : "grantNumberValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "GrantInformationGrantNumber1" + } + }, { + "grantNumberAgency" : { + "typeName" : "grantNumberAgency", + "multiple" : false, + "typeClass" : "primitive", + "value" : "GrantInformationGrantAgency2" + }, + "grantNumberValue" : { + "typeName" : "grantNumberValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "GrantInformationGrantNumber2" + } + } + ] + }, { + "typeName" : "distributor", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "distributorName" : { + "typeName" : "distributorName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastDistributor1, FirstDistributor1" + }, + "distributorAffiliation" : { + "typeName" : "distributorAffiliation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "DistributorAffiliation1" + }, + "distributorAbbreviation" : { + "typeName" : "distributorAbbreviation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "DistributorAbbreviation1" + }, + "distributorURL" : { + "typeName" : "distributorURL", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://DistributorURL1.org" + }, + "distributorLogoURL" : { + "typeName" : "distributorLogoURL", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://DistributorLogoURL1.org" + } + }, { + "distributorName" : { + "typeName" : "distributorName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastDistributor2, FirstDistributor2" + }, + "distributorAffiliation" : { + "typeName" : "distributorAffiliation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "DistributorAffiliation2" + }, + "distributorAbbreviation" : { + "typeName" : "distributorAbbreviation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "DistributorAbbreviation2" + }, + "distributorURL" : { + "typeName" : "distributorURL", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://DistributorURL2.org" + }, + "distributorLogoURL" : { + "typeName" : "distributorLogoURL", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://DistributorLogoURL2.org" + } + } + ] + }, { + "typeName" : "distributionDate", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1004-01-01" + }, { + "typeName" : "depositor", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastDepositor, FirstDepositor" + }, { + "typeName" : "dateOfDeposit", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1002-01-01" + }, { + "typeName" : "timePeriodCovered", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "timePeriodCoveredStart" : { + "typeName" : "timePeriodCoveredStart", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1005-01-01" + }, + "timePeriodCoveredEnd" : { + "typeName" : "timePeriodCoveredEnd", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1005-01-02" + } + }, { + "timePeriodCoveredStart" : { + "typeName" : "timePeriodCoveredStart", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1005-02-01" + }, + "timePeriodCoveredEnd" : { + "typeName" : "timePeriodCoveredEnd", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1005-02-02" + } + } + ] + }, { + "typeName" : "dateOfCollection", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "dateOfCollectionStart" : { + "typeName" : "dateOfCollectionStart", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1006-01-01" + }, + "dateOfCollectionEnd" : { + "typeName" : "dateOfCollectionEnd", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1006-01-01" + } + }, { + "dateOfCollectionStart" : { + "typeName" : "dateOfCollectionStart", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1006-02-01" + }, + "dateOfCollectionEnd" : { + "typeName" : "dateOfCollectionEnd", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1006-02-02" + } + } + ] + }, { + "typeName" : "kindOfData", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "KindOfData1", + "KindOfData2" + ] + }, { + "typeName" : "series", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "seriesName" : { + "typeName" : "seriesName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "SeriesName" + }, + "seriesInformation" : { + "typeName" : "seriesInformation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "SeriesInformation" + } + } + ] + }, { + "typeName" : "software", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "softwareName" : { + "typeName" : "softwareName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "SoftwareName1" + }, + "softwareVersion" : { + "typeName" : "softwareVersion", + "multiple" : false, + "typeClass" : "primitive", + "value" : "SoftwareVersion1" + } + }, { + "softwareName" : { + "typeName" : "softwareName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "SoftwareName2" + }, + "softwareVersion" : { + "typeName" : "softwareVersion", + "multiple" : false, + "typeClass" : "primitive", + "value" : "SoftwareVersion2" + } + } + ] + }, { + "typeName" : "relatedMaterial", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "RelatedMaterial1", + "RelatedMaterial2" + ] + }, { + "typeName" : "relatedDatasets", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "RelatedDatasets1", + "RelatedDatasets2" + ] + }, { + "typeName" : "otherReferences", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "OtherReferences1", + "OtherReferences2" + ] + }, { + "typeName" : "dataSources", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "DataSources1", + "DataSources2" + ] + }, { + "typeName" : "originOfSources", + "multiple" : false, + "typeClass" : "primitive", + "value" : "OriginOfSources" + }, { + "typeName" : "characteristicOfSources", + "multiple" : false, + "typeClass" : "primitive", + "value" : "CharacteristicOfSourcesNoted" + }, { + "typeName" : "accessToSources", + "multiple" : false, + "typeClass" : "primitive", + "value" : "DocumentationAndAccessToSources" + } + ] + }, + "geospatial" : { + "displayName" : "Geospatial Metadata", + "fields" : [{ + "typeName" : "geographicCoverage", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "country" : { + "typeName" : "country", + "multiple" : false, + "typeClass" : "controlledVocabulary", + "value" : "Afghanistan" + }, + "state" : { + "typeName" : "state", + "multiple" : false, + "typeClass" : "primitive", + "value" : "GeographicCoverageStateProvince1" + }, + "city" : { + "typeName" : "city", + "multiple" : false, + "typeClass" : "primitive", + "value" : "GeographicCoverageCity1" + }, + "otherGeographicCoverage" : { + "typeName" : "otherGeographicCoverage", + "multiple" : false, + "typeClass" : "primitive", + "value" : "GeographicCoverageOther1" + } + }, { + "country" : { + "typeName" : "country", + "multiple" : false, + "typeClass" : "controlledVocabulary", + "value" : "Albania" + }, + "state" : { + "typeName" : "state", + "multiple" : false, + "typeClass" : "primitive", + "value" : "GeographicCoverageStateProvince2" + }, + "city" : { + "typeName" : "city", + "multiple" : false, + "typeClass" : "primitive", + "value" : "GeographicCoverageCity2" + }, + "otherGeographicCoverage" : { + "typeName" : "otherGeographicCoverage", + "multiple" : false, + "typeClass" : "primitive", + "value" : "GeographicCoverageOther2" + } + } + ] + }, { + "typeName" : "geographicUnit", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "GeographicUnit1", + "GeographicUnit2" + ] + }, { + "typeName" : "geographicBoundingBox", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "westLongitude" : { + "typeName" : "westLongitude", + "multiple" : false, + "typeClass" : "primitive", + "value" : "-72" + }, + "eastLongitude" : { + "typeName" : "eastLongitude", + "multiple" : false, + "typeClass" : "primitive", + "value" : "-70" + }, + "northLatitude" : { + "typeName" : "northLatitude", + "multiple" : false, + "typeClass" : "primitive", + "value" : "43" + }, + "southLatitude" : { + "typeName" : "southLatitude", + "multiple" : false, + "typeClass" : "primitive", + "value" : "42" + } + }, { + "westLongitude" : { + "typeName" : "westLongitude", + "multiple" : false, + "typeClass" : "primitive", + "value" : "-18" + }, + "eastLongitude" : { + "typeName" : "eastLongitude", + "multiple" : false, + "typeClass" : "primitive", + "value" : "-13" + }, + "northLatitude" : { + "typeName" : "northLatitude", + "multiple" : false, + "typeClass" : "primitive", + "value" : "29" + }, + "southLatitude" : { + "typeName" : "southLatitude", + "multiple" : false, + "typeClass" : "primitive", + "value" : "28" + } + } + ] + } + ] + }, + "socialscience" : { + "displayName" : "Social Science and Humanities Metadata", + "fields" : [{ + "typeName" : "unitOfAnalysis", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "UnitOfAnalysis1", + "UnitOfAnalysis2" + ] + }, { + "typeName" : "universe", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "Universe1", + "Universe2" + ] + }, { + "typeName" : "timeMethod", + "multiple" : false, + "typeClass" : "primitive", + "value" : "TimeMethod" + }, { + "typeName" : "dataCollector", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastDataCollector1, FirstDataCollector1" + }, { + "typeName" : "collectorTraining", + "multiple" : false, + "typeClass" : "primitive", + "value" : "CollectorTraining" + }, { + "typeName" : "frequencyOfDataCollection", + "multiple" : false, + "typeClass" : "primitive", + "value" : "Frequency" + }, { + "typeName" : "samplingProcedure", + "multiple" : false, + "typeClass" : "primitive", + "value" : "SamplingProcedure" + }, { + "typeName" : "targetSampleSize", + "multiple" : false, + "typeClass" : "compound", + "value" : { + "targetSampleActualSize" : { + "typeName" : "targetSampleActualSize", + "multiple" : false, + "typeClass" : "primitive", + "value" : "100" + }, + "targetSampleSizeFormula" : { + "typeName" : "targetSampleSizeFormula", + "multiple" : false, + "typeClass" : "primitive", + "value" : "TargetSampleSizeFormula" + } + } + }, { + "typeName" : "deviationsFromSampleDesign", + "multiple" : false, + "typeClass" : "primitive", + "value" : "MajorDeviationsForSampleDesign" + }, { + "typeName" : "collectionMode", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "CollectionMode" + ] + }, { + "typeName" : "researchInstrument", + "multiple" : false, + "typeClass" : "primitive", + "value" : "TypeOfResearchInstrument" + }, { + "typeName" : "dataCollectionSituation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "CharacteristicsOfDataCollectionSituation" + }, { + "typeName" : "actionsToMinimizeLoss", + "multiple" : false, + "typeClass" : "primitive", + "value" : "ActionsToMinimizeLosses" + }, { + "typeName" : "controlOperations", + "multiple" : false, + "typeClass" : "primitive", + "value" : "ControlOperations" + }, { + "typeName" : "weighting", + "multiple" : false, + "typeClass" : "primitive", + "value" : "Weighting" + }, { + "typeName" : "cleaningOperations", + "multiple" : false, + "typeClass" : "primitive", + "value" : "CleaningOperations" + }, { + "typeName" : "datasetLevelErrorNotes", + "multiple" : false, + "typeClass" : "primitive", + "value" : "StudyLevelErrorNotes" + }, { + "typeName" : "responseRate", + "multiple" : false, + "typeClass" : "primitive", + "value" : "ResponseRate" + }, { + "typeName" : "samplingErrorEstimates", + "multiple" : false, + "typeClass" : "primitive", + "value" : "EstimatesOfSamplingError" + }, { + "typeName" : "otherDataAppraisal", + "multiple" : false, + "typeClass" : "primitive", + "value" : "OtherFormsOfDataAppraisal" + }, { + "typeName" : "socialScienceNotes", + "multiple" : false, + "typeClass" : "compound", + "value" : { + "socialScienceNotesType" : { + "typeName" : "socialScienceNotesType", + "multiple" : false, + "typeClass" : "primitive", + "value" : "NotesType" + }, + "socialScienceNotesSubject" : { + "typeName" : "socialScienceNotesSubject", + "multiple" : false, + "typeClass" : "primitive", + "value" : "NotesSubject" + }, + "socialScienceNotesText" : { + "typeName" : "socialScienceNotesText", + "multiple" : false, + "typeClass" : "primitive", + "value" : "NotesText" + } + } + } + ] + }, + "astrophysics" : { + "displayName" : "Astronomy and Astrophysics Metadata", + "fields" : [{ + "typeName" : "astroType", + "multiple" : true, + "typeClass" : "controlledVocabulary", + "value" : [ + "Image", + "Mosaic", + "EventList", + "Cube" + ] + }, { + "typeName" : "astroFacility", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "Facility1", + "Facility2" + ] + }, { + "typeName" : "astroInstrument", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "Instrument1", + "Instrument2" + ] + }, { + "typeName" : "astroObject", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "Object1", + "Object2" + ] + }, { + "typeName" : "resolution.Spatial", + "multiple" : false, + "typeClass" : "primitive", + "value" : "SpatialResolution" + }, { + "typeName" : "resolution.Spectral", + "multiple" : false, + "typeClass" : "primitive", + "value" : "SpectralResolution" + }, { + "typeName" : "resolution.Temporal", + "multiple" : false, + "typeClass" : "primitive", + "value" : "TimeResolution" + }, { + "typeName" : "coverage.Spectral.Bandpass", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "Bandpass1", + "Bandpass2" + ] + }, { + "typeName" : "coverage.Spectral.CentralWavelength", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "3001", + "3002" + ] + }, { + "typeName" : "coverage.Spectral.Wavelength", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "coverage.Spectral.MinimumWavelength" : { + "typeName" : "coverage.Spectral.MinimumWavelength", + "multiple" : false, + "typeClass" : "primitive", + "value" : "4001" + }, + "coverage.Spectral.MaximumWavelength" : { + "typeName" : "coverage.Spectral.MaximumWavelength", + "multiple" : false, + "typeClass" : "primitive", + "value" : "4002" + } + }, { + "coverage.Spectral.MinimumWavelength" : { + "typeName" : "coverage.Spectral.MinimumWavelength", + "multiple" : false, + "typeClass" : "primitive", + "value" : "4003" + }, + "coverage.Spectral.MaximumWavelength" : { + "typeName" : "coverage.Spectral.MaximumWavelength", + "multiple" : false, + "typeClass" : "primitive", + "value" : "4004" + } + } + ] + }, { + "typeName" : "coverage.Temporal", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "coverage.Temporal.StartTime" : { + "typeName" : "coverage.Temporal.StartTime", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1007-01-01" + }, + "coverage.Temporal.StopTime" : { + "typeName" : "coverage.Temporal.StopTime", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1007-01-02" + } + }, { + "coverage.Temporal.StartTime" : { + "typeName" : "coverage.Temporal.StartTime", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1007-02-01" + }, + "coverage.Temporal.StopTime" : { + "typeName" : "coverage.Temporal.StopTime", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1007-02-02" + } + } + ] + }, { + "typeName" : "coverage.Spatial", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "SkyCoverage1", + "SkyCoverage2" + ] + }, { + "typeName" : "coverage.Depth", + "multiple" : false, + "typeClass" : "primitive", + "value" : "200" + }, { + "typeName" : "coverage.ObjectDensity", + "multiple" : false, + "typeClass" : "primitive", + "value" : "300" + }, { + "typeName" : "coverage.ObjectCount", + "multiple" : false, + "typeClass" : "primitive", + "value" : "400" + }, { + "typeName" : "coverage.SkyFraction", + "multiple" : false, + "typeClass" : "primitive", + "value" : "500" + }, { + "typeName" : "coverage.Polarization", + "multiple" : false, + "typeClass" : "primitive", + "value" : "Polarization" + }, { + "typeName" : "redshiftType", + "multiple" : false, + "typeClass" : "primitive", + "value" : "RedshiftType" + }, { + "typeName" : "resolution.Redshift", + "multiple" : false, + "typeClass" : "primitive", + "value" : "600" + }, { + "typeName" : "coverage.RedshiftValue", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "coverage.Redshift.MinimumValue" : { + "typeName" : "coverage.Redshift.MinimumValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "701" + }, + "coverage.Redshift.MaximumValue" : { + "typeName" : "coverage.Redshift.MaximumValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "702" + } + }, { + "coverage.Redshift.MinimumValue" : { + "typeName" : "coverage.Redshift.MinimumValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "703" + }, + "coverage.Redshift.MaximumValue" : { + "typeName" : "coverage.Redshift.MaximumValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "704" + } + } + ] + } + ] + }, + "biomedical" : { + "displayName" : "Life Sciences Metadata", + "fields" : [{ + "typeName" : "studyDesignType", + "multiple" : true, + "typeClass" : "controlledVocabulary", + "value" : [ + "Case Control", + "Cross Sectional", + "Cohort Study", + "Not Specified" + ] + }, { + "typeName" : "studyFactorType", + "multiple" : true, + "typeClass" : "controlledVocabulary", + "value" : [ + "Age", + "Biomarkers", + "Cell Surface Markers", + "Developmental Stage" + ] + }, { + "typeName" : "studyAssayOrganism", + "multiple" : true, + "typeClass" : "controlledVocabulary", + "value" : [ + "Arabidopsis thaliana", + "Bos taurus", + "Caenorhabditis elegans", + "Danio rerio (zebrafish)" + ] + }, { + "typeName" : "studyAssayOtherOrganism", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "OtherOrganism1", + "OtherOrganism2" + ] + }, { + "typeName" : "studyAssayMeasurementType", + "multiple" : true, + "typeClass" : "controlledVocabulary", + "value" : [ + "genome sequencing", + "cell sorting", + "clinical chemistry analysis", + "DNA methylation profiling" + ] + }, { + "typeName" : "studyAssayOtherMeasurmentType", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "OtherMeasurementType1", + "OtherMeasurementType2" + ] + }, { + "typeName" : "studyAssayTechnologyType", + "multiple" : true, + "typeClass" : "controlledVocabulary", + "value" : [ + "culture based drug susceptibility testing, single concentration", + "culture based drug susceptibility testing, two concentrations", + "culture based drug susceptibility testing, three or more concentrations (minimium inhibitory concentration measurement)", + "flow cytometry" + ] + }, { + "typeName" : "studyAssayPlatform", + "multiple" : true, + "typeClass" : "controlledVocabulary", + "value" : [ + "210-MS GC Ion Trap (Varian)", + "220-MS GC Ion Trap (Varian)", + "225-MS GC Ion Trap (Varian)", + "300-MS quadrupole GC/MS (Varian)" + ] + }, { + "typeName" : "studyAssayCellType", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "CellType1", + "CellType2" + ] + } + ] + }, + "journal" : { + "displayName" : "Journal Metadata", + "fields" : [{ + "typeName" : "journalVolumeIssue", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "journalVolume" : { + "typeName" : "journalVolume", + "multiple" : false, + "typeClass" : "primitive", + "value" : "JournalVolume1" + }, + "journalIssue" : { + "typeName" : "journalIssue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "JournalIssue1" + }, + "journalPubDate" : { + "typeName" : "journalPubDate", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1008-01-01" + } + }, { + "journalVolume" : { + "typeName" : "journalVolume", + "multiple" : false, + "typeClass" : "primitive", + "value" : "JournalVolume2" + }, + "journalIssue" : { + "typeName" : "journalIssue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "JournalIssue2" + }, + "journalPubDate" : { + "typeName" : "journalPubDate", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1008-02-01" + } + } + ] + }, { + "typeName" : "journalArticleType", + "multiple" : false, + "typeClass" : "controlledVocabulary", + "value" : "abstract" + } + ] + } + } + } } \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java index 53ab6c7bef7..abb812d1ba3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java @@ -490,6 +490,8 @@ public String getRestrictions() { return restrictions; } + @Deprecated + //Appears to not be used public String getContact() { return contact; } diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java index eb6fdd4e923..0433c425fd2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java @@ -1357,36 +1357,41 @@ public List getRelatedPublications() { DatasetRelPublication relatedPublication = new DatasetRelPublication(); for (DatasetField subField : publication.getChildDatasetFields()) { switch (subField.getDatasetFieldType().getName()) { - case DatasetFieldConstant.publicationCitation: - relatedPublication.setText(subField.getDisplayValue()); - break; - case DatasetFieldConstant.publicationURL: - // We have to avoid using subField.getDisplayValue() here - because the DisplayFormatType - // for this url metadata field is likely set up so that the display value is automatically - // turned into a clickable HTML HREF block, which we don't want to end in our Schema.org - // JSON-LD output. So we want to use the raw value of the field instead, with minimal HTML - // sanitation, just in case (this would be done on all URLs in getDisplayValue()). - String url = subField.getValue(); - if (StringUtils.isBlank(url) || DatasetField.NA_VALUE.equals(url)) { - relatedPublication.setUrl(""); - } else { - relatedPublication.setUrl(MarkupChecker.sanitizeBasicHTML(url)); - } - break; - case DatasetFieldConstant.publicationIDType: - // QDR idType has a trailing : now (Aug 2021) - // Get sanitized value without any display modifications - subField.getDatasetFieldType().setDisplayFormat("#VALUE"); - relatedPublication.setIdType(subField.getDisplayValue()); - break; - case DatasetFieldConstant.publicationIDNumber: - // Get sanitized value without any display modifications - subField.getDatasetFieldType().setDisplayFormat("#VALUE"); - relatedPublication.setIdNumber(subField.getDisplayValue()); - break; - case DatasetFieldConstant.publicationRelationType: - relatedPublication.setRelationType(subField.getDisplayValue()); - break; + case DatasetFieldConstant.publicationCitation: + relatedPublication.setText(subField.getDisplayValue()); + break; + case DatasetFieldConstant.publicationURL: + // We have to avoid using subField.getDisplayValue() here - because the + // DisplayFormatType + // for this url metadata field is likely set up so that the display value is + // automatically + // turned into a clickable HTML HREF block, which we don't want to end in our + // Schema.org + // JSON-LD output. So we want to use the raw value of the field instead, with + // minimal HTML + // sanitation, just in case (this would be done on all URLs in + // getDisplayValue()). + String url = subField.getValue(); + if (StringUtils.isBlank(url) || DatasetField.NA_VALUE.equals(url)) { + relatedPublication.setUrl(""); + } else { + relatedPublication.setUrl(MarkupChecker.sanitizeBasicHTML(url)); + } + break; + case DatasetFieldConstant.publicationIDType: + // QDR idType has a trailing : now (Aug 2021) + // Get value without any display modifications + subField.getDatasetFieldType().setDisplayFormat("#VALUE"); + relatedPublication.setIdType(subField.getDisplayValue()); + break; + case DatasetFieldConstant.publicationIDNumber: + // Get sanitized value without any display modifications + subField.getDatasetFieldType().setDisplayFormat("#VALUE"); + relatedPublication.setIdNumber(subField.getDisplayValue()); + break; + case DatasetFieldConstant.publicationRelationType: + relatedPublication.setRelationType(subField.getDisplayValue()); + break; } } relatedPublications.add(relatedPublication); diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionUI.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionUI.java index 0a1afe86901..f1ddf2304b7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionUI.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionUI.java @@ -118,21 +118,21 @@ public DatasetVersionUI initDatasetVersionUI(DatasetVersion datasetVersion, boo for (DatasetField subField : relPubVal.getChildDatasetFields()) { String value = subField.getValue(); switch (subField.getDatasetFieldType().getName()) { - case DatasetFieldConstant.publicationCitation: - datasetRelPublication.setText(subField.getValue()); - break; - case DatasetFieldConstant.publicationIDNumber: - datasetRelPublication.setIdNumber(subField.getValue()); - break; - case DatasetFieldConstant.publicationIDType: - datasetRelPublication.setIdType(subField.getValue()); - break; - case DatasetFieldConstant.publicationURL: - datasetRelPublication.setUrl(subField.getValue()); - break; - case DatasetFieldConstant.publicationRelationType: - datasetRelPublication.setRelationType(subField.getValue()); - break; + case DatasetFieldConstant.publicationCitation: + datasetRelPublication.setText(subField.getValue()); + break; + case DatasetFieldConstant.publicationIDNumber: + datasetRelPublication.setIdNumber(subField.getValue()); + break; + case DatasetFieldConstant.publicationIDType: + datasetRelPublication.setIdType(subField.getValue()); + break; + case DatasetFieldConstant.publicationURL: + datasetRelPublication.setUrl(subField.getValue()); + break; + case DatasetFieldConstant.publicationRelationType: + datasetRelPublication.setRelationType(subField.getValue()); + break; } } this.getDatasetRelPublications().add(datasetRelPublication); diff --git a/src/main/java/edu/harvard/iq/dataverse/export/dublincore/DublinCoreExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/dublincore/DublinCoreExportUtil.java index e74a2f26af6..a2ef9120cd7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/dublincore/DublinCoreExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/dublincore/DublinCoreExportUtil.java @@ -307,21 +307,21 @@ private static void writeRelPublElement(XMLStreamWriter xmlw, DatasetVersionDTO for (Iterator iterator = foo.iterator(); iterator.hasNext();) { FieldDTO next = iterator.next(); switch (next.getTypeName()) { - case DatasetFieldConstant.publicationCitation: - citation = next.getSinglePrimitive(); - break; - case DatasetFieldConstant.publicationIDType: - IDType = next.getSinglePrimitive(); - break; - case DatasetFieldConstant.publicationIDNumber: - IDNo = next.getSinglePrimitive(); - break; - case DatasetFieldConstant.publicationURL: - url = next.getSinglePrimitive(); - break; - case DatasetFieldConstant.publicationRelationType: - relationType = next.getSinglePrimitive(); - break; + case DatasetFieldConstant.publicationCitation: + citation = next.getSinglePrimitive(); + break; + case DatasetFieldConstant.publicationIDType: + IDType = next.getSinglePrimitive(); + break; + case DatasetFieldConstant.publicationIDNumber: + IDNo = next.getSinglePrimitive(); + break; + case DatasetFieldConstant.publicationURL: + url = next.getSinglePrimitive(); + break; + case DatasetFieldConstant.publicationRelationType: + relationType = next.getSinglePrimitive(); + break; } } if(StringUtils.isBlank(relationType)) { diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index fe05b0d4537..a74a9f34bc9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -546,15 +546,15 @@ private void writeContributors(XMLStreamWriter xmlw, DvObject dvObject) throws X for (DatasetField subField : contributorFieldValue.getChildDatasetFields()) { switch (subField.getDatasetFieldType().getName()) { - case DatasetFieldConstant.contributorName: - contributor = subField.getValue(); - break; - case DatasetFieldConstant.contributorType: - contributorType = subField.getValue(); - if(contributorType!=null) { - contributorType = contributorType.replace(" ", ""); - } - break; + case DatasetFieldConstant.contributorName: + contributor = subField.getValue(); + break; + case DatasetFieldConstant.contributorType: + contributorType = subField.getValue(); + if (contributorType != null) { + contributorType = contributorType.replace(" ", ""); + } + break; } } // QDR - doesn't have Funder in the contributor type list. @@ -1299,71 +1299,71 @@ private void writeDescriptions(XMLStreamWriter xmlw, DvObject dvObject, boolean for (DatasetField dsf : dsfs) { switch (dsf.getDatasetFieldType().getName()) { - case DatasetFieldConstant.software: - attributes.clear(); - attributes.put("descriptionType", "TechnicalInfo"); - List dsfcvs = dsf.getDatasetFieldCompoundValues(); - for (DatasetFieldCompoundValue dsfcv : dsfcvs) { - - String softwareName = null; - String softwareVersion = null; - List childDsfs = dsfcv.getChildDatasetFields(); - for (DatasetField childDsf : childDsfs) { - if (DatasetFieldConstant.softwareName.equals(childDsf.getDatasetFieldType().getName())) { - softwareName = childDsf.getValue(); - } else if (DatasetFieldConstant.softwareVersion.equals(childDsf.getDatasetFieldType().getName())) { - softwareVersion = childDsf.getValue(); + case DatasetFieldConstant.software: + attributes.clear(); + attributes.put("descriptionType", "TechnicalInfo"); + List dsfcvs = dsf.getDatasetFieldCompoundValues(); + for (DatasetFieldCompoundValue dsfcv : dsfcvs) { + + String softwareName = null; + String softwareVersion = null; + List childDsfs = dsfcv.getChildDatasetFields(); + for (DatasetField childDsf : childDsfs) { + if (DatasetFieldConstant.softwareName.equals(childDsf.getDatasetFieldType().getName())) { + softwareName = childDsf.getValue(); + } else if (DatasetFieldConstant.softwareVersion.equals(childDsf.getDatasetFieldType().getName())) { + softwareVersion = childDsf.getValue(); + } } - } - if (StringUtils.isNotBlank(softwareName)) { - if (StringUtils.isNotBlank(softwareVersion)) { + if (StringUtils.isNotBlank(softwareName)) { + if (StringUtils.isNotBlank(softwareVersion)) { + } + softwareName = softwareName + ", " + softwareVersion; + descriptionsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "descriptions", descriptionsWritten); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "description", attributes, softwareName); } - softwareName = softwareName + ", " + softwareVersion; - descriptionsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "descriptions", descriptionsWritten); - XmlWriterUtil.writeFullElementWithAttributes(xmlw, "description", attributes, softwareName); } - } - break; - case DatasetFieldConstant.originOfSources: - case DatasetFieldConstant.characteristicOfSources: - case DatasetFieldConstant.accessToSources: - attributes.clear(); - attributes.put("descriptionType", "Methods"); - String method = dsf.getValue(); - if (StringUtils.isNotBlank(method)) { - descriptionsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "descriptions", descriptionsWritten); - XmlWriterUtil.writeFullElementWithAttributes(xmlw, "description", attributes, method); - - } - break; - case DatasetFieldConstant.series: - attributes.clear(); - attributes.put("descriptionType", "SeriesInformation"); - dsfcvs = dsf.getDatasetFieldCompoundValues(); - for (DatasetFieldCompoundValue dsfcv : dsfcvs) { - List childDsfs = dsfcv.getChildDatasetFields(); - for (DatasetField childDsf : childDsfs) { + break; + case DatasetFieldConstant.originOfSources: + case DatasetFieldConstant.characteristicOfSources: + case DatasetFieldConstant.accessToSources: + attributes.clear(); + attributes.put("descriptionType", "Methods"); + String method = dsf.getValue(); + if (StringUtils.isNotBlank(method)) { + descriptionsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "descriptions", descriptionsWritten); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "description", attributes, method); - if (DatasetFieldConstant.seriesName.equals(childDsf.getDatasetFieldType().getName())) { - String seriesInformation = childDsf.getValue(); - if (StringUtils.isNotBlank(seriesInformation)) { - descriptionsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "descriptions", descriptionsWritten); - XmlWriterUtil.writeFullElementWithAttributes(xmlw, "description", attributes, seriesInformation); + } + break; + case DatasetFieldConstant.series: + attributes.clear(); + attributes.put("descriptionType", "SeriesInformation"); + dsfcvs = dsf.getDatasetFieldCompoundValues(); + for (DatasetFieldCompoundValue dsfcv : dsfcvs) { + List childDsfs = dsfcv.getChildDatasetFields(); + for (DatasetField childDsf : childDsfs) { + + if (DatasetFieldConstant.seriesName.equals(childDsf.getDatasetFieldType().getName())) { + String seriesInformation = childDsf.getValue(); + if (StringUtils.isNotBlank(seriesInformation)) { + descriptionsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "descriptions", descriptionsWritten); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "description", attributes, seriesInformation); + } + break; } - break; } } - } - break; - case DatasetFieldConstant.notesText: - attributes.clear(); - attributes.put("descriptionType", "Other"); - String notesText = dsf.getValue(); - if (StringUtils.isNotBlank(notesText)) { - descriptionsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "descriptions", descriptionsWritten); - XmlWriterUtil.writeFullElementWithAttributes(xmlw, "description", attributes, notesText); - } - break; + break; + case DatasetFieldConstant.notesText: + attributes.clear(); + attributes.put("descriptionType", "Other"); + String notesText = dsf.getValue(); + if (StringUtils.isNotBlank(notesText)) { + descriptionsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "descriptions", descriptionsWritten); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "description", attributes, notesText); + } + break; } } From 174fcf911e3bb44321e59aaae317ed2eb86aed33 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 17 Sep 2024 13:55:35 -0400 Subject: [PATCH 888/970] Apply suggestions from code review Co-authored-by: Philip Durbin --- .../10632-DataCiteXMLandRelationType.md | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/release-notes/10632-DataCiteXMLandRelationType.md b/doc/release-notes/10632-DataCiteXMLandRelationType.md index 10e9a830714..a3699646314 100644 --- a/doc/release-notes/10632-DataCiteXMLandRelationType.md +++ b/doc/release-notes/10632-DataCiteXMLandRelationType.md @@ -1,21 +1,21 @@ -### Enhanced DataCite Metadata +### Enhanced DataCite Metadata, Relation Type A new field has been added to the citation metadatablock to allow entry of the "Relation Type" between a "Related Publication" and a dataset. The Relation Type is currently limited to the most common 6 values recommended by DataCite: isCitedBy, Cites, IsSupplementTo, IsSupplementedBy, IsReferencedBy, and References. -Additional metadata, including metadata about Related Publications is now being sent to DataCite when DOIs are registered and published and is available in the DataCite XML export. For existing datasets where no "Relation Type" has been specified, "IsSupplementTo" is assumed. The additions are in rough alignment with the OpenAire xml export, but there are some minor differences in addition to the Relation Type addition, including an update to the DataCite 4.5 schema. +Additional metadata, including metadata about Related Publications is now being sent to DataCite when DOIs are registered and published and is available in the DataCite XML export. For existing datasets where no "Relation Type" has been specified, "IsSupplementTo" is assumed. The additions are in rough alignment with the OpenAIRE XML export, but there are some minor differences in addition to the Relation Type addition, including an update to the DataCite 4.5 schema. -For details see https://github.com/IQSS/dataverse/pull/10632 and https://github.com/IQSS/dataverse/pull/10615 and the design document referenced there. +For details see https://github.com/IQSS/dataverse/pull/10632 and https://github.com/IQSS/dataverse/pull/10615 and the [design document](https://docs.google.com/document/d/1JzDo9UOIy9dVvaHvtIbOI8tFU6bWdfDfuQvWWpC0tkA/edit?usp=sharing) referenced there. Multiple backward incompatible changes and bug fixes have been made to API calls (3 of the four of which were not documented) related to updating PID target urls and metadata at the provider service: -[Update Target URL for a Published Dataset at the PID provider](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-target-url-for-a-published-dataset-at-the-pid-provider) -[Update Target URL for all Published Datasets at the PID provider](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-target-url-for-all-published-datasets-at-the-pid-provider) -[Update Metadata for a Published Dataset at the PID provider](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-metadata-for-a-published-dataset-at-the-pid-provider) -[Update Metadata for all Published Datasets at the PID provider](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-metadata-for-all-published-datasets-at-the-pid-provider) +- [Update Target URL for a Published Dataset at the PID provider](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-target-url-for-a-published-dataset-at-the-pid-provider) +- [Update Target URL for all Published Datasets at the PID provider](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-target-url-for-all-published-datasets-at-the-pid-provider) +- [Update Metadata for a Published Dataset at the PID provider](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-metadata-for-a-published-dataset-at-the-pid-provider) +- [Update Metadata for all Published Datasets at the PID provider](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-metadata-for-all-published-datasets-at-the-pid-provider) Upgrade instructions -------------------- -The solr schema has to be updated via the normal mechanism to add the new "relationType" field. +The Solr schema has to be updated via the normal mechanism to add the new "relationType" field. The citation metadatablock has to be reinstalled using the standard instructions. @@ -23,9 +23,9 @@ With these two changes, the "Relation Type" fields will be available and creatio To update existing datasets (and files using DataCite DOIs): -Exports can be updated by running curl http://localhost:8080/api/admin/metadata/reExportAll +Exports can be updated by running `curl http://localhost:8080/api/admin/metadata/reExportAll` -Entries at DataCite for published datasets can be updated by a superuser using an api call (newly documented) +Entries at DataCite for published datasets can be updated by a superuser using an API call (newly documented): `curl -X POST -H 'X-Dataverse-key:' http://localhost:8080/api/datasets/modifyRegistrationPIDMetadataAll` From 4ae0599fb212ee66883435362e7918d3f727ba43 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 17 Sep 2024 14:05:42 -0400 Subject: [PATCH 889/970] cleaner formatting --- ...dataset-create-new-all-default-fields.json | 2523 +++++++++-------- 1 file changed, 1323 insertions(+), 1200 deletions(-) diff --git a/scripts/api/data/dataset-create-new-all-default-fields.json b/scripts/api/data/dataset-create-new-all-default-fields.json index 84b152690f9..151c4732ad7 100644 --- a/scripts/api/data/dataset-create-new-all-default-fields.json +++ b/scripts/api/data/dataset-create-new-all-default-fields.json @@ -1,1410 +1,1533 @@ { - "datasetVersion" : { - "license" : { - "name" : "CC0 1.0", - "uri" : "http://creativecommons.org/publicdomain/zero/1.0" + "datasetVersion": { + "license": { + "name": "CC0 1.0", + "uri": "http://creativecommons.org/publicdomain/zero/1.0" }, - "metadataBlocks" : { - "citation" : { - "displayName" : "Citation Metadata", - "fields" : [{ - "typeName" : "title", - "multiple" : false, - "typeClass" : "primitive", - "value" : "Replication Data for: Title" - }, { - "typeName" : "subtitle", - "multiple" : false, - "typeClass" : "primitive", - "value" : "Subtitle" - }, { - "typeName" : "alternativeTitle", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ - "Alternative Title" - ] - }, { - "typeName" : "alternativeURL", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://AlternativeURL.org" - }, { - "typeName" : "otherId", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "otherIdAgency" : { - "typeName" : "otherIdAgency", - "multiple" : false, - "typeClass" : "primitive", - "value" : "OtherIDAgency1" - }, - "otherIdValue" : { - "typeName" : "otherIdValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "OtherIDIdentifier1" + "metadataBlocks": { + "citation": { + "displayName": "Citation Metadata", + "fields": [ + { + "typeName": "title", + "multiple": false, + "typeClass": "primitive", + "value": "Replication Data for: Title" + }, + { + "typeName": "subtitle", + "multiple": false, + "typeClass": "primitive", + "value": "Subtitle" + }, + { + "typeName": "alternativeTitle", + "multiple": true, + "typeClass": "primitive", + "value": ["Alternative Title"] + }, + { + "typeName": "alternativeURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://AlternativeURL.org" + }, + { + "typeName": "otherId", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "otherIdAgency": { + "typeName": "otherIdAgency", + "multiple": false, + "typeClass": "primitive", + "value": "OtherIDAgency1" + }, + "otherIdValue": { + "typeName": "otherIdValue", + "multiple": false, + "typeClass": "primitive", + "value": "OtherIDIdentifier1" } - }, { - "otherIdAgency" : { - "typeName" : "otherIdAgency", - "multiple" : false, - "typeClass" : "primitive", - "value" : "OtherIDAgency2" - }, - "otherIdValue" : { - "typeName" : "otherIdValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "OtherIDIdentifier2" + }, + { + "otherIdAgency": { + "typeName": "otherIdAgency", + "multiple": false, + "typeClass": "primitive", + "value": "OtherIDAgency2" + }, + "otherIdValue": { + "typeName": "otherIdValue", + "multiple": false, + "typeClass": "primitive", + "value": "OtherIDIdentifier2" } } ] - }, { - "typeName" : "author", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "authorName" : { - "typeName" : "authorName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastAuthor1, FirstAuthor1" - }, - "authorAffiliation" : { - "typeName" : "authorAffiliation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "AuthorAffiliation1" - }, - "authorIdentifierScheme" : { - "typeName" : "authorIdentifierScheme", - "multiple" : false, - "typeClass" : "controlledVocabulary", - "value" : "ORCID" - }, - "authorIdentifier" : { - "typeName" : "authorIdentifier", - "multiple" : false, - "typeClass" : "primitive", - "value" : "AuthorIdentifier1" + }, + { + "typeName": "author", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "authorName": { + "typeName": "authorName", + "multiple": false, + "typeClass": "primitive", + "value": "LastAuthor1, FirstAuthor1" + }, + "authorAffiliation": { + "typeName": "authorAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "AuthorAffiliation1" + }, + "authorIdentifierScheme": { + "typeName": "authorIdentifierScheme", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "ORCID" + }, + "authorIdentifier": { + "typeName": "authorIdentifier", + "multiple": false, + "typeClass": "primitive", + "value": "AuthorIdentifier1" } - }, { - "authorName" : { - "typeName" : "authorName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastAuthor2, FirstAuthor2" - }, - "authorAffiliation" : { - "typeName" : "authorAffiliation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "AuthorAffiliation2" - }, - "authorIdentifierScheme" : { - "typeName" : "authorIdentifierScheme", - "multiple" : false, - "typeClass" : "controlledVocabulary", - "value" : "ISNI" - }, - "authorIdentifier" : { - "typeName" : "authorIdentifier", - "multiple" : false, - "typeClass" : "primitive", - "value" : "AuthorIdentifier2" + }, + { + "authorName": { + "typeName": "authorName", + "multiple": false, + "typeClass": "primitive", + "value": "LastAuthor2, FirstAuthor2" + }, + "authorAffiliation": { + "typeName": "authorAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "AuthorAffiliation2" + }, + "authorIdentifierScheme": { + "typeName": "authorIdentifierScheme", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "ISNI" + }, + "authorIdentifier": { + "typeName": "authorIdentifier", + "multiple": false, + "typeClass": "primitive", + "value": "AuthorIdentifier2" } } ] - }, { - "typeName" : "datasetContact", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "datasetContactName" : { - "typeName" : "datasetContactName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastContact1, FirstContact1" - }, - "datasetContactAffiliation" : { - "typeName" : "datasetContactAffiliation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "ContactAffiliation1" - }, - "datasetContactEmail" : { - "typeName" : "datasetContactEmail", - "multiple" : false, - "typeClass" : "primitive", - "value" : "ContactEmail1@mailinator.com" + }, + { + "typeName": "datasetContact", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "datasetContactName": { + "typeName": "datasetContactName", + "multiple": false, + "typeClass": "primitive", + "value": "LastContact1, FirstContact1" + }, + "datasetContactAffiliation": { + "typeName": "datasetContactAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "ContactAffiliation1" + }, + "datasetContactEmail": { + "typeName": "datasetContactEmail", + "multiple": false, + "typeClass": "primitive", + "value": "ContactEmail1@mailinator.com" } - }, { - "datasetContactName" : { - "typeName" : "datasetContactName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastContact2, FirstContact2" - }, - "datasetContactAffiliation" : { - "typeName" : "datasetContactAffiliation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "ContactAffiliation2" - }, - "datasetContactEmail" : { - "typeName" : "datasetContactEmail", - "multiple" : false, - "typeClass" : "primitive", - "value" : "ContactEmail2@mailinator.com" + }, + { + "datasetContactName": { + "typeName": "datasetContactName", + "multiple": false, + "typeClass": "primitive", + "value": "LastContact2, FirstContact2" + }, + "datasetContactAffiliation": { + "typeName": "datasetContactAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "ContactAffiliation2" + }, + "datasetContactEmail": { + "typeName": "datasetContactEmail", + "multiple": false, + "typeClass": "primitive", + "value": "ContactEmail2@mailinator.com" } } ] - }, { - "typeName" : "dsDescription", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "dsDescriptionValue" : { - "typeName" : "dsDescriptionValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "DescriptionText1" - }, - "dsDescriptionDate" : { - "typeName" : "dsDescriptionDate", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1000-01-01" + }, + { + "typeName": "dsDescription", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "dsDescriptionValue": { + "typeName": "dsDescriptionValue", + "multiple": false, + "typeClass": "primitive", + "value": "DescriptionText1" + }, + "dsDescriptionDate": { + "typeName": "dsDescriptionDate", + "multiple": false, + "typeClass": "primitive", + "value": "1000-01-01" } - }, { - "dsDescriptionValue" : { - "typeName" : "dsDescriptionValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "DescriptionText2" - }, - "dsDescriptionDate" : { - "typeName" : "dsDescriptionDate", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1000-02-02" + }, + { + "dsDescriptionValue": { + "typeName": "dsDescriptionValue", + "multiple": false, + "typeClass": "primitive", + "value": "DescriptionText2" + }, + "dsDescriptionDate": { + "typeName": "dsDescriptionDate", + "multiple": false, + "typeClass": "primitive", + "value": "1000-02-02" } } ] - }, { - "typeName" : "subject", - "multiple" : true, - "typeClass" : "controlledVocabulary", - "value" : [ + }, + { + "typeName": "subject", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ "Agricultural Sciences", "Business and Management", "Engineering", "Law" ] - }, { - "typeName" : "keyword", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "keywordValue" : { - "typeName" : "keywordValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "KeywordTerm1" - }, - "keywordTermURI" : { - "typeName" : "keywordTermURI", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://keywordTermURI1.org" - }, - "keywordVocabulary" : { - "typeName" : "keywordVocabulary", - "multiple" : false, - "typeClass" : "primitive", - "value" : "KeywordVocabulary1" - }, - "keywordVocabularyURI" : { - "typeName" : "keywordVocabularyURI", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://KeywordVocabularyURL1.org" + }, + { + "typeName": "keyword", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "keywordValue": { + "typeName": "keywordValue", + "multiple": false, + "typeClass": "primitive", + "value": "KeywordTerm1" + }, + "keywordTermURI": { + "typeName": "keywordTermURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://keywordTermURI1.org" + }, + "keywordVocabulary": { + "typeName": "keywordVocabulary", + "multiple": false, + "typeClass": "primitive", + "value": "KeywordVocabulary1" + }, + "keywordVocabularyURI": { + "typeName": "keywordVocabularyURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://KeywordVocabularyURL1.org" } - }, { - "keywordValue" : { - "typeName" : "keywordValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "KeywordTerm2" - }, - "keywordTermURI" : { - "typeName" : "keywordTermURI", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://keywordTermURI2.org" - }, - "keywordVocabulary" : { - "typeName" : "keywordVocabulary", - "multiple" : false, - "typeClass" : "primitive", - "value" : "KeywordVocabulary2" - }, - "keywordVocabularyURI" : { - "typeName" : "keywordVocabularyURI", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://KeywordVocabularyURL2.org" + }, + { + "keywordValue": { + "typeName": "keywordValue", + "multiple": false, + "typeClass": "primitive", + "value": "KeywordTerm2" + }, + "keywordTermURI": { + "typeName": "keywordTermURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://keywordTermURI2.org" + }, + "keywordVocabulary": { + "typeName": "keywordVocabulary", + "multiple": false, + "typeClass": "primitive", + "value": "KeywordVocabulary2" + }, + "keywordVocabularyURI": { + "typeName": "keywordVocabularyURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://KeywordVocabularyURL2.org" } } ] - }, { - "typeName" : "topicClassification", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "topicClassValue" : { - "typeName" : "topicClassValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "Topic Classification Term1" - }, - "topicClassVocab" : { - "typeName" : "topicClassVocab", - "multiple" : false, - "typeClass" : "primitive", - "value" : "Topic Classification Vocab1" - }, - "topicClassVocabURI" : { - "typeName" : "topicClassVocabURI", - "multiple" : false, - "typeClass" : "primitive", - "value" : "https://TopicClassificationURL1.com" + }, + { + "typeName": "topicClassification", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "topicClassValue": { + "typeName": "topicClassValue", + "multiple": false, + "typeClass": "primitive", + "value": "Topic Classification Term1" + }, + "topicClassVocab": { + "typeName": "topicClassVocab", + "multiple": false, + "typeClass": "primitive", + "value": "Topic Classification Vocab1" + }, + "topicClassVocabURI": { + "typeName": "topicClassVocabURI", + "multiple": false, + "typeClass": "primitive", + "value": "https://TopicClassificationURL1.com" } - }, { - "topicClassValue" : { - "typeName" : "topicClassValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "Topic Classification Term2" - }, - "topicClassVocab" : { - "typeName" : "topicClassVocab", - "multiple" : false, - "typeClass" : "primitive", - "value" : "Topic Classification Vocab2" - }, - "topicClassVocabURI" : { - "typeName" : "topicClassVocabURI", - "multiple" : false, - "typeClass" : "primitive", - "value" : "https://TopicClassificationURL2.com" + }, + { + "topicClassValue": { + "typeName": "topicClassValue", + "multiple": false, + "typeClass": "primitive", + "value": "Topic Classification Term2" + }, + "topicClassVocab": { + "typeName": "topicClassVocab", + "multiple": false, + "typeClass": "primitive", + "value": "Topic Classification Vocab2" + }, + "topicClassVocabURI": { + "typeName": "topicClassVocabURI", + "multiple": false, + "typeClass": "primitive", + "value": "https://TopicClassificationURL2.com" } } ] - }, { - "typeName" : "publication", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ + }, + { + "typeName": "publication", + "multiple": true, + "typeClass": "compound", + "value": [ + { "publicationRelationType" : { "typeName" : "publicationRelationType", "multiple" : false, "typeClass" : "controlledVocabulary", "value" : "IsSupplementTo" }, - "publicationCitation" : { - "typeName" : "publicationCitation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "RelatedPublicationCitation1" - }, - "publicationIDType" : { - "typeName" : "publicationIDType", - "multiple" : false, - "typeClass" : "controlledVocabulary", - "value" : "ark" - }, - "publicationIDNumber" : { - "typeName" : "publicationIDNumber", - "multiple" : false, - "typeClass" : "primitive", - "value" : "RelatedPublicationIDNumber1" - }, - "publicationURL" : { - "typeName" : "publicationURL", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://RelatedPublicationURL1.org" + "publicationCitation": { + "typeName": "publicationCitation", + "multiple": false, + "typeClass": "primitive", + "value": "RelatedPublicationCitation1" + }, + "publicationIDType": { + "typeName": "publicationIDType", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "ark" + }, + "publicationIDNumber": { + "typeName": "publicationIDNumber", + "multiple": false, + "typeClass": "primitive", + "value": "RelatedPublicationIDNumber1" + }, + "publicationURL": { + "typeName": "publicationURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://RelatedPublicationURL1.org" } - }, { - "publicationCitation" : { - "typeName" : "publicationCitation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "RelatedPublicationCitation2" - }, - "publicationIDType" : { - "typeName" : "publicationIDType", - "multiple" : false, - "typeClass" : "controlledVocabulary", - "value" : "arXiv" - }, - "publicationIDNumber" : { - "typeName" : "publicationIDNumber", - "multiple" : false, - "typeClass" : "primitive", - "value" : "RelatedPublicationIDNumber2" - }, - "publicationURL" : { - "typeName" : "publicationURL", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://RelatedPublicationURL2.org" + }, + { + "publicationCitation": { + "typeName": "publicationCitation", + "multiple": false, + "typeClass": "primitive", + "value": "RelatedPublicationCitation2" + }, + "publicationIDType": { + "typeName": "publicationIDType", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "arXiv" + }, + "publicationIDNumber": { + "typeName": "publicationIDNumber", + "multiple": false, + "typeClass": "primitive", + "value": "RelatedPublicationIDNumber2" + }, + "publicationURL": { + "typeName": "publicationURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://RelatedPublicationURL2.org" } } ] - }, { - "typeName" : "notesText", - "multiple" : false, - "typeClass" : "primitive", - "value" : "Notes1" - }, { - "typeName" : "language", - "multiple" : true, - "typeClass" : "controlledVocabulary", - "value" : [ + }, + { + "typeName": "notesText", + "multiple": false, + "typeClass": "primitive", + "value": "Notes1" + }, + { + "typeName": "language", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ "Abkhaz", "Afar" ] - }, { - "typeName" : "producer", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "producerName" : { - "typeName" : "producerName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastProducer1, FirstProducer1" - }, - "producerAffiliation" : { - "typeName" : "producerAffiliation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "ProducerAffiliation1" - }, - "producerAbbreviation" : { - "typeName" : "producerAbbreviation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "ProducerAbbreviation1" - }, - "producerURL" : { - "typeName" : "producerURL", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://ProducerURL1.org" - }, - "producerLogoURL" : { - "typeName" : "producerLogoURL", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://ProducerLogoURL1.org" + }, + { + "typeName": "producer", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "producerName": { + "typeName": "producerName", + "multiple": false, + "typeClass": "primitive", + "value": "LastProducer1, FirstProducer1" + }, + "producerAffiliation": { + "typeName": "producerAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "ProducerAffiliation1" + }, + "producerAbbreviation": { + "typeName": "producerAbbreviation", + "multiple": false, + "typeClass": "primitive", + "value": "ProducerAbbreviation1" + }, + "producerURL": { + "typeName": "producerURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://ProducerURL1.org" + }, + "producerLogoURL": { + "typeName": "producerLogoURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://ProducerLogoURL1.org" } - }, { - "producerName" : { - "typeName" : "producerName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastProducer2, FirstProducer2" - }, - "producerAffiliation" : { - "typeName" : "producerAffiliation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "ProducerAffiliation2" - }, - "producerAbbreviation" : { - "typeName" : "producerAbbreviation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "ProducerAbbreviation2" - }, - "producerURL" : { - "typeName" : "producerURL", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://ProducerURL2.org" - }, - "producerLogoURL" : { - "typeName" : "producerLogoURL", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://ProducerLogoURL2.org" + }, + { + "producerName": { + "typeName": "producerName", + "multiple": false, + "typeClass": "primitive", + "value": "LastProducer2, FirstProducer2" + }, + "producerAffiliation": { + "typeName": "producerAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "ProducerAffiliation2" + }, + "producerAbbreviation": { + "typeName": "producerAbbreviation", + "multiple": false, + "typeClass": "primitive", + "value": "ProducerAbbreviation2" + }, + "producerURL": { + "typeName": "producerURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://ProducerURL2.org" + }, + "producerLogoURL": { + "typeName": "producerLogoURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://ProducerLogoURL2.org" } } ] - }, { - "typeName" : "productionDate", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1003-01-01" - }, { - "typeName" : "productionPlace", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ - "ProductionPlace" - ] - }, { - "typeName" : "contributor", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "contributorType" : { - "typeName" : "contributorType", - "multiple" : false, - "typeClass" : "controlledVocabulary", - "value" : "Data Collector" - }, - "contributorName" : { - "typeName" : "contributorName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastContributor1, FirstContributor1" + }, + { + "typeName": "productionDate", + "multiple": false, + "typeClass": "primitive", + "value": "1003-01-01" + }, + { + "typeName": "productionPlace", + "multiple": true, + "typeClass": "primitive", + "value": ["ProductionPlace"] + }, + { + "typeName": "contributor", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "contributorType": { + "typeName": "contributorType", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "Data Collector" + }, + "contributorName": { + "typeName": "contributorName", + "multiple": false, + "typeClass": "primitive", + "value": "LastContributor1, FirstContributor1" } - }, { - "contributorType" : { - "typeName" : "contributorType", - "multiple" : false, - "typeClass" : "controlledVocabulary", - "value" : "Data Curator" - }, - "contributorName" : { - "typeName" : "contributorName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastContributor2, FirstContributor2" + }, + { + "contributorType": { + "typeName": "contributorType", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "Data Curator" + }, + "contributorName": { + "typeName": "contributorName", + "multiple": false, + "typeClass": "primitive", + "value": "LastContributor2, FirstContributor2" } } ] - }, { - "typeName" : "grantNumber", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "grantNumberAgency" : { - "typeName" : "grantNumberAgency", - "multiple" : false, - "typeClass" : "primitive", - "value" : "GrantInformationGrantAgency1" - }, - "grantNumberValue" : { - "typeName" : "grantNumberValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "GrantInformationGrantNumber1" + }, + { + "typeName": "grantNumber", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "grantNumberAgency": { + "typeName": "grantNumberAgency", + "multiple": false, + "typeClass": "primitive", + "value": "GrantInformationGrantAgency1" + }, + "grantNumberValue": { + "typeName": "grantNumberValue", + "multiple": false, + "typeClass": "primitive", + "value": "GrantInformationGrantNumber1" } - }, { - "grantNumberAgency" : { - "typeName" : "grantNumberAgency", - "multiple" : false, - "typeClass" : "primitive", - "value" : "GrantInformationGrantAgency2" - }, - "grantNumberValue" : { - "typeName" : "grantNumberValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "GrantInformationGrantNumber2" + }, + { + "grantNumberAgency": { + "typeName": "grantNumberAgency", + "multiple": false, + "typeClass": "primitive", + "value": "GrantInformationGrantAgency2" + }, + "grantNumberValue": { + "typeName": "grantNumberValue", + "multiple": false, + "typeClass": "primitive", + "value": "GrantInformationGrantNumber2" } } ] - }, { - "typeName" : "distributor", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "distributorName" : { - "typeName" : "distributorName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastDistributor1, FirstDistributor1" - }, - "distributorAffiliation" : { - "typeName" : "distributorAffiliation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "DistributorAffiliation1" - }, - "distributorAbbreviation" : { - "typeName" : "distributorAbbreviation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "DistributorAbbreviation1" - }, - "distributorURL" : { - "typeName" : "distributorURL", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://DistributorURL1.org" - }, - "distributorLogoURL" : { - "typeName" : "distributorLogoURL", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://DistributorLogoURL1.org" + }, + { + "typeName": "distributor", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "distributorName": { + "typeName": "distributorName", + "multiple": false, + "typeClass": "primitive", + "value": "LastDistributor1, FirstDistributor1" + }, + "distributorAffiliation": { + "typeName": "distributorAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "DistributorAffiliation1" + }, + "distributorAbbreviation": { + "typeName": "distributorAbbreviation", + "multiple": false, + "typeClass": "primitive", + "value": "DistributorAbbreviation1" + }, + "distributorURL": { + "typeName": "distributorURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://DistributorURL1.org" + }, + "distributorLogoURL": { + "typeName": "distributorLogoURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://DistributorLogoURL1.org" } - }, { - "distributorName" : { - "typeName" : "distributorName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastDistributor2, FirstDistributor2" - }, - "distributorAffiliation" : { - "typeName" : "distributorAffiliation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "DistributorAffiliation2" - }, - "distributorAbbreviation" : { - "typeName" : "distributorAbbreviation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "DistributorAbbreviation2" - }, - "distributorURL" : { - "typeName" : "distributorURL", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://DistributorURL2.org" - }, - "distributorLogoURL" : { - "typeName" : "distributorLogoURL", - "multiple" : false, - "typeClass" : "primitive", - "value" : "http://DistributorLogoURL2.org" + }, + { + "distributorName": { + "typeName": "distributorName", + "multiple": false, + "typeClass": "primitive", + "value": "LastDistributor2, FirstDistributor2" + }, + "distributorAffiliation": { + "typeName": "distributorAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "DistributorAffiliation2" + }, + "distributorAbbreviation": { + "typeName": "distributorAbbreviation", + "multiple": false, + "typeClass": "primitive", + "value": "DistributorAbbreviation2" + }, + "distributorURL": { + "typeName": "distributorURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://DistributorURL2.org" + }, + "distributorLogoURL": { + "typeName": "distributorLogoURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://DistributorLogoURL2.org" } } ] - }, { - "typeName" : "distributionDate", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1004-01-01" - }, { - "typeName" : "depositor", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastDepositor, FirstDepositor" - }, { - "typeName" : "dateOfDeposit", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1002-01-01" - }, { - "typeName" : "timePeriodCovered", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "timePeriodCoveredStart" : { - "typeName" : "timePeriodCoveredStart", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1005-01-01" - }, - "timePeriodCoveredEnd" : { - "typeName" : "timePeriodCoveredEnd", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1005-01-02" + }, + { + "typeName": "distributionDate", + "multiple": false, + "typeClass": "primitive", + "value": "1004-01-01" + }, + { + "typeName": "depositor", + "multiple": false, + "typeClass": "primitive", + "value": "LastDepositor, FirstDepositor" + }, + { + "typeName": "dateOfDeposit", + "multiple": false, + "typeClass": "primitive", + "value": "1002-01-01" + }, + { + "typeName": "timePeriodCovered", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "timePeriodCoveredStart": { + "typeName": "timePeriodCoveredStart", + "multiple": false, + "typeClass": "primitive", + "value": "1005-01-01" + }, + "timePeriodCoveredEnd": { + "typeName": "timePeriodCoveredEnd", + "multiple": false, + "typeClass": "primitive", + "value": "1005-01-02" } - }, { - "timePeriodCoveredStart" : { - "typeName" : "timePeriodCoveredStart", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1005-02-01" - }, - "timePeriodCoveredEnd" : { - "typeName" : "timePeriodCoveredEnd", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1005-02-02" + }, + { + "timePeriodCoveredStart": { + "typeName": "timePeriodCoveredStart", + "multiple": false, + "typeClass": "primitive", + "value": "1005-02-01" + }, + "timePeriodCoveredEnd": { + "typeName": "timePeriodCoveredEnd", + "multiple": false, + "typeClass": "primitive", + "value": "1005-02-02" } } ] - }, { - "typeName" : "dateOfCollection", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "dateOfCollectionStart" : { - "typeName" : "dateOfCollectionStart", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1006-01-01" - }, - "dateOfCollectionEnd" : { - "typeName" : "dateOfCollectionEnd", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1006-01-01" + }, + { + "typeName": "dateOfCollection", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "dateOfCollectionStart": { + "typeName": "dateOfCollectionStart", + "multiple": false, + "typeClass": "primitive", + "value": "1006-01-01" + }, + "dateOfCollectionEnd": { + "typeName": "dateOfCollectionEnd", + "multiple": false, + "typeClass": "primitive", + "value": "1006-01-01" } - }, { - "dateOfCollectionStart" : { - "typeName" : "dateOfCollectionStart", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1006-02-01" - }, - "dateOfCollectionEnd" : { - "typeName" : "dateOfCollectionEnd", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1006-02-02" + }, + { + "dateOfCollectionStart": { + "typeName": "dateOfCollectionStart", + "multiple": false, + "typeClass": "primitive", + "value": "1006-02-01" + }, + "dateOfCollectionEnd": { + "typeName": "dateOfCollectionEnd", + "multiple": false, + "typeClass": "primitive", + "value": "1006-02-02" } } ] - }, { - "typeName" : "kindOfData", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ + }, + { + "typeName": "kindOfData", + "multiple": true, + "typeClass": "primitive", + "value": [ "KindOfData1", "KindOfData2" ] - }, { - "typeName" : "series", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "seriesName" : { - "typeName" : "seriesName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "SeriesName" - }, - "seriesInformation" : { - "typeName" : "seriesInformation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "SeriesInformation" - } + }, + { + "typeName": "series", + "multiple": true, + "typeClass": "compound", + "value": [{ + "seriesName": { + "typeName": "seriesName", + "multiple": false, + "typeClass": "primitive", + "value": "SeriesName" + }, + "seriesInformation": { + "typeName": "seriesInformation", + "multiple": false, + "typeClass": "primitive", + "value": "SeriesInformation" } - ] - }, { - "typeName" : "software", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "softwareName" : { - "typeName" : "softwareName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "SoftwareName1" - }, - "softwareVersion" : { - "typeName" : "softwareVersion", - "multiple" : false, - "typeClass" : "primitive", - "value" : "SoftwareVersion1" + }] + }, + { + "typeName": "software", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "softwareName": { + "typeName": "softwareName", + "multiple": false, + "typeClass": "primitive", + "value": "SoftwareName1" + }, + "softwareVersion": { + "typeName": "softwareVersion", + "multiple": false, + "typeClass": "primitive", + "value": "SoftwareVersion1" } - }, { - "softwareName" : { - "typeName" : "softwareName", - "multiple" : false, - "typeClass" : "primitive", - "value" : "SoftwareName2" - }, - "softwareVersion" : { - "typeName" : "softwareVersion", - "multiple" : false, - "typeClass" : "primitive", - "value" : "SoftwareVersion2" + }, + { + "softwareName": { + "typeName": "softwareName", + "multiple": false, + "typeClass": "primitive", + "value": "SoftwareName2" + }, + "softwareVersion": { + "typeName": "softwareVersion", + "multiple": false, + "typeClass": "primitive", + "value": "SoftwareVersion2" } } ] - }, { - "typeName" : "relatedMaterial", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ + }, + { + "typeName": "relatedMaterial", + "multiple": true, + "typeClass": "primitive", + "value": [ "RelatedMaterial1", "RelatedMaterial2" ] - }, { - "typeName" : "relatedDatasets", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ + }, + { + "typeName": "relatedDatasets", + "multiple": true, + "typeClass": "primitive", + "value": [ "RelatedDatasets1", "RelatedDatasets2" ] - }, { - "typeName" : "otherReferences", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ + }, + { + "typeName": "otherReferences", + "multiple": true, + "typeClass": "primitive", + "value": [ "OtherReferences1", "OtherReferences2" ] - }, { - "typeName" : "dataSources", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ + }, + { + "typeName": "dataSources", + "multiple": true, + "typeClass": "primitive", + "value": [ "DataSources1", "DataSources2" ] - }, { - "typeName" : "originOfSources", - "multiple" : false, - "typeClass" : "primitive", - "value" : "OriginOfSources" - }, { - "typeName" : "characteristicOfSources", - "multiple" : false, - "typeClass" : "primitive", - "value" : "CharacteristicOfSourcesNoted" - }, { - "typeName" : "accessToSources", - "multiple" : false, - "typeClass" : "primitive", - "value" : "DocumentationAndAccessToSources" + }, + { + "typeName": "originOfSources", + "multiple": false, + "typeClass": "primitive", + "value": "OriginOfSources" + }, + { + "typeName": "characteristicOfSources", + "multiple": false, + "typeClass": "primitive", + "value": "CharacteristicOfSourcesNoted" + }, + { + "typeName": "accessToSources", + "multiple": false, + "typeClass": "primitive", + "value": "DocumentationAndAccessToSources" } ] }, - "geospatial" : { - "displayName" : "Geospatial Metadata", - "fields" : [{ - "typeName" : "geographicCoverage", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "country" : { - "typeName" : "country", - "multiple" : false, - "typeClass" : "controlledVocabulary", - "value" : "Afghanistan" - }, - "state" : { - "typeName" : "state", - "multiple" : false, - "typeClass" : "primitive", - "value" : "GeographicCoverageStateProvince1" - }, - "city" : { - "typeName" : "city", - "multiple" : false, - "typeClass" : "primitive", - "value" : "GeographicCoverageCity1" - }, - "otherGeographicCoverage" : { - "typeName" : "otherGeographicCoverage", - "multiple" : false, - "typeClass" : "primitive", - "value" : "GeographicCoverageOther1" + "geospatial": { + "displayName": "Geospatial Metadata", + "fields": [ + { + "typeName": "geographicCoverage", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "country": { + "typeName": "country", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "Afghanistan" + }, + "state": { + "typeName": "state", + "multiple": false, + "typeClass": "primitive", + "value": "GeographicCoverageStateProvince1" + }, + "city": { + "typeName": "city", + "multiple": false, + "typeClass": "primitive", + "value": "GeographicCoverageCity1" + }, + "otherGeographicCoverage": { + "typeName": "otherGeographicCoverage", + "multiple": false, + "typeClass": "primitive", + "value": "GeographicCoverageOther1" } - }, { - "country" : { - "typeName" : "country", - "multiple" : false, - "typeClass" : "controlledVocabulary", - "value" : "Albania" - }, - "state" : { - "typeName" : "state", - "multiple" : false, - "typeClass" : "primitive", - "value" : "GeographicCoverageStateProvince2" - }, - "city" : { - "typeName" : "city", - "multiple" : false, - "typeClass" : "primitive", - "value" : "GeographicCoverageCity2" - }, - "otherGeographicCoverage" : { - "typeName" : "otherGeographicCoverage", - "multiple" : false, - "typeClass" : "primitive", - "value" : "GeographicCoverageOther2" + }, + { + "country": { + "typeName": "country", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "Albania" + }, + "state": { + "typeName": "state", + "multiple": false, + "typeClass": "primitive", + "value": "GeographicCoverageStateProvince2" + }, + "city": { + "typeName": "city", + "multiple": false, + "typeClass": "primitive", + "value": "GeographicCoverageCity2" + }, + "otherGeographicCoverage": { + "typeName": "otherGeographicCoverage", + "multiple": false, + "typeClass": "primitive", + "value": "GeographicCoverageOther2" } } ] - }, { - "typeName" : "geographicUnit", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ + }, + { + "typeName": "geographicUnit", + "multiple": true, + "typeClass": "primitive", + "value": [ "GeographicUnit1", "GeographicUnit2" ] - }, { - "typeName" : "geographicBoundingBox", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "westLongitude" : { - "typeName" : "westLongitude", - "multiple" : false, - "typeClass" : "primitive", - "value" : "-72" - }, - "eastLongitude" : { - "typeName" : "eastLongitude", - "multiple" : false, - "typeClass" : "primitive", - "value" : "-70" - }, - "northLatitude" : { - "typeName" : "northLatitude", - "multiple" : false, - "typeClass" : "primitive", - "value" : "43" - }, - "southLatitude" : { - "typeName" : "southLatitude", - "multiple" : false, - "typeClass" : "primitive", - "value" : "42" + }, + { + "typeName": "geographicBoundingBox", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "westLongitude": { + "typeName": "westLongitude", + "multiple": false, + "typeClass": "primitive", + "value": "-72" + }, + "eastLongitude": { + "typeName": "eastLongitude", + "multiple": false, + "typeClass": "primitive", + "value": "-70" + }, + "northLatitude": { + "typeName": "northLatitude", + "multiple": false, + "typeClass": "primitive", + "value": "43" + }, + "southLatitude": { + "typeName": "southLatitude", + "multiple": false, + "typeClass": "primitive", + "value": "42" } - }, { - "westLongitude" : { - "typeName" : "westLongitude", - "multiple" : false, - "typeClass" : "primitive", - "value" : "-18" - }, - "eastLongitude" : { - "typeName" : "eastLongitude", - "multiple" : false, - "typeClass" : "primitive", - "value" : "-13" - }, - "northLatitude" : { - "typeName" : "northLatitude", - "multiple" : false, - "typeClass" : "primitive", - "value" : "29" - }, - "southLatitude" : { - "typeName" : "southLatitude", - "multiple" : false, - "typeClass" : "primitive", - "value" : "28" + }, + { + "westLongitude": { + "typeName": "westLongitude", + "multiple": false, + "typeClass": "primitive", + "value": "-18" + }, + "eastLongitude": { + "typeName": "eastLongitude", + "multiple": false, + "typeClass": "primitive", + "value": "-13" + }, + "northLatitude": { + "typeName": "northLatitude", + "multiple": false, + "typeClass": "primitive", + "value": "29" + }, + "southLatitude": { + "typeName": "southLatitude", + "multiple": false, + "typeClass": "primitive", + "value": "28" } } ] } ] }, - "socialscience" : { - "displayName" : "Social Science and Humanities Metadata", - "fields" : [{ - "typeName" : "unitOfAnalysis", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ + "socialscience": { + "displayName": "Social Science and Humanities Metadata", + "fields": [ + { + "typeName": "unitOfAnalysis", + "multiple": true, + "typeClass": "primitive", + "value": [ "UnitOfAnalysis1", "UnitOfAnalysis2" ] - }, { - "typeName" : "universe", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ + }, + { + "typeName": "universe", + "multiple": true, + "typeClass": "primitive", + "value": [ "Universe1", "Universe2" ] - }, { - "typeName" : "timeMethod", - "multiple" : false, - "typeClass" : "primitive", - "value" : "TimeMethod" - }, { - "typeName" : "dataCollector", - "multiple" : false, - "typeClass" : "primitive", - "value" : "LastDataCollector1, FirstDataCollector1" - }, { - "typeName" : "collectorTraining", - "multiple" : false, - "typeClass" : "primitive", - "value" : "CollectorTraining" - }, { - "typeName" : "frequencyOfDataCollection", - "multiple" : false, - "typeClass" : "primitive", - "value" : "Frequency" - }, { - "typeName" : "samplingProcedure", - "multiple" : false, - "typeClass" : "primitive", - "value" : "SamplingProcedure" - }, { - "typeName" : "targetSampleSize", - "multiple" : false, - "typeClass" : "compound", - "value" : { - "targetSampleActualSize" : { - "typeName" : "targetSampleActualSize", - "multiple" : false, - "typeClass" : "primitive", - "value" : "100" + }, + { + "typeName": "timeMethod", + "multiple": false, + "typeClass": "primitive", + "value": "TimeMethod" + }, + { + "typeName": "dataCollector", + "multiple": false, + "typeClass": "primitive", + "value": "LastDataCollector1, FirstDataCollector1" + }, + { + "typeName": "collectorTraining", + "multiple": false, + "typeClass": "primitive", + "value": "CollectorTraining" + }, + { + "typeName": "frequencyOfDataCollection", + "multiple": false, + "typeClass": "primitive", + "value": "Frequency" + }, + { + "typeName": "samplingProcedure", + "multiple": false, + "typeClass": "primitive", + "value": "SamplingProcedure" + }, + { + "typeName": "targetSampleSize", + "multiple": false, + "typeClass": "compound", + "value": { + "targetSampleActualSize": { + "typeName": "targetSampleActualSize", + "multiple": false, + "typeClass": "primitive", + "value": "100" }, - "targetSampleSizeFormula" : { - "typeName" : "targetSampleSizeFormula", - "multiple" : false, - "typeClass" : "primitive", - "value" : "TargetSampleSizeFormula" + "targetSampleSizeFormula": { + "typeName": "targetSampleSizeFormula", + "multiple": false, + "typeClass": "primitive", + "value": "TargetSampleSizeFormula" } } - }, { - "typeName" : "deviationsFromSampleDesign", - "multiple" : false, - "typeClass" : "primitive", - "value" : "MajorDeviationsForSampleDesign" - }, { - "typeName" : "collectionMode", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ - "CollectionMode" - ] - }, { - "typeName" : "researchInstrument", - "multiple" : false, - "typeClass" : "primitive", - "value" : "TypeOfResearchInstrument" - }, { - "typeName" : "dataCollectionSituation", - "multiple" : false, - "typeClass" : "primitive", - "value" : "CharacteristicsOfDataCollectionSituation" - }, { - "typeName" : "actionsToMinimizeLoss", - "multiple" : false, - "typeClass" : "primitive", - "value" : "ActionsToMinimizeLosses" - }, { - "typeName" : "controlOperations", - "multiple" : false, - "typeClass" : "primitive", - "value" : "ControlOperations" - }, { - "typeName" : "weighting", - "multiple" : false, - "typeClass" : "primitive", - "value" : "Weighting" - }, { - "typeName" : "cleaningOperations", - "multiple" : false, - "typeClass" : "primitive", - "value" : "CleaningOperations" - }, { - "typeName" : "datasetLevelErrorNotes", - "multiple" : false, - "typeClass" : "primitive", - "value" : "StudyLevelErrorNotes" - }, { - "typeName" : "responseRate", - "multiple" : false, - "typeClass" : "primitive", - "value" : "ResponseRate" - }, { - "typeName" : "samplingErrorEstimates", - "multiple" : false, - "typeClass" : "primitive", - "value" : "EstimatesOfSamplingError" - }, { - "typeName" : "otherDataAppraisal", - "multiple" : false, - "typeClass" : "primitive", - "value" : "OtherFormsOfDataAppraisal" - }, { - "typeName" : "socialScienceNotes", - "multiple" : false, - "typeClass" : "compound", - "value" : { - "socialScienceNotesType" : { - "typeName" : "socialScienceNotesType", - "multiple" : false, - "typeClass" : "primitive", - "value" : "NotesType" + }, + { + "typeName": "deviationsFromSampleDesign", + "multiple": false, + "typeClass": "primitive", + "value": "MajorDeviationsForSampleDesign" + }, + { + "typeName": "collectionMode", + "multiple": true, + "typeClass": "primitive", + "value": ["CollectionMode"] + }, + { + "typeName": "researchInstrument", + "multiple": false, + "typeClass": "primitive", + "value": "TypeOfResearchInstrument" + }, + { + "typeName": "dataCollectionSituation", + "multiple": false, + "typeClass": "primitive", + "value": "CharacteristicsOfDataCollectionSituation" + }, + { + "typeName": "actionsToMinimizeLoss", + "multiple": false, + "typeClass": "primitive", + "value": "ActionsToMinimizeLosses" + }, + { + "typeName": "controlOperations", + "multiple": false, + "typeClass": "primitive", + "value": "ControlOperations" + }, + { + "typeName": "weighting", + "multiple": false, + "typeClass": "primitive", + "value": "Weighting" + }, + { + "typeName": "cleaningOperations", + "multiple": false, + "typeClass": "primitive", + "value": "CleaningOperations" + }, + { + "typeName": "datasetLevelErrorNotes", + "multiple": false, + "typeClass": "primitive", + "value": "StudyLevelErrorNotes" + }, + { + "typeName": "responseRate", + "multiple": false, + "typeClass": "primitive", + "value": "ResponseRate" + }, + { + "typeName": "samplingErrorEstimates", + "multiple": false, + "typeClass": "primitive", + "value": "EstimatesOfSamplingError" + }, + { + "typeName": "otherDataAppraisal", + "multiple": false, + "typeClass": "primitive", + "value": "OtherFormsOfDataAppraisal" + }, + { + "typeName": "socialScienceNotes", + "multiple": false, + "typeClass": "compound", + "value": { + "socialScienceNotesType": { + "typeName": "socialScienceNotesType", + "multiple": false, + "typeClass": "primitive", + "value": "NotesType" }, - "socialScienceNotesSubject" : { - "typeName" : "socialScienceNotesSubject", - "multiple" : false, - "typeClass" : "primitive", - "value" : "NotesSubject" + "socialScienceNotesSubject": { + "typeName": "socialScienceNotesSubject", + "multiple": false, + "typeClass": "primitive", + "value": "NotesSubject" }, - "socialScienceNotesText" : { - "typeName" : "socialScienceNotesText", - "multiple" : false, - "typeClass" : "primitive", - "value" : "NotesText" + "socialScienceNotesText": { + "typeName": "socialScienceNotesText", + "multiple": false, + "typeClass": "primitive", + "value": "NotesText" } } } ] }, - "astrophysics" : { - "displayName" : "Astronomy and Astrophysics Metadata", - "fields" : [{ - "typeName" : "astroType", - "multiple" : true, - "typeClass" : "controlledVocabulary", - "value" : [ + "astrophysics": { + "displayName": "Astronomy and Astrophysics Metadata", + "fields": [ + { + "typeName": "astroType", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ "Image", "Mosaic", "EventList", "Cube" ] - }, { - "typeName" : "astroFacility", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ + }, + { + "typeName": "astroFacility", + "multiple": true, + "typeClass": "primitive", + "value": [ "Facility1", "Facility2" ] - }, { - "typeName" : "astroInstrument", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ + }, + { + "typeName": "astroInstrument", + "multiple": true, + "typeClass": "primitive", + "value": [ "Instrument1", "Instrument2" ] - }, { - "typeName" : "astroObject", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ + }, + { + "typeName": "astroObject", + "multiple": true, + "typeClass": "primitive", + "value": [ "Object1", "Object2" ] - }, { - "typeName" : "resolution.Spatial", - "multiple" : false, - "typeClass" : "primitive", - "value" : "SpatialResolution" - }, { - "typeName" : "resolution.Spectral", - "multiple" : false, - "typeClass" : "primitive", - "value" : "SpectralResolution" - }, { - "typeName" : "resolution.Temporal", - "multiple" : false, - "typeClass" : "primitive", - "value" : "TimeResolution" - }, { - "typeName" : "coverage.Spectral.Bandpass", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ + }, + { + "typeName": "resolution.Spatial", + "multiple": false, + "typeClass": "primitive", + "value": "SpatialResolution" + }, + { + "typeName": "resolution.Spectral", + "multiple": false, + "typeClass": "primitive", + "value": "SpectralResolution" + }, + { + "typeName": "resolution.Temporal", + "multiple": false, + "typeClass": "primitive", + "value": "TimeResolution" + }, + { + "typeName": "coverage.Spectral.Bandpass", + "multiple": true, + "typeClass": "primitive", + "value": [ "Bandpass1", "Bandpass2" ] - }, { - "typeName" : "coverage.Spectral.CentralWavelength", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ + }, + { + "typeName": "coverage.Spectral.CentralWavelength", + "multiple": true, + "typeClass": "primitive", + "value": [ "3001", "3002" ] - }, { - "typeName" : "coverage.Spectral.Wavelength", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "coverage.Spectral.MinimumWavelength" : { - "typeName" : "coverage.Spectral.MinimumWavelength", - "multiple" : false, - "typeClass" : "primitive", - "value" : "4001" - }, - "coverage.Spectral.MaximumWavelength" : { - "typeName" : "coverage.Spectral.MaximumWavelength", - "multiple" : false, - "typeClass" : "primitive", - "value" : "4002" + }, + { + "typeName": "coverage.Spectral.Wavelength", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "coverage.Spectral.MinimumWavelength": { + "typeName": "coverage.Spectral.MinimumWavelength", + "multiple": false, + "typeClass": "primitive", + "value": "4001" + }, + "coverage.Spectral.MaximumWavelength": { + "typeName": "coverage.Spectral.MaximumWavelength", + "multiple": false, + "typeClass": "primitive", + "value": "4002" } - }, { - "coverage.Spectral.MinimumWavelength" : { - "typeName" : "coverage.Spectral.MinimumWavelength", - "multiple" : false, - "typeClass" : "primitive", - "value" : "4003" - }, - "coverage.Spectral.MaximumWavelength" : { - "typeName" : "coverage.Spectral.MaximumWavelength", - "multiple" : false, - "typeClass" : "primitive", - "value" : "4004" + }, + { + "coverage.Spectral.MinimumWavelength": { + "typeName": "coverage.Spectral.MinimumWavelength", + "multiple": false, + "typeClass": "primitive", + "value": "4003" + }, + "coverage.Spectral.MaximumWavelength": { + "typeName": "coverage.Spectral.MaximumWavelength", + "multiple": false, + "typeClass": "primitive", + "value": "4004" } } ] - }, { - "typeName" : "coverage.Temporal", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "coverage.Temporal.StartTime" : { - "typeName" : "coverage.Temporal.StartTime", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1007-01-01" - }, - "coverage.Temporal.StopTime" : { - "typeName" : "coverage.Temporal.StopTime", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1007-01-02" + }, + { + "typeName": "coverage.Temporal", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "coverage.Temporal.StartTime": { + "typeName": "coverage.Temporal.StartTime", + "multiple": false, + "typeClass": "primitive", + "value": "1007-01-01" + }, + "coverage.Temporal.StopTime": { + "typeName": "coverage.Temporal.StopTime", + "multiple": false, + "typeClass": "primitive", + "value": "1007-01-02" } - }, { - "coverage.Temporal.StartTime" : { - "typeName" : "coverage.Temporal.StartTime", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1007-02-01" - }, - "coverage.Temporal.StopTime" : { - "typeName" : "coverage.Temporal.StopTime", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1007-02-02" + }, + { + "coverage.Temporal.StartTime": { + "typeName": "coverage.Temporal.StartTime", + "multiple": false, + "typeClass": "primitive", + "value": "1007-02-01" + }, + "coverage.Temporal.StopTime": { + "typeName": "coverage.Temporal.StopTime", + "multiple": false, + "typeClass": "primitive", + "value": "1007-02-02" } } ] - }, { - "typeName" : "coverage.Spatial", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ + }, + { + "typeName": "coverage.Spatial", + "multiple": true, + "typeClass": "primitive", + "value": [ "SkyCoverage1", "SkyCoverage2" ] - }, { - "typeName" : "coverage.Depth", - "multiple" : false, - "typeClass" : "primitive", - "value" : "200" - }, { - "typeName" : "coverage.ObjectDensity", - "multiple" : false, - "typeClass" : "primitive", - "value" : "300" - }, { - "typeName" : "coverage.ObjectCount", - "multiple" : false, - "typeClass" : "primitive", - "value" : "400" - }, { - "typeName" : "coverage.SkyFraction", - "multiple" : false, - "typeClass" : "primitive", - "value" : "500" - }, { - "typeName" : "coverage.Polarization", - "multiple" : false, - "typeClass" : "primitive", - "value" : "Polarization" - }, { - "typeName" : "redshiftType", - "multiple" : false, - "typeClass" : "primitive", - "value" : "RedshiftType" - }, { - "typeName" : "resolution.Redshift", - "multiple" : false, - "typeClass" : "primitive", - "value" : "600" - }, { - "typeName" : "coverage.RedshiftValue", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "coverage.Redshift.MinimumValue" : { - "typeName" : "coverage.Redshift.MinimumValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "701" - }, - "coverage.Redshift.MaximumValue" : { - "typeName" : "coverage.Redshift.MaximumValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "702" + }, + { + "typeName": "coverage.Depth", + "multiple": false, + "typeClass": "primitive", + "value": "200" + }, + { + "typeName": "coverage.ObjectDensity", + "multiple": false, + "typeClass": "primitive", + "value": "300" + }, + { + "typeName": "coverage.ObjectCount", + "multiple": false, + "typeClass": "primitive", + "value": "400" + }, + { + "typeName": "coverage.SkyFraction", + "multiple": false, + "typeClass": "primitive", + "value": "500" + }, + { + "typeName": "coverage.Polarization", + "multiple": false, + "typeClass": "primitive", + "value": "Polarization" + }, + { + "typeName": "redshiftType", + "multiple": false, + "typeClass": "primitive", + "value": "RedshiftType" + }, + { + "typeName": "resolution.Redshift", + "multiple": false, + "typeClass": "primitive", + "value": "600" + }, + { + "typeName": "coverage.RedshiftValue", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "coverage.Redshift.MinimumValue": { + "typeName": "coverage.Redshift.MinimumValue", + "multiple": false, + "typeClass": "primitive", + "value": "701" + }, + "coverage.Redshift.MaximumValue": { + "typeName": "coverage.Redshift.MaximumValue", + "multiple": false, + "typeClass": "primitive", + "value": "702" } - }, { - "coverage.Redshift.MinimumValue" : { - "typeName" : "coverage.Redshift.MinimumValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "703" - }, - "coverage.Redshift.MaximumValue" : { - "typeName" : "coverage.Redshift.MaximumValue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "704" + }, + { + "coverage.Redshift.MinimumValue": { + "typeName": "coverage.Redshift.MinimumValue", + "multiple": false, + "typeClass": "primitive", + "value": "703" + }, + "coverage.Redshift.MaximumValue": { + "typeName": "coverage.Redshift.MaximumValue", + "multiple": false, + "typeClass": "primitive", + "value": "704" } } ] } ] }, - "biomedical" : { - "displayName" : "Life Sciences Metadata", - "fields" : [{ - "typeName" : "studyDesignType", - "multiple" : true, - "typeClass" : "controlledVocabulary", - "value" : [ + "biomedical": { + "displayName": "Life Sciences Metadata", + "fields": [ + { + "typeName": "studyDesignType", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ "Case Control", "Cross Sectional", "Cohort Study", "Not Specified" ] - }, { - "typeName" : "studyFactorType", - "multiple" : true, - "typeClass" : "controlledVocabulary", - "value" : [ + }, + { + "typeName": "studyFactorType", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ "Age", "Biomarkers", "Cell Surface Markers", "Developmental Stage" ] - }, { - "typeName" : "studyAssayOrganism", - "multiple" : true, - "typeClass" : "controlledVocabulary", - "value" : [ + }, + { + "typeName": "studyAssayOrganism", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ "Arabidopsis thaliana", "Bos taurus", "Caenorhabditis elegans", "Danio rerio (zebrafish)" ] - }, { - "typeName" : "studyAssayOtherOrganism", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ + }, + { + "typeName": "studyAssayOtherOrganism", + "multiple": true, + "typeClass": "primitive", + "value": [ "OtherOrganism1", "OtherOrganism2" ] - }, { - "typeName" : "studyAssayMeasurementType", - "multiple" : true, - "typeClass" : "controlledVocabulary", - "value" : [ + }, + { + "typeName": "studyAssayMeasurementType", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ "genome sequencing", "cell sorting", "clinical chemistry analysis", "DNA methylation profiling" ] - }, { - "typeName" : "studyAssayOtherMeasurmentType", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ + }, + { + "typeName": "studyAssayOtherMeasurmentType", + "multiple": true, + "typeClass": "primitive", + "value": [ "OtherMeasurementType1", "OtherMeasurementType2" ] - }, { - "typeName" : "studyAssayTechnologyType", - "multiple" : true, - "typeClass" : "controlledVocabulary", - "value" : [ + }, + { + "typeName": "studyAssayTechnologyType", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ "culture based drug susceptibility testing, single concentration", "culture based drug susceptibility testing, two concentrations", "culture based drug susceptibility testing, three or more concentrations (minimium inhibitory concentration measurement)", "flow cytometry" ] - }, { - "typeName" : "studyAssayPlatform", - "multiple" : true, - "typeClass" : "controlledVocabulary", - "value" : [ + }, + { + "typeName": "studyAssayPlatform", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ "210-MS GC Ion Trap (Varian)", "220-MS GC Ion Trap (Varian)", "225-MS GC Ion Trap (Varian)", "300-MS quadrupole GC/MS (Varian)" ] - }, { - "typeName" : "studyAssayCellType", - "multiple" : true, - "typeClass" : "primitive", - "value" : [ + }, + { + "typeName": "studyAssayCellType", + "multiple": true, + "typeClass": "primitive", + "value": [ "CellType1", "CellType2" ] } ] }, - "journal" : { - "displayName" : "Journal Metadata", - "fields" : [{ - "typeName" : "journalVolumeIssue", - "multiple" : true, - "typeClass" : "compound", - "value" : [{ - "journalVolume" : { - "typeName" : "journalVolume", - "multiple" : false, - "typeClass" : "primitive", - "value" : "JournalVolume1" - }, - "journalIssue" : { - "typeName" : "journalIssue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "JournalIssue1" - }, - "journalPubDate" : { - "typeName" : "journalPubDate", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1008-01-01" + "journal": { + "displayName": "Journal Metadata", + "fields": [ + { + "typeName": "journalVolumeIssue", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "journalVolume": { + "typeName": "journalVolume", + "multiple": false, + "typeClass": "primitive", + "value": "JournalVolume1" + }, + "journalIssue": { + "typeName": "journalIssue", + "multiple": false, + "typeClass": "primitive", + "value": "JournalIssue1" + }, + "journalPubDate": { + "typeName": "journalPubDate", + "multiple": false, + "typeClass": "primitive", + "value": "1008-01-01" } - }, { - "journalVolume" : { - "typeName" : "journalVolume", - "multiple" : false, - "typeClass" : "primitive", - "value" : "JournalVolume2" - }, - "journalIssue" : { - "typeName" : "journalIssue", - "multiple" : false, - "typeClass" : "primitive", - "value" : "JournalIssue2" - }, - "journalPubDate" : { - "typeName" : "journalPubDate", - "multiple" : false, - "typeClass" : "primitive", - "value" : "1008-02-01" + }, + { + "journalVolume": { + "typeName": "journalVolume", + "multiple": false, + "typeClass": "primitive", + "value": "JournalVolume2" + }, + "journalIssue": { + "typeName": "journalIssue", + "multiple": false, + "typeClass": "primitive", + "value": "JournalIssue2" + }, + "journalPubDate": { + "typeName": "journalPubDate", + "multiple": false, + "typeClass": "primitive", + "value": "1008-02-01" } } ] - }, { - "typeName" : "journalArticleType", - "multiple" : false, - "typeClass" : "controlledVocabulary", - "value" : "abstract" + }, + { + "typeName": "journalArticleType", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "abstract" } ] } } } -} \ No newline at end of file +} From 8141a78c4c446bd0a738ef4616b27f1184959db4 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 17 Sep 2024 14:36:59 -0400 Subject: [PATCH 890/970] minor doc tweak #10632 --- doc/sphinx-guides/source/installation/config.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 7e42c79fa6c..cd56a41c56d 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -232,8 +232,8 @@ Dataverse can be configured with one or more PID providers, each of which can mi to manage an authority/shoulder combination, aka a "prefix" (PermaLinks also support custom separator characters as part of the prefix), along with an optional list of individual PIDs (with different authority/shoulders) than can be managed with that account. -Dataverse automatically manages assigning PIDs and making them findable when datasets are published. There are also :ref:`API calls that -allow updating the PID target URLs and metadata of already-published datasets manually if needed ` , e.g. if a Dataverse instance is +Dataverse automatically manages assigning PIDs and making them findable when datasets are published. There are also :ref:`API calls that +allow updating the PID target URLs and metadata of already-published datasets manually if needed `, e.g. if a Dataverse instance is moved to a new URL or when the software is updated to generate additional metadata or address schema changes at the PID service. Testing PID Providers @@ -250,11 +250,11 @@ configure the credentials as described below. Alternately, you may wish to configure other providers for testing: - - EZID is available to University of California scholars and researchers. Testing can be done using the authority 10.5072 and shoulder FK2 with the "apitest" account (contact EZID for credentials) or an institutional account. Configuration in Dataverse is then analogous to using DataCite. +- EZID is available to University of California scholars and researchers. Testing can be done using the authority 10.5072 and shoulder FK2 with the "apitest" account (contact EZID for credentials) or an institutional account. Configuration in Dataverse is then analogous to using DataCite. - - The PermaLink provider, like the FAKE DOI provider, does not involve an external account. - Unlike the Fake DOI provider, the PermaLink provider creates PIDs that begin with "perma:", making it clearer that they are not DOIs, - and that do resolve to the local dataset/file page in Dataverse, making them useful for some production use cases. See :ref:`permalinks` and (for the FAKE DOI provider) the :doc:`/developers/dev-environment` section of the Developer Guide. +- The PermaLink provider, like the FAKE DOI provider, does not involve an external account. + Unlike the Fake DOI provider, the PermaLink provider creates PIDs that begin with "perma:", making it clearer that they are not DOIs, + and that do resolve to the local dataset/file page in Dataverse, making them useful for some production use cases. See :ref:`permalinks` and (for the FAKE DOI provider) the :doc:`/developers/dev-environment` section of the Developer Guide. Provider-specific configuration is described below. From 695709c200c3fa01f5c291486f8d8ad296ca52b0 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Tue, 17 Sep 2024 15:51:52 -0400 Subject: [PATCH 891/970] standardize image url --- .../iq/dataverse/DataverseServiceBean.java | 10 ++++- .../iq/dataverse/ThumbnailServiceWrapper.java | 20 ++++++++- .../dataverse/search/SearchServiceBean.java | 4 +- .../harvard/iq/dataverse/api/SearchIT.java | 43 +++++++++++++++++++ 4 files changed, 73 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index 31fd775ffdf..65082011eba 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -361,7 +361,15 @@ public String getDataverseLogoThumbnailAsBase64ById(Long dvId) { } return null; } - + + public String getDataverseLogoThumbnailAsUrl(Long dvId) { + File dataverseLogoFile = getLogoById(dvId); + if (dataverseLogoFile != null) { + return SystemConfig.getDataverseSiteUrlStatic() + "/api/access/dvCardImage/" + dvId; + } + return null; + } + private File getLogo(Dataverse dataverse) { if (dataverse.getId() == null) { return null; diff --git a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java index c1127079da4..91d25ba2d38 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java +++ b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java @@ -48,6 +48,18 @@ public class ThumbnailServiceWrapper implements java.io.Serializable { private Map dvobjectViewMap = new HashMap<>(); private Map hasThumbMap = new HashMap<>(); + public String getFileCardImageAsUrl(SolrSearchResult result) { + if (result.isHarvested()) { + return null; + } + + if (result.getEntity() == null) { + return null; + } + + return SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + result.getEntity().getId() + "?imageThumb=true"; + } + // it's the responsibility of the user - to make sure the search result // passed to this method is of the Datafile type! public String getFileCardImageAsBase64Url(SolrSearchResult result) { @@ -208,7 +220,13 @@ public String getDatasetCardImageAsUrl(Dataset dataset, Long versionId, boolean public String getDataverseCardImageAsBase64Url(SolrSearchResult result) { return dataverseService.getDataverseLogoThumbnailAsBase64ById(result.getEntityId()); } - + + // it's the responsibility of the user - to make sure the search result + // passed to this method is of the Dataverse type! + public String getDataverseCardImageAsUrl(SolrSearchResult result) { + return dataverseService.getDataverseLogoThumbnailAsUrl(result.getEntityId()); + } + public void resetObjectMaps() { dvobjectThumbnailsMap = new HashMap<>(); dvobjectViewMap = new HashMap<>(); diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java index 28676caeac5..ee93c49ad34 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java @@ -593,7 +593,7 @@ public SolrQueryResponse search( solrSearchResult.setDataverseAffiliation(dataverseAffiliation); solrSearchResult.setDataverseParentAlias(dataverseParentAlias); solrSearchResult.setDataverseParentName(dataverseParentName); - solrSearchResult.setImageUrl(thumbnailServiceWrapper.getDataverseCardImageAsBase64Url(solrSearchResult)); + solrSearchResult.setImageUrl(thumbnailServiceWrapper.getDataverseCardImageAsUrl(solrSearchResult)); /** * @todo Expose this API URL after "dvs" is changed to * "dataverses". Also, is an API token required for published @@ -652,7 +652,7 @@ public SolrQueryResponse search( } solrSearchResult.setHtmlUrl(baseUrl + "/dataset.xhtml?persistentId=" + parentGlobalId); solrSearchResult.setDownloadUrl(baseUrl + "/api/access/datafile/" + entityid); - solrSearchResult.setImageUrl(thumbnailServiceWrapper.getFileCardImageAsBase64Url(solrSearchResult)); + solrSearchResult.setImageUrl(thumbnailServiceWrapper.getFileCardImageAsUrl(solrSearchResult)); /** * @todo We are not yet setting the API URL for files because * not all files have metadata. Only subsettable files (those diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index c743c12c7de..4b422002cf2 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -1283,4 +1283,47 @@ public void tearDownDataverse() { public static void cleanup() { } + @Test + public void testSearchFiles() { + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.prettyPrint(); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + System.out.println("id: " + datasetId); + String datasetPid = JsonPath.from(createDatasetResponse.getBody().asString()).getString("data.persistentId"); + System.out.println("datasetPid: " + datasetPid); + + String pathToFile = "src/main/webapp/resources/images/dataverseproject.png"; + Response uploadImage = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken); + uploadImage.prettyPrint(); + uploadImage.then().assertThat() + .statusCode(200); + + Response publishDataverse = UtilIT.publishDataverseViaSword(dataverseAlias, apiToken); + publishDataverse.prettyPrint(); + publishDataverse.then().assertThat() + .statusCode(OK.getStatusCode()); + Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken); + publishDataset.prettyPrint(); + publishDataset.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response searchResp = UtilIT.search("dataverseproject", apiToken); + searchResp.prettyPrint(); + searchResp.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.items[0].type", CoreMatchers.is("file")) + .body("data.items[0].file_content_type", CoreMatchers.is("image/png")) + .body("data.items[0].url", CoreMatchers.containsString("/api/access/datafile/")) + .body("data.items[0].image_url", CoreMatchers.containsString("/api/access/datafile/")) + .body("data.items[0].image_url", CoreMatchers.containsString("imageThumb=true")); + } } From c6dd220a8fbcc450ae8cbe9fda84e5e203f7154d Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 17 Sep 2024 16:59:59 -0400 Subject: [PATCH 892/970] No longer needed with use of CDI.current() in XMLMetadataTemplate --- .../iq/dataverse/pidproviders/PidProviderFactoryBean.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java index d7a19355a78..b01fb5e7eba 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java @@ -60,9 +60,6 @@ public class PidProviderFactoryBean { protected DvObjectServiceBean dvObjectService; @Inject SystemConfig systemConfig; - //QDR - adding to support use of ExternalVocabularyValues in DataCite provider/ DataCite XML generation - @EJB - DatasetFieldServiceBean datasetFieldServiceBean; private ServiceLoader loader; private Map pidProviderFactoryMap = new HashMap<>(); From b1e5020d5c44a2c1dd3c8f581482f4db17e1dcd1 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 17 Sep 2024 17:03:43 -0400 Subject: [PATCH 893/970] no longer used and CrossRef ended up using it's own. --- .../doi/datacite_metadata_template.xml | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/pidproviders/doi/datacite_metadata_template.xml diff --git a/src/main/resources/edu/harvard/iq/dataverse/pidproviders/doi/datacite_metadata_template.xml b/src/main/resources/edu/harvard/iq/dataverse/pidproviders/doi/datacite_metadata_template.xml deleted file mode 100644 index 9c311808c87..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/pidproviders/doi/datacite_metadata_template.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - ${identifier} - ${creators} - - ${title} - - ${publisher} - ${publisherYear} - - ${relatedIdentifiers} - - ${description} - - {$contributors} - From 87bd3087aa16f77c5ea0f9c73db48d72d9b5e30b Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 17 Sep 2024 17:21:54 -0400 Subject: [PATCH 894/970] add more info about the scope of changes. --- doc/release-notes/10632-DataCiteXMLandRelationType.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/release-notes/10632-DataCiteXMLandRelationType.md b/doc/release-notes/10632-DataCiteXMLandRelationType.md index a3699646314..42c1cfb6eda 100644 --- a/doc/release-notes/10632-DataCiteXMLandRelationType.md +++ b/doc/release-notes/10632-DataCiteXMLandRelationType.md @@ -1,10 +1,10 @@ ### Enhanced DataCite Metadata, Relation Type -A new field has been added to the citation metadatablock to allow entry of the "Relation Type" between a "Related Publication" and a dataset. The Relation Type is currently limited to the most common 6 values recommended by DataCite: isCitedBy, Cites, IsSupplementTo, IsSupplementedBy, IsReferencedBy, and References. +A new field has been added to the citation metadatablock to allow entry of the "Relation Type" between a "Related Publication" and a dataset. The Relation Type is currently limited to the most common 6 values recommended by DataCite: isCitedBy, Cites, IsSupplementTo, IsSupplementedBy, IsReferencedBy, and References. For existing datasets where no "Relation Type" has been specified, "IsSupplementTo" is assumed. -Additional metadata, including metadata about Related Publications is now being sent to DataCite when DOIs are registered and published and is available in the DataCite XML export. For existing datasets where no "Relation Type" has been specified, "IsSupplementTo" is assumed. The additions are in rough alignment with the OpenAIRE XML export, but there are some minor differences in addition to the Relation Type addition, including an update to the DataCite 4.5 schema. +Dataverse now supports the DataCite v4.5 schema. Additional metadata, including metadata about Related Publications, and files in the dataset are now being sent to DataCite and improvements to how PIDs (ORCID, ROR, DOIs, etc.), license/terms, geospatial, and other metadata is represented have been made. The enhanced metadata will automatically be sent when datasets are created and published and is available in the DataCite XML export after publication. -For details see https://github.com/IQSS/dataverse/pull/10632 and https://github.com/IQSS/dataverse/pull/10615 and the [design document](https://docs.google.com/document/d/1JzDo9UOIy9dVvaHvtIbOI8tFU6bWdfDfuQvWWpC0tkA/edit?usp=sharing) referenced there. +The additions are in rough alignment with the OpenAIRE XML export, but there are some minor differences in addition to the Relation Type addition, including an update to the DataCite 4.5 schema. For details see https://github.com/IQSS/dataverse/pull/10632 and https://github.com/IQSS/dataverse/pull/10615 and the [design document](https://docs.google.com/document/d/1JzDo9UOIy9dVvaHvtIbOI8tFU6bWdfDfuQvWWpC0tkA/edit?usp=sharing) referenced there. Multiple backward incompatible changes and bug fixes have been made to API calls (3 of the four of which were not documented) related to updating PID target urls and metadata at the provider service: - [Update Target URL for a Published Dataset at the PID provider](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-target-url-for-a-published-dataset-at-the-pid-provider) From 8db46f5e832eec2a161a30a1ee6639c687843e67 Mon Sep 17 00:00:00 2001 From: GPortas Date: Wed, 18 Sep 2024 11:52:43 +0100 Subject: [PATCH 895/970] Added: optional query param returnExpiration to token/recreate endpoint --- .../edu/harvard/iq/dataverse/api/Users.java | 16 +++++------- .../edu/harvard/iq/dataverse/api/UsersIT.java | 26 +++++++++++++++---- .../edu/harvard/iq/dataverse/api/UtilIT.java | 11 +++++--- 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Users.java b/src/main/java/edu/harvard/iq/dataverse/api/Users.java index 1f5430340c2..c1a7c95dbff 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Users.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Users.java @@ -24,13 +24,7 @@ import jakarta.ejb.Stateless; import jakarta.json.JsonArray; import jakarta.json.JsonObjectBuilder; -import jakarta.ws.rs.BadRequestException; -import jakarta.ws.rs.DELETE; -import jakarta.ws.rs.GET; -import jakarta.ws.rs.POST; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.PathParam; -import jakarta.ws.rs.Produces; +import jakarta.ws.rs.*; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.MediaType; @@ -157,7 +151,7 @@ public Response getTokenExpirationDate() { @Path("token/recreate") @AuthRequired @POST - public Response recreateToken(@Context ContainerRequestContext crc) { + public Response recreateToken(@Context ContainerRequestContext crc, @QueryParam("returnExpiration") boolean returnExpiration) { User u = getRequestUser(crc); AuthenticatedUser au; @@ -174,8 +168,12 @@ public Response recreateToken(@Context ContainerRequestContext crc) { ApiToken newToken = authSvc.generateApiTokenForUser(au); authSvc.save(newToken); - return ok("New token for " + au.getUserIdentifier() + " is " + newToken.getTokenString()); + String message = "New token for " + au.getUserIdentifier() + " is " + newToken.getTokenString(); + if (returnExpiration) { + message += " and expires on " + newToken.getExpireTime(); + } + return ok(message); } @GET diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java index 0189ffd6e58..1003c1a990c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java @@ -23,6 +23,8 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.contains; import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.hamcrest.CoreMatchers; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -371,23 +373,33 @@ public void testAPITokenEndpoints() { .body("data.message", containsString(userApiToken)) .body("data.message", containsString("expires on")); + // Recreate given a bad API token Response recreateToken = UtilIT.recreateToken("BAD-Token-blah-89234"); recreateToken.prettyPrint(); recreateToken.then().assertThat() .statusCode(UNAUTHORIZED.getStatusCode()); + // Recreate given a valid API token recreateToken = UtilIT.recreateToken(userApiToken); recreateToken.prettyPrint(); recreateToken.then().assertThat() .statusCode(OK.getStatusCode()) - .body("data.message", containsString("New token for")); + .body("data.message", containsString("New token for")) + .body("data.message", CoreMatchers.not(containsString("and expires on"))); + // Recreate given a valid API token and returning expiration createUser = UtilIT.createRandomUser(); - createUser.prettyPrint(); - assertEquals(200, createUser.getStatusCode()); + assertEquals(OK.getStatusCode(), createUser.getStatusCode()); + + userApiToken = UtilIT.getApiTokenFromResponse(createUser); + + recreateToken = UtilIT.recreateToken(userApiToken, true); + recreateToken.prettyPrint(); + recreateToken.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", containsString("New token for")) + .body("data.message", containsString("and expires on")); - String userApiTokenForDelete = UtilIT.getApiTokenFromResponse(createUser); - /* Add tests for Private URL */ @@ -418,6 +430,10 @@ public void testAPITokenEndpoints() { getExpiration.then().assertThat() .statusCode(NOT_FOUND.getStatusCode()); + createUser = UtilIT.createRandomUser(); + assertEquals(OK.getStatusCode(), createUser.getStatusCode()); + + String userApiTokenForDelete = UtilIT.getApiTokenFromResponse(createUser); Response deleteToken = UtilIT.deleteToken(userApiTokenForDelete); deleteToken.prettyPrint(); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 4e20e8e4c33..8aa89bd0dd2 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2813,10 +2813,15 @@ static Response getTokenExpiration( String apiToken) { return response; } - static Response recreateToken( String apiToken) { + static Response recreateToken(String apiToken) { + return recreateToken(apiToken, false); + } + + static Response recreateToken(String apiToken, boolean returnExpiration) { Response response = given() - .header(API_TOKEN_HTTP_HEADER, apiToken) - .post("api/users/token/recreate"); + .header(API_TOKEN_HTTP_HEADER, apiToken) + .queryParam("returnExpiration", returnExpiration) + .post("api/users/token/recreate"); return response; } From bdf626b15381fd61921440fd6e5a673e20447184 Mon Sep 17 00:00:00 2001 From: GPortas Date: Wed, 18 Sep 2024 14:18:21 +0100 Subject: [PATCH 896/970] Added: release notes and docs for #10857 --- .../10857-add-expiration-date-to-recreate-token-api.md | 1 + doc/sphinx-guides/source/api/native-api.rst | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 doc/release-notes/10857-add-expiration-date-to-recreate-token-api.md diff --git a/doc/release-notes/10857-add-expiration-date-to-recreate-token-api.md b/doc/release-notes/10857-add-expiration-date-to-recreate-token-api.md new file mode 100644 index 00000000000..b450867c630 --- /dev/null +++ b/doc/release-notes/10857-add-expiration-date-to-recreate-token-api.md @@ -0,0 +1 @@ +An optional query parameter called 'returnExpiration' has been added to the 'users/token/recreate' endpoint, which, if set to true, returns the expiration time in the response message. diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 117aceb141d..370cd889cfc 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -4412,6 +4412,12 @@ In order to obtain a new token use:: curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/users/token/recreate" +This endpoint by default will return a response message indicating the user identifier and the new token. + +To also include the expiration time in the response message, the query parameter ``returnExpiration`` must be set to true:: + + curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/users/token/recreate?returnExpiration=true" + Delete a Token ~~~~~~~~~~~~~~ From c80a3dffad78474a19972e5cab8e81d13e78a454 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 18 Sep 2024 09:47:41 -0400 Subject: [PATCH 897/970] doc changes --- ...831-standardize-image-url-of-search-api.md | 28 +++++++++++++++++++ doc/sphinx-guides/source/api/changelog.rst | 5 ++++ 2 files changed, 33 insertions(+) create mode 100644 doc/release-notes/10831-standardize-image-url-of-search-api.md diff --git a/doc/release-notes/10831-standardize-image-url-of-search-api.md b/doc/release-notes/10831-standardize-image-url-of-search-api.md new file mode 100644 index 00000000000..3d9216a7c09 --- /dev/null +++ b/doc/release-notes/10831-standardize-image-url-of-search-api.md @@ -0,0 +1,28 @@ +Search API (/api/search) response will now include new image_url format for the Datafile and Dataverse logo. +Note to release not writer: this supersedes the release note 10810-search-api-payload-extensions.md + +For Dataverse: + +- "image_url" (optional) + +```javascript +"items": [ + { + "name": "Darwin's Finches", + ... + "image_url":"/api/access/dvCardImage/{identifier}" +(etc, etc) +``` + +For DataFile: + +- "image_url" (optional) + +```javascript +"items": [ + { + "name": "test.txt", + ... + "image_url":"/api/access/datafile/{identifier}?imageThumb=true" +(etc, etc) +``` diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index a7af3e84b28..8d11ac0c123 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -7,6 +7,11 @@ This API changelog is experimental and we would love feedback on its usefulness. :local: :depth: 1 +v6.4 +---- + +- ** /api/search?q=**: Json values for image_url in DataFiles and Collections have changed from Base64URL ("data:image/png;base64,...) to "/api/access/datafile/{identifier}?imageThumb=true" and "/api/access/dvCardImage/{identifier}" respectively. This was done to match the image_url of Dataset. + v6.3 ---- From b923814e5d1458824a357f6d0984fcbda29cef68 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 18 Sep 2024 09:48:27 -0400 Subject: [PATCH 898/970] doc changes --- doc/release-notes/10831-standardize-image-url-of-search-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/10831-standardize-image-url-of-search-api.md b/doc/release-notes/10831-standardize-image-url-of-search-api.md index 3d9216a7c09..1910091455c 100644 --- a/doc/release-notes/10831-standardize-image-url-of-search-api.md +++ b/doc/release-notes/10831-standardize-image-url-of-search-api.md @@ -1,5 +1,5 @@ Search API (/api/search) response will now include new image_url format for the Datafile and Dataverse logo. -Note to release not writer: this supersedes the release note 10810-search-api-payload-extensions.md +Note to release note writer: this supersedes the release note 10810-search-api-payload-extensions.md For Dataverse: From 26be5c1813f7d9aae501c6872b1eb8b950abe447 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 18 Sep 2024 17:04:47 +0200 Subject: [PATCH 899/970] ci(ct): reshape maintenance workflow into external matrix script Unfortunately, matrix jobs logs and outputs cannot be aggregated in Github Actions. The only way to work around the limitations of GHA is by using a custom build script that create a similar matrix like experience. This commit introduces these scripts, probably also making some custom actions we added obsolete. --- .github/workflows/container_maintenance.yml | 125 ++-------------- .github/workflows/scripts/maintenance-job.sh | 141 +++++++++++++++++++ .github/workflows/scripts/utils.sh | 107 ++++++++++++++ 3 files changed, 260 insertions(+), 113 deletions(-) create mode 100755 .github/workflows/scripts/maintenance-job.sh create mode 100644 .github/workflows/scripts/utils.sh diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 3a28f355e94..65cbcc81d48 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -18,42 +18,14 @@ on: env: PLATFORMS: linux/amd64,linux/arm64 NUM_PAST_RELEASES: 3 - # TODO: change to "develop" in final PR - DEVELOP_BRANCH: 10478-version-base-img jobs: - discover: - name: Discover Release Matrix - runs-on: ubuntu-latest - permissions: - contents: read - packages: read - # TODO: re-enable for final PR - # Only run in upstream repo - avoid unnecessary runs in forks and only for scheduled - #if: ${{ github.repository_owner == 'IQSS' }} - outputs: - branches: ${{ steps.matrix.outputs.branches }} - current_release: ${{ steps.matrix.outputs.current_release }} - steps: - - name: Build branch matrix options - id: matrix - run: | - echo "branches=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | \ - jq '[ .[0:${{ env.NUM_PAST_RELEASES }}] | .[].tag_name, "${{ env.DEVELOP_BRANCH }}" ]')" | tr -d "\n" | tr -s " " | \ - tee -a "$GITHUB_OUTPUT" - echo "current_release=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | jq '.[0].tag_name' )" | tee -a "$GITHUB_OUTPUT" - build: name: Build image runs-on: ubuntu-latest permissions: contents: read packages: read - needs: discover - strategy: - fail-fast: false - matrix: - branch: ${{ fromJson(needs.discover.outputs.branches) }} # TODO: re-enable for final PR # Only run in upstream repo - avoid unnecessary runs in forks #if: ${{ github.repository_owner == 'IQSS' }} @@ -64,7 +36,6 @@ jobs: # Necessary as the checked out release branch might not contain the action as files uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img with: - git-reference: ${{ matrix.branch }} pom-paths: modules/container-base/pom.xml # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and @@ -79,93 +50,21 @@ jobs: with: platforms: ${{ env.PLATFORMS }} - # Try to retrieve backport patches for this git ref (but don't fail if there aren't any) - # and try to apply them if present - - name: Get and apply backported patches - # There might be no patches - ignore errors - continue-on-error: true - run: | - mkdir -p "${GITHUB_WORKSPACE}/patches" - curl -sSL "https://github.com/${GITHUB_REPOSITORY}/archive/${DEVELOP_BRANCH}.tar.gz" | \ - tar -zxf - -C "${GITHUB_WORKSPACE}/patches" --wildcards "*/modules/container-base/src/backports/${{ matrix.branch }}" --strip-components=6 - find "${GITHUB_WORKSPACE}/patches" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -s -i - - # Determine the base image name we are going to use from here on - - name: Determine base image name - run: | - if [[ "${{ matrix.branch }}" = "${{ env.DEVELOP_BRANCH }}" ]]; then - echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" | tee -a "${GITHUB_ENV}" - echo "BASE_IMAGE_UPCOMING=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}" - else - echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}" - fi - - # Figure out if a rebuild is necessary because either there is an updated Java image or our installed packages need updates - - name: Check for recent Temurin image updates - id: temurin-check - # TODO: change to upstream location in final PR - uses: gdcc/wip-dataverse-base-image/.github/actions/check-newer-parent-image@10478-version-base-img - with: - parent: "$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=java.image -q -DforceStdout )" - derived: "${{ env.BASE_IMAGE }}" - # TODO: if we introduce more flavors as a matrix, we need to adapt the install command to check for updates - - name: Check for package updates in base image - id: package-check - if: ${{ steps.temurin-check.outputs.is-more-recent == 'false' }} + # Discover the releases we want to maintain + - name: Discover maintained releases + id: discover run: | - PKGS="$( grep "ARG PKGS" modules/container-base/src/main/docker/Dockerfile | cut -f2 -d= | tr -d '"' )" - if [[ ! $( docker run --rm -u 0 "${BASE_IMAGE}" sh -c "apt update >&2 && apt install -s ${PKGS}" | grep "0 upgraded" ) ]]; then - echo "Base image $BASE_IMAGE needs updates for our custom installed packages" - echo "newer_packages=true" >> "${GITHUB_OUTPUT}" - else - echo "Base image $BASE_IMAGE has no updates for our custom installed packages" - echo "newer_packages=false" >> "${GITHUB_OUTPUT}" - fi + DEVELOPMENT_BRANCH=$( curl -f -sS https://api.github.com/repos/${{ github.repository }} | jq -r '.default_branch' ) + echo "DEVELOPMENT_BRANCH=$DEVELOPMENT_BRANCH" | tee -a "$GITHUB_ENV" + echo "branches=$( curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | jq -r " .[0:${{ env.NUM_PAST_RELEASES }}] | .[].tag_name, \"${DEVELOPMENT_BRANCH}\" " | tr "\n" " " )" | tee -a "${GITHUB_OUTPUT}" + - # TODO: In a future version of this script, we might want to include checking for other security updates, - # not just updates to the packages we installed. - # grep security /etc/apt/sources.list > /tmp/security.list - # apt-get update -oDir::Etc::Sourcelist=/tmp/security.list - # apt-get dist-upgrade -y -oDir::Etc::Sourcelist=/tmp/security.list -oDir::Etc::SourceParts=/bin/false -s - - - name: Calculate revision number for immutable tag (on release branches only) - if: ${{ matrix.branch != env.DEVELOP_BRANCH }} - id: revision-tag - # TODO: change to upstream location in final PR - uses: gdcc/wip-dataverse-base-image/.github/actions/get-image-revision@10478-version-base-img - with: - image-ref: ${{ env.BASE_IMAGE }} - tag-options-prefix: "-Dbase.image.tag.suffix='' -Ddocker.tags.revision=" - - name: Configure update of "latest" tag for development branch - id: develop-tag - if: ${{ matrix.branch == env.DEVELOP_BRANCH }} + # Execute matrix build for the discovered branches + - name: Execute build matrix script + id: execute run: | - echo "tag-options=-Ddocker.tags.develop=unstable -Ddocker.tags.upcoming=${BASE_IMAGE_UPCOMING#*:}" | tee -a "${GITHUB_OUTPUT}" - - name: Deploy multi-arch base container image to Docker Hub - if: ${{ steps.temurin-check.outputs.is-more-recent == 'true' || steps.package-check.outputs.newer_packages == 'true' || inputs.force_build }} - id: build - run: | - mvn -f modules/container-base -Pct deploy -Ddocker.noCache -Ddocker.platforms=${{ env.PLATFORMS }} \ - -Ddocker.imagePropertyConfiguration=override ${{ steps.develop-tag.outputs.tag-options }} ${{ steps.revision-tag.outputs.tag-options }} - echo "rebuild=true" | tee -a "${GITHUB_OUTPUT}" - - # TODO: this is here to not drop the knowledge about matrix output workarounds (for now) - # - if: always() - # name: Save status (workaround for matrix outputs) - # run: | - # # steps.build.outcome is the status BEFORE continue-on-error - # echo "STATUS_$( echo "${{ matrix.branch }}" | tr ".:;,-/ " "_" )=${{ steps.build.outcome }}" | tee -a "${GITHUB_ENV}" - - # TODO: As part of issue #10618 we will need to create this action, shipping updated app images - #- name: Rebuild application container - # if: ${{ steps.build.outputs.rebuild }} - # uses: ./.github/actions/deploy-app-container - # with: - # registry: "" - # registry_token: "" - # ref: "" - # base_image: "" - # base_image_tag: "" + echo "force_build=${{ inputs.force_build }}" + .github/workflows/scripts/maintenance-job.sh ${{ steps.discover.outputs.branches }} # TODO: This job should become part of the matrix as an action, so we don't need to fiddle with matrix outputs hacks #push-app-img: diff --git a/.github/workflows/scripts/maintenance-job.sh b/.github/workflows/scripts/maintenance-job.sh new file mode 100755 index 00000000000..bf1a9748bf7 --- /dev/null +++ b/.github/workflows/scripts/maintenance-job.sh @@ -0,0 +1,141 @@ +#!/bin/bash + +# A matrix-like job to maintain a number of releases as well as the latest snap of Dataverse. + +# PREREQUISITES: +# - You have Java, Maven, QEMU and Docker all setup and ready to go +# - You obviously checked out the develop branch, otherwise you'd not be executing this script +# - You added all the branch names you want to run maintenance for as arguments +# Optional, but recommended: +# - You added a DEVELOPMENT_BRANCH env var to your runner/job env with the name of the development branch +# - You added a FORCE_BUILD=0|1 env var to indicate if the base image build should be forced +# - You added a PLATFORMS env var with all the target platforms you want to build for + +# NOTE: +# This script is a culmination of Github Action steps into a single script. +# The reason to put all of this in here is due to the complexity of the Github Action and the limitation of the +# matrix support in Github actions, where outputs cannot be aggregated or otherwise used further. + +set -euo pipefail + +# Get all the inputs +# If not within a runner, just print to stdout (duplicating the output in case of tee usage, but that's ok for testing) +GITHUB_OUTPUT=${GITHUB_OUTPUT:-"/proc/self/fd/1"} +GITHUB_ENV=${GITHUB_ENV:-"/proc/self/fd/1"} +GITHUB_WORKSPACE=${GITHUB_WORKSPACE:-"$(pwd)"} +GITHUB_SERVER_URL=${GITHUB_SERVER_URL:-"https://github.com"} +GITHUB_REPOSITORY=${GITHUB_REPOSITORY:-"IQSS/dataverse"} + +MAINTENANCE_WORKSPACE="${GITHUB_WORKSPACE}/maintenance-job" + +DEVELOPMENT_BRANCH="${DEVELOPMENT_BRANCH:-"develop"}" +FORCE_BUILD="${FORCE_BUILD:-"0"}" +PLATFORMS="${PLATFORMS:-"linux/amd64,linux/arm64"}" + +# Setup and validation +if [[ -z "$*" ]]; then + >&2 echo "You must give a list of branch names as arguments" + exit 1; +fi + +source "$( dirname "$0" )/utils.sh" + +# Delete old stuff if present +rm -rf "$MAINTENANCE_WORKSPACE" +mkdir -p "$MAINTENANCE_WORKSPACE" + +# Cache the image tags we maintain in this array (same order as branches array!) +# This list will be used to build the support matrix within the Docker Hub image description +SUPPORTED_ROLLING_TAGS=() + +for BRANCH in "$@"; do + echo "::group::Running maintenance for $BRANCH" + + # 0. Determine if this is a development branch and the most current release + IS_DEV=0 + if [[ "$BRANCH" = "$DEVELOPMENT_BRANCH" ]]; then + IS_DEV=1 + fi + IS_CURRENT_RELEASE=0 + if [[ "$BRANCH" = $( curl -f -sS "https://api.github.com/repos/$GITHUB_REPOSITORY/releases" | jq -r '.[0].tag_name' ) ]]; then + IS_CURRENT_RELEASE=1 + fi + + # 1. Let's get the maintained sources + git clone -c advice.detachedHead=false --depth 1 --branch "$BRANCH" "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" "$MAINTENANCE_WORKSPACE/$BRANCH" + # Switch context + cd "$MAINTENANCE_WORKSPACE/$BRANCH" + + # 2. Now let's apply the patches (we have them checked out in $GITHUB_WORKSPACE, not necessarily in this local checkout) + if [[ -d ${GITHUB_WORKSPACE}/modules/container-base/src/backports/$BRANCH ]]; then + find "${GITHUB_WORKSPACE}/modules/container-base/src/backports/$BRANCH" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -s -i + fi + + # 3. Determine the base image ref (/:) + BASE_IMAGE_REF="" + # For the dev branch we want to full flexi stack tag, to detect stack upgrades requiring new build + if (( IS_DEV )); then + BASE_IMAGE_REF=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout ) + else + BASE_IMAGE_REF=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout ) + fi + + # 4. Check for Temurin image updates + JAVA_IMAGE_REF=$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=java.image -q -DforceStdout ) + NEWER_JAVA_IMAGE=0 + if check_newer_parent "$JAVA_IMAGE_REF" "$BASE_IMAGE_REF"; then + NEWER_JAVA_IMAGE=1 + fi + + # 5. Check for package updates in base image + PKGS="$( grep "ARG PKGS" modules/container-base/src/main/docker/Dockerfile | cut -f2 -d= | tr -d '"' )" + NEWER_PKGS=0 + # Don't bother with package checks if the java image is newer already + if ! (( NEWER_JAVA_IMAGE )); then + if check_newer_pkgs "$BASE_IMAGE_REF" "$PKGS"; then + NEWER_PKGS=1 + fi + fi + + # 6. Get current immutable revision tag if not on the dev branch + REV=$( current_revision "$BASE_IMAGE_REF" ) + CURRENT_REV_TAG="${BASE_IMAGE_REF#*:}-r$REV" + NEXT_REV_TAG="${BASE_IMAGE_REF#*:}-r$(( REV + 1 ))" + + # 7. Let's put together what tags we want added to this build run + TAG_OPTIONS="" + if ! (( IS_DEV )); then + TAG_OPTIONS="-Dbase.image=$BASE_IMAGE_REF -Ddocker.tags.revision=$NEXT_REV_TAG" + + # In case of the current release, add the "latest" tag as well. Also add to list of rolling tags. + if (( IS_CURRENT_RELEASE )); then + TAG_OPTIONS="$TAG_OPTIONS -Ddocker.tags.latest=latest" + SUPPORTED_ROLLING_TAGS+=("[\"latest\", \"${BASE_IMAGE_REF#*:}\"]") + else + SUPPORTED_ROLLING_TAGS+=("[\"${BASE_IMAGE_REF#*:}\"]") + fi + else + UPCOMING_TAG=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image.tag -Dbase.image.tag.suffix="" -q -DforceStdout ) + TAG_OPTIONS="-Ddocker.tags.develop=unstable -Ddocker.tags.upcoming=$UPCOMING_TAG" + + SUPPORTED_ROLLING_TAGS+=("[\"unstable\", \"$UPCOMING_TAG\", \"${BASE_IMAGE_REF#*:}\"]") + fi + echo "Determined these additional Maven tag options: $TAG_OPTIONS" + + # 7. Let's build the base image if necessary + NEWER_BASE_IMAGE=0 + if (( NEWER_JAVA_IMAGE + NEWER_PKGS + FORCE_BUILD > 0 )); then + mvn -Pct -f modules/container-base deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ + -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS + NEWER_BASE_IMAGE=1 + else + echo "No rebuild necessary, we're done here." + fi + + if (( NEWER_BASE_IMAGE )); then + echo "Built a new base image, should continue with application images now..." + # TODO: rebuild the app images here + fi + + echo "::endgroup::" +done diff --git a/.github/workflows/scripts/utils.sh b/.github/workflows/scripts/utils.sh new file mode 100644 index 00000000000..b63787105ee --- /dev/null +++ b/.github/workflows/scripts/utils.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +set -euo pipefail + +function check_newer_parent() { + PARENT_IMAGE="$1" + # Get namespace, default to "library" if not found + PARENT_IMAGE_NS="${PARENT_IMAGE%/*}" + if [[ "$PARENT_IMAGE_NS" = "${PARENT_IMAGE}" ]]; then + PARENT_IMAGE_NS="library" + fi + PARENT_IMAGE_REPO="${PARENT_IMAGE%:*}" + PARENT_IMAGE_TAG="${PARENT_IMAGE#*:}" + + PARENT_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${PARENT_IMAGE_NS}/repositories/${PARENT_IMAGE_REPO}/tags/${PARENT_IMAGE_TAG}" | jq -r .last_updated )" + if [[ "$PARENT_IMAGE_LAST_UPDATE" = "null" ]]; then + echo "::error title='Invalid PARENT Image'::Could not find ${PARENT_IMAGE} in the registry" + exit 1 + fi + + DERIVED_IMAGE="$2" + # Get namespace, default to "library" if not found + DERIVED_IMAGE_NS="${DERIVED_IMAGE%/*}" + if [[ "${DERIVED_IMAGE_NS}" = "${DERIVED_IMAGE}" ]]; then + DERIVED_IMAGE_NS="library" + fi + DERIVED_IMAGE_REPO="$( echo "${DERIVED_IMAGE%:*}" | cut -f2 -d/ )" + DERIVED_IMAGE_TAG="${DERIVED_IMAGE#*:}" + + DERIVED_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${DERIVED_IMAGE_NS}/repositories/${DERIVED_IMAGE_REPO}/tags/${DERIVED_IMAGE_TAG}" | jq -r .last_updated )" + if [[ "$DERIVED_IMAGE_LAST_UPDATE" = "null" || "$DERIVED_IMAGE_LAST_UPDATE" < "$PARENT_IMAGE_LAST_UPDATE" ]]; then + echo "Parent image $PARENT_IMAGE has a newer release ($PARENT_IMAGE_LAST_UPDATE), which is more recent than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" + return 0 + else + echo "Parent image $PARENT_IMAGE ($PARENT_IMAGE_LAST_UPDATE) is older than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" + return 1 + fi +} + +function check_newer_pkgs() { + IMAGE="$1" + PKGS="$2" + + docker run --rm -u 0 "${IMAGE}" sh -c "apt update >/dev/null 2>&1 && apt install -s ${PKGS}" | tee /proc/self/fd/2 | grep -q "0 upgraded" + + if [[ ! $? ]]; then + echo "Base image $IMAGE needs updates for our custom installed packages" + return 0 + else + echo "Base image $IMAGE has no updates for our custom installed packages" + return 1 + fi + + # TODO: In a future version of this script, we might want to include checking for other security updates, + # not just updates to the packages we installed. + # grep security /etc/apt/sources.list > /tmp/security.list + # apt-get update -oDir::Etc::Sourcelist=/tmp/security.list + # apt-get dist-upgrade -y -oDir::Etc::Sourcelist=/tmp/security.list -oDir::Etc::SourceParts=/bin/false -s + +} + +function current_revision() { + IMAGE="$1" + IMAGE_NS_REPO="${IMAGE%:*}" + IMAGE_TAG="${IMAGE#*:}" + + if [[ "$IMAGE_TAG" = "$IMAGE_NS_REPO" ]]; then + >&2 echo "You must provide an image reference in the format [/]:" + exit 1 + fi + + case "$IMAGE_NS_REPO" in + */*) :;; # namespace/repository syntax, leave as is + *) IMAGE_NS_REPO="library/$IMAGE_NS_REPO";; # bare repository name (docker official image); must convert to namespace/repository syntax + esac + + # Without such a token we may run into rate limits + # OB 2024-09-16: for some reason using this token stopped working. Let's go without and see if we really fall into rate limits. + # token=$( curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE_NS_REPO:pull" ) + + ALL_TAGS="$( + i=0 + while [ $? == 0 ]; do + i=$((i+1)) + # OB 2024-09-16: for some reason using this token stopped working. Let's go without and see if we really fall into rate limits. + # RESULT=$( curl -s -H "Authorization: Bearer $token" "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) + RESULT=$( curl -s "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) + if [[ $( echo "$RESULT" | jq '.message' ) != "null" ]]; then + # If we run into an error on the first attempt, that means we have a problem. + if [[ "$i" == "1" ]]; then + >&2 echo "Error when retrieving tag data: $( echo "$RESULT" | jq '.message' )" + exit 2 + # Otherwise it will just mean we reached the last page already + else + break + fi + else + echo "$RESULT" | jq -r '."results"[]["name"]' + # DEBUG: + #echo "$RESULT" | >&2 jq -r '."results"[]["name"]' + fi + done + )" + + # Note: if a former tag could not be found, it just might not exist already. Start new series with rev 0 + echo "$ALL_TAGS" | grep "${IMAGE_TAG}-r" | sed -e "s#${IMAGE_TAG}-r##" | sort -h | tail -n1 || echo "-1" +} From ff0498bc2aca5fbef5b53f4786d9eea1d6f4ba35 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 18 Sep 2024 17:30:18 +0200 Subject: [PATCH 900/970] ci(ct): remove obsolete actions for revisions and parent image changes detection --- .../check-newer-parent-image/action.yml | 53 ------------ .github/actions/get-image-revision/action.yml | 81 ------------------- 2 files changed, 134 deletions(-) delete mode 100644 .github/actions/check-newer-parent-image/action.yml delete mode 100644 .github/actions/get-image-revision/action.yml diff --git a/.github/actions/check-newer-parent-image/action.yml b/.github/actions/check-newer-parent-image/action.yml deleted file mode 100644 index 6f8ce950cec..00000000000 --- a/.github/actions/check-newer-parent-image/action.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -name: "Check Newer Parent Image" -description: "Determine if a base image is more recent than the derived." -inputs: - parent: - description: 'The name and (rolling) tag of the parent image to check - or - a shell command to get it' - required: true - derived: - description: 'The name and (rolling) tag of the derived image - or - a shell command to get it' - required: true -outputs: - is-more-recent: - description: "True if base image has a more recent update, false if not." - value: "${{ steps.determine.outputs.is_more_recent }}" - -runs: - using: composite - steps: - - shell: bash - id: determine - run: | - PARENT_IMAGE="${{ inputs.parent }}" - # Get namespace, default to "library" if not found - PARENT_IMAGE_NS="${PARENT_IMAGE%/*}" - if [[ "$PARENT_IMAGE_NS" = "${PARENT_IMAGE}" ]]; then - PARENT_IMAGE_NS="library" - fi - PARENT_IMAGE_REPO="${PARENT_IMAGE%:*}" - PARENT_IMAGE_TAG="${PARENT_IMAGE#*:}" - - PARENT_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${PARENT_IMAGE_NS}/repositories/${PARENT_IMAGE_REPO}/tags/${PARENT_IMAGE_TAG}" | jq -r .last_updated )" - if [[ "$PARENT_IMAGE_LAST_UPDATE" = "null" ]]; then - echo "::error title='Invalid PARENT Image'::Could not find ${PARENT_IMAGE} in the registry" - exit 1 - fi - - DERIVED_IMAGE="${{ inputs.derived }}" - # Get namespace, default to "library" if not found - DERIVED_IMAGE_NS="${DERIVED_IMAGE%/*}" - if [[ "${DERIVED_IMAGE_NS}" = "${DERIVED_IMAGE}" ]]; then - DERIVED_IMAGE_NS="library" - fi - DERIVED_IMAGE_REPO="$( echo "${DERIVED_IMAGE%:*}" | cut -f2 -d/ )" - DERIVED_IMAGE_TAG="${DERIVED_IMAGE#*:}" - - DERIVED_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${DERIVED_IMAGE_NS}/repositories/${DERIVED_IMAGE_REPO}/tags/${DERIVED_IMAGE_TAG}" | jq -r .last_updated )" - if [[ "$DERIVED_IMAGE_LAST_UPDATE" = "null" || "$DERIVED_IMAGE_LAST_UPDATE" < "$PARENT_IMAGE_LAST_UPDATE" ]]; then - echo "Parent image $PARENT_IMAGE has a newer release ($PARENT_IMAGE_LAST_UPDATE), which is more recent than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" - echo "is_more_recent=true" >> $GITHUB_OUTPUT - else - echo "Parent image $PARENT_IMAGE ($PARENT_IMAGE_LAST_UPDATE) is older than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" - echo "is_more_recent=false" >> $GITHUB_OUTPUT - fi diff --git a/.github/actions/get-image-revision/action.yml b/.github/actions/get-image-revision/action.yml deleted file mode 100644 index 6eaf89abba2..00000000000 --- a/.github/actions/get-image-revision/action.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: "Get Incremented Revision Tag" -description: "For a pre-existing rolling tag lookup the latest immutable revision tag, increment and return" -inputs: - image-ref: - description: "The full image reference including namespace, repo and tag" - required: true - revision-separator: - description: "The separator string to use between tag and revision number. Defaults to '-r'" - required: false - default: "-r" - tag-options-prefix: - description: "A string that the revision tag will be appended to and sent to output tag-options" - required: false - default: "" -outputs: - revision-tag: - description: "The updated immutable tag, ready to use" - value: ${{ steps.extract.outputs.revision_tag }} - tag-options: - description: "A string with some command line options (may be empty)" - value: ${{ steps.extract.outputs.tag_options }} -runs: - using: composite - steps: - - shell: bash - id: extract - run: | - IMAGE="${{ inputs.image-ref }}" - IMAGE_NS_REPO="${IMAGE%:*}" - IMAGE_TAG="${IMAGE#*:}" - - if [[ "$IMAGE_TAG" = "$IMAGE_NS_REPO" ]]; then - >&2 echo "You must provide an image reference in the format [/]:" - exit 1 - fi - - case "$IMAGE_NS_REPO" in - */*) :;; # namespace/repository syntax, leave as is - *) IMAGE_NS_REPO="library/$IMAGE_NS_REPO";; # bare repository name (docker official image); must convert to namespace/repository syntax - esac - - # Without such a token we may run into rate limits - # OB 2024-09-16: for some reason using this token stopped working. Let's go without and see if we really fall into rate limits. - token=$( curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE_NS_REPO:pull" ) - - ALL_TAGS="$( - i=0 - while [ $? == 0 ]; do - i=$((i+1)) - # OB 2024-09-16: for some reason using this token stopped working. Let's go without and see if we really fall into rate limits. - # RESULT=$( curl -s -H "Authorization: Bearer $token" "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) - RESULT=$( curl -s "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) - if [[ $( echo "$RESULT" | jq '.message' ) != "null" ]]; then - # If we run into an error on the first attempt, that means we have a problem. - if [[ "$i" == "1" ]]; then - >&2 echo "Error when retrieving tag data: $( echo "$RESULT" | jq '.message' )" - exit 2 - # Otherwise it will just mean we reached the last page already - else - break - fi - else - echo "$RESULT" | jq -r '."results"[]["name"]' - # DEBUG: - #echo "$RESULT" | >&2 jq -r '."results"[]["name"]' - fi - done - )" - - # Note: if a former tag could not be found, it just might not exist already. Start new series with rev 0 - CURRENT=$( echo "$ALL_TAGS" | grep "${IMAGE_TAG}${{ inputs.revision-separator }}" | sed -e "s#${IMAGE_TAG}${{ inputs.revision-separator }}##" | sort -h | tail -n1 ) - if [[ "$CURRENT" ]]; then - REVISION_TAG="${IMAGE_TAG}${{ inputs.revision-separator }}$((CURRENT+1))" - else - REVISION_TAG="${IMAGE_TAG}${{ inputs.revision-separator }}0" - fi - echo "revision_tag=${REVISION_TAG}" | tee -a "$GITHUB_OUTPUT" - - if [[ -n "${{ inputs.tag-options-prefix }}" ]]; then - echo "tag_options=${{ inputs.tag-options-prefix }}${REVISION_TAG}" | tee -a "$GITHUB_OUTPUT" - fi \ No newline at end of file From 2794d3a874012319769f463ca26dcca2b8f1cd49 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 18 Sep 2024 17:44:33 +0200 Subject: [PATCH 901/970] ci(ct): re-enable forced build for maintenance workflow --- .github/workflows/container_maintenance.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 65cbcc81d48..47bbd005764 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -54,16 +54,15 @@ jobs: - name: Discover maintained releases id: discover run: | + echo "FORCE_BUILD=$( [[ "${{ inputs.force_build }}" = "true" ]] && echo 1 || echo 0 )" | tee -a "$GITHUB_ENV" DEVELOPMENT_BRANCH=$( curl -f -sS https://api.github.com/repos/${{ github.repository }} | jq -r '.default_branch' ) echo "DEVELOPMENT_BRANCH=$DEVELOPMENT_BRANCH" | tee -a "$GITHUB_ENV" echo "branches=$( curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | jq -r " .[0:${{ env.NUM_PAST_RELEASES }}] | .[].tag_name, \"${DEVELOPMENT_BRANCH}\" " | tr "\n" " " )" | tee -a "${GITHUB_OUTPUT}" - # Execute matrix build for the discovered branches - name: Execute build matrix script id: execute run: | - echo "force_build=${{ inputs.force_build }}" .github/workflows/scripts/maintenance-job.sh ${{ steps.discover.outputs.branches }} # TODO: This job should become part of the matrix as an action, so we don't need to fiddle with matrix outputs hacks From 18c921107dc57478081132be9718a66cf123164f Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:11:59 -0400 Subject: [PATCH 902/970] doc changes --- doc/sphinx-guides/source/api/search.rst | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/doc/sphinx-guides/source/api/search.rst b/doc/sphinx-guides/source/api/search.rst index 359313ce1b5..7ca9a5abca6 100755 --- a/doc/sphinx-guides/source/api/search.rst +++ b/doc/sphinx-guides/source/api/search.rst @@ -61,7 +61,7 @@ https://demo.dataverse.org/api/search?q=trees "name":"Trees", "type":"dataverse", "url":"https://demo.dataverse.org/dataverse/trees", - "image_url":"data:image/png;base64,iVBORw0...", + "image_url":"https://demo.dataverse.org/api/access/dvCardImage/1", "identifier":"trees", "description":"A tree dataverse with some birds", "published_at":"2016-05-10T12:53:38Z", @@ -76,7 +76,7 @@ https://demo.dataverse.org/api/search?q=trees "name":"Chestnut Trees", "type":"dataverse", "url":"https://demo.dataverse.org/dataverse/chestnuttrees", - "image_url":"data:image/png;base64,iVBORw0...", + "image_url":"https://demo.dataverse.org/api/access/dvCardImage/2", "identifier":"chestnuttrees", "description":"A dataverse with chestnut trees and an oriole", "published_at":"2016-05-10T12:52:38Z", @@ -91,7 +91,7 @@ https://demo.dataverse.org/api/search?q=trees "name":"trees.png", "type":"file", "url":"https://demo.dataverse.org/api/access/datafile/12", - "image_url":"data:image/png;base64,iVBORw0...", + "image_url":"https://demo.dataverse.org/api/access/datafile/12?imageThumb=true", "file_id":"12", "description":"", "published_at":"2016-05-10T12:53:39Z", @@ -113,7 +113,7 @@ https://demo.dataverse.org/api/search?q=trees "name":"Birds", "type":"dataverse", "url":"https://demo.dataverse.org/dataverse/birds", - "image_url":"data:image/png;base64,iVBORw0...", + "image_url":"https://demo.dataverse.org/api/access/dvCardImage/3", "identifier":"birds", "description":"A bird Dataverse collection with some trees", "published_at":"2016-05-10T12:57:27Z", @@ -173,8 +173,6 @@ https://demo.dataverse.org/api/search?q=trees } } -Note that the image_url field, if exists, will be returned as a regular URL for Datasets, while for Files and Dataverses, it will be returned as a Base64 URL. We plan to standardize this behavior so that the field always returns a regular URL. (See: https://github.com/IQSS/dataverse/issues/10831) - .. _advancedsearch-example: Advanced Search Examples @@ -202,7 +200,7 @@ In this example, ``show_relevance=true`` matches per field are shown. Available "name":"Finches", "type":"dataverse", "url":"https://demo.dataverse.org/dataverse/finches", - "image_url":"data:image/png;base64,iVBORw0...", + "image_url":"https://demo.dataverse.org/api/access/dvCardImage/2", "identifier":"finches", "description":"A Dataverse collection with finches", "published_at":"2016-05-10T12:57:38Z", From 47a2233b76ead9b57f9b96d11ae6c4f6d93dfe36 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:55:52 -0400 Subject: [PATCH 903/970] doc changes --- doc/sphinx-guides/source/api/changelog.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index 8d11ac0c123..a7af3e84b28 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -7,11 +7,6 @@ This API changelog is experimental and we would love feedback on its usefulness. :local: :depth: 1 -v6.4 ----- - -- ** /api/search?q=**: Json values for image_url in DataFiles and Collections have changed from Base64URL ("data:image/png;base64,...) to "/api/access/datafile/{identifier}?imageThumb=true" and "/api/access/dvCardImage/{identifier}" respectively. This was done to match the image_url of Dataset. - v6.3 ---- From 585bc8cc603c94ae8a45f6da46994aa2e4d9d85b Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 00:42:56 +0200 Subject: [PATCH 904/970] ci(ct): add outputs to maintenance matrix job Can be picked up by other jobs, e.g. to create textblocks for docs or a job matrix. --- .github/workflows/container_maintenance.yml | 3 ++ .github/workflows/scripts/maintenance-job.sh | 30 +++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 47bbd005764..d6ec6486a59 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -29,6 +29,9 @@ jobs: # TODO: re-enable for final PR # Only run in upstream repo - avoid unnecessary runs in forks #if: ${{ github.repository_owner == 'IQSS' }} + outputs: + supported_tag_matrix: ${{ steps.execute.outputs.supported_tag_matrix }} + rebuilt_base_images: ${{ steps.execute.outputs.rebuilt_base_images }} steps: - name: Checkout and Setup Maven diff --git a/.github/workflows/scripts/maintenance-job.sh b/.github/workflows/scripts/maintenance-job.sh index bf1a9748bf7..df562aac573 100755 --- a/.github/workflows/scripts/maintenance-job.sh +++ b/.github/workflows/scripts/maintenance-job.sh @@ -44,9 +44,12 @@ source "$( dirname "$0" )/utils.sh" rm -rf "$MAINTENANCE_WORKSPACE" mkdir -p "$MAINTENANCE_WORKSPACE" -# Cache the image tags we maintain in this array (same order as branches array!) +# Store the image tags we maintain in this array (same order as branches array!) # This list will be used to build the support matrix within the Docker Hub image description SUPPORTED_ROLLING_TAGS=() +# Store the tags of base images we are actually rebuilding to base new app images upon +# Takes the from "branch-name=base-image-ref" +REBUILT_BASE_IMAGES=() for BRANCH in "$@"; do echo "::group::Running maintenance for $BRANCH" @@ -128,6 +131,13 @@ for BRANCH in "$@"; do mvn -Pct -f modules/container-base deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS NEWER_BASE_IMAGE=1 + + # Save the information about the immutable or rolling tag we just built + if ! (( IS_DEV )); then + REBUILT_BASE_IMAGES+=("$BRANCH=${BASE_IMAGE_REF%:*}:$NEXT_REV_TAG") + else + REBUILT_BASE_IMAGES+=("$BRANCH=$BASE_IMAGE_REF") + fi else echo "No rebuild necessary, we're done here." fi @@ -139,3 +149,21 @@ for BRANCH in "$@"; do echo "::endgroup::" done + +# Built the output which base images have actually been rebuilt as JSON +REBUILT_IMAGES="[" +for IMAGE in "${REBUILT_BASE_IMAGES[@]}"; do + REBUILT_IMAGES+=" \"$IMAGE\" " +done +REBUILT_IMAGES+="]" +echo "rebuilt_base_images=${REBUILT_IMAGES// /, }" | tee -a "${GITHUB_OUTPUT}" + +# Built the supported rolling tags matrix as JSON +SUPPORTED_TAGS="{" +for (( i=0; i < ${#SUPPORTED_ROLLING_TAGS[@]} ; i++ )); do + j=$((i+1)) + SUPPORTED_TAGS+="\"${!j}\": ${SUPPORTED_ROLLING_TAGS[$i]}" + (( i < ${#SUPPORTED_ROLLING_TAGS[@]}-1 )) && SUPPORTED_TAGS+=", " +done +SUPPORTED_TAGS+="}" +echo "supported_tag_matrix=$SUPPORTED_TAGS" | tee -a "$GITHUB_OUTPUT" From 6adc5c46aa2210d04c28fe24d2e27cd55178509a Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 00:43:41 +0200 Subject: [PATCH 905/970] ci(ct): remove draft of building app images in maintenance matrix job --- .github/workflows/scripts/maintenance-job.sh | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/scripts/maintenance-job.sh b/.github/workflows/scripts/maintenance-job.sh index df562aac573..e78bd4979fa 100755 --- a/.github/workflows/scripts/maintenance-job.sh +++ b/.github/workflows/scripts/maintenance-job.sh @@ -125,12 +125,10 @@ for BRANCH in "$@"; do fi echo "Determined these additional Maven tag options: $TAG_OPTIONS" - # 7. Let's build the base image if necessary - NEWER_BASE_IMAGE=0 + # 8. Let's build the base image if necessary if (( NEWER_JAVA_IMAGE + NEWER_PKGS + FORCE_BUILD > 0 )); then mvn -Pct -f modules/container-base deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS - NEWER_BASE_IMAGE=1 # Save the information about the immutable or rolling tag we just built if ! (( IS_DEV )); then @@ -142,11 +140,6 @@ for BRANCH in "$@"; do echo "No rebuild necessary, we're done here." fi - if (( NEWER_BASE_IMAGE )); then - echo "Built a new base image, should continue with application images now..." - # TODO: rebuild the app images here - fi - echo "::endgroup::" done From feeb1265060a9d60945b37b4f90c26c182078ec3 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 16:57:51 +0200 Subject: [PATCH 906/970] style,ci(ct): reword the maintenance build workflow name --- .github/workflows/container_maintenance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index d6ec6486a59..88718b435f7 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -21,7 +21,7 @@ env: jobs: build: - name: Build image + name: Base Image Matrix Build runs-on: ubuntu-latest permissions: contents: read From 130102032ce9edd3f873f10987166b38ed1db3b4 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 16:58:58 +0200 Subject: [PATCH 907/970] ci(ct): make the maintenance workflow push the hub description for the base image #10478 --- .github/workflows/container_base_push.yml | 13 +------ .github/workflows/container_maintenance.yml | 41 ++++++++++++++++----- modules/container-base/README.md | 19 ++++++---- 3 files changed, 45 insertions(+), 28 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 8a9a3212fe9..24b1102d3a7 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -13,6 +13,7 @@ on: paths: - 'modules/container-base/**' - '!modules/container-base/src/backports/**' + - '!modules/container-base/README.md' - 'modules/dataverse-parent/pom.xml' - '.github/workflows/container_base_push.yml' @@ -132,15 +133,3 @@ jobs: uses: ./.github/workflows/container_app_push.yml with: base-image-ref: ${{ needs.build.outputs.base-image-ref }} - - - # TODO: job to update the docker hub description with supported tags and all - #- if: ${{ github.event_name == 'push' && github.ref_name == env.DEVELOPMENT_BRANCH }} - # name: Push description to DockerHub - # uses: peter-evans/dockerhub-description@v3 - # with: - # username: ${{ secrets.DOCKERHUB_USERNAME }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} - # repository: gdcc/base - # short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" - # readme-filepath: ./modules/container-base/README.md diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 88718b435f7..2b1deaded5f 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -87,12 +87,35 @@ jobs: # with: # branch: ${{ matrix.branch }} - # TODO: job to update the docker hub description with supported tags and all - # - name: Push description to DockerHub - # uses: peter-evans/dockerhub-description@v3 - # with: - # username: ${{ secrets.DOCKERHUB_USERNAME }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} - # repository: gdcc/base - # short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" - # readme-filepath: ./modules/container-base/README.md + hub-description: + name: Push description to DockerHub + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + needs: build + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Render README + id: render + run: | + TAGS_JSON='${{ needs.build.outputs.supported_tag_matrix }}' + echo "$TAGS_JSON" | jq -r 'keys | sort | reverse | .[]' | + while IFS= read -r branch; do + echo \ + "- \`$( echo "$TAGS_JSON" | jq --arg v "$branch" -r '.[$v] | join("`, `")' )\`" \ + "([Dockerfile](https://github.com/IQSS/dataverse/blob/${branch}/modules/container-base/src/main/docker/Dockerfile)," \ + "[Patches](https://github.com/IQSS/dataverse/blob/develop/modules/container-base/src/backports/${branch}))" \ + | tee -a "${GITHUB_WORKSPACE}/tags.md" + done + sed -i -e "/<\!-- TAG BLOCK HERE -->/r ${GITHUB_WORKSPACE}/tags.md" "./modules/container-base/README.md" + + - name: Push description to DockerHub + uses: peter-evans/dockerhub-description@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + repository: gdcc/base + short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" + readme-filepath: ./modules/container-base/README.md \ No newline at end of file diff --git a/modules/container-base/README.md b/modules/container-base/README.md index dc4d185bbb5..925a053cf0b 100644 --- a/modules/container-base/README.md +++ b/modules/container-base/README.md @@ -31,17 +31,22 @@ to ask for help and guidance. ## Supported Image Tags This image is sourced within the main upstream code [repository of the Dataverse software](https://github.com/IQSS/dataverse). -Development and maintenance of the [image's code](https://github.com/IQSS/dataverse/tree/develop/modules/container-base) -happens there (again, by the community). Community-supported image tags are based on the two most important branches: +Development and maintenance of the [image's code](https://github.com/IQSS/dataverse/tree/develop/modules/container-base) happens there (again, by the community). +Community-supported image tags are based on the two most important branches: -- The `unstable` tag corresponds to the `develop` branch, where pull requests are merged. - ([`Dockerfile`](https://github.com/IQSS/dataverse/tree/develop/modules/container-base/src/main/docker/Dockerfile)) -- The `alpha` tag corresponds to the `master` branch, where releases are cut from. - ([`Dockerfile`](https://github.com/IQSS/dataverse/tree/master/modules/container-base/src/main/docker/Dockerfile)) +Our tagging is inspired by [Bitnami](https://docs.vmware.com/en/VMware-Tanzu-Application-Catalog/services/tutorials/GUID-understand-rolling-tags-containers-index.html>). +For more detailed information about our tagging policy, please read about our [base image tags](https://guides.dataverse.org/en/latest/container/base-image.html#supported-image-tags) in the Dataverse Containers Guide. + +For ease of use, here is a list of images that are currently maintained. + + + +All of them are rolling tags, except those ending with `-r`, which are immutable tags. +The `unstable` tags are the current development branch snapshot. Within the main repository, you may find the base image files at `/modules/container-base`. This Maven module uses the [Maven Docker Plugin](https://dmp.fabric8.io) to build and ship the image. -You may use, extend, or alter this image to your liking and/or host in some different registry if you want to. +You may use, extend, or alter this image to your liking and/or host in some different registry if you want to under the terms of the Apache 2.0 license. **Supported architectures:** This image is created as a "multi-arch image", supporting the most common architectures Dataverse usually runs on: AMD64 (Windows/Linux/...) and ARM64 (Apple M1/M2). From fbec22df4745e57b9a18a00dd9fe93ada036fbee Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 17:51:32 +0200 Subject: [PATCH 908/970] style(ct): fix simple typo in base image README --- modules/container-base/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/container-base/README.md b/modules/container-base/README.md index 925a053cf0b..da2a54064a4 100644 --- a/modules/container-base/README.md +++ b/modules/container-base/README.md @@ -34,7 +34,7 @@ This image is sourced within the main upstream code [repository of the Dataverse Development and maintenance of the [image's code](https://github.com/IQSS/dataverse/tree/develop/modules/container-base) happens there (again, by the community). Community-supported image tags are based on the two most important branches: -Our tagging is inspired by [Bitnami](https://docs.vmware.com/en/VMware-Tanzu-Application-Catalog/services/tutorials/GUID-understand-rolling-tags-containers-index.html>). +Our tagging is inspired by [Bitnami](https://docs.vmware.com/en/VMware-Tanzu-Application-Catalog/services/tutorials/GUID-understand-rolling-tags-containers-index.html). For more detailed information about our tagging policy, please read about our [base image tags](https://guides.dataverse.org/en/latest/container/base-image.html#supported-image-tags) in the Dataverse Containers Guide. For ease of use, here is a list of images that are currently maintained. From 6b7a9d618aae2fff50742a56c3f760dec3354ddf Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 17:52:49 +0200 Subject: [PATCH 909/970] fix(ct): remove bug from package upgrade detection in maintenance workflow We did not correctly compare the status code of the grep command, breaking the update detection --- .github/workflows/scripts/utils.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/scripts/utils.sh b/.github/workflows/scripts/utils.sh index b63787105ee..987b58d8bb5 100644 --- a/.github/workflows/scripts/utils.sh +++ b/.github/workflows/scripts/utils.sh @@ -42,13 +42,14 @@ function check_newer_pkgs() { PKGS="$2" docker run --rm -u 0 "${IMAGE}" sh -c "apt update >/dev/null 2>&1 && apt install -s ${PKGS}" | tee /proc/self/fd/2 | grep -q "0 upgraded" + STATUS=$? - if [[ ! $? ]]; then - echo "Base image $IMAGE needs updates for our custom installed packages" - return 0 - else + if [[ $STATUS -eq 0 ]]; then echo "Base image $IMAGE has no updates for our custom installed packages" return 1 + else + echo "Base image $IMAGE needs updates for our custom installed packages" + return 0 fi # TODO: In a future version of this script, we might want to include checking for other security updates, From b569fdff945ca30979b3546c11fdb183804c2975 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 19 Sep 2024 12:34:39 -0400 Subject: [PATCH 910/970] update query per review comments to handle all cases --- src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java index 9ec93095cef..98ac8ff387f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java @@ -1407,7 +1407,7 @@ public UploadSessionQuotaLimit getUploadSessionQuotaLimit(DvObjectContainer pare } public boolean isInReleasedVersion(Long id) { - Query query = em.createQuery("SELECT fm.id FROM FileMetadata fm, DvObject dvo WHERE fm.datasetVersion.id=(SELECT max(dv.id) FROM DatasetVersion dv WHERE dv.dataset.id=dvo.owner.id and dv.versionState=edu.harvard.iq.dataverse.DatasetVersion.VersionState.RELEASED) AND dvo.id=fm.dataFile.id AND fm.dataFile.id=:fid"); + Query query = em.createQuery("SELECT fm.id FROM FileMetadata fm, DvObject dvo WHERE fm.datasetVersion.id=(SELECT dv.id FROM DatasetVersion dv WHERE dv.dataset.id=dvo.owner.id and dv.versionState=edu.harvard.iq.dataverse.DatasetVersion.VersionState.RELEASED ORDER BY dv.versionNumber DESC, dv.minorVersionNumber DESC LIMIT 1) AND dvo.id=fm.dataFile.id AND fm.dataFile.id=:fid"); query.setParameter("fid", id); try { From 85731000382f5bda0394275b73f93b4ecd06d45f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 18:22:43 +0200 Subject: [PATCH 911/970] docs,style(ct): small rewording about immutable tags for base image --- modules/container-base/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/container-base/README.md b/modules/container-base/README.md index da2a54064a4..0598d709eac 100644 --- a/modules/container-base/README.md +++ b/modules/container-base/README.md @@ -41,8 +41,9 @@ For ease of use, here is a list of images that are currently maintained. -All of them are rolling tags, except those ending with `-r`, which are immutable tags. +All of them are rolling tags, except those ending with `-r`, which are the most recent immutable tags. The `unstable` tags are the current development branch snapshot. +We strongly recommend using only immutable tags for production use cases. Within the main repository, you may find the base image files at `/modules/container-base`. This Maven module uses the [Maven Docker Plugin](https://dmp.fabric8.io) to build and ship the image. From 9c91798f168e3d77e6347249ea9e053e3bce631a Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 18:23:55 +0200 Subject: [PATCH 912/970] feat,ci(ct): add immutable tags to list of base image tags in maintenance job As discussed during community meeting on 2024-09-19. --- .github/workflows/scripts/maintenance-job.sh | 25 +++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/scripts/maintenance-job.sh b/.github/workflows/scripts/maintenance-job.sh index e78bd4979fa..59e880e7a05 100755 --- a/.github/workflows/scripts/maintenance-job.sh +++ b/.github/workflows/scripts/maintenance-job.sh @@ -109,27 +109,25 @@ for BRANCH in "$@"; do TAG_OPTIONS="" if ! (( IS_DEV )); then TAG_OPTIONS="-Dbase.image=$BASE_IMAGE_REF -Ddocker.tags.revision=$NEXT_REV_TAG" - - # In case of the current release, add the "latest" tag as well. Also add to list of rolling tags. + # In case of the current release, add the "latest" tag as well. if (( IS_CURRENT_RELEASE )); then TAG_OPTIONS="$TAG_OPTIONS -Ddocker.tags.latest=latest" - SUPPORTED_ROLLING_TAGS+=("[\"latest\", \"${BASE_IMAGE_REF#*:}\"]") - else - SUPPORTED_ROLLING_TAGS+=("[\"${BASE_IMAGE_REF#*:}\"]") fi else UPCOMING_TAG=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image.tag -Dbase.image.tag.suffix="" -q -DforceStdout ) TAG_OPTIONS="-Ddocker.tags.develop=unstable -Ddocker.tags.upcoming=$UPCOMING_TAG" + # For the dev branch we only have rolling tags and can add them now already SUPPORTED_ROLLING_TAGS+=("[\"unstable\", \"$UPCOMING_TAG\", \"${BASE_IMAGE_REF#*:}\"]") fi echo "Determined these additional Maven tag options: $TAG_OPTIONS" # 8. Let's build the base image if necessary + NEWER_IMAGE=0 if (( NEWER_JAVA_IMAGE + NEWER_PKGS + FORCE_BUILD > 0 )); then mvn -Pct -f modules/container-base deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS - + NEWER_IMAGE=1 # Save the information about the immutable or rolling tag we just built if ! (( IS_DEV )); then REBUILT_BASE_IMAGES+=("$BRANCH=${BASE_IMAGE_REF%:*}:$NEXT_REV_TAG") @@ -140,6 +138,21 @@ for BRANCH in "$@"; do echo "No rebuild necessary, we're done here." fi + # 9. Add list of rolling and immutable tags for release builds + if ! (( IS_DEV )); then + RELEASE_TAGS_LIST="[" + if (( IS_CURRENT_RELEASE )); then + RELEASE_TAGS_LIST+="\"latest\", " + fi + RELEASE_TAGS_LIST+="\"${BASE_IMAGE_REF#*:}\", " + if (( NEWER_IMAGE )); then + RELEASE_TAGS_LIST+="\"$NEXT_REV_TAG\"]" + else + RELEASE_TAGS_LIST+="\"$CURRENT_REV_TAG\"]" + fi + SUPPORTED_ROLLING_TAGS+=("${RELEASE_TAGS_LIST}") + fi + echo "::endgroup::" done From 452849450e9890dfd54aecb2a4eaaa83e0404a70 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 18:24:24 +0200 Subject: [PATCH 913/970] style,ci(ct): add some more verbosity about progress in maintenance job --- .github/workflows/scripts/maintenance-job.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/scripts/maintenance-job.sh b/.github/workflows/scripts/maintenance-job.sh index 59e880e7a05..370988b9812 100755 --- a/.github/workflows/scripts/maintenance-job.sh +++ b/.github/workflows/scripts/maintenance-job.sh @@ -70,7 +70,9 @@ for BRANCH in "$@"; do cd "$MAINTENANCE_WORKSPACE/$BRANCH" # 2. Now let's apply the patches (we have them checked out in $GITHUB_WORKSPACE, not necessarily in this local checkout) + echo "Checking for patches..." if [[ -d ${GITHUB_WORKSPACE}/modules/container-base/src/backports/$BRANCH ]]; then + echo "Applying patches now." find "${GITHUB_WORKSPACE}/modules/container-base/src/backports/$BRANCH" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -s -i fi @@ -82,9 +84,11 @@ for BRANCH in "$@"; do else BASE_IMAGE_REF=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout ) fi + echo "Determined BASE_IMAGE_REF=$BASE_IMAGE_REF from Maven" # 4. Check for Temurin image updates JAVA_IMAGE_REF=$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=java.image -q -DforceStdout ) + echo "Determined JAVA_IMAGE_REF=$JAVA_IMAGE_REF from Maven" NEWER_JAVA_IMAGE=0 if check_newer_parent "$JAVA_IMAGE_REF" "$BASE_IMAGE_REF"; then NEWER_JAVA_IMAGE=1 @@ -92,6 +96,7 @@ for BRANCH in "$@"; do # 5. Check for package updates in base image PKGS="$( grep "ARG PKGS" modules/container-base/src/main/docker/Dockerfile | cut -f2 -d= | tr -d '"' )" + echo "Determined installed packages=\"$PKGS\" from Maven" NEWER_PKGS=0 # Don't bother with package checks if the java image is newer already if ! (( NEWER_JAVA_IMAGE )); then From 9b9ab1024f25d4634372d3d29ea99d9155f3c869 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 18:55:47 +0200 Subject: [PATCH 914/970] refactor,ci(ct): finishing touches for #10478 Re-enable and change everything necessary to reference the upstream IQSS context as of now. --- .github/workflows/container_app_push.yml | 10 ++- .github/workflows/container_base_push.yml | 67 ++++++++++----------- .github/workflows/container_maintenance.yml | 12 ++-- .github/workflows/maven_unit_test.yml | 4 ++ 4 files changed, 44 insertions(+), 49 deletions(-) diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index e3dbcd4d2c6..8e38a17209c 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -35,13 +35,11 @@ jobs: packages: write pull-requests: write # Only run in upstream repo - avoid unnecessary runs in forks - # TODO: re-enable for final PR - #if: ${{ github.repository_owner == 'IQSS' }} + if: ${{ github.repository_owner == 'IQSS' }} steps: - name: Checkout and Setup Maven - # TODO: change to upstream location in final PR - uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img + uses: IQSS/dataverse/.github/actions/setup-maven@develop with: pom-paths: | pom.xml @@ -51,6 +49,7 @@ jobs: # TODO: Add a filter step here, that avoids building the image if this is a PR and there are other files touched than declared above. # Use https://github.com/dorny/paths-filter to solve this. This will ensure we do not run this twice if this workflow # will be triggered by the other workflows already (base image or java changes) + # To become a part of #10618. - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests) run: > @@ -117,8 +116,7 @@ jobs: ( github.event_name != 'push' || ( github.event_name == 'push' && contains(fromJSON('["develop", "master"]'), github.ref_name))) steps: - name: Checkout and Setup Maven - # TODO: change to upstream location in final PR - uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img + uses: IQSS/dataverse/.github/actions/setup-maven@develop with: pom-paths: | pom.xml diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 24b1102d3a7..c2340576c78 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -7,8 +7,6 @@ on: - 'v[6-9].**' branches: - 'develop' - # TODO: delete for final PR - - '10478-version-base-img' # "Path filters are not evaluated for pushes of tags" https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#onpushpull_requestpull_request_targetpathspaths-ignore paths: - 'modules/container-base/**' @@ -17,13 +15,13 @@ on: - 'modules/dataverse-parent/pom.xml' - '.github/workflows/container_base_push.yml' + # These TODOs are left for #10618 # TODO: we are missing a workflow_call option here, so we can trigger this flow from pr comments and maven tests (keep the secrets availability in mind!) # TODO: we are missing a pull_request option here (filter for stuff that would trigger the maven runs!) so we can trigger preview builds for them when coming from the main repo (keep the secrets availability in mind!) env: PLATFORMS: linux/amd64,linux/arm64 - # TODO: set back to develop for final PR - DEVELOPMENT_BRANCH: 10478-version-base-img + DEVELOPMENT_BRANCH: develop jobs: build: @@ -32,16 +30,14 @@ jobs: permissions: contents: read packages: read - # TODO: re-enable for final PR # Only run in upstream repo - avoid unnecessary runs in forks - #if: ${{ github.repository_owner == 'IQSS' }} + if: ${{ github.repository_owner == 'IQSS' }} outputs: base-image-ref: ${{ steps.finalize.outputs.base-image-ref }} steps: - name: Checkout and Setup Maven - # TODO: change to upstream location in final PR - uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img + uses: IQSS/dataverse/.github/actions/setup-maven@develop with: pom-paths: modules/container-base/pom.xml @@ -55,34 +51,33 @@ jobs: # In case this is a push to develop, we care about buildtime. # Configure a remote ARM64 build host in addition to the local AMD64 in two steps. - # TODO: re-enable for final PR - #- name: Setup SSH agent - # if: ${{ github.event_name != 'schedule' }} - # uses: webfactory/ssh-agent@v0.9.0 - # with: - # ssh-private-key: ${{ secrets.BUILDER_ARM64_SSH_PRIVATE_KEY }} - #- name: Provide the known hosts key and the builder config - # if: ${{ github.event_name != 'schedule' }} - # run: | - # echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts - # mkdir -p modules/container-base/target/buildx-state/buildx/instances - # cat > modules/container-base/target/buildx-state/buildx/instances/maven << EOF - # { "Name": "maven", - # "Driver": "docker-container", - # "Dynamic": false, - # "Nodes": [{"Name": "maven0", - # "Endpoint": "unix:///var/run/docker.sock", - # "Platforms": [{"os": "linux", "architecture": "amd64"}], - # "DriverOpts": null, - # "Flags": ["--allow-insecure-entitlement=network.host"], - # "Files": null}, - # {"Name": "maven1", - # "Endpoint": "ssh://${{ secrets.BUILDER_ARM64_SSH_CONNECTION }}", - # "Platforms": [{"os": "linux", "architecture": "arm64"}], - # "DriverOpts": null, - # "Flags": ["--allow-insecure-entitlement=network.host"], - # "Files": null}]} - # EOF + - name: Setup SSH agent + if: ${{ github.event_name != 'schedule' }} + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.BUILDER_ARM64_SSH_PRIVATE_KEY }} + - name: Provide the known hosts key and the builder config + if: ${{ github.event_name != 'schedule' }} + run: | + echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts + mkdir -p modules/container-base/target/buildx-state/buildx/instances + cat > modules/container-base/target/buildx-state/buildx/instances/maven << EOF + { "Name": "maven", + "Driver": "docker-container", + "Dynamic": false, + "Nodes": [{"Name": "maven0", + "Endpoint": "unix:///var/run/docker.sock", + "Platforms": [{"os": "linux", "architecture": "amd64"}], + "DriverOpts": null, + "Flags": ["--allow-insecure-entitlement=network.host"], + "Files": null}, + {"Name": "maven1", + "Endpoint": "ssh://${{ secrets.BUILDER_ARM64_SSH_CONNECTION }}", + "Platforms": [{"os": "linux", "architecture": "arm64"}], + "DriverOpts": null, + "Flags": ["--allow-insecure-entitlement=network.host"], + "Files": null}]} + EOF # Determine the base image name we are going to use from here on - name: Determine base image name diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 2b1deaded5f..986fe25cdf5 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -26,18 +26,15 @@ jobs: permissions: contents: read packages: read - # TODO: re-enable for final PR # Only run in upstream repo - avoid unnecessary runs in forks - #if: ${{ github.repository_owner == 'IQSS' }} + if: ${{ github.repository_owner == 'IQSS' }} outputs: supported_tag_matrix: ${{ steps.execute.outputs.supported_tag_matrix }} rebuilt_base_images: ${{ steps.execute.outputs.rebuilt_base_images }} steps: - name: Checkout and Setup Maven - # TODO: change to upstream location in final PR - # Necessary as the checked out release branch might not contain the action as files - uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img + uses: IQSS/dataverse/.github/actions/setup-maven@develop with: pom-paths: modules/container-base/pom.xml @@ -68,7 +65,9 @@ jobs: run: | .github/workflows/scripts/maintenance-job.sh ${{ steps.discover.outputs.branches }} - # TODO: This job should become part of the matrix as an action, so we don't need to fiddle with matrix outputs hacks + # TODO: Use the needs.build.outputs.rebuilt_base_images with fromJSON() to create a matrix job. + # Must be a single rank matrix (vector), the branch and base image tag information ships as "branch=tag" string + # Will be part of working on #10618, app image versioned tags. #push-app-img: # name: "Rebase & Publish App Image" # permissions: @@ -77,7 +76,6 @@ jobs: # pull-requests: write # secrets: inherit # needs: - # - discover # - build # strategy: # fail-fast: false diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml index 1c9c6c7d5e3..a94b17a67ba 100644 --- a/.github/workflows/maven_unit_test.yml +++ b/.github/workflows/maven_unit_test.yml @@ -30,6 +30,7 @@ jobs: continue-on-error: ${{ matrix.experimental }} runs-on: ubuntu-latest steps: + # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - uses: actions/checkout@v3 - name: Set up JDK ${{ matrix.jdk }} @@ -95,6 +96,7 @@ jobs: # status: "Experimental" continue-on-error: ${{ matrix.experimental }} steps: + # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - uses: actions/checkout@v3 - name: Set up JDK ${{ matrix.jdk }} @@ -128,6 +130,7 @@ jobs: needs: integration-test name: Coverage Report Submission steps: + # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - uses: actions/checkout@v3 - uses: actions/setup-java@v3 @@ -159,6 +162,7 @@ jobs: # TODO: Add a filter step here, that avoids calling the app image release workflow if there are changes to the base image. # Use https://github.com/dorny/paths-filter to solve this. Will require and additional job or adding to integration-test job. # This way we ensure that we're not running the app image flow with a non-matching base image. + # To become a part of #10618. push-app-img: name: Publish App Image From aa3ea20a40396ce2801da8c7ff95499743db3d14 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 19 Sep 2024 14:43:40 -0400 Subject: [PATCH 915/970] add docs for disable-dataset-thumbnail-autoselect #10819 #10820 --- doc/sphinx-guides/source/installation/config.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index a3d0743216e..1433497063b 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3343,6 +3343,9 @@ please find all known feature flags below. Any of these flags can be activated u * - disable-return-to-author-reason - Removes the reason field in the `Publish/Return To Author` dialog that was added as a required field in v6.2 and makes the reason an optional parameter in the :ref:`return-a-dataset` API call. - ``Off`` + * - disable-dataset-thumbnail-autoselect + - Turns off automatic selection of a dataset thumbnail from image files in that dataset. When set to ``On``, a user can still manually pick a thumbnail image, or upload a dedicated thumbnail image. + - ``Off`` **Note:** Feature flags can be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_FEATURE_XXX`` (e.g. ``DATAVERSE_FEATURE_API_SESSION_AUTH=1``). These environment variables can be set in your shell before starting Payara. If you are using :doc:`Docker for development `, you can set them in the `docker compose `_ file. From 95859251bfab94c0071119fa7e4bbcd4fbff7cc4 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 18 Sep 2024 16:57:58 -0400 Subject: [PATCH 916/970] create 6.4 release notes and add about half the updates #10853 --- .../10169-JSON-schema-validation.md | 3 - ...se-support-address-in-system-email-text.md | 4 - doc/release-notes/10341-croissant.md | 9 - ...3-add-thumbnail-for-featured-dataverses.md | 5 - doc/release-notes/10508-base-image-fixes.md | 12 -- doc/release-notes/10517-datasetType.md | 10 -- ...k-functionality-same-permission-as-link.md | 2 - .../10606-dataverse-in-windows-wsl.md | 1 - .../10633-add-dataverse-api-extension.md | 1 - doc/release-notes/6.4-release-notes.md | 169 ++++++++++++++++++ doc/release-notes/7068-reserve-file-pids.md | 9 - doc/release-notes/8129-harvesting.md | 18 -- .../8578-support-for-iso-639-3-languages.md | 17 -- .../8581-add-crossref-pid-provider.md | 3 - .../8796-fix-license-display-indexing.md | 1 - ...pefiles-under-hidden-directories-in-zip.md | 5 - ...1-CC0-waiver-turned-into-custom-license.md | 6 - doc/release-notes/9317-delete-saved-search.md | 4 - doc/release-notes/api-blocking-filter-json.md | 3 - doc/release-notes/make-data-count-.md | 3 - doc/release-notes/permalink-base-urls.md | 10 -- 21 files changed, 169 insertions(+), 126 deletions(-) delete mode 100644 doc/release-notes/10169-JSON-schema-validation.md delete mode 100644 doc/release-notes/10287-use-support-address-in-system-email-text.md delete mode 100644 doc/release-notes/10341-croissant.md delete mode 100644 doc/release-notes/10433-add-thumbnail-for-featured-dataverses.md delete mode 100644 doc/release-notes/10508-base-image-fixes.md delete mode 100644 doc/release-notes/10517-datasetType.md delete mode 100644 doc/release-notes/10583-dataset-unlink-functionality-same-permission-as-link.md delete mode 100644 doc/release-notes/10606-dataverse-in-windows-wsl.md delete mode 100644 doc/release-notes/10633-add-dataverse-api-extension.md create mode 100644 doc/release-notes/6.4-release-notes.md delete mode 100644 doc/release-notes/7068-reserve-file-pids.md delete mode 100644 doc/release-notes/8129-harvesting.md delete mode 100644 doc/release-notes/8578-support-for-iso-639-3-languages.md delete mode 100644 doc/release-notes/8581-add-crossref-pid-provider.md delete mode 100644 doc/release-notes/8796-fix-license-display-indexing.md delete mode 100644 doc/release-notes/8945-ignore-shapefiles-under-hidden-directories-in-zip.md delete mode 100644 doc/release-notes/9081-CC0-waiver-turned-into-custom-license.md delete mode 100644 doc/release-notes/9317-delete-saved-search.md delete mode 100644 doc/release-notes/api-blocking-filter-json.md delete mode 100644 doc/release-notes/make-data-count-.md delete mode 100644 doc/release-notes/permalink-base-urls.md diff --git a/doc/release-notes/10169-JSON-schema-validation.md b/doc/release-notes/10169-JSON-schema-validation.md deleted file mode 100644 index 92ff4a917d5..00000000000 --- a/doc/release-notes/10169-JSON-schema-validation.md +++ /dev/null @@ -1,3 +0,0 @@ -### Improved JSON Schema validation for datasets - -Enhanced JSON schema validation with checks for required and allowed child objects, type checking for field types including `primitive`, `compound` and `controlledVocabulary`. More user-friendly error messages to help pinpoint the issues in the dataset JSON. See [Retrieve a Dataset JSON Schema for a Collection](https://guides.dataverse.org/en/6.3/api/native-api.html#retrieve-a-dataset-json-schema-for-a-collection) in the API Guide and PR #10543. diff --git a/doc/release-notes/10287-use-support-address-in-system-email-text.md b/doc/release-notes/10287-use-support-address-in-system-email-text.md deleted file mode 100644 index 4c294404298..00000000000 --- a/doc/release-notes/10287-use-support-address-in-system-email-text.md +++ /dev/null @@ -1,4 +0,0 @@ -### Notification Email Improvement - -The system email text has been improved to use the support email address (`dataverse.mail.support-email`) in the text where it states; "contact us for support at", instead of the default system email address. -Using the system email address here was particularly problematic when it was a 'noreply' address. diff --git a/doc/release-notes/10341-croissant.md b/doc/release-notes/10341-croissant.md deleted file mode 100644 index 15bc7029099..00000000000 --- a/doc/release-notes/10341-croissant.md +++ /dev/null @@ -1,9 +0,0 @@ -A new metadata export format called Croissant is now available as an external metadata exporter. It is oriented toward making datasets consumable by machine learning. - -When enabled, Croissant replaces the Schema.org JSON-LD format in the `` of dataset landing pages. For details, see the [Schema.org JSON-LD/Croissant Metadata](https://dataverse-guide--10533.org.readthedocs.build/en/10533/admin/discoverability.html#schema-org-head) under the discoverability section of the Admin Guide. - -For more about the Croissant exporter, see https://github.com/gdcc/exporter-croissant - -For installation instructions, see [Enabling External Exporters](https://dataverse-guide--10533.org.readthedocs.build/en/10533/installation/advanced.html#enabling-external-exporters) in the Installation Guide. - -See also Issue #10341 and PR #10533. diff --git a/doc/release-notes/10433-add-thumbnail-for-featured-dataverses.md b/doc/release-notes/10433-add-thumbnail-for-featured-dataverses.md deleted file mode 100644 index 0ebb84a8eb0..00000000000 --- a/doc/release-notes/10433-add-thumbnail-for-featured-dataverses.md +++ /dev/null @@ -1,5 +0,0 @@ -Add the ability to configure a thumbnail logo that is displayed for a collection when the collection is configured as a featured collection. If present, this thumbnail logo is shown. Otherwise, the collection logo is shown. Configuration is done under the "Theme" for a collection. - -The HTML preview of the documentation can be found [here](https://dataverse-guide--10433.org.readthedocs.build/en/10433/user/dataverse-management.html#theme). - -For more information, see [#10291](https://github.com/IQSS/dataverse/issues/10291). diff --git a/doc/release-notes/10508-base-image-fixes.md b/doc/release-notes/10508-base-image-fixes.md deleted file mode 100644 index 148066435e8..00000000000 --- a/doc/release-notes/10508-base-image-fixes.md +++ /dev/null @@ -1,12 +0,0 @@ -# Security and Compatibility Fixes to the Container Base Image - -- Switch "wait-for" to "wait4x", aligned with the Configbaker Image -- Update "jattach" to v2.2 -- Install AMD64 / ARM64 versions of tools as necessary -- Run base image as unprivileged user by default instead of `root` - this was an oversight from OpenShift changes -- Linux User, Payara Admin and Domain Master passwords: - - Print hints about default, public knowledge passwords in place for - - Enable replacing these passwords at container boot time -- Enable building with updates Temurin JRE image based on Ubuntu 24.04 LTS -- Fix entrypoint script troubles with pre- and postboot script files -- Unify location of files at CONFIG_DIR=/opt/payara/config, avoid writing to other places \ No newline at end of file diff --git a/doc/release-notes/10517-datasetType.md b/doc/release-notes/10517-datasetType.md deleted file mode 100644 index 2e3aff940c7..00000000000 --- a/doc/release-notes/10517-datasetType.md +++ /dev/null @@ -1,10 +0,0 @@ -### Initial Support for Dataset Types - -Out of the box, all datasets have the type "dataset" but superusers can add additional types. At this time the type can only be set at creation time via API. The types "dataset", "software", and "workflow" will be sent to DataCite when the dataset is published. - -For details see and #10517. Please note that this feature is highly experimental and is expected to evolve. - -Upgrade instructions --------------------- - -Update your Solr schema.xml file to pick up the "datasetType" additions and do a full reindex. diff --git a/doc/release-notes/10583-dataset-unlink-functionality-same-permission-as-link.md b/doc/release-notes/10583-dataset-unlink-functionality-same-permission-as-link.md deleted file mode 100644 index f97bd252db3..00000000000 --- a/doc/release-notes/10583-dataset-unlink-functionality-same-permission-as-link.md +++ /dev/null @@ -1,2 +0,0 @@ -New "Unlink Dataset" button has been added to the Dataset Page to allow a user to unlink a dataset from a collection that was previously linked with the "Link Dataset" button. The user must possess the same permissions needed to unlink the Dataset as they would to link the Dataset. -The [existing API](https://guides.dataverse.org/en/6.3/admin/dataverses-datasets.html#unlink-a-dataset) for unlinking datasets has been updated to no longer require superuser access. The "Publish Dataset" permission is now enough. diff --git a/doc/release-notes/10606-dataverse-in-windows-wsl.md b/doc/release-notes/10606-dataverse-in-windows-wsl.md deleted file mode 100644 index 9501d6e3090..00000000000 --- a/doc/release-notes/10606-dataverse-in-windows-wsl.md +++ /dev/null @@ -1 +0,0 @@ -New instructions have been added for developers on Windows trying to run a Dataverse development environment using Windows Subsystem for Linux (WSL). See https://dataverse-guide--10608.org.readthedocs.build/en/10608/developers/windows.html #10606 and #10608. diff --git a/doc/release-notes/10633-add-dataverse-api-extension.md b/doc/release-notes/10633-add-dataverse-api-extension.md deleted file mode 100644 index f5d8030e8ac..00000000000 --- a/doc/release-notes/10633-add-dataverse-api-extension.md +++ /dev/null @@ -1 +0,0 @@ -The addDataverse (/api/dataverses/{identifier}) API endpoint has been extended to allow adding metadata blocks, input levels and facet ids at creation time, as the Dataverse page in create mode does in JSF. diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md new file mode 100644 index 00000000000..2648f478f92 --- /dev/null +++ b/doc/release-notes/6.4-release-notes.md @@ -0,0 +1,169 @@ +# Dataverse 6.4 + +Please note: To read these instructions in full, please go to https://github.com/IQSS/dataverse/releases/tag/v6.4 rather than the list of releases, which will cut them off. + +This release brings new features, enhancements, and bug fixes to Dataverse. Thank you to all of the community members who contributed code, suggestions, bug reports, and other assistance across the project. + +## Release Highlights + +## Features + +### Full List of ISO 639-3 Languages Now Supported + +The controlled vocabulary values list for the metadata field "Language" in the citation block has now been extended to include roughly 7920 ISO 639-3 values. + +Some of the language entries in the pre-6.4 list correspond to "macro languages" in ISO-639-3 and admins/users may wish to update to use the corresponding individual language entries from ISO-639-3. As these cases are expected to be rare (they do not involve major world languages), finding them is not covered in the release notes. Anyone who desires help in this area is encouraged to reach out to the Dataverse community via any of the standard communication channels. + +ISO 639-3 codes were downloaded from [sil.org](https://iso639-3.sil.org/code_tables/download_tables#Complete%20Code%20Tables:~:text=iso%2D639%2D3_Code_Tables_20240415.zip) and the file used for merging with the existing citation.tsv was "iso-639-3.tab". See also #8578 and #10762. + +### Unlink Dataset Button + +New "Unlink Dataset" button has been added to the dataset page to allow a user to unlink a dataset from a collection. The user must possess the same permissions needed to unlink the dataset as they would to link the Dataset. Additionally, the [existing API](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#unlink-a-dataset) for unlinking datasets has been updated to no longer require superuser access as the "Publish Dataset" permission is now enough. See also #10583 and #10689. + +### Pre-Publish File DOI Reservation + +Dataverse installations using DataCite as a persistent identifier (PID) provider (or other providers that support reserving PIDs) will be able to reserve PIDs for files when they are uploaded (rather than at publication time). Note that reserving file DOIs can slow uploads with large numbers of files so administrators may need to adjust timeouts (specifically any Apache "``ProxyPass / ajp://localhost:8009/ timeout=``" setting in the recommended Dataverse configuration). + +### Initial Support for Dataset Types + +Out of the box, all datasets now have the type "dataset" but superusers can add additional types. At this time the type of a dataset can only be set at creation time via API. The types "dataset", "software", and "workflow" (just those three, for now) will be sent to DataCite when the dataset is published. + +For details see [the guides](https://guides.dataverse.org/en/6.4/user/dataset-management.html#dataset-types), #10517 and #10694. Please note that this feature is highly experimental and is expected to [evolve](https://github.com/IQSS/dataverse-pm/issues/307). + +### Collection Thumbnail Logo for Featured Collections + +Collections can now have a thumbnail logo that is displayed when the collection is configured as a featured collection. If present, this thumbnail logo is shown. Otherwise, the collection logo is shown. Configuration is done under the "Theme" for a collection as explained in [the guides](https://guides.dataverse.org/en/6.4/user/dataverse-management.html#theme). See also #10291 and #10433. + +### Croissant + +A new metadata export format called Croissant is now available as an external metadata exporter. It is oriented toward making datasets consumable by machine learning. + +When enabled, Croissant replaces the Schema.org JSON-LD format in the `` of dataset landing pages. For details, see the [Schema.org JSON-LD/Croissant Metadata](https://guides.dataverse.org/en/6.4/admin/discoverability.html#schema-org-head) under the discoverability section of the Admin Guide and #10382. + +For more about the Croissant exporter, including installation instructions, see . See also #10341, #10533, and [discussion](https://groups.google.com/g/dataverse-community/c/JI8HPgGarr8/m/DqEIkiwlAgAJ) on the mailing list. + +### Saved Searches Can Be Deleted + +Saved searches can now be deleted via API. See the [Saved Search](https://guides.dataverse.org/en/6.4/api/native-api.html#saved-search) section of the API Guide, #9317 and #10198. + +### Notification Email Improvement + +When notification emails are sent the part of the closing that says "contact us for support at" will now show the support email address (`dataverse.mail.support-email`), when configured, instead of the default system email address. Using the system email address here was particularly problematic when it was a "noreply" address. See also #10287 and #10504. + +### More Flexible PermaLinks + +The configuration setting `dataverse.pid.*.permalink.base-url`, which is used for PermaLinks, has been updated to support greater flexibility. Previously, the string `/citation?persistentId=` was automatically appended to the configured base URL. With this update, the base URL will now be used exactly as configured, without any automatic additions. See also #10775. + +### Development on Windows + +New instructions have been added for developers on Windows trying to run a Dataverse development environment using Windows Subsystem for Linux (WSL). See [the guides](https://guides.dataverse.org/en/6.4/developers/windows.html), #10606, and #10608. + +### Experimental Crossref PID (DOI) Provider + +Crossref can now be used as a PID (DOI) provider, but this feature is experimental. Please provide feedback through the usual channels. See also the [guides](https://guides.dataverse.org/en/6.4/installation/config.html#crossref-specific-settings), #8581, and #10806. + +### Improved JSON Schema Validation for Datasets + +JSON Schema validation has been enhanced with checks for required and allowed child objects as well as type checking for field types including `primitive`, `compound` and `controlledVocabulary`. More user-friendly error messages help pinpoint the issues in the dataset JSON. See [Retrieve a Dataset JSON Schema for a Collection](https://guides.dataverse.org/en/6.4/api/native-api.html#retrieve-a-dataset-json-schema-for-a-collection) in the API Guide, #10169, and #10543. + +### Counter Processor 1.05 Support (Make Data Count) + +Counter Processor 1.05 is now supported for use with Make Data Count. If you are running Counter Processor, you should reinstall/reconfigure it as described in the latest guides. Note that Counter Processor 1.05 requires Python 3, so you will need to follow the full Counter Processor install. Also note that if you configure the new version the same way, it will reprocess the days in the current month when it is first run. This is normal and will not affect the metrics in Dataverse. See also #10479. + +## Bug Fixes + +### No License Nor Terms of Use + +When datasets have neither a license nor custom terms of use, the dataset page will indicate this. Also, these datasets will no longer be indexed as having custom terms. See also #8796, #10513, and #10614. + +### CC0 License Bug Fix + +At a high level, some datasets have been mislabeled as "Custom License" when they should have been "CC0 1.0". This has been corrected. + +In Dataverse 5.10, datasets with only "CC0 Waiver" in the "termsofuse" field were converted to "Custom License" (instead of the CC0 1.0 license) through a SQL migration script (see #10634). On deployment of Dataverse 6.4, a new SQL migration script will be run automatically to correct this, changing these datasets to CC0. You can review the script in #10634, which only affect the following datasets: + +- The existing "Terms of Use" must be equal to "This dataset is made available under a Creative Commons CC0 license with the following additional/modified terms and conditions: CC0 Waiver" (this was set in #10634). +- The following terms fields must be empty: Confidentiality Declaration, Special Permissions, Restrictions, Citation Requirements, Depositor Requirements, Conditions, and Disclaimer. +- The license ID must not be assigned. + +The script will set the license ID to that of the CC0 1.0 license and remove the contents of "termsofuse" field. See also #9081 and #10634. + +### Remap oai_dc Export and Harvesting Format Fields: dc:type and dc:date + +The `oai_dc` export and harvesting format has had the following fields remapped: + +- dc:type was mapped to the field "Kind of Data". Now it is hard-coded to the word "Dataset". +- dc:date was mapped to the field "Production Date" when available and otherwise to "Publication Date". Now it is mapped the field "Publication Date" or the field used for the citation date, if set (see [Set Citation Date Field Type for a Dataset](https://guides.dataverse.org/en/6.4/api/native-api.html#set-citation-date-field-type-for-a-dataset)). + +In order for these changes to be reflected in existing datasets, a [reexport all](https://guides.dataverse.org/en/6.4/admin/metadataexport.html#batch-exports-through-the-api) should be run (mentioned below). See #8129 and #10737. + +### Zip File No Longer Misdetected as Shapefile (Hidden Directories) + +When detecting files types, Dataverse would previously detect a zip file as a shapefile if it contained [markers of a shapefile](https://guides.dataverse.org/en/6.4/developers/geospatial.html) in hidden directories. These hidden directories are now ignored when deciding if a zip file is a shapefile or not. See also #8945 and #10627. + +### Valid JSON in Error Response + +When any `ApiBlockingFilter` policy applies to a request, the JSON in the body of the error response is now valid JSON. See also #10085. + +### Docker Container Base Image Security and Compatibility + +- Switch "wait-for" to "wait4x", aligned with the Configbaker Image +- Update "jattach" to v2.2 +- Install AMD64 / ARM64 versions of tools as necessary +- Run base image as unprivileged user by default instead of `root` - this was an oversight from OpenShift changes +- Linux User, Payara Admin and Domain Master passwords: + - Print hints about default, public knowledge passwords in place for + - Enable replacing these passwords at container boot time +- Enable building with updates Temurin JRE image based on Ubuntu 24.04 LTS +- Fix entrypoint script troubles with pre- and postboot script files +- Unify location of files at CONFIG_DIR=/opt/payara/config, avoid writing to other places + +See also #10508, #10672 and #10722. + +## API + +### addDataverse + +The addDataverse (`/api/dataverses/{identifier}`) API endpoint has been extended to allow adding metadata blocks, input levels and facet IDs at creation time, as the Dataverse page in create mode does in JSF. See also [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#create-a-dataverse-collection) #10633 and #10644. + +## Major Use Cases + +- Users will have DOIs/PIDs reserved for their files as part of file upload instead of at publication time. (Issue #7068, PR #7334) + +## Settings + +The following settings have been added: + +- dataverse.pid.*.crossref.url +- dataverse.pid.*.crossref.rest-api-url +- dataverse.pid.*.crossref.username +- dataverse.pid.*.crossref.password +- dataverse.pid.*.crossref.depositor +- dataverse.pid.*.crossref.depositor-email + +### Backward incompatible changes + +The oai_dc export format has changed. See the "Remap oai_dc" section above. + +## Complete List of Changes + +## Getting Help + +## Upgrade Instructions + +In order for changes to the `oai_dc` metadata export format to be reflected in existing datasets, a [reexport all](https://guides.dataverse.org/en/6.4/admin/metadataexport.html#batch-exports-through-the-api) should be run. See also "remap oai_dc" above. + +Update your Solr schema.xml file to pick up the "dataset type" additions described above and do a full reindex. + +### Additional Upgrade Steps + +- If you currently use a PermaLink provider with a configured `base-url`: You must manually append + "/citation?persistentId=" to the existing base URL to maintain functionality. +- If you use a PermaLink provider without a configured `base-url`: No changes are required. + +6\. Update the Citation metadata block: + +``` +- `wget https://github.com/IQSS/dataverse/releases/download/v6.4/citation.tsv` +- `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @citation.tsv -H "Content-type: text/tab-separated-values"` +``` diff --git a/doc/release-notes/7068-reserve-file-pids.md b/doc/release-notes/7068-reserve-file-pids.md deleted file mode 100644 index 182a0d7f67b..00000000000 --- a/doc/release-notes/7068-reserve-file-pids.md +++ /dev/null @@ -1,9 +0,0 @@ -## Release Highlights - -### Pre-Publish File DOI Reservation with DataCite - -Dataverse installations using DataCite (or other persistent identifier (PID) Providers that support reserving PIDs) will be able to reserve PIDs for files when they are uploaded (rather than at publication time). Note that reserving file DOIs can slow uploads with large numbers of files so administrators may need to adjust timeouts (specifically any Apache "``ProxyPass / ajp://localhost:8009/ timeout=``" setting in the recommended Dataverse configuration). - -## Major Use Cases - -- Users will have DOIs/PIDs reserved for their files as part of file upload instead of at publication time. (Issue #7068, PR #7334) diff --git a/doc/release-notes/8129-harvesting.md b/doc/release-notes/8129-harvesting.md deleted file mode 100644 index 63ca8744941..00000000000 --- a/doc/release-notes/8129-harvesting.md +++ /dev/null @@ -1,18 +0,0 @@ -### Remap oai_dc export and harvesting format fields: dc:type and dc:date - -The `oai_dc` export and harvesting format has had the following fields remapped: - -- dc:type was mapped to the field "Kind of Data". Now it is hard-coded to the word "Dataset". -- dc:date was mapped to the field "Production Date" when available and otherwise to "Publication Date". Now it is mapped the field "Publication Date" or the field used for the citation date, if set (see [Set Citation Date Field Type for a Dataset](https://guides.dataverse.org/en/6.3/api/native-api.html#set-citation-date-field-type-for-a-dataset)). - -In order for these changes to be reflected in existing datasets, a [reexport all](https://guides.dataverse.org/en/6.3/admin/metadataexport.html#batch-exports-through-the-api) should be run. - -For more information, please see #8129 and #10737. - -### Backward incompatible changes - -See the "Remap oai_dc export" section above. - -### Upgrade instructions - -In order for changes to the `oai_dc` metadata export format to be reflected in existing datasets, a [reexport all](https://guides.dataverse.org/en/6.3/admin/metadataexport.html#batch-exports-through-the-api) should be run. diff --git a/doc/release-notes/8578-support-for-iso-639-3-languages.md b/doc/release-notes/8578-support-for-iso-639-3-languages.md deleted file mode 100644 index c702b6b8a59..00000000000 --- a/doc/release-notes/8578-support-for-iso-639-3-languages.md +++ /dev/null @@ -1,17 +0,0 @@ -The Controlled Vocabulary Values list for the metadata field Language in the Citation block has now been extended to include roughly 7920 ISO 639-3 values. -- Some of the language entries in the pre-v.6.4 list correspond to "macro languages" in ISO-639-3 and admins/users may wish to update to use the corresponding individual language entries from ISO-639-3. As these cases are expected to be rare (they do not involve major world languages), finding them is not covered in the release notes. Anyone who desires help in this area is encouraged to reach out to the Dataverse community via any of the standard communication channels. -- ISO 639-3 codes were downloaded from: -``` -https://iso639-3.sil.org/code_tables/download_tables#Complete%20Code%20Tables:~:text=iso%2D639%2D3_Code_Tables_20240415.zip -``` -- The file used for merging with the existing citation.tsv was iso-639-3.tab - -To be added to the 6.4 release instructions: - -### Additional Upgrade Steps -6\. Update the Citation metadata block: - -``` -- `wget https://github.com/IQSS/dataverse/releases/download/v6.4/citation.tsv` -- `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @citation.tsv -H "Content-type: text/tab-separated-values"` -``` diff --git a/doc/release-notes/8581-add-crossref-pid-provider.md b/doc/release-notes/8581-add-crossref-pid-provider.md deleted file mode 100644 index 3610aa6d7cc..00000000000 --- a/doc/release-notes/8581-add-crossref-pid-provider.md +++ /dev/null @@ -1,3 +0,0 @@ -Added CrossRef DOI Pid Provider - -See Installation Configuration document for JVM Settings to enable CrossRef as a Pid Provider diff --git a/doc/release-notes/8796-fix-license-display-indexing.md b/doc/release-notes/8796-fix-license-display-indexing.md deleted file mode 100644 index ebded088875..00000000000 --- a/doc/release-notes/8796-fix-license-display-indexing.md +++ /dev/null @@ -1 +0,0 @@ -When datasets have neither a license nor custom terms of use the display will indicate this. Also, these datasets will no longer be indexed as having custom terms. diff --git a/doc/release-notes/8945-ignore-shapefiles-under-hidden-directories-in-zip.md b/doc/release-notes/8945-ignore-shapefiles-under-hidden-directories-in-zip.md deleted file mode 100644 index 145ae5f6d55..00000000000 --- a/doc/release-notes/8945-ignore-shapefiles-under-hidden-directories-in-zip.md +++ /dev/null @@ -1,5 +0,0 @@ -### Shapefile Handling will now ignore files under a hidden directory within the zip file - -Directories that are hidden will be ignored when determining if a zip file contains Shapefile files. - -For more information, see #8945. \ No newline at end of file diff --git a/doc/release-notes/9081-CC0-waiver-turned-into-custom-license.md b/doc/release-notes/9081-CC0-waiver-turned-into-custom-license.md deleted file mode 100644 index 042b2ec39fd..00000000000 --- a/doc/release-notes/9081-CC0-waiver-turned-into-custom-license.md +++ /dev/null @@ -1,6 +0,0 @@ -In an earlier Dataverse release, Datasets with only 'CC0 Waiver' in termsofuse field were converted to 'Custom License' instead of CC0 1.0 licenses during an automated process. A new process was added to correct this. Only Datasets with no terms other than the one create by the previous process will be modified. -- The existing 'Terms of Use' must be equal to 'This dataset is made available under a Creative Commons CC0 license with the following additional/modified terms and conditions: CC0 Waiver' -- The following terms fields must be empty: Confidentiality Declaration, Special Permissions, Restrictions, Citation Requirements, Depositor Requirements, Conditions, and Disclaimer. -- The License ID must not be assigned. - -This process will set the License ID to that of the CC0 1.0 license and remove the contents of termsofuse field. diff --git a/doc/release-notes/9317-delete-saved-search.md b/doc/release-notes/9317-delete-saved-search.md deleted file mode 100644 index 34723801036..00000000000 --- a/doc/release-notes/9317-delete-saved-search.md +++ /dev/null @@ -1,4 +0,0 @@ -### Saved search deletion - -Saved searches can now be removed using API `/api/admin/savedsearches/$id`. See PR #10198. -This is reflected in the [Saved Search Native API section](https://dataverse-guide--10198.org.readthedocs.build/en/10198/api/native-api.html#saved-search) of the Guide. \ No newline at end of file diff --git a/doc/release-notes/api-blocking-filter-json.md b/doc/release-notes/api-blocking-filter-json.md deleted file mode 100644 index 337ff82dd8b..00000000000 --- a/doc/release-notes/api-blocking-filter-json.md +++ /dev/null @@ -1,3 +0,0 @@ -* When any `ApiBlockingFilter` policy applies to a request, the JSON in the body of the error response is now valid JSON. - In case an API client did any special processing to allow it to parse the body, that is no longer necessary. - The status code of such responses has not changed. diff --git a/doc/release-notes/make-data-count-.md b/doc/release-notes/make-data-count-.md deleted file mode 100644 index 9022582dddb..00000000000 --- a/doc/release-notes/make-data-count-.md +++ /dev/null @@ -1,3 +0,0 @@ -### Counter Processor 1.05 Support - -This release includes support for counter-processor-1.05 for processing Make Data Count metrics. If you are running Make Data Counts support, you should reinstall/reconfigure counter-processor as described in the latest Guides. (For existing installations, note that counter-processor-1.05 requires a Python3, so you will need to follow the full counter-processor install. Also note that if you configure the new version the same way, it will reprocess the days in the current month when it is first run. This is normal and will not affect the metrics in Dataverse.) diff --git a/doc/release-notes/permalink-base-urls.md b/doc/release-notes/permalink-base-urls.md deleted file mode 100644 index 1dd74057351..00000000000 --- a/doc/release-notes/permalink-base-urls.md +++ /dev/null @@ -1,10 +0,0 @@ -The configuration setting `dataverse.pid.*.permalink.base-url`, which is used for PermaLinks, has been updated to -support greater flexibility. Previously, the string "/citation?persistentId=" was automatically appended to the -configured base URL. With this update, the base URL will now be used exactly as configured, without any automatic -additions. - -**Upgrade instructions:** - -- If you currently use a PermaLink provider with a configured `base-url`: You must manually append - "/citation?persistentId=" to the existing base URL to maintain functionality. -- If you use a PermaLink provider without a configured `base-url`: No changes are required. \ No newline at end of file From c69e78337c6073649928720a3ee1ad6703687847 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 19 Sep 2024 15:12:51 -0400 Subject: [PATCH 917/970] add second half of snippets #10853 --- .../10726-dataverse-facets-api-extension.md | 3 - ...ublication-status-to-search-api-results.md | 14 -- ...st-metadatablocks-display-on-create-fix.md | 1 - doc/release-notes/10744-ro-crate-docs.md | 3 - ...ataverse-user-permissions-api-extension.md | 1 - doc/release-notes/10758-rust-client.md | 3 - .../10797-update-current-version-bug-fix.md | 11 -- .../10800-add-dataverse-request-json-fix.md | 1 - .../10810-search-api-payload-extensions.md | 52 ------ .../10819-publish-thumbnail-bug.md | 6 - doc/release-notes/6.4-release-notes.md | 153 ++++++++++++++++-- 11 files changed, 144 insertions(+), 104 deletions(-) delete mode 100644 doc/release-notes/10726-dataverse-facets-api-extension.md delete mode 100644 doc/release-notes/10733-add-publication-status-to-search-api-results.md delete mode 100644 doc/release-notes/10741-list-metadatablocks-display-on-create-fix.md delete mode 100644 doc/release-notes/10744-ro-crate-docs.md delete mode 100644 doc/release-notes/10749-dataverse-user-permissions-api-extension.md delete mode 100644 doc/release-notes/10758-rust-client.md delete mode 100644 doc/release-notes/10797-update-current-version-bug-fix.md delete mode 100644 doc/release-notes/10800-add-dataverse-request-json-fix.md delete mode 100644 doc/release-notes/10810-search-api-payload-extensions.md delete mode 100644 doc/release-notes/10819-publish-thumbnail-bug.md diff --git a/doc/release-notes/10726-dataverse-facets-api-extension.md b/doc/release-notes/10726-dataverse-facets-api-extension.md deleted file mode 100644 index baf6f798e35..00000000000 --- a/doc/release-notes/10726-dataverse-facets-api-extension.md +++ /dev/null @@ -1,3 +0,0 @@ -New optional query parameter "returnDetails" added to "dataverses/{identifier}/facets/" endpoint to include detailed information of each DataverseFacet. - -New endpoint "datasetfields/facetables" that lists all facetable dataset fields defined in the installation. diff --git a/doc/release-notes/10733-add-publication-status-to-search-api-results.md b/doc/release-notes/10733-add-publication-status-to-search-api-results.md deleted file mode 100644 index d015a50a00d..00000000000 --- a/doc/release-notes/10733-add-publication-status-to-search-api-results.md +++ /dev/null @@ -1,14 +0,0 @@ -Search API (/api/search) response will now include publicationStatuses in the Json response as long as the list is not empty - -Example: -```javascript -"items": [ - { - "name": "Darwin's Finches", - ... - "publicationStatuses": [ - "Unpublished", - "Draft" - ], -(etc, etc) -``` diff --git a/doc/release-notes/10741-list-metadatablocks-display-on-create-fix.md b/doc/release-notes/10741-list-metadatablocks-display-on-create-fix.md deleted file mode 100644 index 4edadcaa1fc..00000000000 --- a/doc/release-notes/10741-list-metadatablocks-display-on-create-fix.md +++ /dev/null @@ -1 +0,0 @@ -Fixed dataverses/{identifier}/metadatablocks endpoint to not return fields marked as displayOnCreate=true if there is an input level with include=false, when query parameters returnDatasetFieldTypes=true and onlyDisplayedOnCreate=true are set. diff --git a/doc/release-notes/10744-ro-crate-docs.md b/doc/release-notes/10744-ro-crate-docs.md deleted file mode 100644 index 9d52b4578b4..00000000000 --- a/doc/release-notes/10744-ro-crate-docs.md +++ /dev/null @@ -1,3 +0,0 @@ -## RO-Crate Support (Metadata Export) - -Dataverse now supports [RO-Crate](https://www.researchobject.org/ro-crate/) in the sense that dataset metadata can be exported in that format. This functionality is not available out of the box but you can enable one or more RO-Crate exporters from the [list of external exporters](https://preview.guides.gdcc.io/en/develop/installation/advanced.html#inventory-of-external-exporters). See also #10744. diff --git a/doc/release-notes/10749-dataverse-user-permissions-api-extension.md b/doc/release-notes/10749-dataverse-user-permissions-api-extension.md deleted file mode 100644 index 706b1f42641..00000000000 --- a/doc/release-notes/10749-dataverse-user-permissions-api-extension.md +++ /dev/null @@ -1 +0,0 @@ -New API endpoint "dataverses/{identifier}/userPermissions" for obtaining the user permissions on a dataverse. diff --git a/doc/release-notes/10758-rust-client.md b/doc/release-notes/10758-rust-client.md deleted file mode 100644 index e206f27ce65..00000000000 --- a/doc/release-notes/10758-rust-client.md +++ /dev/null @@ -1,3 +0,0 @@ -### Rust API client library - -An API client library for the Rust programming language is now available at https://github.com/gdcc/rust-dataverse and has been added to the [list of client libraries](https://dataverse-guide--10758.org.readthedocs.build/en/10758/api/client-libraries.html) in the API Guide. See also #10758. diff --git a/doc/release-notes/10797-update-current-version-bug-fix.md b/doc/release-notes/10797-update-current-version-bug-fix.md deleted file mode 100644 index 2cfaf69cad3..00000000000 --- a/doc/release-notes/10797-update-current-version-bug-fix.md +++ /dev/null @@ -1,11 +0,0 @@ -A significant bug in the superuser-only "Update-Current-Version" publication was found and fixed in this release. If the Update-Current-Version option was used when changes were made to the dataset Terms (rather than to dataset metadata), or if the PID provider service was down/returned an error, the update would fail and render the dataset unusable and require restoration from a backup. The fix in this release allows the update to succeed in both of these cases and redesigns the functionality such that any unknown issues should not make the dataset unusable (i.e. the error would be reported and the dataset would remain in its current state with the last-published version as it was and changes still in the draft version.) - -Users of earlier Dataverse releases are encouraged to alert their superusers to this issue. Those who wish to disable this functionality have two options: -* Change the dataset.updateRelease entry in the Bundle.properties file (or local language version) to "Do Not Use" or similar (doesn't disable but alerts superusers to the issue), or -* Edit the dataset.xhtml file to remove the lines - - - - - -, delete the contents of the generated and osgi-cache directories in the Dataverse Payara domain, and restart the Payara server. diff --git a/doc/release-notes/10800-add-dataverse-request-json-fix.md b/doc/release-notes/10800-add-dataverse-request-json-fix.md deleted file mode 100644 index ddd6c388ec6..00000000000 --- a/doc/release-notes/10800-add-dataverse-request-json-fix.md +++ /dev/null @@ -1 +0,0 @@ -Fixed the "addDataverse" API endpoint (/dataverses/{id} POST) expected request JSON structure to parse facetIds as described in the docs. \ No newline at end of file diff --git a/doc/release-notes/10810-search-api-payload-extensions.md b/doc/release-notes/10810-search-api-payload-extensions.md deleted file mode 100644 index 5112d9f62ee..00000000000 --- a/doc/release-notes/10810-search-api-payload-extensions.md +++ /dev/null @@ -1,52 +0,0 @@ -Search API (/api/search) response will now include new fields for the different entities. - -For Dataverse: - -- "affiliation" -- "parentDataverseName" -- "parentDataverseIdentifier" -- "image_url" (optional) - -```javascript -"items": [ - { - "name": "Darwin's Finches", - ... - "affiliation": "Dataverse.org", - "parentDataverseName": "Root", - "parentDataverseIdentifier": "root", - "image_url":"data:image/png;base64,iVBORw0..." -(etc, etc) -``` - -For DataFile: - -- "releaseOrCreateDate" -- "image_url" (optional) - -```javascript -"items": [ - { - "name": "test.txt", - ... - "releaseOrCreateDate": "2016-05-10T12:53:39Z", - "image_url":"data:image/png;base64,iVBORw0..." -(etc, etc) -``` - -For Dataset: - -- "image_url" (optional) - -```javascript -"items": [ - { - ... - "image_url": "http://localhost:8080/api/datasets/2/logo" - ... -(etc, etc) -``` - -The image_url field was already part of the SolrSearchResult JSON (and incorrectly appeared in Search API documentation), but it wasn’t returned by the API because it was appended only after the Solr query was executed in the SearchIncludeFragment of JSF. Now, the field is set in SearchServiceBean, ensuring it is always returned by the API when an image is available. - -The schema.xml file for Solr has been updated to include a new field called dvParentAlias for supporting the new response field "parentDataverseIdentifier". So for the next Dataverse released version, a Solr reindex will be necessary to apply the new schema.xml version. diff --git a/doc/release-notes/10819-publish-thumbnail-bug.md b/doc/release-notes/10819-publish-thumbnail-bug.md deleted file mode 100644 index 46c9875a6ef..00000000000 --- a/doc/release-notes/10819-publish-thumbnail-bug.md +++ /dev/null @@ -1,6 +0,0 @@ -The initial release of the Dataverse v6.3 introduced a bug where publishing would break the dataset thumbnail, which in turn broke the rendering of the parent Collection ("dataverse") page. This problem was fixed in the PR 10820. - -This bug fix will prevent this from happening in the future, but does not fix any existing broken links. To restore any broken thumbnails caused by this bug, you can call the http://localhost:8080/api/admin/clearThumbnailFailureFlag API, which will attempt to clear the flag on all files (regardless of whether caused by this bug or some other problem with the file) or the http://localhost:8080/api/admin/clearThumbnailFailureFlag/id to clear the flag for individual files. Calling the former, batch API is recommended. - -Additionally, the same PR made it possible to turn off the feature that automatically selects of one of the image datafiles to serve as the thumbnail of the parent dataset. An admin can turn it off by raising the feature flag `-Ddataverse.feature.disable-dataset-thumbnail-autoselect=true`. When the feature is disabled, a user can still manually pick a thumbnail image, or upload a dedicated thumbnail image. - diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 2648f478f92..feb8e54102a 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -6,6 +6,8 @@ This release brings new features, enhancements, and bug fixes to Dataverse. Than ## Release Highlights +- Users will have DOIs/PIDs reserved for their files as part of file upload instead of at publication time. (Issue #7068, PR #7334) + ## Features ### Full List of ISO 639-3 Languages Now Supported @@ -34,14 +36,22 @@ For details see [the guides](https://guides.dataverse.org/en/6.4/user/dataset-ma Collections can now have a thumbnail logo that is displayed when the collection is configured as a featured collection. If present, this thumbnail logo is shown. Otherwise, the collection logo is shown. Configuration is done under the "Theme" for a collection as explained in [the guides](https://guides.dataverse.org/en/6.4/user/dataverse-management.html#theme). See also #10291 and #10433. -### Croissant +### Croissant Support (Metadata Export) -A new metadata export format called Croissant is now available as an external metadata exporter. It is oriented toward making datasets consumable by machine learning. +A new metadata export format called [Croissant](https://github.com/mlcommons/croissant) is now available as an external metadata exporter. It is oriented toward making datasets consumable by machine learning. -When enabled, Croissant replaces the Schema.org JSON-LD format in the `` of dataset landing pages. For details, see the [Schema.org JSON-LD/Croissant Metadata](https://guides.dataverse.org/en/6.4/admin/discoverability.html#schema-org-head) under the discoverability section of the Admin Guide and #10382. +When enabled, Croissant replaces the Schema.org JSON-LD format in the `` of dataset landing pages. For details, see the [Schema.org JSON-LD/Croissant Metadata](https://guides.dataverse.org/en/6.4/admin/discoverability.html#schema-org-head) under the discoverability section of the Admin Guide. This change was made in Dataverse 6.3 in #10382. For more about the Croissant exporter, including installation instructions, see . See also #10341, #10533, and [discussion](https://groups.google.com/g/dataverse-community/c/JI8HPgGarr8/m/DqEIkiwlAgAJ) on the mailing list. +### RO-Crate Support (Metadata Export) + +Dataverse now supports [RO-Crate](https://www.researchobject.org/ro-crate/) as a metadata export format. This functionality is not available out of the box, but you can enable one or more RO-Crate exporters from the [list of external exporters](https://guides.dataverse.org/en/6.4/installation/advanced.html#inventory-of-external-exporters). See also #10744 and #10796. + +### Rust API Client Library + +An Dataverse API client library for the Rust programming language is now available at https://github.com/gdcc/rust-dataverse and has been added to the [list of client libraries](https://guides.dataverse.org/en/6.4/api/client-libraries.html) in the API Guide. See also #10758. + ### Saved Searches Can Be Deleted Saved searches can now be deleted via API. See the [Saved Search](https://guides.dataverse.org/en/6.4/api/native-api.html#saved-search) section of the API Guide, #9317 and #10198. @@ -50,6 +60,10 @@ Saved searches can now be deleted via API. See the [Saved Search](https://guides When notification emails are sent the part of the closing that says "contact us for support at" will now show the support email address (`dataverse.mail.support-email`), when configured, instead of the default system email address. Using the system email address here was particularly problematic when it was a "noreply" address. See also #10287 and #10504. +### Ability to Disable Automatic Thumbnail Selection + +It is now possible to turn off the feature that automatically selects of one of the image datafiles to serve as the thumbnail of the parent dataset. An admin can turn it off by enabling the [feature flag](https://guides.dataverse.org/en/6.4/installation/config.html#feature-flags) `dataverse.feature.disable-dataset-thumbnail-autoselect`. When the feature is disabled, a user can still manually pick a thumbnail image, or upload a dedicated thumbnail image. See also #10820. + ### More Flexible PermaLinks The configuration setting `dataverse.pid.*.permalink.base-url`, which is used for PermaLinks, has been updated to support greater flexibility. Previously, the string `/citation?persistentId=` was automatically appended to the configured base URL. With this update, the base URL will now be used exactly as configured, without any automatic additions. See also #10775. @@ -72,7 +86,34 @@ Counter Processor 1.05 is now supported for use with Make Data Count. If you are ## Bug Fixes -### No License Nor Terms of Use +### Update Current Version + +A significant bug in the superuser-only [Update Current Version](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#make-metadata-updates-without-changing-dataset-version) publication option was fixed. If the "Update Current Version" option was used when changes were made to the dataset terms (rather than to dataset metadata) or if the PID provider service was down or returned an error, the update would fail and render the dataset unusable and require restoration from a backup. The fix in this release allows the update to succeed in both of these cases and redesigns the functionality such that any unknown issues should not make the dataset unusable (i.e. the error would be reported and the dataset would remain in its current state with the last-published version as it was and changes still in the draft version.) + +If you do not plan to upgrade to Dataverse 6.4 right away, you are encouraged to alert your superusers to this issue (see [this post](https://groups.google.com/g/dataverse-community/c/evn5C-pyrS8/m/JrH9vp47DwAJ)). Here are some workarounds for pre-6.4 versions: + +* Change the "dataset.updateRelease" entry in the Bundle.properties file (or local language version) to "Do Not Use" or similar (this doesn't disable the button but alerts superusers to the issue), or +* Edit the dataset.xhtml file to remove the lines below, delete the contents of the generated and osgi-cache directories in the Dataverse Payara domain, and restart the Payara server. This will remove the "Update Current Version" from the UI. + +``` + + + +``` + +Again, the workarounds above are only for pre-6.4 versions. The bug has been fixed in Dataverse 6.4. See also #10797. + +### Broken Thumbnails + +Dataverse 6.3 introduced a bug where publishing would break the dataset thumbnail, which in turn broke the rendering of the parent collection (dataverse) page. + +This bug has been fixed but any existing broken thumbnails must be fixed manually. See "clearThumbnailFailureFlag" in the upgrade instructions below. + +Additionally, it is now possible to turn off the feature that automatically selects of one of the image datafiles to serve as the thumbnail of the parent dataset. An admin can turn it off by raising the feature flag `-Ddataverse.feature.disable-dataset-thumbnail-autoselect=true`. When the feature is disabled, a user can still manually pick a thumbnail image, or upload a dedicated thumbnail image. + +See also #10819, #10820, and [the post](https://groups.google.com/g/dataverse-community/c/evn5C-pyrS8/m/JrH9vp47DwAJ) on the mailing list. + +### No License, No Terms of Use When datasets have neither a license nor custom terms of use, the dataset page will indicate this. Also, these datasets will no longer be indexed as having custom terms. See also #8796, #10513, and #10614. @@ -122,18 +163,108 @@ See also #10508, #10672 and #10722. ## API -### addDataverse +### Search API: affiliation, parentDataverseName, image_url, etc. -The addDataverse (`/api/dataverses/{identifier}`) API endpoint has been extended to allow adding metadata blocks, input levels and facet IDs at creation time, as the Dataverse page in create mode does in JSF. See also [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#create-a-dataverse-collection) #10633 and #10644. +The Search API (`/api/search`) response now includes additional fields, depending on the type. -## Major Use Cases +For collections (dataverses): -- Users will have DOIs/PIDs reserved for their files as part of file upload instead of at publication time. (Issue #7068, PR #7334) +- "affiliation" +- "parentDataverseName" +- "parentDataverseIdentifier" +- "image_url" (optional) + +```javascript +"items": [ + { + "name": "Darwin's Finches", + ... + "affiliation": "Dataverse.org", + "parentDataverseName": "Root", + "parentDataverseIdentifier": "root", + "image_url":"data:image/png;base64,iVBORw0..." +(etc, etc) +``` + +For datasets: + +- "image_url" (optional) + +```javascript +"items": [ + { + ... + "image_url": "http://localhost:8080/api/datasets/2/logo" + ... +(etc, etc) +``` + +For files: + +- "releaseOrCreateDate" +- "image_url" (optional) + +```javascript +"items": [ + { + "name": "test.txt", + ... + "releaseOrCreateDate": "2016-05-10T12:53:39Z", + "image_url":"data:image/png;base64,iVBORw0..." +(etc, etc) +``` + +These examples are also shown in the [Search API](https://guides.dataverse.org/en/6.4/api/search.html) section of the API Guide. + +The image_url field was already part of the SolrSearchResult JSON (and incorrectly appeared in Search API documentation), but it wasn't returned by the API because it was appended only after the Solr query was executed in the SearchIncludeFragment of JSF. Now, the field is set in SearchServiceBean, ensuring it is always returned by the API when an image is available. + +The Solr schema.xml file has been updated to include a new field called "dvParentAlias" for supporting the new response field "parentDataverseIdentifier". See upgrade instructions below. + +See also #10810 and #10811. + +### Search API: publicationStatuses + +The Search API (`/api/search`) response will now include publicationStatuses in the JSON response as long as the list is not empty. + +Example: + +```javascript +"items": [ + { + "name": "Darwin's Finches", + ... + "publicationStatuses": [ + "Unpublished", + "Draft" + ], +(etc, etc) +``` + +See also #10733 and #10738. + +### Search Facet Information Exposed + +A new endpoint `/api/datasetfields/facetables` lists all facetable dataset fields defined in the installation, as described in [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#list-all-facetable-dataset-fields). + +A new optional query parameter "returnDetails" added to `/api/dataverses/{identifier}/facets/` endpoint to include detailed information of each DataverseFacet, as described in [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#list-facets-configured-for-a-dataverse-collection). See also #10726 and #10727. + +### User Permissions on Collections + +A new endpoint at `/api/dataverses/{identifier}/userPermissions` for obtaining the user permissions on a collection (dataverse) has been added. See also #10749 and #10751. + +### addDataverse Extended + +The addDataverse (`/api/dataverses/{identifier}`) API endpoint has been extended to allow adding metadata blocks, input levels and facet IDs at creation time, as the Dataverse page in create mode does in JSF. See also [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#create-a-dataverse-collection), #10633 and #10644. + +### Metadata Blocks and Display on Create + +The `/api/dataverses/{identifier}/metadatablocks` endpoint has been fixed to not return fields marked as displayOnCreate=true if there is an input level with include=false, when query parameters returnDatasetFieldTypes=true and onlyDisplayedOnCreate=true are set. See also #10741 and #10767. ## Settings The following settings have been added: +- dataverse.feature.disable-dataset-thumbnail-autoselect - dataverse.pid.*.crossref.url - dataverse.pid.*.crossref.rest-api-url - dataverse.pid.*.crossref.username @@ -153,7 +284,7 @@ The oai_dc export format has changed. See the "Remap oai_dc" section above. In order for changes to the `oai_dc` metadata export format to be reflected in existing datasets, a [reexport all](https://guides.dataverse.org/en/6.4/admin/metadataexport.html#batch-exports-through-the-api) should be run. See also "remap oai_dc" above. -Update your Solr schema.xml file to pick up the "dataset type" additions described above and do a full reindex. +Update your Solr schema.xml file to pick up the "datasetType" and "dvParentAlias" additions described above and do a full reindex. ### Additional Upgrade Steps @@ -161,6 +292,10 @@ Update your Solr schema.xml file to pick up the "dataset type" additions describ "/citation?persistentId=" to the existing base URL to maintain functionality. - If you use a PermaLink provider without a configured `base-url`: No changes are required. +To restore any broken thumbnails caused by this bug, you can call the http://localhost:8080/api/admin/clearThumbnailFailureFlag API, which will attempt to clear the flag on all files (regardless of whether caused by this bug or some other problem with the file) or the http://localhost:8080/api/admin/clearThumbnailFailureFlag/id to clear the flag for individual files. Calling the former, batch API is recommended. + + + 6\. Update the Citation metadata block: ``` From 45b412b139761bea7404da341be6f55e8db950dd Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 22:46:47 +0200 Subject: [PATCH 918/970] fix,ci(ct): only add base image Maven option when the input is defined Without this in case of the pull_request event the input is null and the build fails because we have base image defined at all. Simply not adding the option if the input is undefined means we stick to what is defined within the POM. --- .github/workflows/container_app_push.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index 8e38a17209c..3b7ce066d73 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -55,7 +55,7 @@ jobs: run: > mvn -B -f modules/dataverse-parent -P ct -pl edu.harvard.iq:dataverse -am - -Dbase.image=${{ inputs.base-image-ref }} + $( [[ -n "${{ inputs.base-image-ref }}" ]] && echo "-Dbase.image=${{ inputs.base-image-ref }}" ) install # TODO: add smoke / integration testing here (add "-Pct -DskipIntegrationTests=false") @@ -157,12 +157,13 @@ jobs: run: > mvn -B -f modules/dataverse-parent -P ct -pl edu.harvard.iq:dataverse -am - -Dbase.image=${{ inputs.base-image-ref }} + $( [[ -n "${{ inputs.base-image-ref }}" ]] && echo "-Dbase.image=${{ inputs.base-image-ref }}" ) install - name: Deploy multi-arch application and configbaker container image run: > mvn - -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image=${{ inputs.base-image-ref }} + -Dapp.image.tag=${{ env.IMAGE_TAG }} + $( [[ -n "${{ inputs.base-image-ref }}" ]] && echo "-Dbase.image=${{ inputs.base-image-ref }}" ) ${{ env.REGISTRY }} -Ddocker.platforms=${{ env.PLATFORMS }} -P ct deploy From c706372fd994df2bbe293949059e21f586d9c81d Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 19 Sep 2024 16:49:45 -0400 Subject: [PATCH 919/970] add highlights and upgrade steps #10853 --- doc/release-notes/6.4-release-notes.md | 198 ++++++++++++++++++++++--- 1 file changed, 178 insertions(+), 20 deletions(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index feb8e54102a..f6013cbe2d6 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -6,9 +6,24 @@ This release brings new features, enhancements, and bug fixes to Dataverse. Than ## Release Highlights -- Users will have DOIs/PIDs reserved for their files as part of file upload instead of at publication time. (Issue #7068, PR #7334) +Dataverse 6.4 fixes a couple important bugs described below and in [a post](https://groups.google.com/g/dataverse-community/c/evn5C-pyrS8/m/JrH9vp47DwAJ) on the mailing list: -## Features +- "Update Current Version" can cause metadata loss +- Publishing breaks designated dataset thumbnail, messes up collection page + +New features include: + +- All ISO 639-3 languages are now supported +- There is now a button for "Unlink Dataset" +- Users will have DOIs/PIDs reserved for their files as part of file upload instead of at publication time +- Datasets can now have types such as "software" or "workflow" +- Croissant support +- RO-Crate support +- A new Rust library + +All this and more can be found below. Read on! + +## Features Added ### Full List of ISO 639-3 Languages Now Supported @@ -32,10 +47,6 @@ Out of the box, all datasets now have the type "dataset" but superusers can add For details see [the guides](https://guides.dataverse.org/en/6.4/user/dataset-management.html#dataset-types), #10517 and #10694. Please note that this feature is highly experimental and is expected to [evolve](https://github.com/IQSS/dataverse-pm/issues/307). -### Collection Thumbnail Logo for Featured Collections - -Collections can now have a thumbnail logo that is displayed when the collection is configured as a featured collection. If present, this thumbnail logo is shown. Otherwise, the collection logo is shown. Configuration is done under the "Theme" for a collection as explained in [the guides](https://guides.dataverse.org/en/6.4/user/dataverse-management.html#theme). See also #10291 and #10433. - ### Croissant Support (Metadata Export) A new metadata export format called [Croissant](https://github.com/mlcommons/croissant) is now available as an external metadata exporter. It is oriented toward making datasets consumable by machine learning. @@ -52,6 +63,10 @@ Dataverse now supports [RO-Crate](https://www.researchobject.org/ro-crate/) as a An Dataverse API client library for the Rust programming language is now available at https://github.com/gdcc/rust-dataverse and has been added to the [list of client libraries](https://guides.dataverse.org/en/6.4/api/client-libraries.html) in the API Guide. See also #10758. +### Collection Thumbnail Logo for Featured Collections + +Collections can now have a thumbnail logo that is displayed when the collection is configured as a featured collection. If present, this thumbnail logo is shown. Otherwise, the collection logo is shown. Configuration is done under the "Theme" for a collection as explained in [the guides](https://guides.dataverse.org/en/6.4/user/dataverse-management.html#theme). See also #10291 and #10433. + ### Saved Searches Can Be Deleted Saved searches can now be deleted via API. See the [Saved Search](https://guides.dataverse.org/en/6.4/api/native-api.html#saved-search) section of the API Guide, #9317 and #10198. @@ -84,7 +99,7 @@ JSON Schema validation has been enhanced with checks for required and allowed ch Counter Processor 1.05 is now supported for use with Make Data Count. If you are running Counter Processor, you should reinstall/reconfigure it as described in the latest guides. Note that Counter Processor 1.05 requires Python 3, so you will need to follow the full Counter Processor install. Also note that if you configure the new version the same way, it will reprocess the days in the current month when it is first run. This is normal and will not affect the metrics in Dataverse. See also #10479. -## Bug Fixes +## Bugs Fixed ### Update Current Version @@ -161,7 +176,7 @@ When any `ApiBlockingFilter` policy applies to a request, the JSON in the body o See also #10508, #10672 and #10722. -## API +## API Updates ### Search API: affiliation, parentDataverseName, image_url, etc. @@ -260,7 +275,7 @@ The addDataverse (`/api/dataverses/{identifier}`) API endpoint has been extended The `/api/dataverses/{identifier}/metadatablocks` endpoint has been fixed to not return fields marked as displayOnCreate=true if there is an input level with include=false, when query parameters returnDatasetFieldTypes=true and onlyDisplayedOnCreate=true are set. See also #10741 and #10767. -## Settings +## Settings Added The following settings have been added: @@ -272,33 +287,176 @@ The following settings have been added: - dataverse.pid.*.crossref.depositor - dataverse.pid.*.crossref.depositor-email -### Backward incompatible changes +## Backward Incompatible Changes The oai_dc export format has changed. See the "Remap oai_dc" section above. ## Complete List of Changes +For the complete list of code changes in this release, see the [6.4 milestone](https://github.com/IQSS/dataverse/issues?q=milestone%3A6.4+is%3Aclosed) in GitHub. + ## Getting Help +For help with upgrading, installing, or general questions please post to the [Dataverse Community Google Group](https://groups.google.com/g/dataverse-community) or email support@dataverse.org. + +## Installation + +If this is a new installation, please follow our [Installation Guide](https://guides.dataverse.org/en/latest/installation/). Please don't be shy about [asking for help](https://guides.dataverse.org/en/latest/installation/intro.html#getting-help) if you need it! + +Once you are in production, we would be delighted to update our [map of Dataverse installations](https://dataverse.org/installations) around the world to include yours! Please [create an issue](https://github.com/IQSS/dataverse-installations/issues) or email us at support@dataverse.org to join the club! + +You are also very welcome to join the [Global Dataverse Community Consortium](https://www.gdcc.io/) (GDCC). + ## Upgrade Instructions -In order for changes to the `oai_dc` metadata export format to be reflected in existing datasets, a [reexport all](https://guides.dataverse.org/en/6.4/admin/metadataexport.html#batch-exports-through-the-api) should be run. See also "remap oai_dc" above. +Upgrading requires a maintenance window and downtime. Please plan accordingly, create backups of your database, etc. -Update your Solr schema.xml file to pick up the "datasetType" and "dvParentAlias" additions described above and do a full reindex. +These instructions assume that you've already upgraded through all the 5.x releases and are now running Dataverse 6.2. -### Additional Upgrade Steps +0\. These instructions assume that you are upgrading from the immediate previous version. If you are running an earlier version, the only supported way to upgrade is to progress through the upgrades to all the releases in between before attempting the upgrade to this version. -- If you currently use a PermaLink provider with a configured `base-url`: You must manually append - "/citation?persistentId=" to the existing base URL to maintain functionality. -- If you use a PermaLink provider without a configured `base-url`: No changes are required. +If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. Use `sudo` to change to that user first. For example, `sudo -i -u dataverse` if `dataverse` is your dedicated application user. -To restore any broken thumbnails caused by this bug, you can call the http://localhost:8080/api/admin/clearThumbnailFailureFlag API, which will attempt to clear the flag on all files (regardless of whether caused by this bug or some other problem with the file) or the http://localhost:8080/api/admin/clearThumbnailFailureFlag/id to clear the flag for individual files. Calling the former, batch API is recommended. +In the following commands, we assume that Payara 6 is installed in `/usr/local/payara6`. If not, adjust as needed. +```shell +export PAYARA=/usr/local/payara6` +``` +(or `setenv PAYARA /usr/local/payara6` if you are using a `csh`-like shell) -6\. Update the Citation metadata block: +1\. Undeploy the previous version +```shell +$PAYARA/bin/asadmin undeploy dataverse-6.3 ``` -- `wget https://github.com/IQSS/dataverse/releases/download/v6.4/citation.tsv` -- `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @citation.tsv -H "Content-type: text/tab-separated-values"` + +2\. Stop and start Payara + +```shell +service payara stop +sudo service payara start +``` + +3\. Deploy this version + +```shell +$PAYARA/bin/asadmin deploy dataverse-6.4.war +``` + +Note: if you have any trouble deploying, stop Payara, remove the following directories, start Payara, and try to deploy again. + +```shell +service payara stop +rm -rf $PAYARA/glassfish/domains/domain1/generated +rm -rf $PAYARA/glassfish/domains/domain1/osgi-cache +rm -rf $PAYARA/glassfish/domains/domain1/lib/databases +``` + +4\. For installations with internationalization: + +Please remember to update translations via [Dataverse language packs](https://github.com/GlobalDataverseCommunityConsortium/dataverse-language-packs). + +5\. Restart Payara + +```shell +service payara stop +service payara start +``` + +6\. Update metadata blocks + +These changes reflect incremental improvements made to the handling of core metadata fields. + +```shell +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.4/scripts/api/data/metadatablocks/citation.tsv + +curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file citation.tsv ``` + +7\. Update Solr schema.xml file. See specific instructions below for those installations without custom metadata blocks (7a) and those with custom metadata blocks (7b). + +7a\. For installations without custom or experimental metadata blocks: + +Stop Solr (usually `service solr stop`, depending on Solr installation/OS, see the [Installation Guide](https://guides.dataverse.org/en/6.4/installation/prerequisites.html#solr-init-script)). + +```shell +service solr stop +``` + +Replace schema.xml + +```shell +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.4/conf/solr/schema.xml +cp schema.xml /usr/local/solr/solr-9.4.1/server/solr/collection1/conf +``` + +Start Solr. + +```shell +service solr start +``` + +7b\. For installations with custom or experimental metadata blocks: + +Stop Solr. + +```shell +service solr stop +``` + +There are two ways to regenerate the schema, either by collecting the output of the Dataverse schema API and feeding it to the `update-fields.sh` script that we supply, as in the example below (modify the command lines as needed)... + +```shell + wget https://raw.githubusercontent.com/IQSS/dataverse/master/conf/solr/9.4.1/update-fields.sh + chmod +x update-fields.sh + curl "http://localhost:8080/api/admin/index/solr/schema" | ./update-fields.sh /usr/local/solr/solr-9.4.1/server/solr/collection1/conf/schema.xml +``` + +...OR, alternatively, you edit your schema.xml by hand and add the following lines from the [6.4 tagged version](https://github.com/IQSS/dataverse/blob/v6.4/conf/solr/schema.xml) of schema.xml. + +``` + + + +``` + +Start Solr. + +```shell +service solr start +``` + +8\. Reindex Solr + +Below is the simple way to reindex Solr. If you have a large installation of Dataverse, you might want to reindex in place, as described in [the guides](https://guides.dataverse.org/en/latest/admin/solr-search-index.html). + +```shell +curl http://localhost:8080/api/admin/index +``` + +9\. Run reExportAll to update dataset metadata exports + +This step is necessary because of changes described above for the `oai_dc` export format. + +Below is the simple way to reexport all dataset metadata. For more advanced usage, please see [the guides](http://guides.dataverse.org/en/6.4/admin/metadataexport.html#batch-exports-through-the-api). + +```shell +curl http://localhost:8080/api/admin/metadata/reExportAll +``` + +### Additional Upgrade Steps + +10\. If there are broken thumbnails + +To restore any broken thumbnails caused by the bug described above, you can call the `http://localhost:8080/api/admin/clearThumbnailFailureFlag` API, which will attempt to clear the flag on all files (regardless of whether caused by this bug or some other problem with the file) or the `http://localhost:8080/api/admin/clearThumbnailFailureFlag/$FILE_ID` to clear the flag for individual files. Calling the former, batch API is recommended. + +```shell +http://localhost:8080/api/admin/clearThumbnailFailureFlag/$FILE_ID +``` + +11\. PermaLinks with custom base-url + +If you currently use PermaLinks with a custom `base-url`: You must manually append `/citation?persistentId=` to the base URL to maintain functionality. + +If you use a PermaLinks without a configured `base-url`, no changes are required. From 52e6408a5a0d33629139c2098253e6ca8f08588a Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 23:00:22 +0200 Subject: [PATCH 920/970] docs(ct): add release note for maintenance workflow #10478 --- doc/release-notes/10478-version-base-image.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/release-notes/10478-version-base-image.md diff --git a/doc/release-notes/10478-version-base-image.md b/doc/release-notes/10478-version-base-image.md new file mode 100644 index 00000000000..34f444a2122 --- /dev/null +++ b/doc/release-notes/10478-version-base-image.md @@ -0,0 +1,7 @@ +### Adding versioned tags to Container Base Images + +With this release we introduce a detailed maintenance workflow for our container images. +As output of the GDCC Containerization Working Group, the community takes another step towards production ready containers available directly from the core project. + +The maintenance workflow regularly updates the Container Base Image, which contains the operating system, Java, Payara Application Server, as well as tools and libraries required by the Dataverse application. +Shipping these rolling releases as well as immutable revisions is the foundation for secure and reliable Dataverse Application Container images. From dc6b59742550106513fb018f9872512ab7cd1039 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 23:34:36 +0200 Subject: [PATCH 921/970] style(ct): add comment explaining what a flavor is in base image Maven props Co-authored-by: Philip Durbin --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index d8fe55ba22a..edf72067976 100644 --- a/pom.xml +++ b/pom.xml @@ -999,6 +999,7 @@ unstable false gdcc/base:${base.image.tag} + noble ${base.image.version}-${base.image.flavor}-p${payara.version}-j${target.java.version} From 79273b0c37985849231b6db0f7fdce20645ceeb4 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 19 Sep 2024 19:34:53 -0400 Subject: [PATCH 922/970] A potential simple fix for #10667 ? --- .../java/edu/harvard/iq/dataverse/EditDatafilesPage.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 993cb02b66b..8953c16aa6e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -2092,6 +2092,12 @@ public void handleFileUpload(FileUploadEvent event) throws IOException { errorMessages.add(cex.getMessage()); uploadComponentId = event.getComponent().getClientId(); return; + } finally { + try { + uFile.delete(); + } catch (IOException ioex) { + logger.warning("Failed to delete temp file uploaded via PrimeFaces " + uFile.getFileName()); + } } /*catch (FileExceedsMaxSizeException ex) { logger.warning("Failed to process and/or save the file " + uFile.getFileName() + "; " + ex.getMessage()); From 79b4204e743e9fc842d3f23fd358c0df76abccd5 Mon Sep 17 00:00:00 2001 From: landreev Date: Fri, 20 Sep 2024 13:17:34 -0400 Subject: [PATCH 923/970] Update 6.4-release-notes.md corrected the schema.xml instructions --- doc/release-notes/6.4-release-notes.md | 31 +++++--------------------- 1 file changed, 5 insertions(+), 26 deletions(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index f6013cbe2d6..8072741033c 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -374,9 +374,7 @@ wget https://raw.githubusercontent.com/IQSS/dataverse/v6.4/scripts/api/data/meta curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file citation.tsv ``` -7\. Update Solr schema.xml file. See specific instructions below for those installations without custom metadata blocks (7a) and those with custom metadata blocks (7b). - -7a\. For installations without custom or experimental metadata blocks: +7\. Update Solr schema.xml file. Start with the standard v6.4 schema.xml, then, if your installation uses any custom or experimental metadata blocks, update it to include the extra fields (step 7a). Stop Solr (usually `service solr stop`, depending on Solr installation/OS, see the [Installation Guide](https://guides.dataverse.org/en/6.4/installation/prerequisites.html#solr-init-script)). @@ -391,41 +389,22 @@ wget https://raw.githubusercontent.com/IQSS/dataverse/v6.4/conf/solr/schema.xml cp schema.xml /usr/local/solr/solr-9.4.1/server/solr/collection1/conf ``` -Start Solr. +Start Solr (but if you use any custom metadata blocks, perform the next step, 7a first). ```shell service solr start ``` -7b\. For installations with custom or experimental metadata blocks: - -Stop Solr. - -```shell -service solr stop -``` +7a\. For installations with custom or experimental metadata blocks: -There are two ways to regenerate the schema, either by collecting the output of the Dataverse schema API and feeding it to the `update-fields.sh` script that we supply, as in the example below (modify the command lines as needed)... +Before starting Solr, update the schema to include all the extra metadata fields that your installation uses. We do this by collecting the output of the Dataverse schema API and feeding it to the `update-fields.sh` script that we supply, as in the example below (modify the command lines as needed to reflect the names of the directories, if different): ```shell wget https://raw.githubusercontent.com/IQSS/dataverse/master/conf/solr/9.4.1/update-fields.sh chmod +x update-fields.sh curl "http://localhost:8080/api/admin/index/solr/schema" | ./update-fields.sh /usr/local/solr/solr-9.4.1/server/solr/collection1/conf/schema.xml -``` - -...OR, alternatively, you edit your schema.xml by hand and add the following lines from the [6.4 tagged version](https://github.com/IQSS/dataverse/blob/v6.4/conf/solr/schema.xml) of schema.xml. -``` - - - -``` - -Start Solr. - -```shell -service solr start -``` +Now start Solr. 8\. Reindex Solr From b9f1c85ad0ae8666403367287790b96a50c5a563 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 20 Sep 2024 13:57:48 -0400 Subject: [PATCH 924/970] add blurb for tagged base images #10853 --- doc/release-notes/10478-version-base-image.md | 7 ------- doc/release-notes/6.4-release-notes.md | 6 ++++++ 2 files changed, 6 insertions(+), 7 deletions(-) delete mode 100644 doc/release-notes/10478-version-base-image.md diff --git a/doc/release-notes/10478-version-base-image.md b/doc/release-notes/10478-version-base-image.md deleted file mode 100644 index 34f444a2122..00000000000 --- a/doc/release-notes/10478-version-base-image.md +++ /dev/null @@ -1,7 +0,0 @@ -### Adding versioned tags to Container Base Images - -With this release we introduce a detailed maintenance workflow for our container images. -As output of the GDCC Containerization Working Group, the community takes another step towards production ready containers available directly from the core project. - -The maintenance workflow regularly updates the Container Base Image, which contains the operating system, Java, Payara Application Server, as well as tools and libraries required by the Dataverse application. -Shipping these rolling releases as well as immutable revisions is the foundation for secure and reliable Dataverse Application Container images. diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 8072741033c..3c7c5fc4e8f 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -99,6 +99,12 @@ JSON Schema validation has been enhanced with checks for required and allowed ch Counter Processor 1.05 is now supported for use with Make Data Count. If you are running Counter Processor, you should reinstall/reconfigure it as described in the latest guides. Note that Counter Processor 1.05 requires Python 3, so you will need to follow the full Counter Processor install. Also note that if you configure the new version the same way, it will reprocess the days in the current month when it is first run. This is normal and will not affect the metrics in Dataverse. See also #10479. +### Version Tags for Container Base Images + +With this release we introduce a detailed maintenance workflow for our container images. As output of the [Containerization Working Group](https://ct.gdcc.io), the community takes another step towards production ready containers available directly from the core project. + +The maintenance workflow regularly updates the [Container Base Image](https://guides.dataverse.org/en/6.4/container/base-image.html), which contains the operating system, Java, Payara, and tools and libraries required by the Dataverse application. Shipping these rolling releases as well as immutable revisions is the foundation for secure and reliable [Dataverse Application Container](https://guides.dataverse.org/en/6.4/container/app-image.html) images. See also #10478 and #10827. + ## Bugs Fixed ### Update Current Version From 6cb3cd3092992d59ea6cf9d7106d211df35e1396 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 20 Sep 2024 16:24:46 -0400 Subject: [PATCH 925/970] doc how to handle develop into develop PRs #9508 --- .../source/developers/version-control.rst | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/version-control.rst b/doc/sphinx-guides/source/developers/version-control.rst index 127955a44ea..02539d5286f 100644 --- a/doc/sphinx-guides/source/developers/version-control.rst +++ b/doc/sphinx-guides/source/developers/version-control.rst @@ -97,10 +97,12 @@ In the issue you can simply leave a comment to say you're working on it. If you tell us your GitHub username we are happy to add you to the "read only" team at https://github.com/orgs/IQSS/teams/dataverse-readonly/members so that we can assign the issue to you while you're working on it. You can also tell us if you'd like to be added to the `Dataverse Community Contributors spreadsheet `_. +.. _create-branch-for-pr: + Create a New Branch Off the develop Branch ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Always create your feature branch from the latest code in develop, pulling the latest code if necessary. As mentioned above, your branch should have a name like "3728-doc-apipolicy-fix" that starts with the issue number you are addressing (e.g. `#3728 `_) and ends with a short, descriptive name. Dashes ("-") and underscores ("_") in your branch name are ok, but please try to avoid other special characters such as ampersands ("&") that have special meaning in Unix shells. +Always create your feature branch from the latest code in develop, pulling the latest code if necessary. As mentioned above, your branch should have a name like "3728-doc-apipolicy-fix" that starts with the issue number you are addressing (e.g. `#3728 `_) and ends with a short, descriptive name. Dashes ("-") and underscores ("_") in your branch name are ok, but please try to avoid other special characters such as ampersands ("&") that have special meaning in Unix shells. Please do not call your branch "develop" as it can cause maintainers :ref:`trouble `. Commit Your Change to Your New Branch ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -299,3 +301,41 @@ GitHub documents how to make changes to a fork at https://help.github.com/articl vim path/to/file.txt git commit git push OdumInstitute 4709-postgresql_96 + +.. _develop-into-develop: + +Handing a Pull Request from a "develop" Branch +---------------------------------------------- + +Note: this is something only maintainers of Dataverse need to worry about, typically. + +From time to time a pull request comes in from a fork of Dataverse that uses "develop" as the branch behind the PR. (We've started asking contributors not to do this. See :ref:`create-branch-for-pr`.) This is problematic because the "develop" branch is the main integration branch for the project. (See :ref:`develop-branch`.) + +If the PR is perfect and can be merged as-is, no problem. Just merge it. However, if you would like to push commits to the PR, you are likely to run into trouble with multiple "develop" branches locally. + +The following is a list of commands oriented toward the simple task of merging the latest from the "develop" branch into the PR but the same technique can be used to push other commits to the PR as well. In this example the PR is coming from username "coder123" on GitHub. At a high level, what we're doing is working in a safe place (/tmp) away from our normal copy of the repo. We clone the main repo from IQSS, check out coder123's version of "develop" (called "dev2" or "false develop"), merge the real "develop" into it, and push to the PR. + +If there's a better way to do this, please get in touch! + + +.. code-block:: bash + + # do all this in /tmp away from your normal code + cd /tmp + git clone git@github.com:IQSS/dataverse.git + cd dataverse + git remote add coder123 git@github.com:coder123/dataverse.git + git fetch coder123 + # check out coder123's "develop" to a branch with a different name ("dev2") + git checkout coder123/develop -b dev2 + # merge IQSS "develop" into coder123's "develop" ("dev2") + git merge origin/develop + # delete the IQSS "develop" branch locally (!) + git branch -d develop + # checkout "dev2" (false "develop") as "develop" for now + git checkout -b develop + # push the false "develop" to coder123's fork (to the PR) + git push coder123 develop + cd .. + # delete the tmp space (done! \o/) + rm -rf /tmp/dataverse From c07f1d0452ed32593ee8ecab18b910e4c36fe3f3 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 20 Sep 2024 16:27:17 -0400 Subject: [PATCH 926/970] remove extra line --- doc/sphinx-guides/source/developers/version-control.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/version-control.rst b/doc/sphinx-guides/source/developers/version-control.rst index 02539d5286f..ecd2db6214d 100644 --- a/doc/sphinx-guides/source/developers/version-control.rst +++ b/doc/sphinx-guides/source/developers/version-control.rst @@ -317,7 +317,6 @@ The following is a list of commands oriented toward the simple task of merging t If there's a better way to do this, please get in touch! - .. code-block:: bash # do all this in /tmp away from your normal code From 48f9f72ac6cb8b6995c10c3375c9aa756e3532ba Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 20 Sep 2024 17:55:24 -0400 Subject: [PATCH 927/970] dont return image_url if there is none --- .../iq/dataverse/DataverseServiceBean.java | 2 +- .../iq/dataverse/ThumbnailServiceWrapper.java | 11 ++++----- .../harvard/iq/dataverse/api/SearchIT.java | 23 ++++++++++++++++++- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index 65082011eba..91b15f77111 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -364,7 +364,7 @@ public String getDataverseLogoThumbnailAsBase64ById(Long dvId) { public String getDataverseLogoThumbnailAsUrl(Long dvId) { File dataverseLogoFile = getLogoById(dvId); - if (dataverseLogoFile != null) { + if (dataverseLogoFile != null && dataverseLogoFile.exists()) { return SystemConfig.getDataverseSiteUrlStatic() + "/api/access/dvCardImage/" + dvId; } return null; diff --git a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java index 91d25ba2d38..36be7c61642 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java +++ b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java @@ -49,15 +49,14 @@ public class ThumbnailServiceWrapper implements java.io.Serializable { private Map hasThumbMap = new HashMap<>(); public String getFileCardImageAsUrl(SolrSearchResult result) { - if (result.isHarvested()) { - return null; - } - if (result.getEntity() == null) { - return null; + if (!result.isHarvested() && result.getEntity() != null && (!((DataFile)result.getEntity()).isRestricted() + || permissionsWrapper.hasDownloadFilePermission(result.getEntity())) + && isThumbnailAvailable((DataFile) result.getEntity())) { + return SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + result.getEntity().getId() + "?imageThumb=true"; } - return SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + result.getEntity().getId() + "?imageThumb=true"; + return null; } // it's the responsibility of the user - to make sure the search result diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index 4b422002cf2..3a2b684c421 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -1284,7 +1284,7 @@ public static void cleanup() { } @Test - public void testSearchFiles() { + public void testSearchFilesAndUrlImages() { Response createUser = UtilIT.createRandomUser(); createUser.prettyPrint(); String username = UtilIT.getUsernameFromResponse(createUser); @@ -1304,6 +1304,11 @@ public void testSearchFiles() { String pathToFile = "src/main/webapp/resources/images/dataverseproject.png"; Response uploadImage = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken); uploadImage.prettyPrint(); + uploadImage.then().assertThat() + .statusCode(200); + pathToFile = "src/main/webapp/resources/js/mydata.js"; + Response uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken); + uploadImage.prettyPrint(); uploadImage.then().assertThat() .statusCode(200); @@ -1325,5 +1330,21 @@ public void testSearchFiles() { .body("data.items[0].url", CoreMatchers.containsString("/api/access/datafile/")) .body("data.items[0].image_url", CoreMatchers.containsString("/api/access/datafile/")) .body("data.items[0].image_url", CoreMatchers.containsString("imageThumb=true")); + + searchResp = UtilIT.search(dataverseAlias, apiToken); + searchResp.prettyPrint(); + searchResp.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.items[0].type", CoreMatchers.is("dataverse")) + .body("data.items[0].url", CoreMatchers.containsString("/dataverse/")) + .body("data.items[0]", CoreMatchers.not(CoreMatchers.hasItem("url_image"))); + + searchResp = UtilIT.search("mydata", apiToken); + searchResp.prettyPrint(); + searchResp.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.items[0].type", CoreMatchers.is("file")) + .body("data.items[0].url", CoreMatchers.containsString("/datafile/")) + .body("data.items[0]", CoreMatchers.not(CoreMatchers.hasItem("url_image"))); } } From c1fa3349062099219ad496eea43367deab5a7b6c Mon Sep 17 00:00:00 2001 From: Ludovic DANIEL Date: Mon, 16 Sep 2024 23:11:22 +0200 Subject: [PATCH 928/970] Fix NPE using CVOC --- .../java/edu/harvard/iq/dataverse/search/IndexServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index fe8f1030f82..a8cf9ed519b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -1128,7 +1128,7 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set childDatasetFields = dsf.getParentDatasetFieldCompoundValue().getChildDatasetFields(); for (DatasetField df : childDatasetFields) { - if(cvocManagedFieldMap.get(dsfType.getId()).contains(df.getDatasetFieldType().getName())) { + if(cvocManagedFieldMap.containsKey(dsfType.getId()) && cvocManagedFieldMap.get(dsfType.getId()).contains(df.getDatasetFieldType().getName())) { String solrManagedFieldSearchable = df.getDatasetFieldType().getSolrField().getNameSearchable(); // Try to get string values from externalvocabularyvalue but for a managed fields of the CVOCConf Set stringsForManagedField = datasetFieldService.getIndexableStringsByTermUri(val, cvocMap.get(dsfType.getId()), df.getDatasetFieldType().getName()); From d265e509e643de1d1dd61cf5273412a34052f24f Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 23 Sep 2024 09:18:38 -0400 Subject: [PATCH 929/970] set dataset type before registering pid (which needs the type) --- .../engine/command/impl/AbstractCreateDatasetCommand.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java index 7b7c5fd0e93..db9dc142506 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java @@ -118,9 +118,6 @@ public Dataset execute(CommandContext ctxt) throws CommandException { pidProvider.generatePid(theDataset); } - // Attempt the registration if importing dataset through the API, or the app (but not harvest) - handlePid(theDataset, ctxt); - DatasetType defaultDatasetType = ctxt.datasetTypes().getByName(DatasetType.DEFAULT_DATASET_TYPE); DatasetType existingDatasetType = theDataset.getDatasetType(); logger.fine("existing dataset type: " + existingDatasetType); @@ -130,6 +127,11 @@ public Dataset execute(CommandContext ctxt) throws CommandException { } else { theDataset.setDatasetType(defaultDatasetType); } + + // Attempt the registration if importing dataset through the API, or the app (but not harvest) + handlePid(theDataset, ctxt); + + ctxt.em().persist(theDataset); From 6d9d6777ae37e0105285a3c73afff771170082b4 Mon Sep 17 00:00:00 2001 From: Ludovic DANIEL Date: Mon, 23 Sep 2024 16:19:03 +0200 Subject: [PATCH 930/970] add release note --- doc/release-notes/10869-fix-npe-using-cvoc.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/release-notes/10869-fix-npe-using-cvoc.md diff --git a/doc/release-notes/10869-fix-npe-using-cvoc.md b/doc/release-notes/10869-fix-npe-using-cvoc.md new file mode 100644 index 00000000000..53214d3789d --- /dev/null +++ b/doc/release-notes/10869-fix-npe-using-cvoc.md @@ -0,0 +1 @@ +This release fixes a bug in the external controlled vocabulary mechanism (introduced in v6.3) that could cause indexing to fail when a script is configured for one child field and no other child fields were managed. \ No newline at end of file From d215221b8b884b55a1ef2c1468deb2e4c799a9b4 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 23 Sep 2024 10:20:51 -0400 Subject: [PATCH 931/970] adding fix from review comment --- .../iq/dataverse/ThumbnailServiceWrapper.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java index 36be7c61642..542cf39cfbe 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java +++ b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java @@ -10,6 +10,7 @@ import edu.harvard.iq.dataverse.dataaccess.StorageIO; import edu.harvard.iq.dataverse.dataset.DatasetUtil; import edu.harvard.iq.dataverse.search.SolrSearchResult; +import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import java.io.IOException; @@ -49,14 +50,16 @@ public class ThumbnailServiceWrapper implements java.io.Serializable { private Map hasThumbMap = new HashMap<>(); public String getFileCardImageAsUrl(SolrSearchResult result) { - - if (!result.isHarvested() && result.getEntity() != null && (!((DataFile)result.getEntity()).isRestricted() - || permissionsWrapper.hasDownloadFilePermission(result.getEntity())) - && isThumbnailAvailable((DataFile) result.getEntity())) { - return SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + result.getEntity().getId() + "?imageThumb=true"; + DataFile dataFile = result != null && result.getEntity() != null ? ((DataFile) result.getEntity()) : null; + if (dataFile == null || result.isHarvested() + || !isThumbnailAvailable(dataFile) + || dataFile.isRestricted() + || !dataFile.isReleased() + || FileUtil.isActivelyEmbargoed(dataFile) + || FileUtil.isRetentionExpired(dataFile)) { + return null; } - - return null; + return SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + dataFile.getId() + "?imageThumb=true"; } // it's the responsibility of the user - to make sure the search result From 7b6f81eb4992c075be22e2debea3ec1b24c9551f Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 23 Sep 2024 11:21:44 -0400 Subject: [PATCH 932/970] remove tabs to make reviewdog happy. woof! #10623 --- .../iq/dataverse/ExternalFileUploadInProgress.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ExternalFileUploadInProgress.java b/src/main/java/edu/harvard/iq/dataverse/ExternalFileUploadInProgress.java index 96bfd3f63f5..c90fdc6edc2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ExternalFileUploadInProgress.java +++ b/src/main/java/edu/harvard/iq/dataverse/ExternalFileUploadInProgress.java @@ -23,10 +23,10 @@ * externally. (?) */ @NamedQueries({ - @NamedQuery( name="ExternalFileUploadInProgress.deleteByTaskId", - query="DELETE FROM ExternalFileUploadInProgress f WHERE f.taskId=:taskId"), - @NamedQuery(name = "ExternalFileUploadInProgress.findByTaskId", - query = "SELECT f FROM ExternalFileUploadInProgress f WHERE f.taskId=:taskId")}) + @NamedQuery(name = "ExternalFileUploadInProgress.deleteByTaskId", + query = "DELETE FROM ExternalFileUploadInProgress f WHERE f.taskId=:taskId"), + @NamedQuery(name = "ExternalFileUploadInProgress.findByTaskId", + query = "SELECT f FROM ExternalFileUploadInProgress f WHERE f.taskId=:taskId")}) @Entity @Table(indexes = {@Index(columnList="taskid")}) public class ExternalFileUploadInProgress implements Serializable { From 2baf62e7cac957979f5d16637acd1b4c39e3713f Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 23 Sep 2024 11:39:54 -0400 Subject: [PATCH 933/970] globus doc tweaks #10623 --- doc/release-notes/10623-globus-improvements.md | 2 +- doc/sphinx-guides/source/developers/big-data-support.rst | 2 +- doc/sphinx-guides/source/developers/globus-api.rst | 2 +- doc/sphinx-guides/source/installation/config.rst | 6 ++++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/release-notes/10623-globus-improvements.md b/doc/release-notes/10623-globus-improvements.md index 03579b59631..c6b9febbb02 100644 --- a/doc/release-notes/10623-globus-improvements.md +++ b/doc/release-notes/10623-globus-improvements.md @@ -1 +1 @@ -A new alternative implementation of Globus polling during upload data transfers has been added in this release. This experimental framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. See the `globus-use-experimental-async-framework` feature flag in the Configuration guide. \ No newline at end of file +A new alternative implementation of Globus polling during upload data transfers has been added in this release. This experimental framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. See the `globus-use-experimental-async-framework` under [Feature Flags](https://dataverse-guide--10781.org.readthedocs.build/en/10781/installation/config.html#feature-flags) in the Installation Guide. diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index 99ea4d2e0ba..02dd31b8472 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -188,4 +188,4 @@ An overview of the control and data transfer interactions between components was See also :ref:`Globus settings <:GlobusSettings>`. -An alternative, experimental implementation of Globus polling of ongoing upload transfers has been added in v6.4. This framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. Due to its experimental nature it is not enabled by default. See the ``globus-use-experimental-async-framework`` feature flag and the JVM option ``dataverse.globus.taskMonitoringServer`` described in the Configuration guide. +An alternative, experimental implementation of Globus polling of ongoing upload transfers has been added in v6.4. This framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. Due to its experimental nature it is not enabled by default. See the ``globus-use-experimental-async-framework`` feature flag (see :ref:`feature-flags`) and the JVM option :ref:`dataverse.globus.taskMonitoringServer`. diff --git a/doc/sphinx-guides/source/developers/globus-api.rst b/doc/sphinx-guides/source/developers/globus-api.rst index 0f16dc704ef..26620fd6a92 100644 --- a/doc/sphinx-guides/source/developers/globus-api.rst +++ b/doc/sphinx-guides/source/developers/globus-api.rst @@ -185,7 +185,7 @@ As the transfer can take significant time and the API call is asynchronous, the Once the transfer completes, Dataverse will remove the write permission for the principal. -An alternative, experimental implementation of Globus polling of ongoing upload transfers has been added in v6.4. This new framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. Due to its experimental nature it is not enabled by default. See the ``globus-use-experimental-async-framework`` feature flag and the JVM option ``dataverse.globus.taskMonitoringServer`` described in the Configuration guide. +An alternative, experimental implementation of Globus polling of ongoing upload transfers has been added in v6.4. This new framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. Due to its experimental nature it is not enabled by default. See the ``globus-use-experimental-async-framework`` feature flag (see :ref:`feature-flags`) and the JVM option :ref:`dataverse.globus.taskMonitoringServer`. Note that when using a managed endpoint that uses the Globus S3 Connector, the checksum should be correct as Dataverse can validate it. For file-based endpoints, the checksum should be included if available but Dataverse cannot verify it. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 23168e12f30..bfbf9190357 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3271,7 +3271,7 @@ Can also be set via *MicroProfile Config API* sources, e.g. the environment vari dataverse.globus.taskMonitoringServer +++++++++++++++++++++++++++++++++++++ -This setting is required in conjunction with the ``globus-use-experimental-async-framework`` feature flag. Setting it to true designates the Dataverse instance to serve as the dedicated polling server. It is needed so that the new framework can be used in a multi-node installation. +This setting is required in conjunction with the ``globus-use-experimental-async-framework`` feature flag (see :ref:`feature-flags`). Setting it to true designates the Dataverse instance to serve as the dedicated polling server. It is needed so that the new framework can be used in a multi-node installation. .. _feature-flags: @@ -3309,7 +3309,7 @@ please find all known feature flags below. Any of these flags can be activated u - Removes the reason field in the `Publish/Return To Author` dialog that was added as a required field in v6.2 and makes the reason an optional parameter in the :ref:`return-a-dataset` API call. - ``Off`` * - globus-use-experimental-async-framework - - Activates a new experimental implementation of Globus polling of ongoing remote data transfers that does not rely on the instance staying up continuously for the duration of the transfers and saves the state information about Globus upload requests in the database. Added in v6.4. Note that the JVM option ``dataverse.globus.taskMonitoringServer`` described above must also be enabled on one (and only one, in a multi-node installation) Dataverse instance. + - Activates a new experimental implementation of Globus polling of ongoing remote data transfers that does not rely on the instance staying up continuously for the duration of the transfers and saves the state information about Globus upload requests in the database. Added in v6.4. Affects :ref:`:GlobusPollingInterval`. Note that the JVM option :ref:`dataverse.globus.taskMonitoringServer` described above must also be enabled on one (and only one, in a multi-node installation) Dataverse instance. - ``Off`` **Note:** Feature flags can be set via any `supported MicroProfile Config API source`_, e.g. the environment variable @@ -4789,6 +4789,8 @@ The list of parent dataset field names for which the LDN Announce workflow step The URL where the `dataverse-globus `_ "transfer" app has been deployed to support Globus integration. See :ref:`globus-support` for details. +.. _:GlobusPollingInterval: + :GlobusPollingInterval ++++++++++++++++++++++ From 14efc334b2b323466387f24d6e860d4e3e0df288 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 23 Sep 2024 15:21:06 -0400 Subject: [PATCH 934/970] add more and better DataCite export #10853 --- .../10632-DataCiteXMLandRelationType.md | 41 ----------------- doc/release-notes/6.4-release-notes.md | 44 +++++++++++++++++-- 2 files changed, 40 insertions(+), 45 deletions(-) delete mode 100644 doc/release-notes/10632-DataCiteXMLandRelationType.md diff --git a/doc/release-notes/10632-DataCiteXMLandRelationType.md b/doc/release-notes/10632-DataCiteXMLandRelationType.md deleted file mode 100644 index 42c1cfb6eda..00000000000 --- a/doc/release-notes/10632-DataCiteXMLandRelationType.md +++ /dev/null @@ -1,41 +0,0 @@ -### Enhanced DataCite Metadata, Relation Type - -A new field has been added to the citation metadatablock to allow entry of the "Relation Type" between a "Related Publication" and a dataset. The Relation Type is currently limited to the most common 6 values recommended by DataCite: isCitedBy, Cites, IsSupplementTo, IsSupplementedBy, IsReferencedBy, and References. For existing datasets where no "Relation Type" has been specified, "IsSupplementTo" is assumed. - -Dataverse now supports the DataCite v4.5 schema. Additional metadata, including metadata about Related Publications, and files in the dataset are now being sent to DataCite and improvements to how PIDs (ORCID, ROR, DOIs, etc.), license/terms, geospatial, and other metadata is represented have been made. The enhanced metadata will automatically be sent when datasets are created and published and is available in the DataCite XML export after publication. - -The additions are in rough alignment with the OpenAIRE XML export, but there are some minor differences in addition to the Relation Type addition, including an update to the DataCite 4.5 schema. For details see https://github.com/IQSS/dataverse/pull/10632 and https://github.com/IQSS/dataverse/pull/10615 and the [design document](https://docs.google.com/document/d/1JzDo9UOIy9dVvaHvtIbOI8tFU6bWdfDfuQvWWpC0tkA/edit?usp=sharing) referenced there. - -Multiple backward incompatible changes and bug fixes have been made to API calls (3 of the four of which were not documented) related to updating PID target urls and metadata at the provider service: -- [Update Target URL for a Published Dataset at the PID provider](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-target-url-for-a-published-dataset-at-the-pid-provider) -- [Update Target URL for all Published Datasets at the PID provider](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-target-url-for-all-published-datasets-at-the-pid-provider) -- [Update Metadata for a Published Dataset at the PID provider](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-metadata-for-a-published-dataset-at-the-pid-provider) -- [Update Metadata for all Published Datasets at the PID provider](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-metadata-for-all-published-datasets-at-the-pid-provider) - -Upgrade instructions --------------------- - -The Solr schema has to be updated via the normal mechanism to add the new "relationType" field. - -The citation metadatablock has to be reinstalled using the standard instructions. - -With these two changes, the "Relation Type" fields will be available and creation/publication of datasets will result in the expanded XML being sent to DataCite. - -To update existing datasets (and files using DataCite DOIs): - -Exports can be updated by running `curl http://localhost:8080/api/admin/metadata/reExportAll` - -Entries at DataCite for published datasets can be updated by a superuser using an API call (newly documented): - -`curl -X POST -H 'X-Dataverse-key:' http://localhost:8080/api/datasets/modifyRegistrationPIDMetadataAll` - -This will loop through all published datasets (and released files with PIDs). As long as the loop completes, the call will return a 200/OK response. Any PIDs for which the update fails can be found using - -`grep 'Failure for id' server.log` - -Failures may occur if PIDs were never registered, or if they were never made findable. Any such cases can be fixed manually in DataCite Fabrica or using the [Reserve a PID](https://guides.dataverse.org/en/latest/api/native-api.html#reserve-a-pid) API call and the newly documented `/api/datasets//modifyRegistration` call respectively. See https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#send-dataset-metadata-to-pid-provider. Please reach out with any questions. - -PIDs can also be updated by a superuser on a per-dataset basis using - -`curl -X POST -H 'X-Dataverse-key:' http://localhost:8080/api/datasets//modifyRegistrationMetadata` - diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 3c7c5fc4e8f..fa50a00391b 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -13,6 +13,7 @@ Dataverse 6.4 fixes a couple important bugs described below and in [a post](http New features include: +- More and better metadata is being sent to DataCite, including "Relation Type" - All ISO 639-3 languages are now supported - There is now a button for "Unlink Dataset" - Users will have DOIs/PIDs reserved for their files as part of file upload instead of at publication time @@ -25,6 +26,20 @@ All this and more can be found below. Read on! ## Features Added +### More and Better Data Sent to DataCite, Including "Relation Type" + +Within the "Related Publication" field, a new subfield has been added called "Relation Type" that allows for the most common [values](https://datacite-metadata-schema.readthedocs.io/en/4.5/appendices/appendix-1/relationType/) recommended by DataCite: isCitedBy, Cites, IsSupplementTo, IsSupplementedBy, IsReferencedBy, and References. For existing datasets where no "Relation Type" has been specified, "IsSupplementTo" is assumed. + +Dataverse now supports the [DataCite v4.5 schema](http://schema.datacite.org/meta/kernel-4/). Additional metadata is now being sent to DataCite including metadata about related publications and files in the dataset. Improvement metadata is being sent including how PIDs (ORCID, ROR, DOIs, etc.), license/terms, geospatial, and other metadata is represented have been made. The enhanced metadata will automatically be sent to DataCite when datasets are created and published. Additionally, after publication, you can inspect what was sent by looking at the DataCite XML export. (Previously the export and what was sent wasn't 100% in sync.) + +The additions are in rough alignment with the OpenAIRE XML export, but there are some minor differences in addition to the Relation Type addition, including an update to the DataCite 4.5 schema. For details see #10632, #10615 and the [design document](https://docs.google.com/document/d/1JzDo9UOIy9dVvaHvtIbOI8tFU6bWdfDfuQvWWpC0tkA/edit?usp=sharing) referenced there. + +Multiple backward incompatible changes and bug fixes have been made to API calls (three of the four of which were not documented) related to updating PID target urls and metadata at the provider service: +- [Update Target URL for a Published Dataset at the PID provider](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#update-target-url-for-a-published-dataset-at-the-pid-provider) +- [Update Target URL for all Published Datasets at the PID provider](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#update-target-url-for-all-published-datasets-at-the-pid-provider) +- [Update Metadata for a Published Dataset at the PID provider](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#update-metadata-for-a-published-dataset-at-the-pid-provider) +- [Update Metadata for all Published Datasets at the PID provider](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#update-metadata-for-all-published-datasets-at-the-pid-provider) + ### Full List of ISO 639-3 Languages Now Supported The controlled vocabulary values list for the metadata field "Language" in the citation block has now been extended to include roughly 7920 ISO 639-3 values. @@ -295,7 +310,8 @@ The following settings have been added: ## Backward Incompatible Changes -The oai_dc export format has changed. See the "Remap oai_dc" section above. +- The oai_dc export format has changed. See the "Remap oai_dc" section above. +- Several APIs related to DataCite have changed. See "More and Better Data Sent to DataCite" above. ## Complete List of Changes @@ -422,7 +438,7 @@ curl http://localhost:8080/api/admin/index 9\. Run reExportAll to update dataset metadata exports -This step is necessary because of changes described above for the `oai_dc` export format. +This step is necessary because of changes described above for the `Datacite` and `oai_dc` export formats. Below is the simple way to reexport all dataset metadata. For more advanced usage, please see [the guides](http://guides.dataverse.org/en/6.4/admin/metadataexport.html#batch-exports-through-the-api). @@ -430,9 +446,29 @@ Below is the simple way to reexport all dataset metadata. For more advanced usag curl http://localhost:8080/api/admin/metadata/reExportAll ``` +10\. Pushing updated metadata to DataCite + +(If you don't use DataCite, you can skip this.) + +Above you updated the citation metadata block and Solr with the new "relationType" field. With these two changes, the "Relation Type" fields will be available and creation/publication of datasets will result in the expanded XML being sent to DataCite. You've also already run "reExportAll" to update the `Datacite` metadata export format. + +Entries at DataCite for published datasets can be updated by a superuser using an API call (newly [documented](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#update-metadata-for-all-published-datasets-at-the-pid-provider)): + +`curl -X POST -H 'X-Dataverse-key:' http://localhost:8080/api/datasets/modifyRegistrationPIDMetadataAll` + +This will loop through all published datasets (and released files with PIDs). As long as the loop completes, the call will return a 200/OK response. Any PIDs for which the update fails can be found using the following command: + +`grep 'Failure for id' server.log` + +Failures may occur if PIDs were never registered, or if they were never made findable. Any such cases can be fixed manually in DataCite Fabrica or using the [Reserve a PID](https://guides.dataverse.org/en/6.4/api/native-api.html#reserve-a-pid) API call and the newly documented `/api/datasets//modifyRegistration` call respectively. See https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#send-dataset-metadata-to-pid-provider. Please reach out with any questions. + +PIDs can also be updated by a superuser on a per-dataset basis using + +`curl -X POST -H 'X-Dataverse-key:' http://localhost:8080/api/datasets//modifyRegistrationMetadata` + ### Additional Upgrade Steps -10\. If there are broken thumbnails +11\. If there are broken thumbnails To restore any broken thumbnails caused by the bug described above, you can call the `http://localhost:8080/api/admin/clearThumbnailFailureFlag` API, which will attempt to clear the flag on all files (regardless of whether caused by this bug or some other problem with the file) or the `http://localhost:8080/api/admin/clearThumbnailFailureFlag/$FILE_ID` to clear the flag for individual files. Calling the former, batch API is recommended. @@ -440,7 +476,7 @@ To restore any broken thumbnails caused by the bug described above, you can call http://localhost:8080/api/admin/clearThumbnailFailureFlag/$FILE_ID ``` -11\. PermaLinks with custom base-url +12\. PermaLinks with custom base-url If you currently use PermaLinks with a custom `base-url`: You must manually append `/citation?persistentId=` to the base URL to maintain functionality. From c3705487f1834a3ac7c8ddf53abf65c06d713fc5 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 23 Sep 2024 15:25:26 -0400 Subject: [PATCH 935/970] add blur for listing feature flags #10853 --- doc/release-notes/6.4-release-notes.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index fa50a00391b..11fd7785a12 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -296,6 +296,10 @@ The addDataverse (`/api/dataverses/{identifier}`) API endpoint has been extended The `/api/dataverses/{identifier}/metadatablocks` endpoint has been fixed to not return fields marked as displayOnCreate=true if there is an input level with include=false, when query parameters returnDatasetFieldTypes=true and onlyDisplayedOnCreate=true are set. See also #10741 and #10767. +### Feature Flags Can Be Listed + +It now possible to list all feature flags and see if they are enabled or not. See #10732 and [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#list-all-feature-flags). + ## Settings Added The following settings have been added: From 09608c1abb28eaab3bf13401b5e282f814a867d4 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 23 Sep 2024 16:27:09 -0400 Subject: [PATCH 936/970] add CVOC bug fix #10853 -6.4-release-notes --- doc/release-notes/10869-fix-npe-using-cvoc.md | 1 - doc/release-notes/6.4-release-notes.md | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) delete mode 100644 doc/release-notes/10869-fix-npe-using-cvoc.md diff --git a/doc/release-notes/10869-fix-npe-using-cvoc.md b/doc/release-notes/10869-fix-npe-using-cvoc.md deleted file mode 100644 index 53214d3789d..00000000000 --- a/doc/release-notes/10869-fix-npe-using-cvoc.md +++ /dev/null @@ -1 +0,0 @@ -This release fixes a bug in the external controlled vocabulary mechanism (introduced in v6.3) that could cause indexing to fail when a script is configured for one child field and no other child fields were managed. \ No newline at end of file diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 11fd7785a12..5ee2d4b4612 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -178,6 +178,10 @@ In order for these changes to be reflected in existing datasets, a [reexport all When detecting files types, Dataverse would previously detect a zip file as a shapefile if it contained [markers of a shapefile](https://guides.dataverse.org/en/6.4/developers/geospatial.html) in hidden directories. These hidden directories are now ignored when deciding if a zip file is a shapefile or not. See also #8945 and #10627. +### External Controlled Vocabulary + +This release fixes a bug (introduced in v6.3) in the external controlled vocabulary mechanism that could cause indexing to fail (with a NullPointerException) when a script is configured for one child field and no other child fields were managed. See also #10869 and #10870. + ### Valid JSON in Error Response When any `ApiBlockingFilter` policy applies to a request, the JSON in the body of the error response is now valid JSON. See also #10085. From 3be6ce44749881eb32b6e71aa4147383961900f8 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 25 Sep 2024 09:50:03 -0400 Subject: [PATCH 937/970] update image_url #10853 --- ...831-standardize-image-url-of-search-api.md | 28 ------------------- doc/release-notes/6.4-release-notes.md | 6 ++-- 2 files changed, 3 insertions(+), 31 deletions(-) delete mode 100644 doc/release-notes/10831-standardize-image-url-of-search-api.md diff --git a/doc/release-notes/10831-standardize-image-url-of-search-api.md b/doc/release-notes/10831-standardize-image-url-of-search-api.md deleted file mode 100644 index 1910091455c..00000000000 --- a/doc/release-notes/10831-standardize-image-url-of-search-api.md +++ /dev/null @@ -1,28 +0,0 @@ -Search API (/api/search) response will now include new image_url format for the Datafile and Dataverse logo. -Note to release note writer: this supersedes the release note 10810-search-api-payload-extensions.md - -For Dataverse: - -- "image_url" (optional) - -```javascript -"items": [ - { - "name": "Darwin's Finches", - ... - "image_url":"/api/access/dvCardImage/{identifier}" -(etc, etc) -``` - -For DataFile: - -- "image_url" (optional) - -```javascript -"items": [ - { - "name": "test.txt", - ... - "image_url":"/api/access/datafile/{identifier}?imageThumb=true" -(etc, etc) -``` diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 5ee2d4b4612..a1d4eee042b 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -222,7 +222,7 @@ For collections (dataverses): "affiliation": "Dataverse.org", "parentDataverseName": "Root", "parentDataverseIdentifier": "root", - "image_url":"data:image/png;base64,iVBORw0..." + "image_url":"/api/access/dvCardImage/{identifier}" (etc, etc) ``` @@ -247,10 +247,10 @@ For files: ```javascript "items": [ { - "name": "test.txt", + "name": "test.png", ... "releaseOrCreateDate": "2016-05-10T12:53:39Z", - "image_url":"data:image/png;base64,iVBORw0..." + "image_url":"/api/access/datafile/42?imageThumb=true" (etc, etc) ``` From 110fbe10dd57d6c881a31ceb602c78470aed2ce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Kir=C3=A1ly?= Date: Tue, 20 Jun 2023 17:35:58 +0200 Subject: [PATCH 938/970] #9650 improve the linked dataverse listing API --- .../java/edu/harvard/iq/dataverse/api/Datasets.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index f26b6d67edf..1700c4fbcf5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -2076,10 +2076,16 @@ public Response getLinks(@Context ContainerRequestContext crc, @PathParam("id") List dvsThatLinkToThisDatasetId = dataverseSvc.findDataversesThatLinkToThisDatasetId(datasetId); JsonArrayBuilder dataversesThatLinkToThisDatasetIdBuilder = Json.createArrayBuilder(); for (Dataverse dataverse : dvsThatLinkToThisDatasetId) { - dataversesThatLinkToThisDatasetIdBuilder.add(dataverse.getAlias() + " (id " + dataverse.getId() + ")"); + JsonObjectBuilder entry = Json.createObjectBuilder(); + entry.add("id", dataverse.getId()); + entry.add("alias", dataverse.getAlias()); + entry.add("displayName", dataverse.getDisplayName()); + dataversesThatLinkToThisDatasetIdBuilder.add(entry); } JsonObjectBuilder response = Json.createObjectBuilder(); - response.add("dataverses that link to dataset id " + datasetId, dataversesThatLinkToThisDatasetIdBuilder); + response.add("id", datasetId); + response.add("identifier", dataset.getIdentifier()); + response.add("linked-dataverses", dataversesThatLinkToThisDatasetIdBuilder); return ok(response); } catch (WrappedResponse wr) { return wr.getResponse(); From 2c2cd86f7a1d0cf994cded37ffbf71e66bb824cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Kir=C3=A1ly?= Date: Tue, 20 Jun 2023 18:29:18 +0200 Subject: [PATCH 939/970] #9650 backwards compatibility, release note --- .../9650-5-improve-list-linked-dataverses-API.md | 5 +++++ src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 doc/release-notes/9650-5-improve-list-linked-dataverses-API.md diff --git a/doc/release-notes/9650-5-improve-list-linked-dataverses-API.md b/doc/release-notes/9650-5-improve-list-linked-dataverses-API.md new file mode 100644 index 00000000000..8c79955891b --- /dev/null +++ b/doc/release-notes/9650-5-improve-list-linked-dataverses-API.md @@ -0,0 +1,5 @@ +The following API have been added: + +/api/datasets/{datasetId}/links + +It lists the linked dataverses to a dataset. It can be executed only by administrators. \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 1700c4fbcf5..24b20b8b882 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -2085,7 +2085,7 @@ public Response getLinks(@Context ContainerRequestContext crc, @PathParam("id") JsonObjectBuilder response = Json.createObjectBuilder(); response.add("id", datasetId); response.add("identifier", dataset.getIdentifier()); - response.add("linked-dataverses", dataversesThatLinkToThisDatasetIdBuilder); + response.add("dataverses that link to dataset id " + datasetId, dataversesThatLinkToThisDatasetIdBuilder); return ok(response); } catch (WrappedResponse wr) { return wr.getResponse(); From 6228d3f030a331487e9237958626f171cb52c623 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Thu, 12 Sep 2024 12:59:33 -0400 Subject: [PATCH 940/970] adding tests, doc changes, and formatting fix --- .../source/admin/dataverses-datasets.rst | 12 ++++++-- .../harvard/iq/dataverse/api/Datasets.java | 12 ++++---- .../harvard/iq/dataverse/api/DatasetsIT.java | 28 +++++++++++++++++++ .../edu/harvard/iq/dataverse/api/MoveIT.java | 7 ++--- 4 files changed, 46 insertions(+), 13 deletions(-) diff --git a/doc/sphinx-guides/source/admin/dataverses-datasets.rst b/doc/sphinx-guides/source/admin/dataverses-datasets.rst index b4d365c4fd4..7c03a6f80c0 100644 --- a/doc/sphinx-guides/source/admin/dataverses-datasets.rst +++ b/doc/sphinx-guides/source/admin/dataverses-datasets.rst @@ -129,15 +129,21 @@ Lists the link(s) created between a dataset and a Dataverse collection (see the curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/datasets/$linked-dataset-id/links -It returns a list in the following format: +It returns a list in the following format (new format as of v6.4): .. code-block:: json { "status": "OK", "data": { - "dataverses that link to dataset id 56782": [ - "crc990 (id 18802)" + "id": 5, + "identifier": "FK2/OTCWMM", + "linked-dataverses": [ + { + "id": 2, + "alias": "dataverse1", + "displayName": "Lab experiments 2023 June" + } ] } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 24b20b8b882..e0d3bf6126e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -2076,16 +2076,16 @@ public Response getLinks(@Context ContainerRequestContext crc, @PathParam("id") List dvsThatLinkToThisDatasetId = dataverseSvc.findDataversesThatLinkToThisDatasetId(datasetId); JsonArrayBuilder dataversesThatLinkToThisDatasetIdBuilder = Json.createArrayBuilder(); for (Dataverse dataverse : dvsThatLinkToThisDatasetId) { - JsonObjectBuilder entry = Json.createObjectBuilder(); - entry.add("id", dataverse.getId()); - entry.add("alias", dataverse.getAlias()); - entry.add("displayName", dataverse.getDisplayName()); - dataversesThatLinkToThisDatasetIdBuilder.add(entry); + JsonObjectBuilder datasetBuilder = Json.createObjectBuilder(); + datasetBuilder.add("id", dataverse.getId()); + datasetBuilder.add("alias", dataverse.getAlias()); + datasetBuilder.add("displayName", dataverse.getDisplayName()); + dataversesThatLinkToThisDatasetIdBuilder.add(datasetBuilder.build()); } JsonObjectBuilder response = Json.createObjectBuilder(); response.add("id", datasetId); response.add("identifier", dataset.getIdentifier()); - response.add("dataverses that link to dataset id " + datasetId, dataversesThatLinkToThisDatasetIdBuilder); + response.add("linked-dataverses", dataversesThatLinkToThisDatasetIdBuilder); return ok(response); } catch (WrappedResponse wr) { return wr.getResponse(); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index f52aa4fe9bd..b7bdaeebbfc 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -2963,6 +2963,34 @@ public void testLinkingDatasets() { linkDataset.then().assertThat() .statusCode(OK.getStatusCode()); + // Link another to test the list of linked datasets + Response createDataverse3 = UtilIT.createRandomDataverse(apiToken); + createDataverse3.prettyPrint(); + createDataverse3.then().assertThat() + .statusCode(CREATED.getStatusCode()); + String dataverse3Alias = UtilIT.getAliasFromResponse(createDataverse3); + Integer dataverse3Id = UtilIT.getDatasetIdFromResponse(createDataverse3); + linkDataset = UtilIT.linkDataset(datasetPid, dataverse3Alias, superuserApiToken); + linkDataset.prettyPrint(); + linkDataset.then().assertThat() + .statusCode(OK.getStatusCode()); + // get the list in Json format + Response linkDatasetsResponse = UtilIT.getDatasetLinks(datasetPid, superuserApiToken); + linkDatasetsResponse.prettyPrint(); + linkDatasetsResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + JsonObject linkDatasets = Json.createReader(new StringReader(linkDatasetsResponse.asString())).readObject(); + JsonArray lst = linkDatasets.getJsonObject("data").getJsonArray("linked-dataverses"); + List ids = List.of(dataverse2Id, dataverse3Id); + List uniqueids = new ArrayList<>(); + assertEquals(ids.size(), lst.size()); + for (int i = 0; i < lst.size(); i++) { + int id = lst.getJsonObject(i).getInt("id"); + assertTrue(ids.contains(id)); + assertFalse(uniqueids.contains(id)); + uniqueids.add(id); + } + //Experimental code for trying to trick test into thinking the dataset has been harvested /* createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverse1Alias, apiToken); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/MoveIT.java b/src/test/java/edu/harvard/iq/dataverse/api/MoveIT.java index f7135ce7f3b..8951b0bd42e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/MoveIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/MoveIT.java @@ -278,8 +278,8 @@ public void testMoveLinkedDataset() { .body("message", equalTo("Use the query parameter forceMove=true to complete the move. This dataset is linked to the new host dataverse or one of its parents. This move would remove the link to this dataset. ")); JsonObject linksBeforeData = Json.createReader(new StringReader(getLinksBefore.asString())).readObject(); - assertEquals("OK", linksBeforeData.getString("status")); - assertEquals(dataverse2Alias + " (id " + dataverse2Id + ")", linksBeforeData.getJsonObject("data").getJsonArray("dataverses that link to dataset id " + datasetId).getString(0)); + assertEquals(datasetId, linksBeforeData.getJsonObject("data").getInt("id")); + assertEquals(dataverse2Id, linksBeforeData.getJsonObject("data").getJsonArray("linked-dataverses").get(0).asJsonObject().getInt("id")); boolean forceMove = true; Response forceMoveLinkedDataset = UtilIT.moveDataset(datasetId.toString(), dataverse2Alias, forceMove, superuserApiToken); @@ -308,8 +308,7 @@ public void testMoveLinkedDataset() { JsonObject linksAfterData = Json.createReader(new StringReader(getLinksAfter.asString())).readObject(); assertEquals("OK", linksAfterData.getString("status")); - assertEquals(0, linksAfterData.getJsonObject("data").getJsonArray("dataverses that link to dataset id " + datasetId).size()); - + assertEquals(0, linksAfterData.getJsonObject("data").getJsonArray("linked-dataverses").size()); } @Test From d610094c7fc557b06fad2115f44291057d72cc3d Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 25 Sep 2024 10:20:17 -0400 Subject: [PATCH 941/970] Updated the docs to reflect the new name of a JVM option (#10623) --- doc/sphinx-guides/source/developers/big-data-support.rst | 2 +- doc/sphinx-guides/source/developers/globus-api.rst | 2 +- doc/sphinx-guides/source/installation/config.rst | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index 99ea4d2e0ba..dc4006b7d89 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -188,4 +188,4 @@ An overview of the control and data transfer interactions between components was See also :ref:`Globus settings <:GlobusSettings>`. -An alternative, experimental implementation of Globus polling of ongoing upload transfers has been added in v6.4. This framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. Due to its experimental nature it is not enabled by default. See the ``globus-use-experimental-async-framework`` feature flag and the JVM option ``dataverse.globus.taskMonitoringServer`` described in the Configuration guide. +An alternative, experimental implementation of Globus polling of ongoing upload transfers has been added in v6.4. This framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. Due to its experimental nature it is not enabled by default. See the ``globus-use-experimental-async-framework`` feature flag and the JVM option ``dataverse.files.globus-monitoring-server`` described in the Configuration guide. diff --git a/doc/sphinx-guides/source/developers/globus-api.rst b/doc/sphinx-guides/source/developers/globus-api.rst index 0f16dc704ef..a1bde106f92 100644 --- a/doc/sphinx-guides/source/developers/globus-api.rst +++ b/doc/sphinx-guides/source/developers/globus-api.rst @@ -185,7 +185,7 @@ As the transfer can take significant time and the API call is asynchronous, the Once the transfer completes, Dataverse will remove the write permission for the principal. -An alternative, experimental implementation of Globus polling of ongoing upload transfers has been added in v6.4. This new framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. Due to its experimental nature it is not enabled by default. See the ``globus-use-experimental-async-framework`` feature flag and the JVM option ``dataverse.globus.taskMonitoringServer`` described in the Configuration guide. +An alternative, experimental implementation of Globus polling of ongoing upload transfers has been added in v6.4. This new framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. Due to its experimental nature it is not enabled by default. See the ``globus-use-experimental-async-framework`` feature flag and the JVM option ``dataverse.files.globus-monitoring-server`` described in the Configuration guide. Note that when using a managed endpoint that uses the Globus S3 Connector, the checksum should be correct as Dataverse can validate it. For file-based endpoints, the checksum should be included if available but Dataverse cannot verify it. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 23168e12f30..bf64793e385 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3266,10 +3266,10 @@ The email for your institution that you'd like to appear in bag-info.txt. See :r Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_BAGIT_SOURCEORG_EMAIL``. -.. _dataverse.globus.taskMonitoringServer: +.. _dataverse.files.globus-monitoring-server: -dataverse.globus.taskMonitoringServer -+++++++++++++++++++++++++++++++++++++ +dataverse.files.globus-monitoring-server +++++++++++++++++++++++++++++++++++++++++ This setting is required in conjunction with the ``globus-use-experimental-async-framework`` feature flag. Setting it to true designates the Dataverse instance to serve as the dedicated polling server. It is needed so that the new framework can be used in a multi-node installation. @@ -3309,7 +3309,7 @@ please find all known feature flags below. Any of these flags can be activated u - Removes the reason field in the `Publish/Return To Author` dialog that was added as a required field in v6.2 and makes the reason an optional parameter in the :ref:`return-a-dataset` API call. - ``Off`` * - globus-use-experimental-async-framework - - Activates a new experimental implementation of Globus polling of ongoing remote data transfers that does not rely on the instance staying up continuously for the duration of the transfers and saves the state information about Globus upload requests in the database. Added in v6.4. Note that the JVM option ``dataverse.globus.taskMonitoringServer`` described above must also be enabled on one (and only one, in a multi-node installation) Dataverse instance. + - Activates a new experimental implementation of Globus polling of ongoing remote data transfers that does not rely on the instance staying up continuously for the duration of the transfers and saves the state information about Globus upload requests in the database. Added in v6.4. Note that the JVM option ``dataverse.files.globus-monitoring-server`` described above must also be enabled on one (and only one, in a multi-node installation) Dataverse instance. - ``Off`` **Note:** Feature flags can be set via any `supported MicroProfile Config API source`_, e.g. the environment variable From 54fad55ed773dab96f023ee95dc980294289cec5 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Thu, 12 Sep 2024 14:42:30 -0400 Subject: [PATCH 942/970] update doc --- doc/sphinx-guides/source/api/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index c5890fd9996..92cd4fc941b 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -12,7 +12,7 @@ v6.4 - **/api/datasets/$dataset-id/modifyRegistration**: Changed from GET to POST - **/api/datasets/modifyRegistrationPIDMetadataAll**: Changed from GET to POST - +- **/api/datasets/{identifier}/links**: The GET endpoint returns a list of Dataverses linked to the given Dataset. The format of the response has changes for v6.4 making it backward incompatible. v6.3 ---- From ea02478879820b0780122c4c85982f3c1f21c9c4 Mon Sep 17 00:00:00 2001 From: ofahimIQSS Date: Wed, 25 Sep 2024 11:39:14 -0400 Subject: [PATCH 943/970] I fixed anchor links #10876 (#10877) --- doc/sphinx-guides/source/contributor/documentation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/contributor/documentation.md b/doc/sphinx-guides/source/contributor/documentation.md index 96277c3b373..2a8d6794921 100644 --- a/doc/sphinx-guides/source/contributor/documentation.md +++ b/doc/sphinx-guides/source/contributor/documentation.md @@ -56,7 +56,7 @@ In case you decide to use a Sphinx Docker container to build the guides, you can ### Installing Sphinx -First, make a fork of https://github.com/IQSS/dataverse and clone your fork locally. Then change to the ``doc/sphinx-guides`` directory. +First, make a fork of and clone your fork locally. Then change to the ``doc/sphinx-guides`` directory. ``cd doc/sphinx-guides`` @@ -83,7 +83,7 @@ On a Mac we recommend installing GraphViz through [Homebrew](). To edit the existing documentation: -- Create a branch (see :ref:`how-to-make-a-pull-request`). +- Create a branch (see {ref}`how-to-make-a-pull-request`). - In ``doc/sphinx-guides/source`` you will find the .rst files that correspond to https://guides.dataverse.org. - Using your preferred text editor, open and edit the necessary files, or create new ones. From 682c89fc069f71c5cd6477bdbffc58107fb01720 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 25 Sep 2024 11:43:33 -0400 Subject: [PATCH 944/970] improve release note #10623 --- doc/release-notes/10623-globus-improvements.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/10623-globus-improvements.md b/doc/release-notes/10623-globus-improvements.md index c6b9febbb02..9eb529bc4f7 100644 --- a/doc/release-notes/10623-globus-improvements.md +++ b/doc/release-notes/10623-globus-improvements.md @@ -1 +1 @@ -A new alternative implementation of Globus polling during upload data transfers has been added in this release. This experimental framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. See the `globus-use-experimental-async-framework` under [Feature Flags](https://dataverse-guide--10781.org.readthedocs.build/en/10781/installation/config.html#feature-flags) in the Installation Guide. +A new alternative implementation of Globus polling during upload data transfers has been added in this release. This experimental framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. See `globus-use-experimental-async-framework` under [Feature Flags](https://dataverse-guide--10781.org.readthedocs.build/en/10781/installation/config.html#feature-flags) and [dataverse.files.globus-monitoring-server](https://dataverse-guide--10781.org.readthedocs.build/en/10781/installation/config.html#dataverse-files-globus-monitoring-server) in the Installation Guide. See also #10623 and #10781. From def801ef45cb8a6ff639bb096410f8710af6e785 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 25 Sep 2024 14:54:23 -0400 Subject: [PATCH 945/970] add globus async #10853 --- doc/release-notes/10623-globus-improvements.md | 1 - doc/release-notes/6.4-release-notes.md | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) delete mode 100644 doc/release-notes/10623-globus-improvements.md diff --git a/doc/release-notes/10623-globus-improvements.md b/doc/release-notes/10623-globus-improvements.md deleted file mode 100644 index 9eb529bc4f7..00000000000 --- a/doc/release-notes/10623-globus-improvements.md +++ /dev/null @@ -1 +0,0 @@ -A new alternative implementation of Globus polling during upload data transfers has been added in this release. This experimental framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. See `globus-use-experimental-async-framework` under [Feature Flags](https://dataverse-guide--10781.org.readthedocs.build/en/10781/installation/config.html#feature-flags) and [dataverse.files.globus-monitoring-server](https://dataverse-guide--10781.org.readthedocs.build/en/10781/installation/config.html#dataverse-files-globus-monitoring-server) in the Installation Guide. See also #10623 and #10781. diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index a1d4eee042b..b0dcdd1fcf7 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -98,6 +98,10 @@ It is now possible to turn off the feature that automatically selects of one of The configuration setting `dataverse.pid.*.permalink.base-url`, which is used for PermaLinks, has been updated to support greater flexibility. Previously, the string `/citation?persistentId=` was automatically appended to the configured base URL. With this update, the base URL will now be used exactly as configured, without any automatic additions. See also #10775. +### Globus Async Framework + +A new alternative implementation of Globus polling during upload data transfers has been added in this release. This experimental framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. See `globus-use-experimental-async-framework` under [Feature Flags](https://guides.dataverse.org/en/6.4/installation/config.html#feature-flags) and [dataverse.files.globus-monitoring-server](https://guides.dataverse.org/en/6.4/installation/config.html#dataverse-files-globus-monitoring-server) in the Installation Guide. See also #10623 and #10781. + ### Development on Windows New instructions have been added for developers on Windows trying to run a Dataverse development environment using Windows Subsystem for Linux (WSL). See [the guides](https://guides.dataverse.org/en/6.4/developers/windows.html), #10606, and #10608. From c2865280cac9ae67a8f01e8b1c6308dacbb0f924 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 25 Sep 2024 16:27:47 -0400 Subject: [PATCH 946/970] add cvoc update #10853 --- doc/release-notes/10711-CVoc Updates.md | 1 - doc/release-notes/6.4-release-notes.md | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) delete mode 100644 doc/release-notes/10711-CVoc Updates.md diff --git a/doc/release-notes/10711-CVoc Updates.md b/doc/release-notes/10711-CVoc Updates.md deleted file mode 100644 index f747bedb049..00000000000 --- a/doc/release-notes/10711-CVoc Updates.md +++ /dev/null @@ -1 +0,0 @@ -Changes in Dataverse and updates to the ORCID and ROR external vocabulary scripts support deploying these for the citation block author field (and others). diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index b0dcdd1fcf7..7115d26a505 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -102,6 +102,10 @@ The configuration setting `dataverse.pid.*.permalink.base-url`, which is used fo A new alternative implementation of Globus polling during upload data transfers has been added in this release. This experimental framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. See `globus-use-experimental-async-framework` under [Feature Flags](https://guides.dataverse.org/en/6.4/installation/config.html#feature-flags) and [dataverse.files.globus-monitoring-server](https://guides.dataverse.org/en/6.4/installation/config.html#dataverse-files-globus-monitoring-server) in the Installation Guide. See also #10623 and #10781. +### CVoc (Controlled Vocabulary): Allow ORCID and ROR to be Used Together in Author Field + +Changes in Dataverse and updates to the ORCID and ROR external vocabulary scripts support deploying these for the citation block author field (and others). See also #10711, #10712, and . + ### Development on Windows New instructions have been added for developers on Windows trying to run a Dataverse development environment using Windows Subsystem for Linux (WSL). See [the guides](https://guides.dataverse.org/en/6.4/developers/windows.html), #10606, and #10608. From 62dd3486b42fc41eec9760cfcea917e4398ccefb Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 26 Sep 2024 10:08:50 -0400 Subject: [PATCH 947/970] add new globus settings under settings section #10853 --- doc/release-notes/6.4-release-notes.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 7115d26a505..6bdc2625c46 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -317,6 +317,8 @@ It now possible to list all feature flags and see if they are enabled or not. Se The following settings have been added: - dataverse.feature.disable-dataset-thumbnail-autoselect +- dataverse.feature.globus-use-experimental-async-framework +- dataverse.files.globus-monitoring-server - dataverse.pid.*.crossref.url - dataverse.pid.*.crossref.rest-api-url - dataverse.pid.*.crossref.username From a01a0cbe989085b7dfa881ac6f5c69bea2a1be56 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 26 Sep 2024 10:44:50 -0400 Subject: [PATCH 948/970] various tweaks #10853 --- doc/release-notes/6.4-release-notes.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 6bdc2625c46..a632e5e3d19 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -30,11 +30,11 @@ All this and more can be found below. Read on! Within the "Related Publication" field, a new subfield has been added called "Relation Type" that allows for the most common [values](https://datacite-metadata-schema.readthedocs.io/en/4.5/appendices/appendix-1/relationType/) recommended by DataCite: isCitedBy, Cites, IsSupplementTo, IsSupplementedBy, IsReferencedBy, and References. For existing datasets where no "Relation Type" has been specified, "IsSupplementTo" is assumed. -Dataverse now supports the [DataCite v4.5 schema](http://schema.datacite.org/meta/kernel-4/). Additional metadata is now being sent to DataCite including metadata about related publications and files in the dataset. Improvement metadata is being sent including how PIDs (ORCID, ROR, DOIs, etc.), license/terms, geospatial, and other metadata is represented have been made. The enhanced metadata will automatically be sent to DataCite when datasets are created and published. Additionally, after publication, you can inspect what was sent by looking at the DataCite XML export. (Previously the export and what was sent wasn't 100% in sync.) +Dataverse now supports the [DataCite v4.5 schema](http://schema.datacite.org/meta/kernel-4/). Additional metadata is now being sent to DataCite including metadata about related publications and files in the dataset. Improved metadata is being sent including how PIDs (ORCID, ROR, DOIs, etc.), license/terms, geospatial, and other metadata are represented. The enhanced metadata will automatically be sent to DataCite when datasets are created and published. Additionally, after publication, you can inspect what was sent by looking at the DataCite XML export. The additions are in rough alignment with the OpenAIRE XML export, but there are some minor differences in addition to the Relation Type addition, including an update to the DataCite 4.5 schema. For details see #10632, #10615 and the [design document](https://docs.google.com/document/d/1JzDo9UOIy9dVvaHvtIbOI8tFU6bWdfDfuQvWWpC0tkA/edit?usp=sharing) referenced there. -Multiple backward incompatible changes and bug fixes have been made to API calls (three of the four of which were not documented) related to updating PID target urls and metadata at the provider service: +Multiple backward incompatible changes and bug fixes have been made to API calls (three of four of which were not documented) related to updating PID target URLs and metadata at the provider service: - [Update Target URL for a Published Dataset at the PID provider](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#update-target-url-for-a-published-dataset-at-the-pid-provider) - [Update Target URL for all Published Datasets at the PID provider](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#update-target-url-for-all-published-datasets-at-the-pid-provider) - [Update Metadata for a Published Dataset at the PID provider](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#update-metadata-for-a-published-dataset-at-the-pid-provider) @@ -50,7 +50,7 @@ ISO 639-3 codes were downloaded from [sil.org](https://iso639-3.sil.org/code_tab ### Unlink Dataset Button -New "Unlink Dataset" button has been added to the dataset page to allow a user to unlink a dataset from a collection. The user must possess the same permissions needed to unlink the dataset as they would to link the Dataset. Additionally, the [existing API](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#unlink-a-dataset) for unlinking datasets has been updated to no longer require superuser access as the "Publish Dataset" permission is now enough. See also #10583 and #10689. +A new "Unlink Dataset" button has been added to the dataset page to allow a user to unlink a dataset from a collection. To unlink a dataset the user must have permission to link the dataset. Additionally, the [existing API](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#unlink-a-dataset) for unlinking datasets has been updated to no longer require superuser access as the "Publish Dataset" permission is now enough. See also #10583 and #10689. ### Pre-Publish File DOI Reservation @@ -58,7 +58,7 @@ Dataverse installations using DataCite as a persistent identifier (PID) provider ### Initial Support for Dataset Types -Out of the box, all datasets now have the type "dataset" but superusers can add additional types. At this time the type of a dataset can only be set at creation time via API. The types "dataset", "software", and "workflow" (just those three, for now) will be sent to DataCite when the dataset is published. +Out of the box, all datasets now have the type "dataset" but superusers can add additional types. At this time the type of a dataset can only be set at creation time via API. The types "dataset", "software", and "workflow" (just those three, for now) will be sent to DataCite (as `resourceTypeGeneral`) when the dataset is published. For details see [the guides](https://guides.dataverse.org/en/6.4/user/dataset-management.html#dataset-types), #10517 and #10694. Please note that this feature is highly experimental and is expected to [evolve](https://github.com/IQSS/dataverse-pm/issues/307). @@ -102,7 +102,7 @@ The configuration setting `dataverse.pid.*.permalink.base-url`, which is used fo A new alternative implementation of Globus polling during upload data transfers has been added in this release. This experimental framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. See `globus-use-experimental-async-framework` under [Feature Flags](https://guides.dataverse.org/en/6.4/installation/config.html#feature-flags) and [dataverse.files.globus-monitoring-server](https://guides.dataverse.org/en/6.4/installation/config.html#dataverse-files-globus-monitoring-server) in the Installation Guide. See also #10623 and #10781. -### CVoc (Controlled Vocabulary): Allow ORCID and ROR to be Used Together in Author Field +### CVoc (Controlled Vocabulary): Allow ORCID and ROR to Be Used Together in Author Field Changes in Dataverse and updates to the ORCID and ROR external vocabulary scripts support deploying these for the citation block author field (and others). See also #10711, #10712, and . @@ -159,7 +159,7 @@ See also #10819, #10820, and [the post](https://groups.google.com/g/dataverse-co ### No License, No Terms of Use -When datasets have neither a license nor custom terms of use, the dataset page will indicate this. Also, these datasets will no longer be indexed as having custom terms. See also #8796, #10513, and #10614. +When datasets have neither a license nor custom terms of use, the dataset page will now indicate this. Also, these datasets will no longer be indexed as having custom terms. See also #8796, #10513, and #10614. ### CC0 License Bug Fix @@ -264,7 +264,7 @@ For files: These examples are also shown in the [Search API](https://guides.dataverse.org/en/6.4/api/search.html) section of the API Guide. -The image_url field was already part of the SolrSearchResult JSON (and incorrectly appeared in Search API documentation), but it wasn't returned by the API because it was appended only after the Solr query was executed in the SearchIncludeFragment of JSF. Now, the field is set in SearchServiceBean, ensuring it is always returned by the API when an image is available. +The image_url field was already part of the SolrSearchResult JSON (and incorrectly appeared in Search API documentation), but it wasn't returned by the API because it was appended only after the Solr query was executed in the SearchIncludeFragment of JSF (the old/current UI framework). Now, the field is set in SearchServiceBean, ensuring it is always returned by the API when an image is available. The Solr schema.xml file has been updated to include a new field called "dvParentAlias" for supporting the new response field "parentDataverseIdentifier". See upgrade instructions below. @@ -298,7 +298,7 @@ A new optional query parameter "returnDetails" added to `/api/dataverses/{identi ### User Permissions on Collections -A new endpoint at `/api/dataverses/{identifier}/userPermissions` for obtaining the user permissions on a collection (dataverse) has been added. See also #10749 and #10751. +A new endpoint at `/api/dataverses/{identifier}/userPermissions` for obtaining the user permissions on a collection (dataverse) has been added. See also [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#get-user-permissions-on-a-dataverse), #10749 and #10751. ### addDataverse Extended @@ -310,7 +310,7 @@ The `/api/dataverses/{identifier}/metadatablocks` endpoint has been fixed to not ### Feature Flags Can Be Listed -It now possible to list all feature flags and see if they are enabled or not. See #10732 and [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#list-all-feature-flags). +It now possible to list all feature flags and see if they are enabled or not. See also [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#list-all-feature-flags) and #10732. ## Settings Added From 0177062665dc22c25a479caced80bfab7ef79fb6 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 26 Sep 2024 11:00:20 -0400 Subject: [PATCH 949/970] typo #10853 --- doc/release-notes/6.4-release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index a632e5e3d19..7ab2aafcd95 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -310,7 +310,7 @@ The `/api/dataverses/{identifier}/metadatablocks` endpoint has been fixed to not ### Feature Flags Can Be Listed -It now possible to list all feature flags and see if they are enabled or not. See also [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#list-all-feature-flags) and #10732. +It is now possible to list all feature flags and see if they are enabled or not. See also [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#list-all-feature-flags) and #10732. ## Settings Added From 7b0c5115679de1b71e9a0eb573eb3a2db8f6841a Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 26 Sep 2024 14:06:11 -0400 Subject: [PATCH 950/970] put features before bug fixes #10853 --- doc/release-notes/6.4-release-notes.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 7ab2aafcd95..5ed3341998f 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -6,12 +6,7 @@ This release brings new features, enhancements, and bug fixes to Dataverse. Than ## Release Highlights -Dataverse 6.4 fixes a couple important bugs described below and in [a post](https://groups.google.com/g/dataverse-community/c/evn5C-pyrS8/m/JrH9vp47DwAJ) on the mailing list: - -- "Update Current Version" can cause metadata loss -- Publishing breaks designated dataset thumbnail, messes up collection page - -New features include: +Dataverse 6.4 includes many new features, including: - More and better metadata is being sent to DataCite, including "Relation Type" - All ISO 639-3 languages are now supported @@ -22,7 +17,12 @@ New features include: - RO-Crate support - A new Rust library -All this and more can be found below. Read on! +This release also fixes a couple important bugs described below and in [a post](https://groups.google.com/g/dataverse-community/c/evn5C-pyrS8/m/JrH9vp47DwAJ) on the mailing list: + +- "Update Current Version" can cause metadata loss +- Publishing breaks designated dataset thumbnail, messes up collection page + +Additional details on the above as well as many more features and bug fixes included in the release are described below. Read on! ## Features Added From e54e066ad4e4275fb1797753403533f3f04bd67e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 26 Sep 2024 14:15:55 -0400 Subject: [PATCH 951/970] reword --- doc/release-notes/6.4-release-notes.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 5ed3341998f..c7c8856843e 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -6,7 +6,7 @@ This release brings new features, enhancements, and bug fixes to Dataverse. Than ## Release Highlights -Dataverse 6.4 includes many new features, including: +New features in Dataverse 6.4: - More and better metadata is being sent to DataCite, including "Relation Type" - All ISO 639-3 languages are now supported @@ -15,9 +15,13 @@ Dataverse 6.4 includes many new features, including: - Datasets can now have types such as "software" or "workflow" - Croissant support - RO-Crate support -- A new Rust library +- and more! Please see below. -This release also fixes a couple important bugs described below and in [a post](https://groups.google.com/g/dataverse-community/c/evn5C-pyrS8/m/JrH9vp47DwAJ) on the mailing list: +New client library: + +- Rust + +This release also fixes two important bugs described below and in [a post](https://groups.google.com/g/dataverse-community/c/evn5C-pyrS8/m/JrH9vp47DwAJ) on the mailing list: - "Update Current Version" can cause metadata loss - Publishing breaks designated dataset thumbnail, messes up collection page From c6d626dea0e93f110c35ac2016a36922f534dfc1 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 26 Sep 2024 14:20:06 -0400 Subject: [PATCH 952/970] datacite title --- doc/release-notes/6.4-release-notes.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index c7c8856843e..60c9ab95adf 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -8,7 +8,7 @@ This release brings new features, enhancements, and bug fixes to Dataverse. Than New features in Dataverse 6.4: -- More and better metadata is being sent to DataCite, including "Relation Type" +- Enhanced DataCite Metadata, including "Relation Type" - All ISO 639-3 languages are now supported - There is now a button for "Unlink Dataset" - Users will have DOIs/PIDs reserved for their files as part of file upload instead of at publication time @@ -30,7 +30,7 @@ Additional details on the above as well as many more features and bug fixes incl ## Features Added -### More and Better Data Sent to DataCite, Including "Relation Type" +### Enhanced DataCite Metadata, Including "Relation Type" Within the "Related Publication" field, a new subfield has been added called "Relation Type" that allows for the most common [values](https://datacite-metadata-schema.readthedocs.io/en/4.5/appendices/appendix-1/relationType/) recommended by DataCite: isCitedBy, Cites, IsSupplementTo, IsSupplementedBy, IsReferencedBy, and References. For existing datasets where no "Relation Type" has been specified, "IsSupplementTo" is assumed. From 1654313d28596478a49e38c4f2640f4a26440bc5 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 26 Sep 2024 14:27:45 -0400 Subject: [PATCH 953/970] croissant update #10853 --- doc/release-notes/6.4-release-notes.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 60c9ab95adf..648e628864d 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -70,10 +70,10 @@ For details see [the guides](https://guides.dataverse.org/en/6.4/user/dataset-ma A new metadata export format called [Croissant](https://github.com/mlcommons/croissant) is now available as an external metadata exporter. It is oriented toward making datasets consumable by machine learning. -When enabled, Croissant replaces the Schema.org JSON-LD format in the `` of dataset landing pages. For details, see the [Schema.org JSON-LD/Croissant Metadata](https://guides.dataverse.org/en/6.4/admin/discoverability.html#schema-org-head) under the discoverability section of the Admin Guide. This change was made in Dataverse 6.3 in #10382. - For more about the Croissant exporter, including installation instructions, see . See also #10341, #10533, and [discussion](https://groups.google.com/g/dataverse-community/c/JI8HPgGarr8/m/DqEIkiwlAgAJ) on the mailing list. +Please note: the Croissant exporter works best with Dataverse 6.2 and higher (where it updates the content of `` as [described](https://guides.dataverse.org/en/6.4/admin/discoverability.html#schema-org-head) in the guides) but can be used with 6.0 and higher (to get the export functionality). + ### RO-Crate Support (Metadata Export) Dataverse now supports [RO-Crate](https://www.researchobject.org/ro-crate/) as a metadata export format. This functionality is not available out of the box, but you can enable one or more RO-Crate exporters from the [list of external exporters](https://guides.dataverse.org/en/6.4/installation/advanced.html#inventory-of-external-exporters). See also #10744 and #10796. From 1b2a52b5ed158686719afaee6c351535d33e4073 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 27 Sep 2024 09:43:06 -0400 Subject: [PATCH 954/970] bump version to 6.4 #10852 (#10871) --- doc/sphinx-guides/source/conf.py | 4 ++-- doc/sphinx-guides/source/versions.rst | 3 ++- modules/dataverse-parent/pom.xml | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py index c719fb05e3c..7ee355302d8 100755 --- a/doc/sphinx-guides/source/conf.py +++ b/doc/sphinx-guides/source/conf.py @@ -68,9 +68,9 @@ # built documents. # # The short X.Y version. -version = '6.3' +version = '6.4' # The full version, including alpha/beta/rc tags. -release = '6.3' +release = '6.4' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/sphinx-guides/source/versions.rst b/doc/sphinx-guides/source/versions.rst index 952eba72616..800bdc6e0f9 100755 --- a/doc/sphinx-guides/source/versions.rst +++ b/doc/sphinx-guides/source/versions.rst @@ -7,7 +7,8 @@ Dataverse Software Documentation Versions This list provides a way to refer to the documentation for previous and future versions of the Dataverse Software. In order to learn more about the updates delivered from one version to another, visit the `Releases `__ page in our GitHub repo. - pre-release `HTML (not final!) `__ and `PDF (experimental!) `__ built from the :doc:`develop ` branch :doc:`(how to contribute!) ` -- 6.3 +- 6.4 +- `6.3 `__ - `6.2 `__ - `6.1 `__ - `6.0 `__ diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 6bea02569ec..76f0ddbc033 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -131,7 +131,7 @@ - 6.3 + 6.4 17 UTF-8 @@ -446,8 +446,8 @@ Once the release has been made (tag created), change this back to "${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion}" (These properties are provided by the build-helper plugin below.) --> - ${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion} - + + ${revision} From 068607793b70d6fdd0b0ee1b1a3d2a5bfc2c2574 Mon Sep 17 00:00:00 2001 From: Guillermo Portas Date: Fri, 27 Sep 2024 16:05:32 +0200 Subject: [PATCH 955/970] displayOnCreate set to true for depositor and dateOfDeposit in Citation metadata block (#10884) * Changed: displayOnCreate set to true for depositor and dateOfDeposit in citation.tsv * Changed: MetadataBlocksIT test assertion for new total number of displayOnCreate fields * Added: release notes for #10850 * Added: minor tweak to release notes --- ...50-citation-tsv-displayoncreate-depositor-dateofdeposit.md | 1 + scripts/api/data/metadatablocks/citation.tsv | 4 ++-- .../java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 doc/release-notes/10850-citation-tsv-displayoncreate-depositor-dateofdeposit.md diff --git a/doc/release-notes/10850-citation-tsv-displayoncreate-depositor-dateofdeposit.md b/doc/release-notes/10850-citation-tsv-displayoncreate-depositor-dateofdeposit.md new file mode 100644 index 00000000000..2e404fec75b --- /dev/null +++ b/doc/release-notes/10850-citation-tsv-displayoncreate-depositor-dateofdeposit.md @@ -0,0 +1 @@ +The fields depositor and dateOfDeposit in the citation.tsv metadata block file have been updated to have the property displayOnCreate set to TRUE. \ No newline at end of file diff --git a/scripts/api/data/metadatablocks/citation.tsv b/scripts/api/data/metadatablocks/citation.tsv index a9ea2b9ca0e..abc09465603 100644 --- a/scripts/api/data/metadatablocks/citation.tsv +++ b/scripts/api/data/metadatablocks/citation.tsv @@ -59,8 +59,8 @@ distributorURL URL The URL of the distributor's webpage https:// url 55 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE distributor citation distributorLogoURL Logo URL The URL of the distributor's logo image, used to show the image on the Dataset's page https:// url 56
FALSE FALSE FALSE FALSE FALSE FALSE distributor citation distributionDate Distribution Date The date when the Dataset was made available for distribution/presentation YYYY-MM-DD date 57 TRUE FALSE FALSE TRUE FALSE FALSE citation - depositor Depositor The entity, such as a person or organization, that deposited the Dataset in the repository 1) FamilyName, GivenName or 2) Organization text 58 FALSE FALSE FALSE FALSE FALSE FALSE citation - dateOfDeposit Deposit Date The date when the Dataset was deposited into the repository YYYY-MM-DD date 59 FALSE FALSE FALSE TRUE FALSE FALSE citation http://purl.org/dc/terms/dateSubmitted + depositor Depositor The entity, such as a person or organization, that deposited the Dataset in the repository 1) FamilyName, GivenName or 2) Organization text 58 FALSE FALSE FALSE FALSE TRUE FALSE citation + dateOfDeposit Deposit Date The date when the Dataset was deposited into the repository YYYY-MM-DD date 59 FALSE FALSE FALSE TRUE TRUE FALSE citation http://purl.org/dc/terms/dateSubmitted timePeriodCovered Time Period The time period that the data refer to. Also known as span. This is the time period covered by the data, not the dates of coding, collecting data, or making documents machine-readable none 60 ; FALSE FALSE TRUE FALSE FALSE FALSE citation https://schema.org/temporalCoverage timePeriodCoveredStart Start Date The start date of the time period that the data refer to YYYY-MM-DD date 61 #NAME: #VALUE TRUE FALSE FALSE TRUE FALSE FALSE timePeriodCovered citation timePeriodCoveredEnd End Date The end date of the time period that the data refer to YYYY-MM-DD date 62 #NAME: #VALUE TRUE FALSE FALSE TRUE FALSE FALSE timePeriodCovered citation diff --git a/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java b/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java index 0153d8dc893..6e7061961f0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java @@ -51,7 +51,7 @@ void testListMetadataBlocks() { // onlyDisplayedOnCreate=true and returnDatasetFieldTypes=true listMetadataBlocksResponse = UtilIT.listMetadataBlocks(true, true); - expectedNumberOfMetadataFields = 26; + expectedNumberOfMetadataFields = 28; listMetadataBlocksResponse.then().assertThat() .statusCode(OK.getStatusCode()) .body("data[0].fields", not(equalTo(null))) From 39b6aa28c0befcf316a84117e2ef9af5a250f028 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 27 Sep 2024 10:22:19 -0400 Subject: [PATCH 956/970] #10853 fix typo version number --- doc/release-notes/6.4-release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 648e628864d..7d64ce898c7 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -355,7 +355,7 @@ You are also very welcome to join the [Global Dataverse Community Consortium](ht Upgrading requires a maintenance window and downtime. Please plan accordingly, create backups of your database, etc. -These instructions assume that you've already upgraded through all the 5.x releases and are now running Dataverse 6.2. +These instructions assume that you've already upgraded through all the 5.x releases and are now running Dataverse 6.3. 0\. These instructions assume that you are upgrading from the immediate previous version. If you are running an earlier version, the only supported way to upgrade is to progress through the upgrades to all the releases in between before attempting the upgrade to this version. From 56118add8d464c1574b752f42486e255cc75e7c7 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 27 Sep 2024 10:34:01 -0400 Subject: [PATCH 957/970] Add release note change for fields depositor and dateOfDeposit in the citation.tsv --- doc/release-notes/6.4-release-notes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 7d64ce898c7..693ec4521e1 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -311,6 +311,7 @@ The addDataverse (`/api/dataverses/{identifier}`) API endpoint has been extended ### Metadata Blocks and Display on Create The `/api/dataverses/{identifier}/metadatablocks` endpoint has been fixed to not return fields marked as displayOnCreate=true if there is an input level with include=false, when query parameters returnDatasetFieldTypes=true and onlyDisplayedOnCreate=true are set. See also #10741 and #10767. +The fields depositor and dateOfDeposit in the citation.tsv metadata block file have been updated to have the property displayOnCreate set to TRUE. ### Feature Flags Can Be Listed From 423d4f3707c27cee16ee96fe19892836a619dccd Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 27 Sep 2024 10:52:20 -0400 Subject: [PATCH 958/970] remove old release note --- ...10850-citation-tsv-displayoncreate-depositor-dateofdeposit.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 doc/release-notes/10850-citation-tsv-displayoncreate-depositor-dateofdeposit.md diff --git a/doc/release-notes/10850-citation-tsv-displayoncreate-depositor-dateofdeposit.md b/doc/release-notes/10850-citation-tsv-displayoncreate-depositor-dateofdeposit.md deleted file mode 100644 index 2e404fec75b..00000000000 --- a/doc/release-notes/10850-citation-tsv-displayoncreate-depositor-dateofdeposit.md +++ /dev/null @@ -1 +0,0 @@ -The fields depositor and dateOfDeposit in the citation.tsv metadata block file have been updated to have the property displayOnCreate set to TRUE. \ No newline at end of file From b6abba9e056e5a3ef4ba3f3f7d36be64bc047774 Mon Sep 17 00:00:00 2001 From: landreev Date: Fri, 27 Sep 2024 10:54:36 -0400 Subject: [PATCH 959/970] formatting fix fixed formatting of the shell block in the upgrade instruction --- doc/release-notes/6.4-release-notes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 693ec4521e1..a3027ece5d8 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -448,6 +448,7 @@ Before starting Solr, update the schema to include all the extra metadata fields wget https://raw.githubusercontent.com/IQSS/dataverse/master/conf/solr/9.4.1/update-fields.sh chmod +x update-fields.sh curl "http://localhost:8080/api/admin/index/solr/schema" | ./update-fields.sh /usr/local/solr/solr-9.4.1/server/solr/collection1/conf/schema.xml +``` Now start Solr. From 8e0496d2d16d43670b9a55aca6c18f8efe352159 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 27 Sep 2024 11:07:13 -0400 Subject: [PATCH 960/970] tweak depositor and dateOfDeposit #10853 --- doc/release-notes/6.4-release-notes.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index a3027ece5d8..4268f6fb56e 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -311,7 +311,8 @@ The addDataverse (`/api/dataverses/{identifier}`) API endpoint has been extended ### Metadata Blocks and Display on Create The `/api/dataverses/{identifier}/metadatablocks` endpoint has been fixed to not return fields marked as displayOnCreate=true if there is an input level with include=false, when query parameters returnDatasetFieldTypes=true and onlyDisplayedOnCreate=true are set. See also #10741 and #10767. -The fields depositor and dateOfDeposit in the citation.tsv metadata block file have been updated to have the property displayOnCreate set to TRUE. + +The fields "depositor" and "dateOfDeposit" in the citation.tsv metadata block file have been updated to have the property "displayOnCreate" set to TRUE. In practice, only the API is affected because the UI has special logic that already shows these fields when datasets are created. See also and #10850 and #10884. ### Feature Flags Can Be Listed From 7be0b678612cbd1e1735717c917d4bd3cd19146e Mon Sep 17 00:00:00 2001 From: landreev Date: Fri, 27 Sep 2024 11:28:46 -0400 Subject: [PATCH 961/970] fixed update-fields.sh url (it had "9.4.1" in it; and we probably don't want to get it from the master branch either) --- doc/release-notes/6.4-release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 4268f6fb56e..7ef1aa9fae1 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -446,7 +446,7 @@ service solr start Before starting Solr, update the schema to include all the extra metadata fields that your installation uses. We do this by collecting the output of the Dataverse schema API and feeding it to the `update-fields.sh` script that we supply, as in the example below (modify the command lines as needed to reflect the names of the directories, if different): ```shell - wget https://raw.githubusercontent.com/IQSS/dataverse/master/conf/solr/9.4.1/update-fields.sh + wget https://raw.githubusercontent.com/IQSS/dataverse/v6.4/conf/solr/update-fields.sh chmod +x update-fields.sh curl "http://localhost:8080/api/admin/index/solr/schema" | ./update-fields.sh /usr/local/solr/solr-9.4.1/server/solr/collection1/conf/schema.xml ``` From e8e5127fdbbe57f455c8a934d8d54fec370cd39d Mon Sep 17 00:00:00 2001 From: landreev Date: Fri, 27 Sep 2024 12:10:48 -0400 Subject: [PATCH 962/970] reindex instruction --- doc/release-notes/6.4-release-notes.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 7ef1aa9fae1..0bce6ee1b2c 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -455,12 +455,14 @@ Now start Solr. 8\. Reindex Solr -Below is the simple way to reindex Solr. If you have a large installation of Dataverse, you might want to reindex in place, as described in [the guides](https://guides.dataverse.org/en/latest/admin/solr-search-index.html). +Below is the simplest way to reindex Solr: ```shell curl http://localhost:8080/api/admin/index ``` +The API above rebuilds the existing index "in place". If you want to be absolutely sure that your index is up-to-date and consistent, you may consider wiping it clean and reindexing everything from scratch (see [the guides](https://guides.dataverse.org/en/latest/admin/solr-search-index.html)). Just note that, depending on the size of your database, a full reindex may take a while and the users will be seeing incomplete search results during that window. + 9\. Run reExportAll to update dataset metadata exports This step is necessary because of changes described above for the `Datacite` and `oai_dc` export formats. From 453e1992b28c470acb664ba365b2fbf0c539da61 Mon Sep 17 00:00:00 2001 From: landreev Date: Fri, 27 Sep 2024 12:14:47 -0400 Subject: [PATCH 963/970] removed a superfluous command line --- doc/release-notes/6.4-release-notes.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 0bce6ee1b2c..58ea4ab016e 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -499,10 +499,6 @@ PIDs can also be updated by a superuser on a per-dataset basis using To restore any broken thumbnails caused by the bug described above, you can call the `http://localhost:8080/api/admin/clearThumbnailFailureFlag` API, which will attempt to clear the flag on all files (regardless of whether caused by this bug or some other problem with the file) or the `http://localhost:8080/api/admin/clearThumbnailFailureFlag/$FILE_ID` to clear the flag for individual files. Calling the former, batch API is recommended. -```shell -http://localhost:8080/api/admin/clearThumbnailFailureFlag/$FILE_ID -``` - 12\. PermaLinks with custom base-url If you currently use PermaLinks with a custom `base-url`: You must manually append `/citation?persistentId=` to the base URL to maintain functionality. From 0b48c048967fcd41d114936d9533906b7231f83a Mon Sep 17 00:00:00 2001 From: landreev Date: Fri, 27 Sep 2024 12:39:42 -0400 Subject: [PATCH 964/970] temp dir cleanup --- doc/release-notes/6.4-release-notes.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 58ea4ab016e..e682d28512b 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -213,6 +213,22 @@ When any `ApiBlockingFilter` policy applies to a request, the JSON in the body o See also #10508, #10672 and #10722. +### Cleanup of Temp Directories + +In this release we addressed an issue where copies of files uploaded via the UI were left in one specific temp directory (`.../domain1/uploads` by default). We would like to remind all the installation admins that it is strongly recommended to have some automated (and aggressive) cleanup mechanisms in place for all the temp directories used by Dataverse. For example, at Harvard/IQSS we have the following configuration for the PrimeFaces uploads directory above: (note that, even with this fix in place, PrimeFaces will be leaving a large number of small log files in that location) + +Instead of the default location (`.../domain1/uploads`) we use a directory on a dedicated partition, outside of the filesystem where Dataverse is installed, via the following JVM option: + +``` +-Ddataverse.files.uploads=/uploads/web +``` + +and we have a dedicated cronjob that runs every 30 minutes and deletes everything older than 2 hours in that directory: + +``` +15,45 * * * * /bin/find /uploads/web/ -mmin +119 -type f -name "upload*" -exec rm -f {} \; > /dev/null 2>&1 +``` + ## API Updates ### Search API: affiliation, parentDataverseName, image_url, etc. From 3a4e7c609a2528d3dfc780809a168318ab456520 Mon Sep 17 00:00:00 2001 From: landreev Date: Fri, 27 Sep 2024 13:49:40 -0400 Subject: [PATCH 965/970] typo --- doc/release-notes/6.4-release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index e682d28512b..49936e3260d 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -96,7 +96,7 @@ When notification emails are sent the part of the closing that says "contact us ### Ability to Disable Automatic Thumbnail Selection -It is now possible to turn off the feature that automatically selects of one of the image datafiles to serve as the thumbnail of the parent dataset. An admin can turn it off by enabling the [feature flag](https://guides.dataverse.org/en/6.4/installation/config.html#feature-flags) `dataverse.feature.disable-dataset-thumbnail-autoselect`. When the feature is disabled, a user can still manually pick a thumbnail image, or upload a dedicated thumbnail image. See also #10820. +It is now possible to turn off the feature that automatically selects one of the image datafiles to serve as the thumbnail of the parent dataset. An admin can turn it off by enabling the [feature flag](https://guides.dataverse.org/en/6.4/installation/config.html#feature-flags) `dataverse.feature.disable-dataset-thumbnail-autoselect`. When the feature is disabled, a user can still manually pick a thumbnail image, or upload a dedicated thumbnail image. See also #10820. ### More Flexible PermaLinks From 8fc75cceba599cfc25fca61825a2394ae4de61fa Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 30 Sep 2024 16:00:24 +0200 Subject: [PATCH 966/970] docs: update release notes from #10343 --- doc/release-notes/10343-trailing-comma.md | 5 ----- doc/release-notes/6.4-release-notes.md | 6 ++++++ 2 files changed, 6 insertions(+), 5 deletions(-) delete mode 100644 doc/release-notes/10343-trailing-comma.md diff --git a/doc/release-notes/10343-trailing-comma.md b/doc/release-notes/10343-trailing-comma.md deleted file mode 100644 index 03bd18715d7..00000000000 --- a/doc/release-notes/10343-trailing-comma.md +++ /dev/null @@ -1,5 +0,0 @@ -### Trailing commas in author name now permitted - -When an author name ends on a comma (e.g. "Smith,") a dataset cannot be properly loaded when using json-ld. A null check fixes this. - -For more information, see #10343. \ No newline at end of file diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 49936e3260d..cb34c6d13df 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -229,6 +229,12 @@ and we have a dedicated cronjob that runs every 30 minutes and deletes everythin 15,45 * * * * /bin/find /uploads/web/ -mmin +119 -type f -name "upload*" -exec rm -f {} \; > /dev/null 2>&1 ``` +### Trailing commas in author name now permitted + +When an author name ends on a comma (e.g. `Smith,` or `Smith, `), the dataset page display was broken after publishing. +Instead, an "Error 500" page was presented to the user. +The underlying issue causing the JSON-LD schema.org output on the page to break was fixed in #10343. + ## API Updates ### Search API: affiliation, parentDataverseName, image_url, etc. From c3aeeb0367a5080d8fce822fbe3e0b7ead22099c Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 30 Sep 2024 10:16:00 -0400 Subject: [PATCH 967/970] tweaks #10343 --- doc/release-notes/6.4-release-notes.md | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index cb34c6d13df..979fd16bf9e 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -213,27 +213,25 @@ When any `ApiBlockingFilter` policy applies to a request, the JSON in the body o See also #10508, #10672 and #10722. -### Cleanup of Temp Directories +### Cleanup of Temp Directories -In this release we addressed an issue where copies of files uploaded via the UI were left in one specific temp directory (`.../domain1/uploads` by default). We would like to remind all the installation admins that it is strongly recommended to have some automated (and aggressive) cleanup mechanisms in place for all the temp directories used by Dataverse. For example, at Harvard/IQSS we have the following configuration for the PrimeFaces uploads directory above: (note that, even with this fix in place, PrimeFaces will be leaving a large number of small log files in that location) +In this release we addressed an issue where copies of files uploaded via the UI were left in one specific temp directory (`.../domain1/uploads` by default). We would like to remind all the installation admins that it is strongly recommended to have some automated (and aggressive) cleanup mechanisms in place for all the temp directories used by Dataverse. For example, at Harvard/IQSS we have the following configuration for the PrimeFaces uploads directory above: (note that, even with this fix in place, PrimeFaces will be leaving a large number of small log files in that location) -Instead of the default location (`.../domain1/uploads`) we use a directory on a dedicated partition, outside of the filesystem where Dataverse is installed, via the following JVM option: +Instead of the default location (`.../domain1/uploads`) we use a directory on a dedicated partition, outside of the filesystem where Dataverse is installed, via the following JVM option: ``` -Ddataverse.files.uploads=/uploads/web ``` -and we have a dedicated cronjob that runs every 30 minutes and deletes everything older than 2 hours in that directory: +and we have a dedicated cronjob that runs every 30 minutes and deletes everything older than 2 hours in that directory: ``` 15,45 * * * * /bin/find /uploads/web/ -mmin +119 -type f -name "upload*" -exec rm -f {} \; > /dev/null 2>&1 ``` -### Trailing commas in author name now permitted +### Trailing Commas in Author Name Now Permitted -When an author name ends on a comma (e.g. `Smith,` or `Smith, `), the dataset page display was broken after publishing. -Instead, an "Error 500" page was presented to the user. -The underlying issue causing the JSON-LD schema.org output on the page to break was fixed in #10343. +When an author name ended in a comma (e.g. `Smith,` or `Smith, `), the dataset page was broken after publishing (a "500" error page was presented to the user). The underlying issue causing the JSON-LD Schema.org output on the page to break was fixed. See #10343 and #10776. ## API Updates @@ -483,7 +481,7 @@ Below is the simplest way to reindex Solr: curl http://localhost:8080/api/admin/index ``` -The API above rebuilds the existing index "in place". If you want to be absolutely sure that your index is up-to-date and consistent, you may consider wiping it clean and reindexing everything from scratch (see [the guides](https://guides.dataverse.org/en/latest/admin/solr-search-index.html)). Just note that, depending on the size of your database, a full reindex may take a while and the users will be seeing incomplete search results during that window. +The API above rebuilds the existing index "in place". If you want to be absolutely sure that your index is up-to-date and consistent, you may consider wiping it clean and reindexing everything from scratch (see [the guides](https://guides.dataverse.org/en/latest/admin/solr-search-index.html)). Just note that, depending on the size of your database, a full reindex may take a while and the users will be seeing incomplete search results during that window. 9\. Run reExportAll to update dataset metadata exports From 6bc565c15d2c7c1cfe99e4703d1683c42f69bb6f Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 30 Sep 2024 12:09:30 -0400 Subject: [PATCH 968/970] post 6.4 container version --- modules/dataverse-parent/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 76f0ddbc033..5abf2763128 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -446,8 +446,8 @@ Once the release has been made (tag created), change this back to "${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion}" (These properties are provided by the build-helper plugin below.) --> - - ${revision} + ${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion} + From c59746d2176f1d8a7e1b567f12143175281e094a Mon Sep 17 00:00:00 2001 From: Vera Clemens <16904069+vera@users.noreply.github.com> Date: Mon, 30 Sep 2024 19:19:33 +0200 Subject: [PATCH 969/970] Importing unmanaged pids (#10805) * fix: send proper error response when trying to import dataset with unmanaged PID * test: add tests for importing datasets as JSON * docs: state that importing datasets with unmanaged PIDs is not supported --- doc/sphinx-guides/source/api/native-api.rst | 4 +- .../harvard/iq/dataverse/api/Dataverses.java | 6 + .../iq/dataverse/api/DataversesIT.java | 173 ++++++++++++++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 29 +++ 4 files changed, 210 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 117aceb141d..acb6131c9d2 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -887,7 +887,7 @@ Before calling the API, make sure the data files referenced by the ``POST``\ ed * This API does not cover staging files (with correct contents, checksums, sizes, etc.) in the corresponding places in the Dataverse installation's filestore. * This API endpoint does not support importing *files'* persistent identifiers. - * A Dataverse installation can import datasets with a valid PID that uses a different protocol or authority than said server is configured for. However, the server will not update the PID metadata on subsequent update and publish actions. + * A Dataverse installation can only import datasets with a valid PID that is managed by one of the PID providers that said installation is configured for. .. _import-dataset-with-type: @@ -935,7 +935,7 @@ Note that DDI XML does not have a field that corresponds to the "Subject" field .. warning:: * This API does not handle files related to the DDI file. - * A Dataverse installation can import datasets with a valid PID that uses a different protocol or authority than said server is configured for. However, the server will not update the PID metadata on subsequent update and publish actions. + * A Dataverse installation can only import datasets with a valid PID that is managed by one of the PID providers that said installation is configured for. .. _publish-dataverse-api: diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 17e3086f184..0ee146ed99b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -407,6 +407,12 @@ public Response importDataset(@Context ContainerRequestContext crc, String jsonB if (ds.getIdentifier() == null) { return badRequest("Please provide a persistent identifier, either by including it in the JSON, or by using the pid query parameter."); } + + PidProvider pidProvider = PidUtil.getPidProvider(ds.getGlobalId().getProviderId()); + if (pidProvider == null || !pidProvider.canManagePID()) { + return badRequest("Cannot import a dataset that has a PID that doesn't match the server's settings"); + } + boolean shouldRelease = StringUtil.isTrue(releaseParam); DataverseRequest request = createDataverseRequest(u); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 6fbe91c8405..8c6a8244af1 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -646,6 +646,179 @@ public void testImportDDI() throws IOException, InterruptedException { Response deleteUserResponse = UtilIT.deleteUser(username); assertEquals(200, deleteUserResponse.getStatusCode()); } + + @Test + public void testImport() throws IOException, InterruptedException { + + Response createUser = UtilIT.createRandomUser(); + String username = UtilIT.getUsernameFromResponse(createUser); + Response makeSuperUser = UtilIT.makeSuperUser(username); + assertEquals(200, makeSuperUser.getStatusCode()); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken); + assertEquals(200, publishDataverse.getStatusCode()); + + JsonObjectBuilder datasetJson = Json.createObjectBuilder() + .add("datasetVersion", Json.createObjectBuilder() + .add("license", Json.createObjectBuilder() + .add("name", "CC0 1.0") + ) + .add("metadataBlocks", Json.createObjectBuilder() + .add("citation", Json.createObjectBuilder() + .add("fields", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("typeName", "title") + .add("value", "Test Dataset") + .add("typeClass", "primitive") + .add("multiple", false) + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("authorName", + Json.createObjectBuilder() + .add("value", "Simpson, Homer") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "authorName")) + ) + ) + .add("typeClass", "compound") + .add("multiple", true) + .add("typeName", "author") + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("datasetContactEmail", + Json.createObjectBuilder() + .add("value", "hsimpson@mailinator.com") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "datasetContactEmail")) + ) + ) + .add("typeClass", "compound") + .add("multiple", true) + .add("typeName", "datasetContact") + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("dsDescriptionValue", + Json.createObjectBuilder() + .add("value", "This a test dataset.") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "dsDescriptionValue")) + ) + ) + .add("typeClass", "compound") + .add("multiple", true) + .add("typeName", "dsDescription") + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add("Other") + ) + .add("typeClass", "controlledVocabulary") + .add("multiple", true) + .add("typeName", "subject") + ) + ) + ) + )); + + String json = datasetJson.build().toString(); + + Response importJSONNoPid = UtilIT.importDatasetViaNativeApi(apiToken, dataverseAlias, json, null, "no"); + logger.info(importJSONNoPid.prettyPrint()); + assertEquals(400, importJSONNoPid.getStatusCode()); + + String body = importJSONNoPid.getBody().asString(); + String status = JsonPath.from(body).getString("status"); + assertEquals("ERROR", status); + + String message = JsonPath.from(body).getString("message"); + assertEquals( + "Please provide a persistent identifier, either by including it in the JSON, or by using the pid query parameter.", + message + ); + + Response importJSONNoPidRelease = UtilIT.importDatasetViaNativeApi(apiToken, dataverseAlias, json, null, "yes"); + logger.info( importJSONNoPidRelease.prettyPrint()); + assertEquals(400, importJSONNoPidRelease.getStatusCode()); + + body = importJSONNoPidRelease.getBody().asString(); + status = JsonPath.from(body).getString("status"); + assertEquals("ERROR", status); + + message = JsonPath.from(body).getString("message"); + assertEquals( + "Please provide a persistent identifier, either by including it in the JSON, or by using the pid query parameter.", + message + ); + + Response importJSONUnmanagedPid = UtilIT.importDatasetViaNativeApi(apiToken, dataverseAlias, json, "doi:10.5073/FK2/ABCD11", "no"); + logger.info(importJSONUnmanagedPid.prettyPrint()); + assertEquals(400, importJSONUnmanagedPid.getStatusCode()); + + body = importJSONUnmanagedPid.getBody().asString(); + status = JsonPath.from(body).getString("status"); + assertEquals("ERROR", status); + + message = JsonPath.from(body).getString("message"); + assertEquals( + "Cannot import a dataset that has a PID that doesn't match the server's settings", + message + ); + + // Under normal conditions, you shouldn't need to destroy these datasets. + // Uncomment if they're still around from a previous failed run. +// Response destroy1 = UtilIT.destroyDataset("doi:10.5072/FK2/ABCD11", apiToken); +// destroy1.prettyPrint(); +// Response destroy2 = UtilIT.destroyDataset("doi:10.5072/FK2/ABCD22", apiToken); +// destroy2.prettyPrint(); + + Response importJSONPid = UtilIT.importDatasetViaNativeApi(apiToken, dataverseAlias, json, "doi:10.5072/FK2/ABCD11", "no"); + logger.info(importJSONPid.prettyPrint()); + assertEquals(201, importJSONPid.getStatusCode()); + + Response importJSONPidRel = UtilIT.importDatasetViaNativeApi(apiToken, dataverseAlias, json, "doi:10.5072/FK2/ABCD22", "yes"); + logger.info(importJSONPidRel.prettyPrint()); + assertEquals(201, importJSONPidRel.getStatusCode()); + + Integer datasetIdInt = JsonPath.from(importJSONPid.body().asString()).getInt("data.id"); + + Response search1 = UtilIT.search("id:dataset_" + datasetIdInt + "_draft", apiToken); // santity check, can find it + search1.prettyPrint(); + search1.then().assertThat() + .body("data.total_count", CoreMatchers.is(1)) + .body("data.count_in_response", CoreMatchers.is(1)) + .body("data.items[0].name", CoreMatchers.is("Test Dataset")) + .statusCode(OK.getStatusCode()); + + //cleanup + + Response destroyDatasetResponse = UtilIT.destroyDataset(datasetIdInt, apiToken); + assertEquals(200, destroyDatasetResponse.getStatusCode()); + + Integer datasetIdIntPidRel = JsonPath.from(importJSONPidRel.body().asString()).getInt("data.id"); + Response destroyDatasetResponsePidRel = UtilIT.destroyDataset(datasetIdIntPidRel, apiToken); + assertEquals(200, destroyDatasetResponsePidRel.getStatusCode()); + + UtilIT.sleepForDeadlock(UtilIT.MAXIMUM_IMPORT_DURATION); + + Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); + assertEquals(200, deleteDataverseResponse.getStatusCode()); + + Response deleteUserResponse = UtilIT.deleteUser(username); + assertEquals(200, deleteUserResponse.getStatusCode()); + } @Test public void testAttributesApi() throws Exception { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 4e20e8e4c33..4fbe84bcfcf 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -3672,6 +3672,35 @@ static Response importDatasetDDIViaNativeApi(String apiToken, String dataverseAl return importDDI.post(postString); } + + static Response importDatasetViaNativeApi(String apiToken, String dataverseAlias, String json, String pid, String release) { + String postString = "/api/dataverses/" + dataverseAlias + "/datasets/:import"; + if (pid != null || release != null ) { + //postString = postString + "?"; + if (pid != null) { + postString = postString + "?pid=" + pid; + if (release != null && release.compareTo("yes") == 0) { + postString = postString + "&release=" + release; + } + } else { + if (release != null && release.compareTo("yes") == 0) { + postString = postString + "?release=" + release; + } + } + } + logger.info("Here importDatasetViaNativeApi"); + logger.info(postString); + + RequestSpecification importJSON = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .urlEncodingEnabled(false) + .body(json) + .contentType("application/json"); + + return importJSON.post(postString); + } + + static Response retrieveMyDataAsJsonString(String apiToken, String userIdentifier, ArrayList roleIds) { Response response = given() .header(API_TOKEN_HTTP_HEADER, apiToken) From d9139effc5f37330b939e8a72eec96bde90004cf Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 30 Sep 2024 14:35:15 -0400 Subject: [PATCH 970/970] #10879 remove access request from render logic --- src/main/webapp/dataset-license-terms.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/dataset-license-terms.xhtml b/src/main/webapp/dataset-license-terms.xhtml index 255e63fbfc2..03173faf989 100644 --- a/src/main/webapp/dataset-license-terms.xhtml +++ b/src/main/webapp/dataset-license-terms.xhtml @@ -12,7 +12,7 @@ or !empty termsOfUseAndAccess.originalArchive or !empty termsOfUseAndAccess.availabilityStatus or !empty termsOfUseAndAccess.contactForAccess or !empty termsOfUseAndAccess.sizeOfCollection or !empty termsOfUseAndAccess.studyCompletion - or termsOfUseAndAccess.fileAccessRequest}"/> + }"/>