From 385174065ed4b095bbcc6ab36873c9d036bf56b7 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 31 Jan 2019 18:31:18 -0500 Subject: [PATCH 001/145] IQSS-5505 --- pom.xml | 5 +++ .../AbstractGlobalIdServiceBean.java | 4 +++ .../dataverse/DOIDataCiteRegisterService.java | 35 +++++++++++++++++++ .../iq/dataverse/GlobalIdServiceBean.java | 2 ++ .../harvard/iq/dataverse/api/Datasets.java | 12 +++++-- .../UpdateDvObjectPIDMetadataCommand.java | 4 +-- 6 files changed, 58 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 380fbbbe402..224ab957fd7 100644 --- a/pom.xml +++ b/pom.xml @@ -585,6 +585,11 @@ tika-parsers 1.19 + + org.xmlunit + xmlunit-core + 2.6.2 + diff --git a/src/main/java/edu/harvard/iq/dataverse/AbstractGlobalIdServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/AbstractGlobalIdServiceBean.java index 98310a136b5..4e632dce27f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/AbstractGlobalIdServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/AbstractGlobalIdServiceBean.java @@ -428,5 +428,9 @@ public String getMetadataFromDvObject(String identifier, Map met logger.log(Level.FINE, "XML to send to DataCite: {0}", xmlMetadata); return xmlMetadata; } + + public boolean updateIdentifier(DvObject dvObject) { + return publicizeIdentifier(dvObject); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java index 50f92f81fb5..99cf4f3694d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java +++ b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java @@ -23,10 +23,17 @@ import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; import javax.persistence.TypedQuery; +import javax.xml.transform.Source; + import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import org.xmlunit.builder.DiffBuilder; +import org.xmlunit.builder.Input; +import org.xmlunit.builder.Input.Builder; +import org.xmlunit.diff.Diff; +import org.xmlunit.diff.Difference; /** * @@ -114,6 +121,34 @@ public String registerIdentifier(String identifier, Map metadata } return retString; } + + public String reRegisterIdentifier(String identifier, Map metadata, DvObject dvObject) throws IOException { + String retString = ""; + String numericIdentifier = identifier.substring(identifier.indexOf(":") + 1); + String xmlMetadata = getMetadataFromDvObject(identifier, metadata, dvObject); + String target = metadata.get("_target"); + DataCiteRESTfullClient client = getClient(); + String currentMetadata = client.getMetadata(numericIdentifier); + Diff myDiff = DiffBuilder.compare(xmlMetadata) + .withTest(currentMetadata).ignoreWhitespace().checkForSimilar() + .build(); + + if (myDiff.hasDifferences()) { + for(Difference d : myDiff.getDifferences()) { + + logger.fine(d.toString()); + } + retString = "metadata:\\r" + client.postMetadata(xmlMetadata) + "\\r"; + } + if (!target.equals(client.getUrl(numericIdentifier))) { + logger.fine("updating target URl to " + target); + client.postUrl(numericIdentifier, target); + retString = retString + "url:\\r" + target; + + } + + return retString; + } public String deactivateIdentifier(String identifier, HashMap metadata, DvObject dvObject) { String retString = ""; diff --git a/src/main/java/edu/harvard/iq/dataverse/GlobalIdServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/GlobalIdServiceBean.java index 0d64c1050b8..cd0c2c04c73 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GlobalIdServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/GlobalIdServiceBean.java @@ -50,6 +50,8 @@ public interface GlobalIdServiceBean { boolean publicizeIdentifier(DvObject studyIn); + boolean updateIdentifier(DvObject dvObject); + static GlobalIdServiceBean getBean(String protocol, CommandContext ctxt) { final Function protocolHandler = BeanDispatcher.DISPATCHER.get(protocol); if ( protocolHandler != null ) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 4f868d90ae7..42f01f67ace 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -386,8 +386,9 @@ public Response updateDatasetPIDMetadata(@PathParam("id") String id) { } return response(req -> { - execCommand(new UpdateDvObjectPIDMetadataCommand(findDatasetOrDie(id), req)); - List args = Arrays.asList(id); + Dataset dataset = findDatasetOrDie(id); + execCommand(new UpdateDvObjectPIDMetadataCommand(dataset, req)); + List args = Arrays.asList(dataset.getIdentifier()); return ok(BundleUtil.getStringFromBundle("datasets.api.updatePIDMetadata.success.for.single.dataset", args)); }); } @@ -398,7 +399,14 @@ public Response updateDatasetPIDMetadataAll() { return response( req -> { datasetService.findAll().forEach( ds -> { try { + logger.fine("ReRegistering: " + ds.getId() + " : " + ds.getIdentifier()); + if (!ds.isReleased() || (!ds.isIdentifierRegistered() || (ds.getIdentifier() == null))) { + if (ds.isReleased()) { + logger.warning("Dataset id=" + ds.getId() + " is in an inconsistent state (publicationdate but no identifier/identifier not registered"); + } + } else { execCommand(new UpdateDvObjectPIDMetadataCommand(findDatasetOrDie(ds.getId().toString()), req)); + } } catch (WrappedResponse ex) { Logger.getLogger(Datasets.class.getName()).log(Level.SEVERE, null, ex); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDvObjectPIDMetadataCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDvObjectPIDMetadataCommand.java index e36fe06b863..99d0a183c9d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDvObjectPIDMetadataCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDvObjectPIDMetadataCommand.java @@ -47,13 +47,13 @@ protected void executeImpl(CommandContext ctxt) throws CommandException { } GlobalIdServiceBean idServiceBean = GlobalIdServiceBean.getBean(target.getProtocol(), ctxt); try { - Boolean doiRetString = idServiceBean.publicizeIdentifier(target); + Boolean doiRetString = idServiceBean.updateIdentifier(target); if (doiRetString) { target.setGlobalIdCreateTime(new Timestamp(new Date().getTime())); ctxt.em().merge(target); ctxt.em().flush(); for (DataFile df : target.getFiles()) { - doiRetString = idServiceBean.publicizeIdentifier(df); + doiRetString = idServiceBean.updateIdentifier(df); if (doiRetString) { df.setGlobalIdCreateTime(new Timestamp(new Date().getTime())); ctxt.em().merge(df); From d5dec010424435bdcea65c76dd2b1e524473d8a9 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 22 Jun 2020 17:56:28 -0400 Subject: [PATCH 002/145] xml escape dataset description --- .../edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java index f9107d8565b..fa04b710819 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java +++ b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java @@ -228,7 +228,8 @@ public static String getMetadataFromDvObject(String identifier, Map from HTML, it leaves '&' (at least so we need to xml escape as well + String description = StringEscapeUtils.escapeXml(dataset.getLatestVersion().getDescriptionPlainText()); if (description.isEmpty() || description.equals(DatasetField.NA_VALUE)) { description = AbstractGlobalIdServiceBean.UNAVAILABLE; } From 72fca2c7e30cce95c8c4446399d32c6a418630f0 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Mon, 27 Mar 2023 13:26:21 -0400 Subject: [PATCH 003/145] Pdf export function --- pom.xml | 6 +++ .../dataverse/export/ddi/DdiExportUtil.java | 46 ++++++++++++++++++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e028771d24b..780bb7bdc0c 100644 --- a/pom.xml +++ b/pom.xml @@ -429,6 +429,12 @@ commons-compress + + + org.apache.xmlgraphics + fop + 2.8 + org.duracloud common diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index c78bb02d5c5..ee9231a9d53 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -74,7 +74,19 @@ import javax.xml.transform.stream.StreamResult; import java.io.File; import java.io.InputStream; -import java.io.InputStreamReader; + +import java.io.OutputStream; +import javax.xml.transform.Result; +import javax.xml.transform.Source; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.sax.SAXResult; +import javax.xml.transform.stream.StreamSource; + +import org.apache.fop.apps.FOUserAgent; +import org.apache.fop.apps.Fop; +import org.apache.fop.apps.FopFactory; +import org.apache.fop.apps.MimeConstants; public class DdiExportUtil { @@ -1948,6 +1960,38 @@ private static boolean checkParentElement(XMLStreamWriter xmlw, String elementNa return true; } + public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { + try { + File xsltfile = new File("ddi-to-fo.xsl"); + + final FopFactory fopFactory = FopFactory.newInstance(new File(".").toURI)); + FOUserAgent foUserAgent = fopFactory.newFOUserAgent(); + + try { + Fop fop = fopFactory.newFop(MimeConstants.MIME_PDF, foUserAgent, outputStream); + // Setup XSLT + TransformerFactory factory = TransformerFactory.newInstance(); + Transformer transformer = factory.newTransformer(new StreamSource(xsltfile)); + + // Set the value of a in the stylesheet + transformer.setParameter("versionParam", "2.0"); + + // Setup input for XSLT transformation + Source src = new StreamSource(datafile); + + // Resulting SAX events (the generated FO) must be piped through to FOP + Result res = new SAXResult(fop.getDefaultHandler()); + + // Start XSLT transformation and FOP processing + transformer.transform(src, res); + } finally { + outputStream.close(); + } + } catch (Exception e) { + logger.severe(e.getMessage()); + } + } + public static void datasetHtmlDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); From a5821b9fd0d53137918c329fb79590bc6431dbe1 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Mon, 3 Apr 2023 17:35:28 -0400 Subject: [PATCH 004/145] pdf codebook --- .../harvard/iq/dataverse/api/Datasets.java | 2 +- .../dataverse/export/PdfCodeBookExporter.java | 91 + .../dataverse/export/ddi/DdiExportUtil.java | 7 +- src/main/java/propertyFiles/Bundle.properties | 1 + .../from-ddi-2.5/ddi-pdf/i18n.inc.xslt | 5 + .../ddi-pdf/messages_en.properties.xml | 174 + .../ddi-pdf/messages_es.properties.xml | 170 + .../ddi-pdf/messages_fr.properties.xml | 173 + .../ddi-pdf/messages_ja.properties.xml | 161 + .../ddi-pdf/messages_nn.properties.xml | 174 + .../ddi-pdf/messages_no.properties.xml | 174 + .../ddi-pdf/messages_ru.properties.xml | 169 + .../iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl | 4473 +++++++++++++++++ 13 files changed, 5771 insertions(+), 3 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index d40bc153141..3b3326611dc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -272,7 +272,7 @@ public Response getDataset(@Context ContainerRequestContext crc, @PathParam("id" @GET @Path("/export") - @Produces({"application/xml", "application/json", "application/html" }) + @Produces({"application/xml", "application/json", "application/html", "*/*" }) public Response exportDataset(@QueryParam("persistentId") String persistentId, @QueryParam("exporter") String exporter, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) { try { diff --git a/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java new file mode 100644 index 00000000000..8f3bb78f8d3 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java @@ -0,0 +1,91 @@ +package edu.harvard.iq.dataverse.export; + +import com.google.auto.service.AutoService; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.export.ddi.DdiExportUtil; +import edu.harvard.iq.dataverse.export.spi.Exporter; +import edu.harvard.iq.dataverse.util.BundleUtil; + +import javax.json.JsonObject; +import javax.ws.rs.core.MediaType; +import javax.xml.stream.XMLStreamException; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.nio.file.Paths; + +@AutoService(Exporter.class) +public class PdfCodeBookExporter implements Exporter { + + @Override + public String getProviderName() { + return "pdf"; + } + + @Override + public String getDisplayName() { + return BundleUtil.getStringFromBundle("dataset.exportBtn.itemLabel.pdf") != null ? BundleUtil.getStringFromBundle("dataset.exportBtn.itemLabel.pdf") : "DDI PDF Codebook"; + } + + @Override + public void exportDataset(DatasetVersion version, JsonObject json, OutputStream outputStream) throws ExportException { + try { + InputStream ddiInputStream; + try { + ddiInputStream = ExportService.getInstance().getExport(version.getDataset(), "ddi"); + } catch(ExportException | IOException e) { + throw new ExportException ("Cannot open export_ddi cached file"); + } + DdiExportUtil.datasetPdfDDI(ddiInputStream, outputStream); + } catch (XMLStreamException xse) { + throw new ExportException ("Caught XMLStreamException performing DDI export"); + } + } + + @Override + public Boolean isXMLFormat() { + return false; + } + + @Override + public Boolean isHarvestable() { + // No, we don't want this format to be harvested! + // For datasets with tabular data the portions of the DDIs + // become huge and expensive to parse; even as they don't contain any + // metadata useful to remote harvesters. -- L.A. 4.5 + return false; + } + + @Override + public Boolean isAvailableToUsers() { + return true; + } + + @Override + public String getXMLNameSpace() throws ExportException { + return null; + } + + @Override + public String getXMLSchemaLocation() throws ExportException { + return null; + } + + @Override + public String getXMLSchemaVersion() throws ExportException { + return null; + } + + @Override + public void setParam(String name, Object value) { + // this exporter does not uses or supports any parameters as of now. + } + + @Override + public String getMediaType() { + return MediaType.WILDCARD; + }; +} diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index ee9231a9d53..3912b28a886 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -1962,9 +1962,12 @@ private static boolean checkParentElement(XMLStreamWriter xmlw, String elementNa public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { try { - File xsltfile = new File("ddi-to-fo.xsl"); + File xsltfile = new File("/home/victoria/ddi-to-fo.xsl"); + logger.info("start datasetPdfDDI"); + //InputStream xsltfile = DdiExportUtil.class.getClassLoader().getResourceAsStream( + // "edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl"); - final FopFactory fopFactory = FopFactory.newInstance(new File(".").toURI)); + final FopFactory fopFactory = FopFactory.newInstance(new File(".").toURI()); FOUserAgent foUserAgent = fopFactory.newFOUserAgent(); try { diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 45807dc7cde..3ba8d3fc8dd 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1395,6 +1395,7 @@ dataset.exportBtn.itemLabel.json=JSON dataset.exportBtn.itemLabel.oai_ore=OAI_ORE dataset.exportBtn.itemLabel.dataciteOpenAIRE=OpenAIRE dataset.exportBtn.itemLabel.html=DDI HTML Codebook +dataset.exportBtn.itemLabel.pdf=DDI PDF Codebook license.custom=Custom Dataset Terms license.custom.description=Custom terms specific to this dataset metrics.title=Metrics diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt new file mode 100644 index 00000000000..edf876f3b04 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt @@ -0,0 +1,5 @@ + + + + + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml new file mode 100644 index 00000000000..d8e98dfd3c6 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml @@ -0,0 +1,174 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 +Valid +Frequency table not shown +Derivation +discrete +Data Collection Mode +Other Processing +Other Acknowledgment(s) +Untitled +Identification +Click here to access/export data files from Nesstar format +Value +Percentage +Weighting +Primary Investigator(s) +This document was generated using the +Sampling +Cases +Access Conditions +Source +Modalities +Rights & Disclaimer +Definition +Estimates of Sampling Error +Data Files +Geographic Coverage +April +Mean +Metadata Production +Data Collection +Missing Data +Scripts and programs +Variable(s) +Interviewer instructions +Funding Agency/ies +November +Missing +Version +Universe +Contributor(s) +Access Authority +Data Processing & Appraisal +Scope +Administrative documents +StdDev +Contact(s) +Label +Technical documents +Decimal(s) +Type +Literal question +Concepts +Range +Abstract +June +Supervision +Other Forms of Data Appraisal +References +Accessibility +Data Collection Dates +Data Editing +Questionnaires +Valid case(s) +Reports and analytical documents +Copyright +Documentation +Deviations from Sample Design +Publisher(s) +February +Dataset contains +Acknowledgment(s) +Continuous +Standard deviation +Variables Description +Producer +Production Date + +The Explorer allows you to view data files and export them to common statistical formats +Discrete +Group +July +Filename +Cases +Name +Warning: these figures indicate the number of cases found in the data file. They cannot be interpreted as summary statistics of the population of interest. +Statistical tables +December +Subjects +Processing Checks +software +Interviewer's instructions +Table of Contents +Document Information +Subgroup(s) +Keywords +group(s) +W +Weight +Files Description +Notes +Data Collection Notes +file(s) +continuous +Disclaimer +Content +variable(s) +Other Producer(s) +Producers & Sponsors +Data Cleaning Notes +Distributor(s) +Overview +Citation Requirements +September +Category +Confidentiality +Statistics +May +Undetermined +Structure +file +Pre-question +Response Rate +Width +Recoding and Derivation +Series +October +Unit of Analysis +Data Processing Notes +Kind of Data +File +Time Period(s) +File Content +Invalid +Vars +cont. +Key(s) +Question +Source of information +Imputation +Security +To open this file, you will need the free +Other resources +Data Dictionnary +Information +January +Other documents +Minimum +Scope & Coverage +Metadata Producer(s) +Show more info +Data Collector(s) +Post-question +Topics +Sampling Procedure +File Structure +Variables List +Format +Sampling Notes +Variables Group(s) +Description +Categories +Maximum +Depositor(s) +August +NW +Cover Page +Weighted +March + total - showing a subset of +Countries +question details + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml new file mode 100644 index 00000000000..9cfcdaf6e7e --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml @@ -0,0 +1,170 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 +Válido +No se presentan las tablas de frecuencias +Derivación +discreta +Método de Recolección +Otros relacionados al procesamiento +Otros Reconocimientos +Sin título +Identificación +Presione aquí para acceder/exportar al archivo(s) de datos +Valor +Porcentaje +Ponderando +Investigadores Principales +Este documento fue producido utilizando el +Muestreo +Casos +Condiciones de uso +Fuente +Modalidades +Derechos y Notas Legales +Definición +Estimaciones del Error Muestral +Archivo de Datos +Cobertura Geográfica +Abril +Media +Producción de los Metadatos +Recolección de Datos +Datos perdidos +Programas informáticos +Variable(s) +Manual del encuestador +Agencias Auspiciadoras +Noviembre +Valores perdidos +Versión +Universo +Contribuidor(es) +Institución Propietaria +Tratamiento y Validación de Datos +Dominio Temático +Documentos Administrativos +Desviación +Contacto(s) +Etiqueta +Documentos Técnicos +Decimal(es) +Tipo +Pregunta textual +Conceptos +Rango +Resumen +Junio +Supervisión +Otras Formas de Validación de los Datos +Referencias +Accesibilidad +Fechas de Recolección de Datos +Procesamiento de Datos +Cuestionarios +Casos válidos +Reportes y documentos analíticos +Derechos de Autor +Documentación +Modificaciones al Diseño Muestral +Editor(es) +Febrero +Contenido de la Base de Datos +Reconocimiento(s) +Contínua +Desviación estándar +Descripción de la variable +Productor +Fecha de Producción +El Explorador NESSTAR permite visualizar los archivos de datos y exportarlos a diferentes formatos estadísticos +Discreta +Grupo +Julio +Nombre del Archivo +Casos +Nombre +Cuadros estadísticos +Diciembre +Temas +Controles de Tratamiento +software +Manual del encuestador +Indice +Información acerca de la Documentación +Subgrupo(s) +Palabra Clave +grupo(s) +P +Ponderador +Descripción de los Archivos +Notas +Notas sobre la Recolección de Datos +archivo(s) +continua +Nota Legal +Contenido +variable(s) +Otros Productores +Productores y Auspiciadores +Notas acerca de la Depuración de los Datos +Distribuidor(es) +Resumen General +Forma de citar +Septiembre +Categoría +Confidencialidad +Estadística + +Mayo +Indeterminado +Estructura +archivo +Pre-pregunta +Tasa de Respuesta +Ancho +Recodificación y Derivación +Series +Octubre +Unidad de Análisis +Notas sobre el Procesamiento de Datos +Tipo de Datos +Archivo +Periodo de Referencia +Contenido del Archivo +Inválido +Vars. +cont. +Clave(s) +Pregunta +Fuente de información +Imputación +Seguridad +Para abrir este archivo se necesita el software gratuito +Otros recursos +Diccionario de Datos +Información +Enero +Otros documentos +Mínimo +Cobertura y Dominio Temático +Productor de los Metadatos +Mostrar más información +Entrevistador(es) +Pos-pregunta +Temas +Procedimiento de Muestreo +Estructura del Archivo +Lista de variables +Formato +Notas sobre el Muestreo +Grupo(s) de Variables +Descripción +Categorías +Máximo +Depositante(s) +Agosto +NP +Carátula +Ponderado +Marzo + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml new file mode 100644 index 00000000000..9fa4d2178b1 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml @@ -0,0 +1,173 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 +Valide +Tableau de fréquences non-affiché +Mode de calcul +discrète +Méthode de collecte +Autre traitement +Autre(s) remerciement(s) +Sans titre +Identification +Cliquer ici pour accéder/exporter les fichiers de données du format Nesstar +Valeur +Pourcentage +Pondération +Enquêteur(s) principal/aux +Ce document a été généré à l'aide du +Echantillonage +Enregistrements +Conditions d'accès +Source +Modalités +Responsabilité et droits d'auteurs +Définition +Estimation des erreurs d'échantillonage +Fichiers de données +Couverture géographique +Avril +Moyenne +Production des méta-données +Collecte des données +Valeures manquantes +Programmes informatiques +Variable(s) +Instructions aux enquêteurs +Financement +Novembre +Manquant +Version +Univers +Contributeur(s) +Agence propriétaire +Traitement et évaluation des données +Domaines thématiques +Documents administratifs +Ecart type +Contact(s) +Libellé +Documents techniques +Décimale(s) +Type +Formulation de la question +Concepts +Gamme +Résumé +Juin +Supervision +Autres formes d'évaluation des données +Références +Accessibilité +Dates de collecte +Edition des données +Questionnaires +Cas valide(s) +Rapports et documents analytiques +Droits d'auteurs +Documentation +Déviations par rapport à l'échantillon initial +Editeur(s) +Février +Le jeu de données contient +Remerciement(s) +Continue +Ecart type +Description des variables +Producteur +Date de production + +L'Explorer vous permet d'accéder aux données et de les exporter vers les formats statistiques les plus courants +Discrète +Groupe +Juillet +Nom du fichier +Enreg. +Nom +Avertissement: Ces chiffres indiquent le nombre de cas identifiés dans le fichier de données. Ils ne peuvent pas être interpretés comme étant représentatifs de la population concernée. +Tableaux statistiques +Décembre +Sujets +Contrôles de traitement + +Instructions aux enquêteurs +Table des matières +Informations sur le document +Sous-groupe(s) +Mots-clé +groupe(s) +P +Pondération +Description des fichiers +Notes +Notes sur la collecte +fichier(s) +continue +Responsabilité(s) +Contenu +variable(s) +Autre(s) producteur(s) +Producteurs et sponsors +Notes sur l'apurement des données +Distributeur(s) +Aperçu +Citation +Septembre +Catégorie +Confidentialité +Statistiques +Mai +Indéterminé +Structure +fichier +Pré-question +Taux de réponse +Taille +Formulation de la question +Recodage et dérivation +Série +Octobre +Unité d'analyse +Notes sur le traitement des données +Type d'étude +Fichier +Période(s) de référence +Contenu du fichier +Non-valide +Vars +suite +Clé(s) +Question +Source d'information +Imputation +Sécurité +Pour ouvrir ce fichier, vous avez besoin du logiciel gratuit +Autres resources +Dictionnaire des variables +Information +Janvier +Autres documents +Minimum +Domaines thématiques et couverture +Producteur(s) des méta-données +Information complémentaire +Enquêteurs +Post-question +Thèmes +Procédure d'échantillonage +Structure du fichier +Liste des variables +Format +Notes sur l'échantillonage +Groupe(s) de variables +Description +Catégories +Maximum +Dépositaire(s) +Août +NP +Couverture +Pondéré +Mars +question details + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml new file mode 100644 index 00000000000..bc5dbb06154 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml @@ -0,0 +1,161 @@ + + +Generated by Properties2Xml on Tue Feb 13 13:55:43 EST 2007 +有効な +度数表(Frequency table)は表示されません +由来 +不連続な +データ収集モード +その他の確認事項 +識別番号 +データファイルにアクセスするにはここをクリックしてください +無題 +値 +割合 +ウェイティング +第一次調査官 +この文書はToolkitを使用して作られました +サンプリング +ケース +アクセス条件 +情報源 +様相 +権利及び声明文 +定義 +サンプルエラーの見積もり +データファイル +地理的な適用範囲 +４月 +平均 +メタデータ製作 +データ収集 +損失データ +スクリプトおよびプログラム +可変的 +面接者の指示 +出資機関 +１１月 +バージョン +共通の +貢献者 +アクセス権限 +データ処理、評価 +範囲, 領域 +管理用文章 +連絡先 +ラベル +技術的な文書 +小数点 +タイプ +文字の質問 +概念 +範囲 +要約 +６月 +監督 +その他ファーマットのデータ評価 +参照 +アクセス、入手法 +データ収集日 +データ編集 +質問 +レポートおよび分析的な文書 +有効な場合 +コピーライト +書類 +サンプルデザインによる偏差 +発行者 +２月 +データセットに含まれる +確認事項 +連続的な +標準偏差 +変数の記述 +製作者 +製作日 +” Explorer”によってデータファイルを参照することも一般的に使えわれている統計データフォーマットに変換。抽出することも可能です +不連続性 +グループ +７月 +ファイルの名前 +ケース +名前 +統計表 +１２月 +主題, 内容 +工程監査 +ソフト +面接者への指示 +目録 +書類の情報 +サブグループ +キーワード + +グループ +ウェイト +ファイルの詳細 +メモ +データ収集メモ +ファイル +継続的な +声明文 +内容 +変数 +その他の製作者 +製作者とスポンサー +データクリーニングメモ +分配者 +概略 +引用する場合の必要条件 +９月 +カテゴリー +機密性、コンフィデンシャリティー +５月 +未定 +構造 +ファイル +調査前の質問 +回答比率 +幅 +記録と誘導 +シリー +１０月 +分析の単位 +データ処理メモ +データの種類 + +ファイル +期間 +ファイルの内容 +無効 +キー +情報源 +非難 +セキュリティー +このファイルを開けるには、無料で配布されているNesstar Explorer が必要です。 +その他の資料 +データ辞典 +情報 +１月 +その他の書類 +最小値 +規模及び適用範囲 +メタデータ製作者 +さらにインフォメーションを表示 +データ収集者 +調査後の質問 +サンプリングの手順 +ファイルの構造 +変数のリスト +フォーマット +サンプリングメモ +変数のグループ +詳細 +カテゴリー +最大値 +デポジター、提供者、供託者 +８月 +表紙 +ウェイトされた +３月 + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml new file mode 100644 index 00000000000..fdf14f5dfcd --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml @@ -0,0 +1,174 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 +Gyldige +Frekvenstabell ikke vist +Avledning +diskret +Type datainnsamling +Annen prosessering +Andre identifikatorer og krediteringer +Uten tittel +Identifisering +Click here to access/export data files from Nesstar format +Verdi +Prosent +Vekting +Primary Investigator(s) +Dette dokumentet ble generert av +Utvalg +Enheter +Tilgangsbetingelser +Kilde +Modaliteter +Rights & Disclaimer +Definisjon +Estimert utvalgsfeil +Datafiler +Geografisk omfang +April +Mean +Metadata-produksjon +Datainnsamling +Manglende data +Script og programmer +Variable(r) +Instruksjoner til intervjueren +Sponsor/finansierende institusjon(er) +November +Missing +Versjon +Univers +Bidragsyter(e) +Tilgangskontrollør +Dataprosessering og -evaluering +Omfang +Administrative dokumenter +Standardavvik +Kontaktperson(er) +Merkelapp +Tekniske dokumenter +Desimal(er) +Type +Spørsmålstekst +Begrep(er) +Rekkevidde +Sammendrag +Juni +Supervision +Andre former for dataevaluering +Referanser +Tilgjengelighet +Datainnsamlingsdatoer +Dataredigering +Spørreskjema +Gyldige enheter +Rapporter og analysedokumenter +Copyright +Dokumentasjon +Avvik fra utvalgsdesign +Utgiver(e) +Februar +Datasettet inneholder +Krediteringer +Kontinuerlig +Standardavvik +Variabelbeskrivelse +Produsent +Produksjonsdato + +The Explorer allows you to view data files and export them to common statistical formats +Diskret +Gruppe +Juli +Filnavn +Enheter +Navn +Advarsel: disse tallene indikerer antall enheter (cases) i datafilen. De kan ikke tolkes som oppsummert statistikk for populasjonen. +Statistiske tabeller +Desember +Emner +Prosesseringssjekk +programvare +Instruksjoner til intervjueren +Innholdsfortegnelse +Dokumentinformasjon +Undergruppe(r) +Nøkkelord +gruppe(r) +W +Vekt +Filbeskrivelse +Kommentarer +Datainnsamlingskommentarer +file(r) +kontinuerlig +Fraskrivelse +Innhold +variable(r) +Andre produsenter +Produsenter og sponsorer +Kommentarer om datarensing +Distributør(er) +Oversikt +Sitatkrav +September +Kategori +Konfidensialitet +Statistikk +Mai +Uavklart +Struktur +fil +Tekst før spørsmål +Responsrate +Bredde +Omkodinger og utledninger +Serie +Oktober +Analyseenhet +Dataprosesseringskommentarer +Datatype +Fil +Tidsperiode(r) +Filinnhold +Ugyldig +Variabler +kont. +Nøkler +Spørsmål +Kilde for informasjon +Imputasjon +Sikkerhet +For å åpne denne filen trenger du følgende gratisverktøy +Andre ressurser +Dataordbok +Informasjon +Januar +Andre dokumenter +Minimum +Omfang +Metadataprodusenter +Vis mer informasjon +Datainnsamler(e) +Tekst etter spørsmål +Emner +Utvalgsprosedyre +Filstruktur +Variabelliste +Format +Utvalgskommentarer +Variabelgrupper +Beskrivelse +Kategorier +Maksimum +Utgiver(e) +August +NW +Forside +Vektet +Mars + total - viser et utvalg av +Land +spørsmålsdetaljer + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml new file mode 100644 index 00000000000..fdf14f5dfcd --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml @@ -0,0 +1,174 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 +Gyldige +Frekvenstabell ikke vist +Avledning +diskret +Type datainnsamling +Annen prosessering +Andre identifikatorer og krediteringer +Uten tittel +Identifisering +Click here to access/export data files from Nesstar format +Verdi +Prosent +Vekting +Primary Investigator(s) +Dette dokumentet ble generert av +Utvalg +Enheter +Tilgangsbetingelser +Kilde +Modaliteter +Rights & Disclaimer +Definisjon +Estimert utvalgsfeil +Datafiler +Geografisk omfang +April +Mean +Metadata-produksjon +Datainnsamling +Manglende data +Script og programmer +Variable(r) +Instruksjoner til intervjueren +Sponsor/finansierende institusjon(er) +November +Missing +Versjon +Univers +Bidragsyter(e) +Tilgangskontrollør +Dataprosessering og -evaluering +Omfang +Administrative dokumenter +Standardavvik +Kontaktperson(er) +Merkelapp +Tekniske dokumenter +Desimal(er) +Type +Spørsmålstekst +Begrep(er) +Rekkevidde +Sammendrag +Juni +Supervision +Andre former for dataevaluering +Referanser +Tilgjengelighet +Datainnsamlingsdatoer +Dataredigering +Spørreskjema +Gyldige enheter +Rapporter og analysedokumenter +Copyright +Dokumentasjon +Avvik fra utvalgsdesign +Utgiver(e) +Februar +Datasettet inneholder +Krediteringer +Kontinuerlig +Standardavvik +Variabelbeskrivelse +Produsent +Produksjonsdato + +The Explorer allows you to view data files and export them to common statistical formats +Diskret +Gruppe +Juli +Filnavn +Enheter +Navn +Advarsel: disse tallene indikerer antall enheter (cases) i datafilen. De kan ikke tolkes som oppsummert statistikk for populasjonen. +Statistiske tabeller +Desember +Emner +Prosesseringssjekk +programvare +Instruksjoner til intervjueren +Innholdsfortegnelse +Dokumentinformasjon +Undergruppe(r) +Nøkkelord +gruppe(r) +W +Vekt +Filbeskrivelse +Kommentarer +Datainnsamlingskommentarer +file(r) +kontinuerlig +Fraskrivelse +Innhold +variable(r) +Andre produsenter +Produsenter og sponsorer +Kommentarer om datarensing +Distributør(er) +Oversikt +Sitatkrav +September +Kategori +Konfidensialitet +Statistikk +Mai +Uavklart +Struktur +fil +Tekst før spørsmål +Responsrate +Bredde +Omkodinger og utledninger +Serie +Oktober +Analyseenhet +Dataprosesseringskommentarer +Datatype +Fil +Tidsperiode(r) +Filinnhold +Ugyldig +Variabler +kont. +Nøkler +Spørsmål +Kilde for informasjon +Imputasjon +Sikkerhet +For å åpne denne filen trenger du følgende gratisverktøy +Andre ressurser +Dataordbok +Informasjon +Januar +Andre dokumenter +Minimum +Omfang +Metadataprodusenter +Vis mer informasjon +Datainnsamler(e) +Tekst etter spørsmål +Emner +Utvalgsprosedyre +Filstruktur +Variabelliste +Format +Utvalgskommentarer +Variabelgrupper +Beskrivelse +Kategorier +Maksimum +Utgiver(e) +August +NW +Forside +Vektet +Mars + total - viser et utvalg av +Land +spørsmålsdetaljer + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml new file mode 100644 index 00000000000..06fde85af5e --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml @@ -0,0 +1,169 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 +Валидный +Частотная таблица не выводится +Расчет +дискретная +Способ сбора данных +Прочая обработка +Другие участники +Безымянный +Индентификация +Щелкните здесь, чтобы получить доступ к файлам или экспортировать их +Значение +Процент +Взвешивание +Первичный(е) исследователь(и) +Документ был сгенерирован с использованием +Выборка +Наблюдения +Условия доступа +Источник +Модальности +Авторские права и ограничения ответственности +Определение +Оценки ошибок выборки +Файлы данных +Географический охват +Апрель +Среднее +Разработка метаданных +Сбор данных +Пропущенные данные +Скрипты и программы +Переменная(ые) +Инструкции интервьюеру +Кто финансировал +Ноябрь +Пропущенные +Версия +Генеральная совокупность +Участник(и) +Права доступа +Обработка и инспекция данных +Охват +Административные документы +СтдОткл +Контак(ы) +Метка +Технические документы +Десятичные +Тип +Текст вопроса +Концепции +Диапазон +Резюме +Июнь +Контроль +Другие формы инспекции данных +Установки +Доступность +Даты сбора данных +Редактирование данных +Вопросники +Валидное(ые) наблюдение(я) +Отчеты и аналитические документы +Авторские права +Документация +Отклонения от дизайна выборки +Издатель(и) +Февраль +Набор данных содержит +Участник(и) +Непрерывная +Стандартное отклонение +Описание переменных +Разработчик +Дата разработки +Проводник позволяет просматривать файлы данных и экспортировать их в распространенные статистические форматы +Дикретная +Группа +Июль +Имя файла +Наблюдения +Имя +Статистичсекие таблицы +Декабрь +Темы +Контроль обработки +программное обеспечение +Инструкции интервьюеру +Оглавление +Информация о документе +Подгруппа(ы) +Ключевые слова +группа(ы) +B +Вес +Описание файла +Примечания +Примечания по сбору данных +файл(ы) +непрерывная +Ограничения ответственности +Содержание +переменная(ые) +Другие разработчики +Разработчики и спонсоры +Примечания по чистке данных +Дистрибьютор(ы) +Обзор +Требования по цитированию +Сентябрь +Категория +Конфиденциальность +Статистики +Май +Неопределенный +Структура +файл +Текст, предваряющий вопрос +Доля ответов +Ширина +Перекодировка и расчеты +Серия +Октябрь +Единица анализа +Примечания по обработке данных +Тип данных +Файл +Период(ы) времени +Содержание файла +Некорректный +Переменные +непр. +Ключ(и) +Вопрос +Источник информации +Импутация +Безопасность +Чтобы открыть этот файл, необходимо иметь свободным +Прочие источники +Словарь данных +Информация +Январь +Прочие документы +Минимум +Охват и покрытие +Разработчик(и) метаданных +Показать дополнительную информацию +Кто собирал данные +Текст после вопроса +Разделы +Процедура выборки +Структура файла +Список переменных +Формат +Примечания по выборке +Группа(ы) переменных +Описание +Категории +Максимум +Депозитор(ы) +Август +HB +Титульный лист +взвешенные +Март + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl new file mode 100644 index 00000000000..64849846481 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl @@ -0,0 +1,4473 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + default-page + + + + + + + + + + + + + + + + + + + + + + Times + + + + + + + + + + + + + + + + + + + + + + + + + 50 + + + + + + + 0 + 0 + 0 + 1 + + + + + + + 1 + 0 + + + + + + + 0 + 1 + + + + + + ddp + toolkit + toolkit + toolkit + + + + + + + + () + + + + + + , + + + + + + + + + + + + + + + + - + + + + + + + + + + + + 1 + 1 + 1 + 1 + 0 + + + + + 1 + 1 + 1 + 1 + 0 + + + + + 1 + 1 + 1 + 1 + 1 + 0 + + + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 0 + + + + + 1 + 1 + 1 + 0 + + + + + 1 + 1 + 1 + 1 + 1 + 1 + 0 + + + + + 1 + 1 + 0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + : + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + : + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + *** + + **** + + + + + + + + + + + + + + + , + + + + + + + + : + + + + + + + + + + + + + () + , + + + + , + + + + + + + + + + + () + , + + + + + + + : + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + # + + + + + + + + + + + # + + + + + + + + + + + + + + + + + : + + + + + + :  + + + + + ,  + + +  () + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + : + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + () + , + + + + + + + + + + , + + , + + + + + + + + + + () + , + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + .. + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + - + + + + - + + + + . + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + +   + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + # + + + + + : + + + + + + + + + + + + + + + + + + + + + [= + + + + + + ] + + + + [= + + - + + + + . + + + ] + + + + [= - + ] + + + [ + + =* + + / + + + ] + + + + + + + + + + + + + + [ + + / + + ] + + + + + + + + + + + + + + + + + + + + + + + + + + + [ + + = + + + / + + + + + + - + + + ] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + () + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ( + ) + + + + + + + + + + + + + + + + + + + + : + + + + + + + + + + + + + + + + + + *** + *** + + + + + , + + + + + + + , + + + + + + , + + + + + + + + + + + + + , + + + + + , + + + + + , " + + " + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + # + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <h2> + + + </h2> + + + <br/> + + + + + + + + + + + + + + + + + + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Z + + + + + + + + + + + + + + + + + + + - + + + + + + + + NaN + + + + + + + + + + + From 84caa2a85e2f86d1c6f82725673bb4a60ec19f3d Mon Sep 17 00:00:00 2001 From: lubitchv Date: Mon, 12 Jun 2023 17:52:04 -0400 Subject: [PATCH 005/145] q --- .../dataverse/export/PdfCodeBookExporter.java | 88 +++++++++---------- 1 file changed, 40 insertions(+), 48 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java index 8f3bb78f8d3..c4f1d3ff2cc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java @@ -1,55 +1,58 @@ + + package edu.harvard.iq.dataverse.export; import com.google.auto.service.AutoService; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.export.ddi.DdiExportUtil; -import edu.harvard.iq.dataverse.export.spi.Exporter; -import edu.harvard.iq.dataverse.util.BundleUtil; + import edu.harvard.iq.dataverse.export.ddi.DdiExportUtil; + import io.gdcc.spi.export.ExportDataProvider; + import io.gdcc.spi.export.ExportException; + import io.gdcc.spi.export.Exporter; + import edu.harvard.iq.dataverse.util.BundleUtil; -import javax.json.JsonObject; -import javax.ws.rs.core.MediaType; -import javax.xml.stream.XMLStreamException; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Path; -import java.nio.file.Paths; + import javax.json.JsonObject; + import javax.ws.rs.core.MediaType; + import javax.xml.stream.XMLStreamException; + import java.io.File; + import java.io.IOException; + import java.io.InputStream; + import java.io.OutputStream; + import java.nio.file.Path; + import java.nio.file.Paths; + import java.util.Locale; + import java.util.Optional; @AutoService(Exporter.class) public class PdfCodeBookExporter implements Exporter { @Override - public String getProviderName() { + public String getFormatName() { return "pdf"; } @Override - public String getDisplayName() { - return BundleUtil.getStringFromBundle("dataset.exportBtn.itemLabel.pdf") != null ? BundleUtil.getStringFromBundle("dataset.exportBtn.itemLabel.pdf") : "DDI PDF Codebook"; + public String getDisplayName(Locale locale) { + String displayName = BundleUtil.getStringFromBundle("dataset.exportBtn.itemLabel.pdf", locale); + return Optional.ofNullable(displayName).orElse("DDI pdf codebook"); } @Override - public void exportDataset(DatasetVersion version, JsonObject json, OutputStream outputStream) throws ExportException { - try { - InputStream ddiInputStream; - try { - ddiInputStream = ExportService.getInstance().getExport(version.getDataset(), "ddi"); - } catch(ExportException | IOException e) { - throw new ExportException ("Cannot open export_ddi cached file"); + public void exportDataset(ExportDataProvider dataProvider, OutputStream outputStream) throws ExportException { + Optional ddiInputStreamOptional = dataProvider.getPrerequisiteInputStream(); + if (ddiInputStreamOptional.isPresent()) { + try (InputStream ddiInputStream = ddiInputStreamOptional.get()) { + DdiExportUtil.datasetPdfDDI(ddiInputStream, outputStream); + } catch (IOException e) { + throw new ExportException("Cannot open export_ddi cached file"); + } catch (XMLStreamException xse) { + throw new ExportException("Caught XMLStreamException performing DDI export"); } - DdiExportUtil.datasetPdfDDI(ddiInputStream, outputStream); - } catch (XMLStreamException xse) { - throw new ExportException ("Caught XMLStreamException performing DDI export"); + } else { + throw new ExportException("No prerequisite input stream found"); } } - @Override - public Boolean isXMLFormat() { - return false; - } - @Override public Boolean isHarvestable() { // No, we don't want this format to be harvested! @@ -65,27 +68,16 @@ public Boolean isAvailableToUsers() { } @Override - public String getXMLNameSpace() throws ExportException { - return null; - } - - @Override - public String getXMLSchemaLocation() throws ExportException { - return null; - } - - @Override - public String getXMLSchemaVersion() throws ExportException { - return null; - } - - @Override - public void setParam(String name, Object value) { - // this exporter does not uses or supports any parameters as of now. + public Optional getPrerequisiteFormatName() { + //This exporter relies on being able to get the output of the ddi exporter + return Optional.of("ddi"); } @Override public String getMediaType() { - return MediaType.WILDCARD; + return MediaType.TEXT_HTML; }; } + + + From cb85fd678945ed206cbcca7fc378bbb7d1771394 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Thu, 15 Jun 2023 17:54:09 -0400 Subject: [PATCH 006/145] english --- .../dataverse/export/PdfCodeBookExporter.java | 2 +- .../dataverse/export/ddi/DdiExportUtil.java | 15 +- .../{from-ddi-2.5 => }/ddi-to-fo.xsl | 444 +++++++++--------- .../from-ddi-2.5/ddi-pdf/i18n.inc.xslt | 5 - .../ddi-pdf/messages_en.properties.xml | 174 ------- .../ddi-pdf/messages_es.properties.xml | 170 ------- .../ddi-pdf/messages_fr.properties.xml | 173 ------- .../ddi-pdf/messages_ja.properties.xml | 161 ------- .../ddi-pdf/messages_nn.properties.xml | 174 ------- .../ddi-pdf/messages_no.properties.xml | 174 ------- .../ddi-pdf/messages_ru.properties.xml | 169 ------- 11 files changed, 234 insertions(+), 1427 deletions(-) rename src/main/resources/edu/harvard/iq/dataverse/{from-ddi-2.5 => }/ddi-to-fo.xsl (91%) delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml diff --git a/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java index c4f1d3ff2cc..e0d5171e30c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java @@ -75,7 +75,7 @@ public Optional getPrerequisiteFormatName() { @Override public String getMediaType() { - return MediaType.TEXT_HTML; + return MediaType.WILDCARD; }; } diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 00ec3c4d5c9..c77be3515f7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -43,6 +43,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -2110,7 +2111,10 @@ private static boolean checkParentElement(XMLStreamWriter xmlw, String elementNa public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { try { - File xsltfile = new File("/home/victoria/ddi-to-fo.xsl"); + //File xsltfile = new File("/home/victoria/fop-2.8/fop/ddi-to-fo.xsl"); + //URL resource = DdiExportUtil.class.getResource("edu/harvard/iq/dataverse/ddi-to-fo.xsl"); + //File xsltfile = new File(resource.toURI()); + InputStream styleSheetInput = DdiExportUtil.class.getClassLoader().getResourceAsStream("edu/harvard/iq/dataverse/ddi-to-fo.xsl"); logger.info("start datasetPdfDDI"); //InputStream xsltfile = DdiExportUtil.class.getClassLoader().getResourceAsStream( // "edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl"); @@ -2122,7 +2126,7 @@ public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream Fop fop = fopFactory.newFop(MimeConstants.MIME_PDF, foUserAgent, outputStream); // Setup XSLT TransformerFactory factory = TransformerFactory.newInstance(); - Transformer transformer = factory.newTransformer(new StreamSource(xsltfile)); + Transformer transformer = factory.newTransformer(new StreamSource(styleSheetInput)); // Set the value of a in the stylesheet transformer.setParameter("versionParam", "2.0"); @@ -2135,10 +2139,13 @@ public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream // Start XSLT transformation and FOP processing transformer.transform(src, res); - } finally { - outputStream.close(); + + } catch (Exception e) { + logger.info("First try"); + logger.severe(e.getMessage()); } } catch (Exception e) { + logger.info("Second try"); logger.severe(e.getMessage()); } } diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl similarity index 91% rename from src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl rename to src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl index 64849846481..1c03d5caf34 100644 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl @@ -66,7 +66,8 @@ Report optional text --> - + + @@ -123,7 +124,6 @@ Times - - + - + - + - + - + @@ -437,21 +437,21 @@ + select="'Data_Processing_and_Appraisal'"/> - + - + @@ -461,7 +461,7 @@ - + @@ -486,7 +486,7 @@ - + @@ -503,7 +503,7 @@ - + @@ -528,7 +528,7 @@ - + @@ -569,7 +569,7 @@ - + @@ -642,7 +642,7 @@ - + @@ -657,7 +657,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="'Metadata_Producers'"/> + select="'Production_Date'"/> + select="'Version'"/> + select="'Identification'"/> - + @@ -745,7 +745,7 @@ - + @@ -764,14 +764,14 @@ - + - + @@ -782,7 +782,7 @@ + select="'Scope_and_Coverage'"/> @@ -793,7 +793,7 @@ + select="'Producers_and_Sponsors'"/> @@ -803,7 +803,7 @@ - + @@ -813,7 +813,7 @@ - + @@ -825,7 +825,7 @@ internal-destination="data-processing-and-appraisal" text-decoration="underline" color="blue"> + select="'Data_Processing_and_Appraisal'"/> @@ -836,7 +836,7 @@ - + @@ -847,7 +847,7 @@ + select="'Rights_and_Disclaimer'"/> @@ -858,7 +858,7 @@ + select="'Files_Description'"/> @@ -891,7 +891,7 @@ - + @@ -924,7 +924,7 @@ - + @@ -949,7 +949,7 @@ + select="'Variables_Description'"/> @@ -982,7 +982,7 @@ - + @@ -1002,7 +1002,7 @@ - + @@ -1065,7 +1065,7 @@ - + @@ -1078,7 +1078,7 @@ - + + select="'Identification'"/> - @@ -1130,7 +1130,7 @@ select="/ddi:codeBook/ddi:stdyDscr/ddi:citation/ddi:verStmt/ddi:version"> : + select="'Production_Date'"/>: @@ -1139,7 +1139,7 @@ - @@ -1156,7 +1156,7 @@ - + - + select="'Kind_of_Data'"/> + select="'Unit_of_Analysis'"/> + select="'Scope_and_Coverage'"/> @@ -1257,7 +1257,7 @@ - + - @@ -1299,7 +1299,7 @@ - + - + @@ -1344,7 +1344,7 @@ - + @@ -1361,7 +1361,7 @@ border="{$default-border}" padding="{$cell-padding}"> - @@ -1422,7 +1422,7 @@ padding="{$cell-padding}"> @@ -1444,7 +1444,7 @@ padding="{$cell-padding}"> + select="'Other_Producers'"/> + select="'Funding_Agencies'"/> @@ -1514,7 +1514,7 @@ - @@ -1534,7 +1534,7 @@ padding="{$cell-padding}"> + select="'Sampling'"/> + select="'Sampling_Notes'"/> + select="'Sampling_Procedure'"/> + select="'Response_Rate'"/> + select="'Weighting'"/> + select="'Data_Collection'"/> @@ -1672,7 +1672,7 @@ padding="{$cell-padding}"> @@ -1694,7 +1694,7 @@ padding="{$cell-padding}"> + select="'Time_Periods'"/> @@ -1738,7 +1738,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="'Questionnaires'"/> + select="'Data_Collectors'"/> + select="'Supervision'"/> - + @@ -1885,7 +1885,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="'Data_Editing'"/> + select="'Other_Processing'"/> + select="'Accessibility'"/> @@ -1980,7 +1980,7 @@ padding="{$cell-padding}"> + select="'Access_Authority'"/> - @@ -2022,7 +2022,7 @@ padding="{$cell-padding}"> + select="'Distributors'"/> + select="'Depositors'"/> + select="'Confidentiality'"/> + select="'Access_Conditions'"/> @@ -2142,7 +2142,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="'Disclaimer'"/> + select="'Copyright'"/> - + @@ -2194,15 +2194,15 @@ - + - + - + @@ -2218,7 +2218,7 @@ - + @@ -2227,16 +2227,16 @@ - + - + - - + + - + @@ -2262,17 +2262,17 @@ - + - + - + - + @@ -2290,7 +2290,7 @@ - + @@ -2299,17 +2299,17 @@ - + - + - + - + @@ -2327,7 +2327,7 @@ - + @@ -2336,7 +2336,7 @@ - + @@ -2372,7 +2372,7 @@ + select="'Reports_and_analytical_documents'"/> @@ -2389,7 +2389,7 @@ space-after="0.03in"> - + @@ -2407,7 +2407,7 @@ + select="'Technical_documents'"/> @@ -2425,7 +2425,7 @@ + select="'Administrative_documents'"/> @@ -2442,7 +2442,7 @@ space-after="0.03in"> - + @@ -2459,7 +2459,7 @@ space-after="0.03in"> - + @@ -2477,7 +2477,7 @@ + select="'Statistical_tables'"/> @@ -2495,7 +2495,7 @@ + select="'Scripts_and_programs'"/> @@ -2512,7 +2512,7 @@ space-after="0.03in"> - + @@ -2532,7 +2532,7 @@ + select="'Reports_and_analytical_documents'"/> - + - + - - + - + - + @@ -2602,7 +2602,7 @@ - + @@ -2611,7 +2611,7 @@ - + *** - + **** @@ -2749,7 +2749,7 @@ - # # @@ -2762,7 +2762,7 @@ - # # @@ -2776,18 +2776,18 @@ - + - : + : - :  + :  @@ -2811,7 +2811,7 @@ - + @@ -2823,7 +2823,7 @@ - + @@ -2835,7 +2835,7 @@ - + @@ -2847,7 +2847,7 @@ - + @@ -2859,7 +2859,7 @@ - + @@ -2871,7 +2871,7 @@ - + @@ -2905,10 +2905,10 @@ - + - + @@ -2955,7 +2955,7 @@ - + : @@ -3059,7 +3059,7 @@ - + @@ -3080,7 +3080,7 @@ - + @@ -3092,7 +3092,7 @@ - + @@ -3104,7 +3104,7 @@ - + @@ -3116,7 +3116,7 @@ - + @@ -3239,13 +3239,13 @@ - + - + - + @@ -3322,7 +3322,7 @@   - + @@ -3369,7 +3369,7 @@ : - + @@ -3382,7 +3382,7 @@ - + [= + select="'Type'"/>= + select="'discrete'"/> + select="'continuous'"/> ] @@ -3404,7 +3404,7 @@ [== - @@ -3419,13 +3419,13 @@ [= = - ] [ - + =* / @@ -3437,7 +3437,7 @@ @@ -3452,12 +3452,12 @@ - + [ + select="'Abbrev_NotWeighted'"/> / - + ] @@ -3470,18 +3470,18 @@ - + select="'Invalid'"/> - + - @@ -3517,7 +3517,7 @@ - + - + - + - + - + - + @@ -3638,7 +3638,7 @@ - + + select="'Recoding_and_Derivation'"/> - + - + - + - - + + - @@ -3798,7 +3798,7 @@ padding="{$cell-padding}" text-align="center"> + select="'Cases_Abbreviation'"/> @@ -3806,7 +3806,7 @@ padding="{$cell-padding}" text-align="center"> + select="'Weighted'"/> @@ -3814,9 +3814,9 @@ padding="{$cell-padding}" text-align="center"> + select="'Percentage'"/> () + select="'Weighted'"/>) @@ -3950,16 +3950,16 @@ + select="'SumStat_Warning'"/> - + ( - ) + ) @@ -4003,7 +4003,7 @@ - *** + *** *** @@ -4055,7 +4055,7 @@ - + @@ -4070,7 +4070,7 @@ - + @@ -4085,7 +4085,7 @@ - + @@ -4100,7 +4100,7 @@ - + @@ -4162,38 +4162,38 @@ - + - + - + - + - + - + - + @@ -4257,40 +4257,40 @@ - + - + - + - + - + - + - + - + - + - + - + - + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt deleted file mode 100644 index edf876f3b04..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml deleted file mode 100644 index d8e98dfd3c6..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml +++ /dev/null @@ -1,174 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 -Valid -Frequency table not shown -Derivation -discrete -Data Collection Mode -Other Processing -Other Acknowledgment(s) -Untitled -Identification -Click here to access/export data files from Nesstar format -Value -Percentage -Weighting -Primary Investigator(s) -This document was generated using the -Sampling -Cases -Access Conditions -Source -Modalities -Rights & Disclaimer -Definition -Estimates of Sampling Error -Data Files -Geographic Coverage -April -Mean -Metadata Production -Data Collection -Missing Data -Scripts and programs -Variable(s) -Interviewer instructions -Funding Agency/ies -November -Missing -Version -Universe -Contributor(s) -Access Authority -Data Processing & Appraisal -Scope -Administrative documents -StdDev -Contact(s) -Label -Technical documents -Decimal(s) -Type -Literal question -Concepts -Range -Abstract -June -Supervision -Other Forms of Data Appraisal -References -Accessibility -Data Collection Dates -Data Editing -Questionnaires -Valid case(s) -Reports and analytical documents -Copyright -Documentation -Deviations from Sample Design -Publisher(s) -February -Dataset contains -Acknowledgment(s) -Continuous -Standard deviation -Variables Description -Producer -Production Date - -The Explorer allows you to view data files and export them to common statistical formats -Discrete -Group -July -Filename -Cases -Name -Warning: these figures indicate the number of cases found in the data file. They cannot be interpreted as summary statistics of the population of interest. -Statistical tables -December -Subjects -Processing Checks -software -Interviewer's instructions -Table of Contents -Document Information -Subgroup(s) -Keywords -group(s) -W -Weight -Files Description -Notes -Data Collection Notes -file(s) -continuous -Disclaimer -Content -variable(s) -Other Producer(s) -Producers & Sponsors -Data Cleaning Notes -Distributor(s) -Overview -Citation Requirements -September -Category -Confidentiality -Statistics -May -Undetermined -Structure -file -Pre-question -Response Rate -Width -Recoding and Derivation -Series -October -Unit of Analysis -Data Processing Notes -Kind of Data -File -Time Period(s) -File Content -Invalid -Vars -cont. -Key(s) -Question -Source of information -Imputation -Security -To open this file, you will need the free -Other resources -Data Dictionnary -Information -January -Other documents -Minimum -Scope & Coverage -Metadata Producer(s) -Show more info -Data Collector(s) -Post-question -Topics -Sampling Procedure -File Structure -Variables List -Format -Sampling Notes -Variables Group(s) -Description -Categories -Maximum -Depositor(s) -August -NW -Cover Page -Weighted -March - total - showing a subset of -Countries -question details - diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml deleted file mode 100644 index 9cfcdaf6e7e..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml +++ /dev/null @@ -1,170 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 -Válido -No se presentan las tablas de frecuencias -Derivación -discreta -Método de Recolección -Otros relacionados al procesamiento -Otros Reconocimientos -Sin título -Identificación -Presione aquí para acceder/exportar al archivo(s) de datos -Valor -Porcentaje -Ponderando -Investigadores Principales -Este documento fue producido utilizando el -Muestreo -Casos -Condiciones de uso -Fuente -Modalidades -Derechos y Notas Legales -Definición -Estimaciones del Error Muestral -Archivo de Datos -Cobertura Geográfica -Abril -Media -Producción de los Metadatos -Recolección de Datos -Datos perdidos -Programas informáticos -Variable(s) -Manual del encuestador -Agencias Auspiciadoras -Noviembre -Valores perdidos -Versión -Universo -Contribuidor(es) -Institución Propietaria -Tratamiento y Validación de Datos -Dominio Temático -Documentos Administrativos -Desviación -Contacto(s) -Etiqueta -Documentos Técnicos -Decimal(es) -Tipo -Pregunta textual -Conceptos -Rango -Resumen -Junio -Supervisión -Otras Formas de Validación de los Datos -Referencias -Accesibilidad -Fechas de Recolección de Datos -Procesamiento de Datos -Cuestionarios -Casos válidos -Reportes y documentos analíticos -Derechos de Autor -Documentación -Modificaciones al Diseño Muestral -Editor(es) -Febrero -Contenido de la Base de Datos -Reconocimiento(s) -Contínua -Desviación estándar -Descripción de la variable -Productor -Fecha de Producción -El Explorador NESSTAR permite visualizar los archivos de datos y exportarlos a diferentes formatos estadísticos -Discreta -Grupo -Julio -Nombre del Archivo -Casos -Nombre -Cuadros estadísticos -Diciembre -Temas -Controles de Tratamiento -software -Manual del encuestador -Indice -Información acerca de la Documentación -Subgrupo(s) -Palabra Clave -grupo(s) -P -Ponderador -Descripción de los Archivos -Notas -Notas sobre la Recolección de Datos -archivo(s) -continua -Nota Legal -Contenido -variable(s) -Otros Productores -Productores y Auspiciadores -Notas acerca de la Depuración de los Datos -Distribuidor(es) -Resumen General -Forma de citar -Septiembre -Categoría -Confidencialidad -Estadística - -Mayo -Indeterminado -Estructura -archivo -Pre-pregunta -Tasa de Respuesta -Ancho -Recodificación y Derivación -Series -Octubre -Unidad de Análisis -Notas sobre el Procesamiento de Datos -Tipo de Datos -Archivo -Periodo de Referencia -Contenido del Archivo -Inválido -Vars. -cont. -Clave(s) -Pregunta -Fuente de información -Imputación -Seguridad -Para abrir este archivo se necesita el software gratuito -Otros recursos -Diccionario de Datos -Información -Enero -Otros documentos -Mínimo -Cobertura y Dominio Temático -Productor de los Metadatos -Mostrar más información -Entrevistador(es) -Pos-pregunta -Temas -Procedimiento de Muestreo -Estructura del Archivo -Lista de variables -Formato -Notas sobre el Muestreo -Grupo(s) de Variables -Descripción -Categorías -Máximo -Depositante(s) -Agosto -NP -Carátula -Ponderado -Marzo - diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml deleted file mode 100644 index 9fa4d2178b1..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml +++ /dev/null @@ -1,173 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 -Valide -Tableau de fréquences non-affiché -Mode de calcul -discrète -Méthode de collecte -Autre traitement -Autre(s) remerciement(s) -Sans titre -Identification -Cliquer ici pour accéder/exporter les fichiers de données du format Nesstar -Valeur -Pourcentage -Pondération -Enquêteur(s) principal/aux -Ce document a été généré à l'aide du -Echantillonage -Enregistrements -Conditions d'accès -Source -Modalités -Responsabilité et droits d'auteurs -Définition -Estimation des erreurs d'échantillonage -Fichiers de données -Couverture géographique -Avril -Moyenne -Production des méta-données -Collecte des données -Valeures manquantes -Programmes informatiques -Variable(s) -Instructions aux enquêteurs -Financement -Novembre -Manquant -Version -Univers -Contributeur(s) -Agence propriétaire -Traitement et évaluation des données -Domaines thématiques -Documents administratifs -Ecart type -Contact(s) -Libellé -Documents techniques -Décimale(s) -Type -Formulation de la question -Concepts -Gamme -Résumé -Juin -Supervision -Autres formes d'évaluation des données -Références -Accessibilité -Dates de collecte -Edition des données -Questionnaires -Cas valide(s) -Rapports et documents analytiques -Droits d'auteurs -Documentation -Déviations par rapport à l'échantillon initial -Editeur(s) -Février -Le jeu de données contient -Remerciement(s) -Continue -Ecart type -Description des variables -Producteur -Date de production - -L'Explorer vous permet d'accéder aux données et de les exporter vers les formats statistiques les plus courants -Discrète -Groupe -Juillet -Nom du fichier -Enreg. -Nom -Avertissement: Ces chiffres indiquent le nombre de cas identifiés dans le fichier de données. Ils ne peuvent pas être interpretés comme étant représentatifs de la population concernée. -Tableaux statistiques -Décembre -Sujets -Contrôles de traitement - -Instructions aux enquêteurs -Table des matières -Informations sur le document -Sous-groupe(s) -Mots-clé -groupe(s) -P -Pondération -Description des fichiers -Notes -Notes sur la collecte -fichier(s) -continue -Responsabilité(s) -Contenu -variable(s) -Autre(s) producteur(s) -Producteurs et sponsors -Notes sur l'apurement des données -Distributeur(s) -Aperçu -Citation -Septembre -Catégorie -Confidentialité -Statistiques -Mai -Indéterminé -Structure -fichier -Pré-question -Taux de réponse -Taille -Formulation de la question -Recodage et dérivation -Série -Octobre -Unité d'analyse -Notes sur le traitement des données -Type d'étude -Fichier -Période(s) de référence -Contenu du fichier -Non-valide -Vars -suite -Clé(s) -Question -Source d'information -Imputation -Sécurité -Pour ouvrir ce fichier, vous avez besoin du logiciel gratuit -Autres resources -Dictionnaire des variables -Information -Janvier -Autres documents -Minimum -Domaines thématiques et couverture -Producteur(s) des méta-données -Information complémentaire -Enquêteurs -Post-question -Thèmes -Procédure d'échantillonage -Structure du fichier -Liste des variables -Format -Notes sur l'échantillonage -Groupe(s) de variables -Description -Catégories -Maximum -Dépositaire(s) -Août -NP -Couverture -Pondéré -Mars -question details - diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml deleted file mode 100644 index bc5dbb06154..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml +++ /dev/null @@ -1,161 +0,0 @@ - - -Generated by Properties2Xml on Tue Feb 13 13:55:43 EST 2007 -有効な -度数表(Frequency table)は表示されません -由来 -不連続な -データ収集モード -その他の確認事項 -識別番号 -データファイルにアクセスするにはここをクリックしてください -無題 -値 -割合 -ウェイティング -第一次調査官 -この文書はToolkitを使用して作られました -サンプリング -ケース -アクセス条件 -情報源 -様相 -権利及び声明文 -定義 -サンプルエラーの見積もり -データファイル -地理的な適用範囲 -４月 -平均 -メタデータ製作 -データ収集 -損失データ -スクリプトおよびプログラム -可変的 -面接者の指示 -出資機関 -１１月 -バージョン -共通の -貢献者 -アクセス権限 -データ処理、評価 -範囲, 領域 -管理用文章 -連絡先 -ラベル -技術的な文書 -小数点 -タイプ -文字の質問 -概念 -範囲 -要約 -６月 -監督 -その他ファーマットのデータ評価 -参照 -アクセス、入手法 -データ収集日 -データ編集 -質問 -レポートおよび分析的な文書 -有効な場合 -コピーライト -書類 -サンプルデザインによる偏差 -発行者 -２月 -データセットに含まれる -確認事項 -連続的な -標準偏差 -変数の記述 -製作者 -製作日 -” Explorer”によってデータファイルを参照することも一般的に使えわれている統計データフォーマットに変換。抽出することも可能です -不連続性 -グループ -７月 -ファイルの名前 -ケース -名前 -統計表 -１２月 -主題, 内容 -工程監査 -ソフト -面接者への指示 -目録 -書類の情報 -サブグループ -キーワード - -グループ -ウェイト -ファイルの詳細 -メモ -データ収集メモ -ファイル -継続的な -声明文 -内容 -変数 -その他の製作者 -製作者とスポンサー -データクリーニングメモ -分配者 -概略 -引用する場合の必要条件 -９月 -カテゴリー -機密性、コンフィデンシャリティー -５月 -未定 -構造 -ファイル -調査前の質問 -回答比率 -幅 -記録と誘導 -シリー -１０月 -分析の単位 -データ処理メモ -データの種類 - -ファイル -期間 -ファイルの内容 -無効 -キー -情報源 -非難 -セキュリティー -このファイルを開けるには、無料で配布されているNesstar Explorer が必要です。 -その他の資料 -データ辞典 -情報 -１月 -その他の書類 -最小値 -規模及び適用範囲 -メタデータ製作者 -さらにインフォメーションを表示 -データ収集者 -調査後の質問 -サンプリングの手順 -ファイルの構造 -変数のリスト -フォーマット -サンプリングメモ -変数のグループ -詳細 -カテゴリー -最大値 -デポジター、提供者、供託者 -８月 -表紙 -ウェイトされた -３月 - diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml deleted file mode 100644 index fdf14f5dfcd..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml +++ /dev/null @@ -1,174 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 -Gyldige -Frekvenstabell ikke vist -Avledning -diskret -Type datainnsamling -Annen prosessering -Andre identifikatorer og krediteringer -Uten tittel -Identifisering -Click here to access/export data files from Nesstar format -Verdi -Prosent -Vekting -Primary Investigator(s) -Dette dokumentet ble generert av -Utvalg -Enheter -Tilgangsbetingelser -Kilde -Modaliteter -Rights & Disclaimer -Definisjon -Estimert utvalgsfeil -Datafiler -Geografisk omfang -April -Mean -Metadata-produksjon -Datainnsamling -Manglende data -Script og programmer -Variable(r) -Instruksjoner til intervjueren -Sponsor/finansierende institusjon(er) -November -Missing -Versjon -Univers -Bidragsyter(e) -Tilgangskontrollør -Dataprosessering og -evaluering -Omfang -Administrative dokumenter -Standardavvik -Kontaktperson(er) -Merkelapp -Tekniske dokumenter -Desimal(er) -Type -Spørsmålstekst -Begrep(er) -Rekkevidde -Sammendrag -Juni -Supervision -Andre former for dataevaluering -Referanser -Tilgjengelighet -Datainnsamlingsdatoer -Dataredigering -Spørreskjema -Gyldige enheter -Rapporter og analysedokumenter -Copyright -Dokumentasjon -Avvik fra utvalgsdesign -Utgiver(e) -Februar -Datasettet inneholder -Krediteringer -Kontinuerlig -Standardavvik -Variabelbeskrivelse -Produsent -Produksjonsdato - -The Explorer allows you to view data files and export them to common statistical formats -Diskret -Gruppe -Juli -Filnavn -Enheter -Navn -Advarsel: disse tallene indikerer antall enheter (cases) i datafilen. De kan ikke tolkes som oppsummert statistikk for populasjonen. -Statistiske tabeller -Desember -Emner -Prosesseringssjekk -programvare -Instruksjoner til intervjueren -Innholdsfortegnelse -Dokumentinformasjon -Undergruppe(r) -Nøkkelord -gruppe(r) -W -Vekt -Filbeskrivelse -Kommentarer -Datainnsamlingskommentarer -file(r) -kontinuerlig -Fraskrivelse -Innhold -variable(r) -Andre produsenter -Produsenter og sponsorer -Kommentarer om datarensing -Distributør(er) -Oversikt -Sitatkrav -September -Kategori -Konfidensialitet -Statistikk -Mai -Uavklart -Struktur -fil -Tekst før spørsmål -Responsrate -Bredde -Omkodinger og utledninger -Serie -Oktober -Analyseenhet -Dataprosesseringskommentarer -Datatype -Fil -Tidsperiode(r) -Filinnhold -Ugyldig -Variabler -kont. -Nøkler -Spørsmål -Kilde for informasjon -Imputasjon -Sikkerhet -For å åpne denne filen trenger du følgende gratisverktøy -Andre ressurser -Dataordbok -Informasjon -Januar -Andre dokumenter -Minimum -Omfang -Metadataprodusenter -Vis mer informasjon -Datainnsamler(e) -Tekst etter spørsmål -Emner -Utvalgsprosedyre -Filstruktur -Variabelliste -Format -Utvalgskommentarer -Variabelgrupper -Beskrivelse -Kategorier -Maksimum -Utgiver(e) -August -NW -Forside -Vektet -Mars - total - viser et utvalg av -Land -spørsmålsdetaljer - diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml deleted file mode 100644 index fdf14f5dfcd..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml +++ /dev/null @@ -1,174 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 -Gyldige -Frekvenstabell ikke vist -Avledning -diskret -Type datainnsamling -Annen prosessering -Andre identifikatorer og krediteringer -Uten tittel -Identifisering -Click here to access/export data files from Nesstar format -Verdi -Prosent -Vekting -Primary Investigator(s) -Dette dokumentet ble generert av -Utvalg -Enheter -Tilgangsbetingelser -Kilde -Modaliteter -Rights & Disclaimer -Definisjon -Estimert utvalgsfeil -Datafiler -Geografisk omfang -April -Mean -Metadata-produksjon -Datainnsamling -Manglende data -Script og programmer -Variable(r) -Instruksjoner til intervjueren -Sponsor/finansierende institusjon(er) -November -Missing -Versjon -Univers -Bidragsyter(e) -Tilgangskontrollør -Dataprosessering og -evaluering -Omfang -Administrative dokumenter -Standardavvik -Kontaktperson(er) -Merkelapp -Tekniske dokumenter -Desimal(er) -Type -Spørsmålstekst -Begrep(er) -Rekkevidde -Sammendrag -Juni -Supervision -Andre former for dataevaluering -Referanser -Tilgjengelighet -Datainnsamlingsdatoer -Dataredigering -Spørreskjema -Gyldige enheter -Rapporter og analysedokumenter -Copyright -Dokumentasjon -Avvik fra utvalgsdesign -Utgiver(e) -Februar -Datasettet inneholder -Krediteringer -Kontinuerlig -Standardavvik -Variabelbeskrivelse -Produsent -Produksjonsdato - -The Explorer allows you to view data files and export them to common statistical formats -Diskret -Gruppe -Juli -Filnavn -Enheter -Navn -Advarsel: disse tallene indikerer antall enheter (cases) i datafilen. De kan ikke tolkes som oppsummert statistikk for populasjonen. -Statistiske tabeller -Desember -Emner -Prosesseringssjekk -programvare -Instruksjoner til intervjueren -Innholdsfortegnelse -Dokumentinformasjon -Undergruppe(r) -Nøkkelord -gruppe(r) -W -Vekt -Filbeskrivelse -Kommentarer -Datainnsamlingskommentarer -file(r) -kontinuerlig -Fraskrivelse -Innhold -variable(r) -Andre produsenter -Produsenter og sponsorer -Kommentarer om datarensing -Distributør(er) -Oversikt -Sitatkrav -September -Kategori -Konfidensialitet -Statistikk -Mai -Uavklart -Struktur -fil -Tekst før spørsmål -Responsrate -Bredde -Omkodinger og utledninger -Serie -Oktober -Analyseenhet -Dataprosesseringskommentarer -Datatype -Fil -Tidsperiode(r) -Filinnhold -Ugyldig -Variabler -kont. -Nøkler -Spørsmål -Kilde for informasjon -Imputasjon -Sikkerhet -For å åpne denne filen trenger du følgende gratisverktøy -Andre ressurser -Dataordbok -Informasjon -Januar -Andre dokumenter -Minimum -Omfang -Metadataprodusenter -Vis mer informasjon -Datainnsamler(e) -Tekst etter spørsmål -Emner -Utvalgsprosedyre -Filstruktur -Variabelliste -Format -Utvalgskommentarer -Variabelgrupper -Beskrivelse -Kategorier -Maksimum -Utgiver(e) -August -NW -Forside -Vektet -Mars - total - viser et utvalg av -Land -spørsmålsdetaljer - diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml deleted file mode 100644 index 06fde85af5e..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml +++ /dev/null @@ -1,169 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 -Валидный -Частотная таблица не выводится -Расчет -дискретная -Способ сбора данных -Прочая обработка -Другие участники -Безымянный -Индентификация -Щелкните здесь, чтобы получить доступ к файлам или экспортировать их -Значение -Процент -Взвешивание -Первичный(е) исследователь(и) -Документ был сгенерирован с использованием -Выборка -Наблюдения -Условия доступа -Источник -Модальности -Авторские права и ограничения ответственности -Определение -Оценки ошибок выборки -Файлы данных -Географический охват -Апрель -Среднее -Разработка метаданных -Сбор данных -Пропущенные данные -Скрипты и программы -Переменная(ые) -Инструкции интервьюеру -Кто финансировал -Ноябрь -Пропущенные -Версия -Генеральная совокупность -Участник(и) -Права доступа -Обработка и инспекция данных -Охват -Административные документы -СтдОткл -Контак(ы) -Метка -Технические документы -Десятичные -Тип -Текст вопроса -Концепции -Диапазон -Резюме -Июнь -Контроль -Другие формы инспекции данных -Установки -Доступность -Даты сбора данных -Редактирование данных -Вопросники -Валидное(ые) наблюдение(я) -Отчеты и аналитические документы -Авторские права -Документация -Отклонения от дизайна выборки -Издатель(и) -Февраль -Набор данных содержит -Участник(и) -Непрерывная -Стандартное отклонение -Описание переменных -Разработчик -Дата разработки -Проводник позволяет просматривать файлы данных и экспортировать их в распространенные статистические форматы -Дикретная -Группа -Июль -Имя файла -Наблюдения -Имя -Статистичсекие таблицы -Декабрь -Темы -Контроль обработки -программное обеспечение -Инструкции интервьюеру -Оглавление -Информация о документе -Подгруппа(ы) -Ключевые слова -группа(ы) -B -Вес -Описание файла -Примечания -Примечания по сбору данных -файл(ы) -непрерывная -Ограничения ответственности -Содержание -переменная(ые) -Другие разработчики -Разработчики и спонсоры -Примечания по чистке данных -Дистрибьютор(ы) -Обзор -Требования по цитированию -Сентябрь -Категория -Конфиденциальность -Статистики -Май -Неопределенный -Структура -файл -Текст, предваряющий вопрос -Доля ответов -Ширина -Перекодировка и расчеты -Серия -Октябрь -Единица анализа -Примечания по обработке данных -Тип данных -Файл -Период(ы) времени -Содержание файла -Некорректный -Переменные -непр. -Ключ(и) -Вопрос -Источник информации -Импутация -Безопасность -Чтобы открыть этот файл, необходимо иметь свободным -Прочие источники -Словарь данных -Информация -Январь -Прочие документы -Минимум -Охват и покрытие -Разработчик(и) метаданных -Показать дополнительную информацию -Кто собирал данные -Текст после вопроса -Разделы -Процедура выборки -Структура файла -Список переменных -Формат -Примечания по выборке -Группа(ы) переменных -Описание -Категории -Максимум -Депозитор(ы) -Август -HB -Титульный лист -взвешенные -Март - From 685bc8be429e6f65765e66668de372ee2c8bb2b5 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Fri, 1 Sep 2023 14:09:06 -0400 Subject: [PATCH 007/145] xslt --- .../edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 7739408e408..9076680754e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -2123,13 +2123,7 @@ private static boolean checkParentElement(XMLStreamWriter xmlw, String elementNa public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { try { - //File xsltfile = new File("/home/victoria/fop-2.8/fop/ddi-to-fo.xsl"); - //URL resource = DdiExportUtil.class.getResource("edu/harvard/iq/dataverse/ddi-to-fo.xsl"); - //File xsltfile = new File(resource.toURI()); InputStream styleSheetInput = DdiExportUtil.class.getClassLoader().getResourceAsStream("edu/harvard/iq/dataverse/ddi-to-fo.xsl"); - logger.info("start datasetPdfDDI"); - //InputStream xsltfile = DdiExportUtil.class.getClassLoader().getResourceAsStream( - // "edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl"); final FopFactory fopFactory = FopFactory.newInstance(new File(".").toURI()); FOUserAgent foUserAgent = fopFactory.newFOUserAgent(); @@ -2153,7 +2147,6 @@ public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream transformer.transform(src, res); } catch (Exception e) { - logger.info("First try"); logger.severe(e.getMessage()); } } catch (Exception e) { From 12c463a8fa41c44ee3a94926299ceffaa7d6a114 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Tue, 5 Sep 2023 12:19:17 -0400 Subject: [PATCH 008/145] inc --- .../dataverse/export/PdfCodeBookExporter.java | 36 +- .../dataverse/export/ddi/DdiExportUtil.java | 7 +- .../harvard/iq/dataverse/ddi-pdf/i18n.inc.xsl | 5 + .../ddi-pdf/messages_en.properties.xml | 174 +++++++ .../ddi-pdf/messages_es.properties.xml | 170 +++++++ .../ddi-pdf/messages_fr.properties.xml | 173 +++++++ .../ddi-pdf/messages_ja.properties.xml | 161 +++++++ .../ddi-pdf/messages_nn.properties.xml | 174 +++++++ .../ddi-pdf/messages_no.properties.xml | 174 +++++++ .../ddi-pdf/messages_ru.properties.xml | 169 +++++++ .../edu/harvard/iq/dataverse/ddi-to-fo.xsl | 448 +++++++++--------- 11 files changed, 1446 insertions(+), 245 deletions(-) create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/i18n.inc.xsl create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_en.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_es.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_fr.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ja.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_nn.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_no.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ru.properties.xml diff --git a/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java index e0d5171e30c..4772c09ffd5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java @@ -3,25 +3,23 @@ package edu.harvard.iq.dataverse.export; import com.google.auto.service.AutoService; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetVersion; - import edu.harvard.iq.dataverse.export.ddi.DdiExportUtil; - import io.gdcc.spi.export.ExportDataProvider; - import io.gdcc.spi.export.ExportException; - import io.gdcc.spi.export.Exporter; - import edu.harvard.iq.dataverse.util.BundleUtil; - - import javax.json.JsonObject; - import javax.ws.rs.core.MediaType; - import javax.xml.stream.XMLStreamException; - import java.io.File; - import java.io.IOException; - import java.io.InputStream; - import java.io.OutputStream; - import java.nio.file.Path; - import java.nio.file.Paths; - import java.util.Locale; - import java.util.Optional; + +import edu.harvard.iq.dataverse.export.ddi.DdiExportUtil; +import io.gdcc.spi.export.ExportDataProvider; +import io.gdcc.spi.export.ExportException; +import io.gdcc.spi.export.Exporter; +import edu.harvard.iq.dataverse.util.BundleUtil; +import jakarta.ws.rs.core.MediaType; + +import javax.xml.stream.XMLStreamException; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Locale; +import java.util.Optional; @AutoService(Exporter.class) public class PdfCodeBookExporter implements Exporter { diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 9ed89a6fe94..aab728c7773 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -76,6 +76,7 @@ import org.apache.fop.apps.Fop; import org.apache.fop.apps.FopFactory; import org.apache.fop.apps.MimeConstants; +import java.io.File; public class DdiExportUtil { @@ -2108,6 +2109,7 @@ private static boolean checkParentElement(XMLStreamWriter xmlw, String elementNa public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { try { + String sysId = DdiExportUtil.class.getClassLoader().getResource("edu/harvard/iq/dataverse/ddi-to-fo.xsl").toURI().toString(); InputStream styleSheetInput = DdiExportUtil.class.getClassLoader().getResourceAsStream("edu/harvard/iq/dataverse/ddi-to-fo.xsl"); final FopFactory fopFactory = FopFactory.newInstance(new File(".").toURI()); @@ -2117,7 +2119,9 @@ public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream Fop fop = fopFactory.newFop(MimeConstants.MIME_PDF, foUserAgent, outputStream); // Setup XSLT TransformerFactory factory = TransformerFactory.newInstance(); - Transformer transformer = factory.newTransformer(new StreamSource(styleSheetInput)); + Source mySrc = new StreamSource(styleSheetInput); + mySrc.setSystemId(sysId); + Transformer transformer = factory.newTransformer(mySrc); // Set the value of a in the stylesheet transformer.setParameter("versionParam", "2.0"); @@ -2135,7 +2139,6 @@ public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream logger.severe(e.getMessage()); } } catch (Exception e) { - logger.info("Second try"); logger.severe(e.getMessage()); } } diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/i18n.inc.xsl b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/i18n.inc.xsl new file mode 100644 index 00000000000..edf876f3b04 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/i18n.inc.xsl @@ -0,0 +1,5 @@ + + + + + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_en.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_en.properties.xml new file mode 100644 index 00000000000..d8e98dfd3c6 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_en.properties.xml @@ -0,0 +1,174 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 +Valid +Frequency table not shown +Derivation +discrete +Data Collection Mode +Other Processing +Other Acknowledgment(s) +Untitled +Identification +Click here to access/export data files from Nesstar format +Value +Percentage +Weighting +Primary Investigator(s) +This document was generated using the +Sampling +Cases +Access Conditions +Source +Modalities +Rights & Disclaimer +Definition +Estimates of Sampling Error +Data Files +Geographic Coverage +April +Mean +Metadata Production +Data Collection +Missing Data +Scripts and programs +Variable(s) +Interviewer instructions +Funding Agency/ies +November +Missing +Version +Universe +Contributor(s) +Access Authority +Data Processing & Appraisal +Scope +Administrative documents +StdDev +Contact(s) +Label +Technical documents +Decimal(s) +Type +Literal question +Concepts +Range +Abstract +June +Supervision +Other Forms of Data Appraisal +References +Accessibility +Data Collection Dates +Data Editing +Questionnaires +Valid case(s) +Reports and analytical documents +Copyright +Documentation +Deviations from Sample Design +Publisher(s) +February +Dataset contains +Acknowledgment(s) +Continuous +Standard deviation +Variables Description +Producer +Production Date + +The Explorer allows you to view data files and export them to common statistical formats +Discrete +Group +July +Filename +Cases +Name +Warning: these figures indicate the number of cases found in the data file. They cannot be interpreted as summary statistics of the population of interest. +Statistical tables +December +Subjects +Processing Checks +software +Interviewer's instructions +Table of Contents +Document Information +Subgroup(s) +Keywords +group(s) +W +Weight +Files Description +Notes +Data Collection Notes +file(s) +continuous +Disclaimer +Content +variable(s) +Other Producer(s) +Producers & Sponsors +Data Cleaning Notes +Distributor(s) +Overview +Citation Requirements +September +Category +Confidentiality +Statistics +May +Undetermined +Structure +file +Pre-question +Response Rate +Width +Recoding and Derivation +Series +October +Unit of Analysis +Data Processing Notes +Kind of Data +File +Time Period(s) +File Content +Invalid +Vars +cont. +Key(s) +Question +Source of information +Imputation +Security +To open this file, you will need the free +Other resources +Data Dictionnary +Information +January +Other documents +Minimum +Scope & Coverage +Metadata Producer(s) +Show more info +Data Collector(s) +Post-question +Topics +Sampling Procedure +File Structure +Variables List +Format +Sampling Notes +Variables Group(s) +Description +Categories +Maximum +Depositor(s) +August +NW +Cover Page +Weighted +March + total - showing a subset of +Countries +question details + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_es.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_es.properties.xml new file mode 100644 index 00000000000..9cfcdaf6e7e --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_es.properties.xml @@ -0,0 +1,170 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 +Válido +No se presentan las tablas de frecuencias +Derivación +discreta +Método de Recolección +Otros relacionados al procesamiento +Otros Reconocimientos +Sin título +Identificación +Presione aquí para acceder/exportar al archivo(s) de datos +Valor +Porcentaje +Ponderando +Investigadores Principales +Este documento fue producido utilizando el +Muestreo +Casos +Condiciones de uso +Fuente +Modalidades +Derechos y Notas Legales +Definición +Estimaciones del Error Muestral +Archivo de Datos +Cobertura Geográfica +Abril +Media +Producción de los Metadatos +Recolección de Datos +Datos perdidos +Programas informáticos +Variable(s) +Manual del encuestador +Agencias Auspiciadoras +Noviembre +Valores perdidos +Versión +Universo +Contribuidor(es) +Institución Propietaria +Tratamiento y Validación de Datos +Dominio Temático +Documentos Administrativos +Desviación +Contacto(s) +Etiqueta +Documentos Técnicos +Decimal(es) +Tipo +Pregunta textual +Conceptos +Rango +Resumen +Junio +Supervisión +Otras Formas de Validación de los Datos +Referencias +Accesibilidad +Fechas de Recolección de Datos +Procesamiento de Datos +Cuestionarios +Casos válidos +Reportes y documentos analíticos +Derechos de Autor +Documentación +Modificaciones al Diseño Muestral +Editor(es) +Febrero +Contenido de la Base de Datos +Reconocimiento(s) +Contínua +Desviación estándar +Descripción de la variable +Productor +Fecha de Producción +El Explorador NESSTAR permite visualizar los archivos de datos y exportarlos a diferentes formatos estadísticos +Discreta +Grupo +Julio +Nombre del Archivo +Casos +Nombre +Cuadros estadísticos +Diciembre +Temas +Controles de Tratamiento +software +Manual del encuestador +Indice +Información acerca de la Documentación +Subgrupo(s) +Palabra Clave +grupo(s) +P +Ponderador +Descripción de los Archivos +Notas +Notas sobre la Recolección de Datos +archivo(s) +continua +Nota Legal +Contenido +variable(s) +Otros Productores +Productores y Auspiciadores +Notas acerca de la Depuración de los Datos +Distribuidor(es) +Resumen General +Forma de citar +Septiembre +Categoría +Confidencialidad +Estadística + +Mayo +Indeterminado +Estructura +archivo +Pre-pregunta +Tasa de Respuesta +Ancho +Recodificación y Derivación +Series +Octubre +Unidad de Análisis +Notas sobre el Procesamiento de Datos +Tipo de Datos +Archivo +Periodo de Referencia +Contenido del Archivo +Inválido +Vars. +cont. +Clave(s) +Pregunta +Fuente de información +Imputación +Seguridad +Para abrir este archivo se necesita el software gratuito +Otros recursos +Diccionario de Datos +Información +Enero +Otros documentos +Mínimo +Cobertura y Dominio Temático +Productor de los Metadatos +Mostrar más información +Entrevistador(es) +Pos-pregunta +Temas +Procedimiento de Muestreo +Estructura del Archivo +Lista de variables +Formato +Notas sobre el Muestreo +Grupo(s) de Variables +Descripción +Categorías +Máximo +Depositante(s) +Agosto +NP +Carátula +Ponderado +Marzo + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_fr.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_fr.properties.xml new file mode 100644 index 00000000000..9fa4d2178b1 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_fr.properties.xml @@ -0,0 +1,173 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 +Valide +Tableau de fréquences non-affiché +Mode de calcul +discrète +Méthode de collecte +Autre traitement +Autre(s) remerciement(s) +Sans titre +Identification +Cliquer ici pour accéder/exporter les fichiers de données du format Nesstar +Valeur +Pourcentage +Pondération +Enquêteur(s) principal/aux +Ce document a été généré à l'aide du +Echantillonage +Enregistrements +Conditions d'accès +Source +Modalités +Responsabilité et droits d'auteurs +Définition +Estimation des erreurs d'échantillonage +Fichiers de données +Couverture géographique +Avril +Moyenne +Production des méta-données +Collecte des données +Valeures manquantes +Programmes informatiques +Variable(s) +Instructions aux enquêteurs +Financement +Novembre +Manquant +Version +Univers +Contributeur(s) +Agence propriétaire +Traitement et évaluation des données +Domaines thématiques +Documents administratifs +Ecart type +Contact(s) +Libellé +Documents techniques +Décimale(s) +Type +Formulation de la question +Concepts +Gamme +Résumé +Juin +Supervision +Autres formes d'évaluation des données +Références +Accessibilité +Dates de collecte +Edition des données +Questionnaires +Cas valide(s) +Rapports et documents analytiques +Droits d'auteurs +Documentation +Déviations par rapport à l'échantillon initial +Editeur(s) +Février +Le jeu de données contient +Remerciement(s) +Continue +Ecart type +Description des variables +Producteur +Date de production + +L'Explorer vous permet d'accéder aux données et de les exporter vers les formats statistiques les plus courants +Discrète +Groupe +Juillet +Nom du fichier +Enreg. +Nom +Avertissement: Ces chiffres indiquent le nombre de cas identifiés dans le fichier de données. Ils ne peuvent pas être interpretés comme étant représentatifs de la population concernée. +Tableaux statistiques +Décembre +Sujets +Contrôles de traitement + +Instructions aux enquêteurs +Table des matières +Informations sur le document +Sous-groupe(s) +Mots-clé +groupe(s) +P +Pondération +Description des fichiers +Notes +Notes sur la collecte +fichier(s) +continue +Responsabilité(s) +Contenu +variable(s) +Autre(s) producteur(s) +Producteurs et sponsors +Notes sur l'apurement des données +Distributeur(s) +Aperçu +Citation +Septembre +Catégorie +Confidentialité +Statistiques +Mai +Indéterminé +Structure +fichier +Pré-question +Taux de réponse +Taille +Formulation de la question +Recodage et dérivation +Série +Octobre +Unité d'analyse +Notes sur le traitement des données +Type d'étude +Fichier +Période(s) de référence +Contenu du fichier +Non-valide +Vars +suite +Clé(s) +Question +Source d'information +Imputation +Sécurité +Pour ouvrir ce fichier, vous avez besoin du logiciel gratuit +Autres resources +Dictionnaire des variables +Information +Janvier +Autres documents +Minimum +Domaines thématiques et couverture +Producteur(s) des méta-données +Information complémentaire +Enquêteurs +Post-question +Thèmes +Procédure d'échantillonage +Structure du fichier +Liste des variables +Format +Notes sur l'échantillonage +Groupe(s) de variables +Description +Catégories +Maximum +Dépositaire(s) +Août +NP +Couverture +Pondéré +Mars +question details + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ja.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ja.properties.xml new file mode 100644 index 00000000000..bc5dbb06154 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ja.properties.xml @@ -0,0 +1,161 @@ + + +Generated by Properties2Xml on Tue Feb 13 13:55:43 EST 2007 +有効な +度数表(Frequency table)は表示されません +由来 +不連続な +データ収集モード +その他の確認事項 +識別番号 +データファイルにアクセスするにはここをクリックしてください +無題 +値 +割合 +ウェイティング +第一次調査官 +この文書はToolkitを使用して作られました +サンプリング +ケース +アクセス条件 +情報源 +様相 +権利及び声明文 +定義 +サンプルエラーの見積もり +データファイル +地理的な適用範囲 +４月 +平均 +メタデータ製作 +データ収集 +損失データ +スクリプトおよびプログラム +可変的 +面接者の指示 +出資機関 +１１月 +バージョン +共通の +貢献者 +アクセス権限 +データ処理、評価 +範囲, 領域 +管理用文章 +連絡先 +ラベル +技術的な文書 +小数点 +タイプ +文字の質問 +概念 +範囲 +要約 +６月 +監督 +その他ファーマットのデータ評価 +参照 +アクセス、入手法 +データ収集日 +データ編集 +質問 +レポートおよび分析的な文書 +有効な場合 +コピーライト +書類 +サンプルデザインによる偏差 +発行者 +２月 +データセットに含まれる +確認事項 +連続的な +標準偏差 +変数の記述 +製作者 +製作日 +” Explorer”によってデータファイルを参照することも一般的に使えわれている統計データフォーマットに変換。抽出することも可能です +不連続性 +グループ +７月 +ファイルの名前 +ケース +名前 +統計表 +１２月 +主題, 内容 +工程監査 +ソフト +面接者への指示 +目録 +書類の情報 +サブグループ +キーワード + +グループ +ウェイト +ファイルの詳細 +メモ +データ収集メモ +ファイル +継続的な +声明文 +内容 +変数 +その他の製作者 +製作者とスポンサー +データクリーニングメモ +分配者 +概略 +引用する場合の必要条件 +９月 +カテゴリー +機密性、コンフィデンシャリティー +５月 +未定 +構造 +ファイル +調査前の質問 +回答比率 +幅 +記録と誘導 +シリー +１０月 +分析の単位 +データ処理メモ +データの種類 + +ファイル +期間 +ファイルの内容 +無効 +キー +情報源 +非難 +セキュリティー +このファイルを開けるには、無料で配布されているNesstar Explorer が必要です。 +その他の資料 +データ辞典 +情報 +１月 +その他の書類 +最小値 +規模及び適用範囲 +メタデータ製作者 +さらにインフォメーションを表示 +データ収集者 +調査後の質問 +サンプリングの手順 +ファイルの構造 +変数のリスト +フォーマット +サンプリングメモ +変数のグループ +詳細 +カテゴリー +最大値 +デポジター、提供者、供託者 +８月 +表紙 +ウェイトされた +３月 + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_nn.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_nn.properties.xml new file mode 100644 index 00000000000..fdf14f5dfcd --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_nn.properties.xml @@ -0,0 +1,174 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 +Gyldige +Frekvenstabell ikke vist +Avledning +diskret +Type datainnsamling +Annen prosessering +Andre identifikatorer og krediteringer +Uten tittel +Identifisering +Click here to access/export data files from Nesstar format +Verdi +Prosent +Vekting +Primary Investigator(s) +Dette dokumentet ble generert av +Utvalg +Enheter +Tilgangsbetingelser +Kilde +Modaliteter +Rights & Disclaimer +Definisjon +Estimert utvalgsfeil +Datafiler +Geografisk omfang +April +Mean +Metadata-produksjon +Datainnsamling +Manglende data +Script og programmer +Variable(r) +Instruksjoner til intervjueren +Sponsor/finansierende institusjon(er) +November +Missing +Versjon +Univers +Bidragsyter(e) +Tilgangskontrollør +Dataprosessering og -evaluering +Omfang +Administrative dokumenter +Standardavvik +Kontaktperson(er) +Merkelapp +Tekniske dokumenter +Desimal(er) +Type +Spørsmålstekst +Begrep(er) +Rekkevidde +Sammendrag +Juni +Supervision +Andre former for dataevaluering +Referanser +Tilgjengelighet +Datainnsamlingsdatoer +Dataredigering +Spørreskjema +Gyldige enheter +Rapporter og analysedokumenter +Copyright +Dokumentasjon +Avvik fra utvalgsdesign +Utgiver(e) +Februar +Datasettet inneholder +Krediteringer +Kontinuerlig +Standardavvik +Variabelbeskrivelse +Produsent +Produksjonsdato + +The Explorer allows you to view data files and export them to common statistical formats +Diskret +Gruppe +Juli +Filnavn +Enheter +Navn +Advarsel: disse tallene indikerer antall enheter (cases) i datafilen. De kan ikke tolkes som oppsummert statistikk for populasjonen. +Statistiske tabeller +Desember +Emner +Prosesseringssjekk +programvare +Instruksjoner til intervjueren +Innholdsfortegnelse +Dokumentinformasjon +Undergruppe(r) +Nøkkelord +gruppe(r) +W +Vekt +Filbeskrivelse +Kommentarer +Datainnsamlingskommentarer +file(r) +kontinuerlig +Fraskrivelse +Innhold +variable(r) +Andre produsenter +Produsenter og sponsorer +Kommentarer om datarensing +Distributør(er) +Oversikt +Sitatkrav +September +Kategori +Konfidensialitet +Statistikk +Mai +Uavklart +Struktur +fil +Tekst før spørsmål +Responsrate +Bredde +Omkodinger og utledninger +Serie +Oktober +Analyseenhet +Dataprosesseringskommentarer +Datatype +Fil +Tidsperiode(r) +Filinnhold +Ugyldig +Variabler +kont. +Nøkler +Spørsmål +Kilde for informasjon +Imputasjon +Sikkerhet +For å åpne denne filen trenger du følgende gratisverktøy +Andre ressurser +Dataordbok +Informasjon +Januar +Andre dokumenter +Minimum +Omfang +Metadataprodusenter +Vis mer informasjon +Datainnsamler(e) +Tekst etter spørsmål +Emner +Utvalgsprosedyre +Filstruktur +Variabelliste +Format +Utvalgskommentarer +Variabelgrupper +Beskrivelse +Kategorier +Maksimum +Utgiver(e) +August +NW +Forside +Vektet +Mars + total - viser et utvalg av +Land +spørsmålsdetaljer + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_no.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_no.properties.xml new file mode 100644 index 00000000000..fdf14f5dfcd --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_no.properties.xml @@ -0,0 +1,174 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 +Gyldige +Frekvenstabell ikke vist +Avledning +diskret +Type datainnsamling +Annen prosessering +Andre identifikatorer og krediteringer +Uten tittel +Identifisering +Click here to access/export data files from Nesstar format +Verdi +Prosent +Vekting +Primary Investigator(s) +Dette dokumentet ble generert av +Utvalg +Enheter +Tilgangsbetingelser +Kilde +Modaliteter +Rights & Disclaimer +Definisjon +Estimert utvalgsfeil +Datafiler +Geografisk omfang +April +Mean +Metadata-produksjon +Datainnsamling +Manglende data +Script og programmer +Variable(r) +Instruksjoner til intervjueren +Sponsor/finansierende institusjon(er) +November +Missing +Versjon +Univers +Bidragsyter(e) +Tilgangskontrollør +Dataprosessering og -evaluering +Omfang +Administrative dokumenter +Standardavvik +Kontaktperson(er) +Merkelapp +Tekniske dokumenter +Desimal(er) +Type +Spørsmålstekst +Begrep(er) +Rekkevidde +Sammendrag +Juni +Supervision +Andre former for dataevaluering +Referanser +Tilgjengelighet +Datainnsamlingsdatoer +Dataredigering +Spørreskjema +Gyldige enheter +Rapporter og analysedokumenter +Copyright +Dokumentasjon +Avvik fra utvalgsdesign +Utgiver(e) +Februar +Datasettet inneholder +Krediteringer +Kontinuerlig +Standardavvik +Variabelbeskrivelse +Produsent +Produksjonsdato + +The Explorer allows you to view data files and export them to common statistical formats +Diskret +Gruppe +Juli +Filnavn +Enheter +Navn +Advarsel: disse tallene indikerer antall enheter (cases) i datafilen. De kan ikke tolkes som oppsummert statistikk for populasjonen. +Statistiske tabeller +Desember +Emner +Prosesseringssjekk +programvare +Instruksjoner til intervjueren +Innholdsfortegnelse +Dokumentinformasjon +Undergruppe(r) +Nøkkelord +gruppe(r) +W +Vekt +Filbeskrivelse +Kommentarer +Datainnsamlingskommentarer +file(r) +kontinuerlig +Fraskrivelse +Innhold +variable(r) +Andre produsenter +Produsenter og sponsorer +Kommentarer om datarensing +Distributør(er) +Oversikt +Sitatkrav +September +Kategori +Konfidensialitet +Statistikk +Mai +Uavklart +Struktur +fil +Tekst før spørsmål +Responsrate +Bredde +Omkodinger og utledninger +Serie +Oktober +Analyseenhet +Dataprosesseringskommentarer +Datatype +Fil +Tidsperiode(r) +Filinnhold +Ugyldig +Variabler +kont. +Nøkler +Spørsmål +Kilde for informasjon +Imputasjon +Sikkerhet +For å åpne denne filen trenger du følgende gratisverktøy +Andre ressurser +Dataordbok +Informasjon +Januar +Andre dokumenter +Minimum +Omfang +Metadataprodusenter +Vis mer informasjon +Datainnsamler(e) +Tekst etter spørsmål +Emner +Utvalgsprosedyre +Filstruktur +Variabelliste +Format +Utvalgskommentarer +Variabelgrupper +Beskrivelse +Kategorier +Maksimum +Utgiver(e) +August +NW +Forside +Vektet +Mars + total - viser et utvalg av +Land +spørsmålsdetaljer + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ru.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ru.properties.xml new file mode 100644 index 00000000000..06fde85af5e --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ru.properties.xml @@ -0,0 +1,169 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 +Валидный +Частотная таблица не выводится +Расчет +дискретная +Способ сбора данных +Прочая обработка +Другие участники +Безымянный +Индентификация +Щелкните здесь, чтобы получить доступ к файлам или экспортировать их +Значение +Процент +Взвешивание +Первичный(е) исследователь(и) +Документ был сгенерирован с использованием +Выборка +Наблюдения +Условия доступа +Источник +Модальности +Авторские права и ограничения ответственности +Определение +Оценки ошибок выборки +Файлы данных +Географический охват +Апрель +Среднее +Разработка метаданных +Сбор данных +Пропущенные данные +Скрипты и программы +Переменная(ые) +Инструкции интервьюеру +Кто финансировал +Ноябрь +Пропущенные +Версия +Генеральная совокупность +Участник(и) +Права доступа +Обработка и инспекция данных +Охват +Административные документы +СтдОткл +Контак(ы) +Метка +Технические документы +Десятичные +Тип +Текст вопроса +Концепции +Диапазон +Резюме +Июнь +Контроль +Другие формы инспекции данных +Установки +Доступность +Даты сбора данных +Редактирование данных +Вопросники +Валидное(ые) наблюдение(я) +Отчеты и аналитические документы +Авторские права +Документация +Отклонения от дизайна выборки +Издатель(и) +Февраль +Набор данных содержит +Участник(и) +Непрерывная +Стандартное отклонение +Описание переменных +Разработчик +Дата разработки +Проводник позволяет просматривать файлы данных и экспортировать их в распространенные статистические форматы +Дикретная +Группа +Июль +Имя файла +Наблюдения +Имя +Статистичсекие таблицы +Декабрь +Темы +Контроль обработки +программное обеспечение +Инструкции интервьюеру +Оглавление +Информация о документе +Подгруппа(ы) +Ключевые слова +группа(ы) +B +Вес +Описание файла +Примечания +Примечания по сбору данных +файл(ы) +непрерывная +Ограничения ответственности +Содержание +переменная(ые) +Другие разработчики +Разработчики и спонсоры +Примечания по чистке данных +Дистрибьютор(ы) +Обзор +Требования по цитированию +Сентябрь +Категория +Конфиденциальность +Статистики +Май +Неопределенный +Структура +файл +Текст, предваряющий вопрос +Доля ответов +Ширина +Перекодировка и расчеты +Серия +Октябрь +Единица анализа +Примечания по обработке данных +Тип данных +Файл +Период(ы) времени +Содержание файла +Некорректный +Переменные +непр. +Ключ(и) +Вопрос +Источник информации +Импутация +Безопасность +Чтобы открыть этот файл, необходимо иметь свободным +Прочие источники +Словарь данных +Информация +Январь +Прочие документы +Минимум +Охват и покрытие +Разработчик(и) метаданных +Показать дополнительную информацию +Кто собирал данные +Текст после вопроса +Разделы +Процедура выборки +Структура файла +Список переменных +Формат +Примечания по выборке +Группа(ы) переменных +Описание +Категории +Максимум +Депозитор(ы) +Август +HB +Титульный лист +взвешенные +Март + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl index 1c03d5caf34..c25c76246d6 100644 --- a/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl @@ -66,8 +66,7 @@ Report optional text --> - - + @@ -110,7 +109,7 @@ - + @@ -124,6 +123,7 @@ Times + - 1 + 1 0 @@ -381,21 +381,21 @@ - + - + - + @@ -403,33 +403,33 @@ - + - + - + - + - + @@ -437,21 +437,21 @@ + select="$msg/*/entry[@key='Data_Processing_and_Appraisal']"/> - + - + @@ -461,7 +461,7 @@ - + @@ -486,7 +486,7 @@ - + @@ -503,7 +503,7 @@ - + @@ -528,7 +528,7 @@ - + @@ -569,7 +569,7 @@ - + @@ -642,7 +642,7 @@ - + @@ -657,7 +657,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="$msg/*/entry[@key='Metadata_Producers']"/> + select="$msg/*/entry[@key='Production_Date']"/> + select="$msg/*/entry[@key='Version']"/> + select="$msg/*/entry[@key='Identification']"/> - + @@ -745,7 +745,7 @@ - + @@ -764,14 +764,14 @@ - + - + @@ -782,7 +782,7 @@ + select="$msg/*/entry[@key='Scope_and_Coverage']"/> @@ -793,7 +793,7 @@ + select="$msg/*/entry[@key='Producers_and_Sponsors']"/> @@ -803,7 +803,7 @@ - + @@ -813,7 +813,7 @@ - + @@ -825,7 +825,7 @@ internal-destination="data-processing-and-appraisal" text-decoration="underline" color="blue"> + select="$msg/*/entry[@key='Data_Processing_and_Appraisal']"/> @@ -836,7 +836,7 @@ - + @@ -847,7 +847,7 @@ + select="$msg/*/entry[@key='Rights_and_Disclaimer']"/> @@ -858,7 +858,7 @@ + select="$msg/*/entry[@key='Files_Description']"/> @@ -891,7 +891,7 @@ - + @@ -924,7 +924,7 @@ - + @@ -949,7 +949,7 @@ + select="$msg/*/entry[@key='Variables_Description']"/> @@ -982,7 +982,7 @@ - + @@ -1002,7 +1002,7 @@ - + @@ -1065,7 +1065,7 @@ - + @@ -1078,7 +1078,7 @@ - + + select="$msg/*/entry[@key='Identification']"/> - @@ -1130,7 +1130,7 @@ select="/ddi:codeBook/ddi:stdyDscr/ddi:citation/ddi:verStmt/ddi:version"> : + select="$msg/*/entry[@key='Production_Date']"/>: @@ -1139,7 +1139,7 @@ - @@ -1156,7 +1156,7 @@ - + - + select="$msg/*/entry[@key='Kind_of_Data']"/> + select="$msg/*/entry[@key='Unit_of_Analysis']"/> + select="$msg/*/entry[@key='Scope_and_Coverage']"/> @@ -1257,7 +1257,7 @@ - + - @@ -1299,7 +1299,7 @@ - + - + @@ -1344,7 +1344,7 @@ - + @@ -1361,7 +1361,7 @@ border="{$default-border}" padding="{$cell-padding}"> - @@ -1422,7 +1422,7 @@ padding="{$cell-padding}"> @@ -1444,7 +1444,7 @@ padding="{$cell-padding}"> + select="$msg/*/entry[@key='Other_Producers']"/> + select="$msg/*/entry[@key='Funding_Agencies']"/> @@ -1514,7 +1514,7 @@ - @@ -1534,7 +1534,7 @@ padding="{$cell-padding}"> + select="$msg/*/entry[@key='Sampling']"/> + select="$msg/*/entry[@key='Sampling_Notes']"/> + select="$msg/*/entry[@key='Sampling_Procedure']"/> + select="$msg/*/entry[@key='Response_Rate']"/> + select="$msg/*/entry[@key='Weighting']"/> + select="$msg/*/entry[@key='Data_Collection']"/> @@ -1672,7 +1672,7 @@ padding="{$cell-padding}"> @@ -1694,7 +1694,7 @@ padding="{$cell-padding}"> + select="$msg/*/entry[@key='Time_Periods']"/> @@ -1738,7 +1738,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="$msg/*/entry[@key='Questionnaires']"/> + select="$msg/*/entry[@key='Data_Collectors']"/> + select="$msg/*/entry[@key='Supervision']"/> - + @@ -1885,7 +1885,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="$msg/*/entry[@key='Data_Editing']"/> + select="$msg/*/entry[@key='Other_Processing']"/> + select="$msg/*/entry[@key='Accessibility']"/> @@ -1980,7 +1980,7 @@ padding="{$cell-padding}"> + select="$msg/*/entry[@key='Access_Authority']"/> - @@ -2022,7 +2022,7 @@ padding="{$cell-padding}"> + select="$msg/*/entry[@key='Distributors']"/> + select="$msg/*/entry[@key='Depositors']"/> + select="$msg/*/entry[@key='Confidentiality']"/> + select="$msg/*/entry[@key='Access_Conditions']"/> @@ -2142,7 +2142,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="$msg/*/entry[@key='Disclaimer']"/> + select="$msg/*/entry[@key='Copyright']"/> - + @@ -2194,15 +2194,15 @@ - + - + - + @@ -2218,7 +2218,7 @@ - + @@ -2227,16 +2227,16 @@ - + - + - - + + - + @@ -2262,17 +2262,17 @@ - + - + - + - + @@ -2290,7 +2290,7 @@ - + @@ -2299,17 +2299,17 @@ - + - + - + - + @@ -2327,7 +2327,7 @@ - + @@ -2336,7 +2336,7 @@ - + @@ -2372,7 +2372,7 @@ + select="$msg/*/entry[@key='Reports_and_analytical_documents']"/> @@ -2389,7 +2389,7 @@ space-after="0.03in"> - + @@ -2407,7 +2407,7 @@ + select="$msg/*/entry[@key='Technical_documents']"/> @@ -2425,7 +2425,7 @@ + select="$msg/*/entry[@key='Administrative_documents']"/> @@ -2442,7 +2442,7 @@ space-after="0.03in"> - + @@ -2459,7 +2459,7 @@ space-after="0.03in"> - + @@ -2477,7 +2477,7 @@ + select="$msg/*/entry[@key='Statistical_tables']"/> @@ -2495,7 +2495,7 @@ + select="$msg/*/entry[@key='Scripts_and_programs']"/> @@ -2512,7 +2512,7 @@ space-after="0.03in"> - + @@ -2532,7 +2532,7 @@ + select="$msg/*/entry[@key='Reports_and_analytical_documents']"/> - + - + - - + - + - + @@ -2602,7 +2602,7 @@ - + @@ -2611,7 +2611,7 @@ - + *** - + **** @@ -2749,7 +2749,7 @@ - # # @@ -2762,7 +2762,7 @@ - # # @@ -2776,18 +2776,18 @@ - + - : + : - :  + :  @@ -2811,7 +2811,7 @@ - + @@ -2823,7 +2823,7 @@ - + @@ -2835,7 +2835,7 @@ - + @@ -2847,7 +2847,7 @@ - + @@ -2859,7 +2859,7 @@ - + @@ -2871,7 +2871,7 @@ - + @@ -2905,10 +2905,10 @@ - + - + @@ -2955,7 +2955,7 @@ - + : @@ -3059,7 +3059,7 @@ - + @@ -3080,7 +3080,7 @@ - + @@ -3092,7 +3092,7 @@ - + @@ -3104,7 +3104,7 @@ - + @@ -3116,7 +3116,7 @@ - + @@ -3239,13 +3239,13 @@ - + - + - + @@ -3322,7 +3322,7 @@   - + @@ -3369,7 +3369,7 @@ : - + @@ -3382,7 +3382,7 @@ - + [= + select="$msg/*/entry[@key='Type']"/>= + select="$msg/*/entry[@key='discrete']"/> + select="$msg/*/entry[@key='continuous']"/> ] @@ -3404,7 +3404,7 @@ [== - @@ -3419,13 +3419,13 @@ [= = - ] [ - + =* / @@ -3437,7 +3437,7 @@ @@ -3452,12 +3452,12 @@ - + [ + select="$msg/*/entry[@key='Abbrev_NotWeighted']"/> / - + ] @@ -3470,18 +3470,18 @@ - + select="$msg/*/entry[@key='Invalid']"/> - + - @@ -3517,7 +3517,7 @@ - + - + - + - + - + - + @@ -3638,7 +3638,7 @@ - + + select="$msg/*/entry[@key='Recoding_and_Derivation']"/> - + - + - + - - + + - @@ -3798,7 +3798,7 @@ padding="{$cell-padding}" text-align="center"> + select="$msg/*/entry[@key='Cases_Abbreviation']"/> @@ -3806,7 +3806,7 @@ padding="{$cell-padding}" text-align="center"> + select="$msg/*/entry[@key='Weighted']"/> @@ -3814,9 +3814,9 @@ padding="{$cell-padding}" text-align="center"> + select="$msg/*/entry[@key='Percentage']"/> () + select="$msg/*/entry[@key='Weighted']"/>) @@ -3950,16 +3950,16 @@ + select="$msg/*/entry[@key='SumStat_Warning']"/> - + ( - ) + ) @@ -4003,7 +4003,7 @@ - *** + *** *** @@ -4055,7 +4055,7 @@ - + @@ -4070,7 +4070,7 @@ - + @@ -4085,7 +4085,7 @@ - + @@ -4100,7 +4100,7 @@ - + @@ -4162,38 +4162,38 @@ - + - + - + - + - + - + - + @@ -4257,40 +4257,40 @@ - + - + - + - + - + - + - + - + - + - + - + - + From 3764bb2358aac756ed7bf97e052441836d5512e8 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 8 Nov 2023 14:34:48 -0500 Subject: [PATCH 009/145] fix guestbook preview stop refresh of datasetForm --- src/main/webapp/dataset-license-terms.xhtml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/webapp/dataset-license-terms.xhtml b/src/main/webapp/dataset-license-terms.xhtml index c54d94442ea..6d5b0a5fe4f 100644 --- a/src/main/webapp/dataset-license-terms.xhtml +++ b/src/main/webapp/dataset-license-terms.xhtml @@ -482,7 +482,7 @@
- #{bundle['file.dataFilesTab.terms.list.guestbook']}   + #{bundle['file.dataFilesTab.terms.list.guestbook']}  
@@ -524,7 +524,7 @@ + update=":datasetForm:previewGuestbook" oncomplete="PF('viewGuestbook').show();"/>
@@ -563,7 +563,7 @@ + update=":datasetForm:previewGuestbook" oncomplete="PF('viewGuestbook').show();"/> From 528e25a50c74976449ad306c7e61b39906671ea1 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Fri, 2 Feb 2024 14:22:58 -0500 Subject: [PATCH 010/145] xsl --- .../edu/harvard/iq/dataverse/ddi-to-fo.xsl | 52 +++---------------- 1 file changed, 7 insertions(+), 45 deletions(-) diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl index c25c76246d6..cb622912960 100644 --- a/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl @@ -109,7 +109,7 @@ - + @@ -123,7 +123,7 @@ Times - + - 1 + 1 0 @@ -3884,50 +3884,17 @@ select="$catgry-freq div $catgry-sum-freq"/> - - - - - - - - - - - - - - - - - - - - 0 - - - - - + + + test="not(@missing='Y')"> - - - - - - + @@ -3947,11 +3914,6 @@ - - - From 60a2795951aa665819cfecea92c4b9e4e0a63b07 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Fri, 2 Feb 2024 17:09:24 -0500 Subject: [PATCH 011/145] xsl instead xslt --- src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl index cb622912960..26fd7c23479 100644 --- a/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl @@ -123,7 +123,7 @@ Times - + - ${payara.version} From 56e2f9c7b0d8d437a9aed465ab1ef0cdbc98365b Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 15 Apr 2024 14:30:46 +0200 Subject: [PATCH 015/145] docs: fix link to Payara docs to not include a version We should avoid hardcoding the version into the link to avoid stale information. --- doc/sphinx-guides/source/admin/monitoring.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/admin/monitoring.rst b/doc/sphinx-guides/source/admin/monitoring.rst index ef306c88c6f..16bb18b7ad2 100644 --- a/doc/sphinx-guides/source/admin/monitoring.rst +++ b/doc/sphinx-guides/source/admin/monitoring.rst @@ -154,7 +154,7 @@ Tips: MicroProfile Metrics endpoint ----------------------------- -Payara provides the metrics endpoint: _ +Payara provides the metrics endpoint: _ The metrics you can retrieve that way: - `index_permit_wait_time_seconds_mean` displays how long does it take to receive a permit to index a dataset. - `index_time_seconds` displays how long does it take to index a dataset. From f7b6d94cedbe7b3d9f71279e99b546cb631f933d Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 15 Apr 2024 14:32:08 +0200 Subject: [PATCH 016/145] docs: update to reference Payara 6.2024.4 as requirement --- doc/sphinx-guides/source/developers/classic-dev-env.rst | 6 +++--- doc/sphinx-guides/source/installation/prerequisites.rst | 6 +++--- doc/sphinx-guides/source/qa/test-automation.md | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/sphinx-guides/source/developers/classic-dev-env.rst b/doc/sphinx-guides/source/developers/classic-dev-env.rst index 82e10b727ef..a2f35acfe70 100755 --- a/doc/sphinx-guides/source/developers/classic-dev-env.rst +++ b/doc/sphinx-guides/source/developers/classic-dev-env.rst @@ -93,15 +93,15 @@ On Linux, install ``jq`` from your package manager or download a binary from htt Install Payara ~~~~~~~~~~~~~~ -Payara 6.2023.8 or higher is required. +Payara 6.2024.4 or higher is required. To install Payara, run the following commands: ``cd /usr/local`` -``sudo curl -O -L https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2023.8/payara-6.2023.8.zip`` +``sudo curl -O -L https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2024.4/payara-6.2024.4.zip`` -``sudo unzip payara-6.2023.8.zip`` +``sudo unzip payara-6.2024.4.zip`` ``sudo chown -R $USER /usr/local/payara6`` diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index a56f4811ace..9f88dd7ecfb 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -44,7 +44,7 @@ On RHEL/derivative you can make Java 17 the default with the ``alternatives`` co Payara ------ -Payara 6.2023.8 is recommended. Newer versions might work fine. Regular updates are recommended. +Payara 6.2024.4 is recommended. Newer versions might work fine. Regular updates are recommended. Installing Payara ================= @@ -55,8 +55,8 @@ Installing Payara - Download and install Payara (installed in ``/usr/local/payara6`` in the example commands below):: - # wget https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2023.8/payara-6.2023.8.zip - # unzip payara-6.2023.8.zip + # wget https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2024.4/payara-6.2024.4.zip + # unzip payara-6.2024.4.zip # mv payara6 /usr/local If nexus.payara.fish is ever down for maintenance, Payara distributions are also available from https://repo1.maven.org/maven2/fish/payara/distributions/payara/ diff --git a/doc/sphinx-guides/source/qa/test-automation.md b/doc/sphinx-guides/source/qa/test-automation.md index 708d0f88e23..d2de33b48a5 100644 --- a/doc/sphinx-guides/source/qa/test-automation.md +++ b/doc/sphinx-guides/source/qa/test-automation.md @@ -52,7 +52,7 @@ Go to the end of the log and then scroll up, looking for the failure. A failed A ``` TASK [dataverse : download payara zip] ***************************************** -fatal: [localhost]: FAILED! => {"changed": false, "dest": "/tmp/payara.zip", "elapsed": 10, "msg": "Request failed: ", "url": "https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2023.8/payara-6.2023.8.zip"} +fatal: [localhost]: FAILED! => {"changed": false, "dest": "/tmp/payara.zip", "elapsed": 10, "msg": "Request failed: ", "url": "https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2024.4/payara-6.2024.4.zip"} ``` In the example above, if Payara can't be downloaded, we're obviously going to have problems deploying Dataverse to it! From b061b91ec711511e1be5b0d97ef9b08a56e59367 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 15 Apr 2024 14:32:50 +0200 Subject: [PATCH 017/145] build: upgrade to Payara 6.2024.4 #10494 --- modules/dataverse-parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 612902b47a4..f857c105a33 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -148,7 +148,7 @@ -Duser.timezone=${project.timezone} -Dfile.encoding=${project.build.sourceEncoding} -Duser.language=${project.language} -Duser.region=${project.region} - 6.2023.8 + 6.2024.4 42.7.2 9.3.0 1.12.290 From 76a9ef3f485dc8d70ec8bc0afcdfedd1daa2755e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 15 Apr 2024 14:36:08 +0200 Subject: [PATCH 018/145] build: remove phased out Payara dependency repo on GitHub --- modules/dataverse-parent/pom.xml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index f857c105a33..b2ce8c2069c 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -386,18 +386,6 @@ false - - - payara-patched-externals - Payara Patched Externals - https://raw.github.com/payara/Payara_PatchedProjects/master - - true - - - false - - central-repo Central Repository From 77a5c41939856ff7a91f611d32b88e7b65c3efc1 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 15 Apr 2024 15:21:17 +0200 Subject: [PATCH 019/145] refactor(install): remove workaround for FISH-7722 No longer necessary as the required add-opens has been included in upstream --- scripts/installer/as-setup.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/installer/as-setup.sh b/scripts/installer/as-setup.sh index 34deddf51a3..eb8c72973c8 100755 --- a/scripts/installer/as-setup.sh +++ b/scripts/installer/as-setup.sh @@ -117,9 +117,6 @@ function preliminary_setup() ./asadmin $ASADMIN_OPTS create-jvm-options "-Ddataverse.timerServer=true" - # Workaround for FISH-7722: Failed to deploy war with @Stateless https://github.com/payara/Payara/issues/6337 - ./asadmin $ASADMIN_OPTS create-jvm-options --add-opens=java.base/java.io=ALL-UNNAMED - # enable comet support ./asadmin $ASADMIN_OPTS set server-config.network-config.protocols.protocol.http-listener-1.http.comet-support-enabled="true" From 222b326aab6be19be9d7bc8907504801bc362343 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 15 Apr 2024 15:46:57 +0200 Subject: [PATCH 020/145] doc(deps): add release note about Payara upgrade #10494 --- doc/release-notes/10494-payara-upgrade.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 doc/release-notes/10494-payara-upgrade.md diff --git a/doc/release-notes/10494-payara-upgrade.md b/doc/release-notes/10494-payara-upgrade.md new file mode 100644 index 00000000000..050faee1f3e --- /dev/null +++ b/doc/release-notes/10494-payara-upgrade.md @@ -0,0 +1,6 @@ +# Upgrade Payara to v6.2024.4 + +With this version of Dataverse, we encourage you to upgrade to version 6.2024.4. +This will address security issues accumulated since the release of 6.2023.8, which was required since Dataverse release 6.0. + +If you are using GDCC containers, this upgrade is included when pulling new release images. From e9bf15effbd1ea8d985b10aa46934d0f03303d98 Mon Sep 17 00:00:00 2001 From: Ludovic DANIEL Date: Wed, 3 Apr 2024 14:24:25 +0200 Subject: [PATCH 021/145] 9276 - CVOC : allow customized mapping of indexed fields of cvoc configuration + handle ontoportal json formats from externalvocabularyvalue --- doc/release-notes/9276-doc-cvoc-index-in.md | 8 ++ .../iq/dataverse/DatasetFieldServiceBean.java | 93 ++++++++++++------- .../iq/dataverse/search/IndexServiceBean.java | 40 +++++++- 3 files changed, 107 insertions(+), 34 deletions(-) create mode 100644 doc/release-notes/9276-doc-cvoc-index-in.md diff --git a/doc/release-notes/9276-doc-cvoc-index-in.md b/doc/release-notes/9276-doc-cvoc-index-in.md new file mode 100644 index 00000000000..5c4dd4ca10f --- /dev/null +++ b/doc/release-notes/9276-doc-cvoc-index-in.md @@ -0,0 +1,8 @@ +## Release Highlights + +### Updates on Support for External Vocabulary Services + +#### Indexed field accuracy + +For more relevant indexing, you can now map external vocabulary values to a `managed-fields` of a [:CVocConf setting](https://guides.dataverse.org/en/6.3/installation/config.html#cvocconf) by adding the key `indexIn` in `retrieval-filtering`. +For more information, please check [GDCC/dataverse-external-vocab-support documentation](https://github.com/gdcc/dataverse-external-vocab-support/tree/main/docs). \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java index 6223cd83773..b1717431e41 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java @@ -41,6 +41,7 @@ import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.httpclient.HttpException; +import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpResponse; import org.apache.http.HttpResponseInterceptor; import org.apache.http.client.methods.HttpGet; @@ -321,14 +322,15 @@ public Map getCVocConf(boolean byTermUriField){ + jo.getString("term-uri-field")); } } - if (jo.containsKey("child-fields")) { - JsonArray childFields = jo.getJsonArray("child-fields"); - for (JsonString elm : childFields.getValuesAs(JsonString.class)) { - dft = findByNameOpt(elm.getString()); - logger.info("Found: " + dft.getName()); + if (jo.containsKey("managed-fields")) { + JsonObject managedFields = jo.getJsonObject("managed-fields"); + for (String s : managedFields.keySet()) { + dft = findByNameOpt(managedFields.getString(s)); if (dft == null) { logger.warning("Ignoring External Vocabulary setting for non-existent child field: " - + elm.getString()); + + managedFields.getString(s)); + } else { + logger.info("Found: " + dft.getName()); } } } @@ -345,7 +347,7 @@ public Map getCVocConf(boolean byTermUriField){ * @param df - the primitive/parent compound field containing a newly saved value */ public void registerExternalVocabValues(DatasetField df) { - DatasetFieldType dft =df.getDatasetFieldType(); + DatasetFieldType dft = df.getDatasetFieldType(); logger.fine("Registering for field: " + dft.getName()); JsonObject cvocEntry = getCVocConf(true).get(dft.getId()); if (dft.isPrimitive()) { @@ -366,38 +368,48 @@ public void registerExternalVocabValues(DatasetField df) { } } } - + /** - * Retrieves indexable strings from a cached externalvocabularyvalue entry. - * - * This method assumes externalvocabularyvalue entries have been filtered and - * the externalvocabularyvalue entry contain a single JsonObject whose "personName" or "termName" values - * are either Strings or an array of objects with "lang" and ("value" or "content") keys. The - * string, or the "value/content"s for each language are added to the set. - * + * Retrieves indexable strings from a cached externalvocabularyvalue entry filtered through retrieval-filtering configuration. + *

+ * This method externalvocabularyvalue entries have been filtered and contains a single JsonObject. + * Is handled : Strings, Array of Objects with "lang" and ("value" or "content") keys, Object with Strings as value or Object with Array of Strings as value. + * The string, or the "value/content"s for each language are added to the set. + * This method can retrieve string values to be indexed in term-uri-field (parameter defined in CVOC configuration) or in "indexIn" field (optional parameter of retrieval-filtering defined in CVOC configuration). + *

* Any parsing error results in no entries (there can be unfiltered entries with * unknown structure - getting some strings from such an entry could give fairly * random info that would be bad to addd for searches, etc.) - * - * @param termUri + * + * @param termUri unique identifier to search in database + * @param cvocEntry related cvoc configuration + * @param indexingField name of solr field that will be filled with getStringsFor while indexing * @return - a set of indexable strings */ - public Set getStringsFor(String termUri) { - Set strings = new HashSet(); + public Set getIndexableStringsByTermUri(String termUri, JsonObject cvocEntry, String indexingField) { + Set strings = new HashSet<>(); JsonObject jo = getExternalVocabularyValue(termUri); + JsonObject filtering = cvocEntry.getJsonObject("retrieval-filtering"); + String termUriField = cvocEntry.getJsonString("term-uri-field").getString(); if (jo != null) { try { for (String key : jo.keySet()) { - if (key.equals("termName") || key.equals("personName")) { + String indexIn = filtering.getJsonObject(key).getString("indexIn", null); + // Either we are in mapping mode so indexingField (solr field) equals indexIn (cvoc config) + // Or we are in default mode indexingField is termUriField, indexIn is not defined then only termName and personName keys are used + if (indexingField.equals(indexIn) || + (indexIn == null && termUriField.equals(indexingField) && (key.equals("termName")) || key.equals("personName"))) { JsonValue jv = jo.get(key); if (jv.getValueType().equals(JsonValue.ValueType.STRING)) { logger.fine("adding " + jo.getString(key) + " for " + termUri); strings.add(jo.getString(key)); - } else { - if (jv.getValueType().equals(JsonValue.ValueType.ARRAY)) { - JsonArray jarr = jv.asJsonArray(); - for (int i = 0; i < jarr.size(); i++) { + } else if (jv.getValueType().equals(JsonValue.ValueType.ARRAY)) { + JsonArray jarr = jv.asJsonArray(); + for (int i = 0; i < jarr.size(); i++) { + if (jarr.get(i).getValueType().equals(JsonValue.ValueType.STRING)) { + strings.add(jarr.getString(i)); + } else if (jarr.get(i).getValueType().equals(ValueType.OBJECT)) { // This condition handles SKOMOS format like [{"lang": "en","value": "non-apis bee"},{"lang": "fr","value": "abeille non apis"}] JsonObject entry = jarr.getJsonObject(i); if (entry.containsKey("value")) { logger.fine("adding " + entry.getString("value") + " for " + termUri); @@ -409,6 +421,22 @@ public Set getStringsFor(String termUri) { } } } + } else if (jv.getValueType().equals(JsonValue.ValueType.OBJECT)) { + JsonObject joo = jv.asJsonObject(); + for (Map.Entry entry : joo.entrySet()) { + if (entry.getValue().getValueType().equals(JsonValue.ValueType.STRING)) { // This condition handles format like { "fr": "association de quartier", "en": "neighborhood associations"} + logger.fine("adding " + joo.getString(entry.getKey()) + " for " + termUri); + strings.add(joo.getString(entry.getKey())); + } else if (entry.getValue().getValueType().equals(ValueType.ARRAY)) { // This condition handles format like {"en": ["neighbourhood societies"]} + JsonArray jarr = entry.getValue().asJsonArray(); + for (int i = 0; i < jarr.size(); i++) { + if (jarr.get(i).getValueType().equals(JsonValue.ValueType.STRING)) { + logger.fine("adding " + jarr.getString(i) + " for " + termUri); + strings.add(jarr.getString(i)); + } + } + } + } } } } @@ -420,7 +448,7 @@ public Set getStringsFor(String termUri) { } logger.fine("Returning " + String.join(",", strings) + " for " + termUri); return strings; - } + } /** * Perform a query to retrieve a cached value from the externalvocabularvalue table @@ -454,10 +482,11 @@ public JsonObject getExternalVocabularyValue(String termUri) { public void registerExternalTerm(JsonObject cvocEntry, String term) { String retrievalUri = cvocEntry.getString("retrieval-uri"); String prefix = cvocEntry.getString("prefix", null); - if(term.isBlank()) { - logger.fine("Ingoring blank term"); + if(StringUtils.isBlank(term)) { + logger.fine("Ignoring blank term"); return; } + boolean isExternal = false; JsonObject vocabs = cvocEntry.getJsonObject("vocabs"); for (String key: vocabs.keySet()) { @@ -512,7 +541,7 @@ public void process(HttpResponse response, HttpContext context) throws HttpExcep if (statusCode == 200) { logger.fine("Returned data: " + data); try (JsonReader jsonReader = Json.createReader(new StringReader(data))) { - String dataObj =filterResponse(cvocEntry, jsonReader.readObject(), term).toString(); + String dataObj = filterResponse(cvocEntry, jsonReader.readObject(), term).toString(); evv.setValue(dataObj); evv.setLastUpdateDate(Timestamp.from(Instant.now())); logger.fine("JsonObject: " + dataObj); @@ -543,7 +572,7 @@ public void process(HttpResponse response, HttpContext context) throws HttpExcep * Parse the raw value returned by an external service for a give term uri and * filter it according to the 'retrieval-filtering' configuration for this * DatasetFieldType, creating a Json value with the specified structure - * + * * @param cvocEntry - the config for this DatasetFieldType * @param readObject - the raw response from the service * @param termUri - the term uri @@ -602,6 +631,8 @@ private JsonObject filterResponse(JsonObject cvocEntry, JsonObject readObject, S if (pattern.equals("{0}")) { if (vals.get(0) instanceof JsonArray) { job.add(filterKey, (JsonArray) vals.get(0)); + } else if (vals.get(0) instanceof JsonObject) { + job.add(filterKey, (JsonObject) vals.get(0)); } else { job.add(filterKey, (String) vals.get(0)); } @@ -639,7 +670,7 @@ Object processPathSegment(int index, String[] pathParts, JsonValue curPath, Stri String[] keyVal = pathParts[index].split("="); logger.fine("Looking for object where " + keyVal[0] + " is " + keyVal[1]); String expected = keyVal[1]; - + if (!expected.equals("*")) { if (expected.equals("@id")) { expected = termUri; @@ -668,7 +699,7 @@ Object processPathSegment(int index, String[] pathParts, JsonValue curPath, Stri } return parts.build(); } - + } else { curPath = ((JsonObject) curPath).get(pathParts[index]); logger.fine("Found next Path object " + curPath.toString()); diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index d6b3fd8c339..cf8a37e0a80 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -922,6 +922,20 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set langs = settingsService.getConfiguredLanguages(); Map cvocMap = datasetFieldService.getCVocConf(true); + Map> cvocManagedFieldMap = new HashMap<>(); + for (Map.Entry cvocEntry : cvocMap.entrySet()) { + if(cvocEntry.getValue().containsKey("managed-fields")) { + JsonObject managedFields = cvocEntry.getValue().getJsonObject("managed-fields"); + Set managedFieldValues = new HashSet<>(); + for (String s : managedFields.keySet()) { + managedFieldValues.add(managedFields.getString(s)); + } + cvocManagedFieldMap.put(cvocEntry.getKey(), managedFieldValues); + } + } + + + Set metadataBlocksWithValue = new HashSet<>(); for (DatasetField dsf : datasetVersion.getFlatDatasetFields()) { @@ -996,19 +1010,39 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set vals = dsf.getValues_nondisplay(); - Set searchStrings = new HashSet(); + Set searchStrings = new HashSet<>(); for (String val: vals) { searchStrings.add(val); - searchStrings.addAll(datasetFieldService.getStringsFor(val)); + // Try to get string values from externalvocabularyvalue using val as termUri + searchStrings.addAll(datasetFieldService.getIndexableStringsByTermUri(val, cvocMap.get(dsfType.getId()), dsfType.getName())); + + if(dsfType.getParentDatasetFieldType()!=null) { + List childDatasetFields = dsf.getParentDatasetFieldCompoundValue().getChildDatasetFields(); + for (DatasetField df : childDatasetFields) { + if(cvocManagedFieldMap.get(dsfType.getId()).contains(df.getDatasetFieldType().getName())) { + String solrManagedFieldSearchable = df.getDatasetFieldType().getSolrField().getNameSearchable(); + // Try to get string values from externalvocabularyvalue but for a managed fields of the CVOCConf + Set stringsForManagedField = datasetFieldService.getIndexableStringsByTermUri(val, cvocMap.get(dsfType.getId()), df.getDatasetFieldType().getName()); + logger.fine(solrManagedFieldSearchable + " filled with externalvocabularyvalue : " + stringsForManagedField); + //.addField works as addition of value not a replace of value + // it allows to add mapped values by CVOCConf before or after indexing real DatasetField value(s) of solrManagedFieldSearchable + solrInputDocument.addField(solrManagedFieldSearchable, stringsForManagedField); + } + } + } } + logger.fine(solrFieldSearchable + " filled with externalvocabularyvalue : " + searchStrings); solrInputDocument.addField(solrFieldSearchable, searchStrings); if (dsfType.getSolrField().isFacetable()) { + logger.fine(solrFieldFacetable + " gets " + vals); solrInputDocument.addField(solrFieldFacetable, vals); } } + if (dsfType.isControlledVocabulary()) { /** If the cvv list is empty but the dfv list is not then it is assumed this was harvested * from an installation that had controlled vocabulary entries that don't exist in our this db From 6b2e111142a39083db578d429394375dc061ce60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20ROUCOU?= Date: Tue, 23 Apr 2024 12:08:42 +0200 Subject: [PATCH 022/145] Add unit tests for "getIndexableStringsByTermUri" method --- .../DatasetFieldServiceBeanTest.java | 179 ++++++++++++++++++ src/test/resources/json/cvoc-agroportal.json | 76 ++++++++ src/test/resources/json/cvoc-orcid.json | 43 +++++ src/test/resources/json/cvoc-skosmos.json | 69 +++++++ 4 files changed, 367 insertions(+) create mode 100644 src/test/java/edu/harvard/iq/dataverse/DatasetFieldServiceBeanTest.java create mode 100644 src/test/resources/json/cvoc-agroportal.json create mode 100644 src/test/resources/json/cvoc-orcid.json create mode 100644 src/test/resources/json/cvoc-skosmos.json diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldServiceBeanTest.java new file mode 100644 index 00000000000..873d417131d --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldServiceBeanTest.java @@ -0,0 +1,179 @@ +package edu.harvard.iq.dataverse; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.Set; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.AdditionalMatchers; +import org.mockito.Mockito; + +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import jakarta.json.Json; +import jakarta.json.JsonObject; + +public class DatasetFieldServiceBeanTest { + + private DatasetFieldServiceBean datasetFieldServiceBean; + + static String getCvocJson(String pathToJsonFile) throws IOException { + final File datasetVersionJson = new File(pathToJsonFile); + return new String(Files.readAllBytes(Paths.get(datasetVersionJson.getAbsolutePath()))); + } + + @BeforeEach + void setUp() { + this.datasetFieldServiceBean = Mockito.spy(new DatasetFieldServiceBean()); + } + + @AfterEach + void tearDown() { + this.datasetFieldServiceBean = null; + } + + @Test + void getIndexableStringsByTermUriSkosmos() throws IOException { + String fieldName = "keyword"; + String termURI = "http://aims.fao.org/aos/agrovoc/c_2389"; + + JsonObject cvocEntry = prepare(fieldName, "src/test/resources/json/cvoc-skosmos.json"); + + JsonObject getExtVocabValueReturnedValue = Json.createObjectBuilder() + .add("@id", termURI) + .add("termName", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("lang", "fr") + .add("value", "faux bourdon")) + .add(Json.createObjectBuilder() + .add("lang", "en") + .add("value", "drone (insects)"))) + .add("vocabularyUri", "http://aims.fao.org/aos/agrovoc") + .add("synonyms", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("lang", "fr") + .add("value", "Abeille mâle")) + .add(Json.createObjectBuilder() + .add("lang", "en") + .add("value", "drone honey bees"))) + .add("genericTerm", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("lang", "fr") + .add("value", "Colonie d'abeilles")) + .add(Json.createObjectBuilder() + .add("lang", "en") + .add("value", "bee colonies"))) + .build(); + Mockito.doReturn(getExtVocabValueReturnedValue).when(datasetFieldServiceBean).getExternalVocabularyValue(termURI); + Mockito.doReturn(null).when(datasetFieldServiceBean).getExternalVocabularyValue(AdditionalMatchers.not(Mockito.eq(termURI))); + + // keywordTermURL + Set result = datasetFieldServiceBean.getIndexableStringsByTermUri(termURI, cvocEntry, "keywordTermURL"); + assertEquals(Set.of("faux bourdon", "drone (insects)"), result); + + // keywordValue + result = datasetFieldServiceBean.getIndexableStringsByTermUri(termURI, cvocEntry, "keywordValue"); + assertEquals(Collections.emptySet(), result, "Only 'keywordTermURL' must return values for Skosmos"); + + // Any others field + result = datasetFieldServiceBean.getIndexableStringsByTermUri(termURI, cvocEntry, ""); + assertEquals(Collections.emptySet(), result, "Only 'keywordTermURL' must return values for Skosmos"); + + // Another termURI not in database + result = datasetFieldServiceBean.getIndexableStringsByTermUri("http://example.org/uuid", cvocEntry, "keywordTermURL"); + assertEquals(Collections.emptySet(), result); + } + + @Test + void getIndexableStringsByTermUriAgroportal() throws IOException { + String fieldName = "keyword"; + String termURI = "http://aims.fao.org/aos/agrovoc/c_50265"; + + JsonObject cvocEntry = prepare(fieldName, "src/test/resources/json/cvoc-agroportal.json"); + + JsonObject getExtVocabValueReturnedValue = Json.createObjectBuilder() + .add("@id", termURI) + .add("termName", Json.createObjectBuilder() + .add("fr", "association de quartier") + .add("en", "neighborhood associations")) + .add("vocabularyName", "https://data.agroportal.lirmm.fr/ontologies/AGROVOC") + .add("vocabularyUri", "https://data.agroportal.lirmm.fr/ontologies/AGROVOC") + .add("synonyms", Json.createObjectBuilder() + .add("en", Json.createArrayBuilder().add("neighborhood societies"))) + .build(); + Mockito.doReturn(getExtVocabValueReturnedValue).when(datasetFieldServiceBean).getExternalVocabularyValue(termURI); + Mockito.doReturn(null).when(datasetFieldServiceBean).getExternalVocabularyValue(AdditionalMatchers.not(Mockito.eq(termURI))); + + // keywordValue + Set result = datasetFieldServiceBean.getIndexableStringsByTermUri(termURI, cvocEntry, "keywordValue"); + assertEquals(Set.of("association de quartier", "neighborhood associations", "neighborhood societies"), result); + + // keywordTermURL + result = datasetFieldServiceBean.getIndexableStringsByTermUri(termURI, cvocEntry, "keywordTermURL"); + assertEquals(Collections.emptySet(), result, "Only 'keywordValue' must return values for Agroportal"); + + // Any others field + result = datasetFieldServiceBean.getIndexableStringsByTermUri(termURI, cvocEntry, ""); + assertEquals(Collections.emptySet(), result, "Only 'keywordValue' must return values for Agroportal"); + + // Another termURI not in database + result = datasetFieldServiceBean.getIndexableStringsByTermUri("http://example.org/uuid", cvocEntry, "keywordValue"); + assertEquals(Collections.emptySet(), result); + } + + @Test + void getIndexableStringsByTermUriOrcid() throws IOException { + String fieldName = "creator"; + String termURI = "https://orcid.org/0000-0003-4217-153X"; + + JsonObject cvocEntry = prepare(fieldName, "src/test/resources/json/cvoc-orcid.json"); + + JsonObject getExtVocabValueReturnedValue = Json.createObjectBuilder() + .add("@id", termURI) + .add("scheme", "ORCID") + .add("@type", "https://schema.org/Person") + .add("personName", "Doe, John") + .build(); + Mockito.doReturn(getExtVocabValueReturnedValue).when(datasetFieldServiceBean).getExternalVocabularyValue(termURI); + Mockito.doReturn(null).when(datasetFieldServiceBean).getExternalVocabularyValue(AdditionalMatchers.not(Mockito.eq(termURI))); + + // ORCID match with "personName" field into "getIndexableStringsByTermUri" method + Set result = datasetFieldServiceBean.getIndexableStringsByTermUri(termURI, cvocEntry, ""); + assertEquals(Set.of("Doe, John"), result); + + // Another termURI not in database + result = datasetFieldServiceBean.getIndexableStringsByTermUri("http://example.org/uuid", cvocEntry, fieldName); + assertEquals(Collections.emptySet(), result); + } + + /** + * Prepare unit tests with mock methods. + * + * @param fieldName "field-name" into cvoc configuration file + * @param jsonPath path of the JSON configuration file: src/test/resources/json/... + * @return {@link JsonObject} representing the configuration file + * @throws IOException in case on read error on configuration file + */ + JsonObject prepare(String fieldName, String jsonPath) throws IOException { + Long dftId = Long.parseLong("1"); + // DatasetFieldType name corresponding to "field-name" into cvoc configuration file + DatasetFieldType dft = new DatasetFieldType(fieldName, DatasetFieldType.FieldType.NONE, true); + dft.setId(dftId); + + Mockito.doReturn(dft).when(datasetFieldServiceBean).findByNameOpt(fieldName); + Mockito.doReturn(null).when(datasetFieldServiceBean).findByNameOpt(AdditionalMatchers.not(Mockito.eq(fieldName))); + + SettingsServiceBean settingsService = Mockito.mock(SettingsServiceBean.class); + Mockito.when(settingsService.getValueForKey(SettingsServiceBean.Key.CVocConf)).thenReturn(getCvocJson(jsonPath)); + datasetFieldServiceBean.settingsService = settingsService; + + return datasetFieldServiceBean.getCVocConf(false).get(dftId); + } + +} diff --git a/src/test/resources/json/cvoc-agroportal.json b/src/test/resources/json/cvoc-agroportal.json new file mode 100644 index 00000000000..03c9e2f4d07 --- /dev/null +++ b/src/test/resources/json/cvoc-agroportal.json @@ -0,0 +1,76 @@ +[ + { + "field-name": "keyword", + "term-uri-field": "keywordTermURL", + "cvoc-url": "https://data.agroportal.lirmm.fr/", + "js-url": "https://domain.tld/assets/cvoc/ontoportal.js", + "headers": { + "Authorization": "apikey token=XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX" + }, + "protocol": "ontoportal", + "retrieval-uri": "https://data.agroportal.lirmm.fr/ontologies/{keywordVocabulary}/classes/{encodeUrl:keywordTermURL}?language=en,fr", + "term-parent-uri": "", + "allow-free-text": true, + "languages": "en, fr", + "vocabs": { + "AGROVOC": { + "vocabularyUri": "https://data.agroportal.lirmm.fr/ontologies/AGROVOC", + "uriSpace": "http" + }, + "ONTOBIOTOPE": { + "vocabularyUri": "https://data.agroportal.lirmm.fr/ontologies/ONTOBIOTOPE", + "uriSpace": "http" + }, + "CROPUSAGE": { + "vocabularyUri": "https://data.agroportal.lirmm.fr/ontologies/CROPUSAGE", + "uriSpace": "http" + } + }, + "managed-fields": { + "vocabularyName": "keywordVocabulary", + "termName": "keywordValue", + "vocabularyUri": "keywordVocabularyURI" + }, + "retrieval-filtering": { + "@context": { + "termName": "https://schema.org/name", + "vocabularyName": "https://dataverse.org/schema/vocabularyName", + "vocabularyUri": "https://dataverse.org/schema/vocabularyUri", + "lang": "@language", + "value": "@value" + }, + "@id": { + "pattern": "{0}", + "params": [ + "@id" + ] + }, + "termName": { + "pattern": "{0}", + "params": [ + "/prefLabel" + ], + "indexIn": "keywordValue" + }, + "vocabularyName": { + "pattern": "{0}", + "params": [ + "/links/ontology" + ] + }, + "vocabularyUri": { + "pattern": "{0}", + "params": [ + "/links/ontology" + ] + }, + "synonyms": { + "pattern": "{0}", + "params": [ + "/synonym" + ], + "indexIn": "keywordValue" + } + } + } +] diff --git a/src/test/resources/json/cvoc-orcid.json b/src/test/resources/json/cvoc-orcid.json new file mode 100644 index 00000000000..6b904aefc3f --- /dev/null +++ b/src/test/resources/json/cvoc-orcid.json @@ -0,0 +1,43 @@ +[ + { + "field-name": "creator", + "term-uri-field": "creator", + "js-url": "https://gdcc.github.io/dataverse-external-vocab-support/scripts/people.js", + "protocol": "orcid", + "retrieval-uri": "https://pub.orcid.org/v3.0/{0}/person", + "allow-free-text": true, + "prefix": "https://orcid.org/", + "managed-fields": {}, + "languages": "", + "vocabs": { + "orcid": { + "uriSpace": "https://orcid.org/" + } + }, + "retrieval-filtering": { + "@context": { + "personName": "https://schema.org/name", + "scheme": "http://www.w3.org/2004/02/skos/core#inScheme" + }, + "personName": { + "pattern": "{0}, {1}", + "params": [ + "/name/family-name/value", + "/name/given-names/value" + ] + }, + "@id": { + "pattern": "{0}", + "params": [ + "@id" + ] + }, + "scheme": { + "pattern": "ORCID" + }, + "@type": { + "pattern": "https://schema.org/Person" + } + } + } +] diff --git a/src/test/resources/json/cvoc-skosmos.json b/src/test/resources/json/cvoc-skosmos.json new file mode 100644 index 00000000000..6d32b29f054 --- /dev/null +++ b/src/test/resources/json/cvoc-skosmos.json @@ -0,0 +1,69 @@ +[ + { + "field-name": "keyword", + "term-uri-field": "keywordTermURL", + "cvoc-url": "https://demo.skosmos.org/", + "js-url": "https://github.com/gdcc/dataverse-external-vocab-support/blob/main/scripts/skosmos.js", + "protocol": "skosmos", + "retrieval-uri": "https://demo.skosmos.org/rest/v1/data?uri={0}", + "term-parent-uri": "", + "allow-free-text": true, + "languages": "en, fr", + "vocabs": { + "agrovoc": { + "vocabularyUri": "http://aims.fao.org/vest-registry/kos/agrovoc", + "uriSpace": "http://aims.fao.org/aos/agrovoc/" + } + }, + "managed-fields": { + "vocabularyName": "keywordVocabulary", + "termName": "keywordValue", + "vocabularyUri": "keywordVocabularyURI" + }, + "retrieval-filtering": { + "@context": { + "termName": "https://schema.org/name", + "vocabularyName": "https://dataverse.org/schema/vocabularyName", + "vocabularyUri": "https://dataverse.org/schema/vocabularyUri", + "lang": "@language", + "value": "@value" + }, + "@id": { + "pattern": "{0}", + "params": [ + "@id" + ] + }, + "termName": { + "pattern": "{0}", + "params": [ + "/graph/uri=@id/prefLabel" + ] + }, + "vocabularyName": { + "pattern": "{0}", + "params": [ + "/graph/type=skos:ConceptScheme/prefLabel" + ] + }, + "vocabularyUri": { + "pattern": "{0}", + "params": [ + "/graph/type=skos:ConceptScheme/uri" + ] + }, + "synonyms": { + "pattern": "{0}", + "params": [ + "/graph/uri=@id/altLabel" + ] + }, + "genericTerm": { + "pattern": "{0}", + "params": [ + "/graph/type=skos:Concept/prefLabel" + ] + } + } + } +] From 6b5ed03e5716d320d2a6f98178c41727b07ba6c2 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 26 Apr 2024 11:36:47 -0400 Subject: [PATCH 023/145] stub out Contributor Guide #10531 --- doc/sphinx-guides/source/contributor/index.md | 5 ++ doc/sphinx-guides/source/contributor/intro.md | 70 +++++++++++++++++++ doc/sphinx-guides/source/index.rst | 1 + 3 files changed, 76 insertions(+) create mode 100644 doc/sphinx-guides/source/contributor/index.md create mode 100644 doc/sphinx-guides/source/contributor/intro.md diff --git a/doc/sphinx-guides/source/contributor/index.md b/doc/sphinx-guides/source/contributor/index.md new file mode 100644 index 00000000000..8f49785780b --- /dev/null +++ b/doc/sphinx-guides/source/contributor/index.md @@ -0,0 +1,5 @@ +# Contributor Guide + +```{toctree} +intro.md +``` diff --git a/doc/sphinx-guides/source/contributor/intro.md b/doc/sphinx-guides/source/contributor/intro.md new file mode 100644 index 00000000000..08de9c642a2 --- /dev/null +++ b/doc/sphinx-guides/source/contributor/intro.md @@ -0,0 +1,70 @@ +# Introduction + +Please note: this is a copy/paste from but we intend to split the content on this intro page into sub-pages. + +Thank you for your interest in contributing to Dataverse! We are open to contributions from everyone. You don't need permission to participate. Just jump in. If you have questions, please reach out using one or more of the channels described below. + +We aren't just looking for developers. There are many ways to contribute to Dataverse. We welcome contributions of ideas, bug reports, usability research/feedback, documentation, code, and more! + +```{contents} Contents: +:local: +:depth: 3 +``` + +## Ideas/Feature Requests + +Your idea or feature request might already be captured in the Dataverse [issue tracker] on GitHub but if not, the best way to bring it to the community's attention is by posting on the [dataverse-community Google Group][] or bringing it up on a [Community Call][]. You're also welcome to make some noise in [chat.dataverse.org][] or cram your idea into 280 characters and mention [@dataverseorg][] on Twitter. To discuss your idea privately, please email it to + +There's a chance your idea is already on our roadmap, which is available at + +[chat.dataverse.org]: http://chat.dataverse.org +[issue tracker]: https://github.com/IQSS/dataverse/issues +[@dataverseorg]: https://twitter.com/dataverseorg + +## Usability Testing + +Please email us at if you are interested in participating in usability testing. + +## Bug Reports/Issues + +An issue is a bug (a feature is no longer behaving the way it should) or a feature (something new to Dataverse that helps users complete tasks). You can browse the Dataverse [issue tracker] on GitHub by open or closed issues or by milestones. + +Before submitting an issue, please search the existing issues by using the search bar at the top of the page. If there is an existing open issue that matches the issue you want to report, please add a comment to it. + +If there is no pre-existing issue or it has been closed, please click on the "New Issue" button, log in, and write in what the issue is (unless it is a security issue which should be reported privately to security@dataverse.org). + +If you do not receive a reply to your new issue or comment in a timely manner, please email with a link to the issue. + +### Writing an Issue + +For the subject of an issue, please start it by writing the feature or functionality it relates to, i.e. "Create Account:..." or "Dataset Page:...". In the body of the issue, please outline the issue you are reporting with as much detail as possible. In order for the Dataverse development team to best respond to the issue, we need as much information about the issue as you can provide. Include steps to reproduce bugs. Indicate which version you're using, which is shown at the bottom of the page. We love screenshots! + +### Issue Attachments + +You can attach certain files (images, screenshots, logs, etc.) by dragging and dropping, selecting them, or pasting from the clipboard. Files must be one of GitHub's [supported attachment formats] such as png, gif, jpg, txt, pdf, zip, etc. (Pro tip: A file ending in .log can be renamed to .txt so you can upload it.) If there's no easy way to attach your file, please include a URL that points to the file in question. + +[supported attachment formats]: https://help.github.com/articles/file-attachments-on-issues-and-pull-requests/ + +## Documentation + +The source for the documentation is in the GitHub repo under the "[doc][]" folder. If you find a typo or inaccuracy or something to clarify, please send us a pull request! For more on the tools used to write docs, please see the {doc}`/developers/documentation` section of the Developer Guide. + +[doc]: https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source + +## Code/Pull Requests + +We love code contributions. Developers are not limited to the main Dataverse code in this git repo. You can help with API client libraries in your favorite language that are mentioned in the {doc}`/api/index` or create a new library. You can help work on configuration management code that's mentioned in the {doc}`/installation/index` . The Installation Guide also covers a relatively new concept called "external tools" that allows developers to create their own tools that are available from within an installation of Dataverse. + +If you are interested in working on the main Dataverse code, great! Before you start coding, please reach out to us either on the [dataverse-community Google Group][], the [dataverse-dev Google Group][], [chat.dataverse.org][], or via to make sure the effort is well coordinated and we avoid merge conflicts. We maintain a list of [community contributors][] and [dev efforts][] the community is working on so please let us know if you'd like to be added or removed from either list. + +Please read {doc}`/developers/version-control` in the Developer Guide to understand how we use the "git flow" model of development and how we will encourage you to create a GitHub issue (if it doesn't exist already) to associate with your pull request. That page also includes tips on making a pull request. + +After making your pull request, your goal should be to help it advance through our kanban board at . If no one has moved your pull request to the code review column in a timely manner, please reach out. Note that once a pull request is created for an issue, we'll remove the issue from the board so that we only track one card (the pull request). + +Thanks for your contribution! + +[dataverse-community Google Group]: https://groups.google.com/group/dataverse-community +[Community Call]: https://dataverse.org/community-calls +[dataverse-dev Google Group]: https://groups.google.com/group/dataverse-dev +[community contributors]: https://docs.google.com/spreadsheets/d/1o9DD-MQ0WkrYaEFTD5rF_NtyL8aUISgURsAXSL7Budk/edit?usp=sharing +[dev efforts]: https://github.com/orgs/IQSS/projects/34/views/6 diff --git a/doc/sphinx-guides/source/index.rst b/doc/sphinx-guides/source/index.rst index 3184160b387..af394108eea 100755 --- a/doc/sphinx-guides/source/index.rst +++ b/doc/sphinx-guides/source/index.rst @@ -17,6 +17,7 @@ These documentation guides are for the |version| version of Dataverse. To find g admin/index api/index installation/index + contributor/index developers/index container/index style/index From adf50744e42e44ad5f4f259e43c490859b7e8e0e Mon Sep 17 00:00:00 2001 From: Ludovic DANIEL Date: Thu, 2 May 2024 11:36:28 +0200 Subject: [PATCH 024/145] Update documentations related to PR 'CVOC : Indexed field accuracy (Ontoportal integration) #10505' --- doc/release-notes/9276-doc-cvoc-index-in.md | 16 +++++++++++++--- .../source/admin/metadatacustomization.rst | 6 ++++-- .../iq/dataverse/DatasetFieldServiceBean.java | 10 +++++----- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/doc/release-notes/9276-doc-cvoc-index-in.md b/doc/release-notes/9276-doc-cvoc-index-in.md index 5c4dd4ca10f..78289201511 100644 --- a/doc/release-notes/9276-doc-cvoc-index-in.md +++ b/doc/release-notes/9276-doc-cvoc-index-in.md @@ -2,7 +2,17 @@ ### Updates on Support for External Vocabulary Services -#### Indexed field accuracy +Multiple extensions of the External Vocabulary mechanism have been added. These extensions allow interaction with services based on the Ontoportal software and are expected to be generally useful for other service types. -For more relevant indexing, you can now map external vocabulary values to a `managed-fields` of a [:CVocConf setting](https://guides.dataverse.org/en/6.3/installation/config.html#cvocconf) by adding the key `indexIn` in `retrieval-filtering`. -For more information, please check [GDCC/dataverse-external-vocab-support documentation](https://github.com/gdcc/dataverse-external-vocab-support/tree/main/docs). \ No newline at end of file +These changes include: + +#### Improved Indexing with Compound Fields + +When using an external vocabulary service with compound fields, you can now specify which field(s) will include additional indexed information, such as translations of an entry into other languages. This is done by adding the `indexIn` in `retrieval-filtering`. (#10505) +For more information, please check [GDCC/dataverse-external-vocab-support documentation](https://github.com/gdcc/dataverse-external-vocab-support/tree/main/docs). + +#### Broader Support for Indexing Service Responses + +Indexing of the results from `retrieval-filtering` responses can now handle additional formats including Json Arrays of Strings and values from arbitrary keys within a JSON Object. (#10505) + +**** This documentation must be merged with 9276-allow-flexible-params-in-retrievaluri-cvoc.md (#10404) \ No newline at end of file diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index 66911aa0ad1..e70cf0e0897 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -552,6 +552,8 @@ Great care must be taken when reloading a metadata block. Matching is done on fi The ability to reload metadata blocks means that SQL update scripts don't need to be written for these changes. See also the :doc:`/developers/sql-upgrade-scripts` section of the Developer Guide. +.. _using-external-vocabulary-services: + Using External Vocabulary Services ---------------------------------- @@ -577,9 +579,9 @@ In general, the external vocabulary support mechanism may be a better choice for The specifics of the user interface for entering/selecting a vocabulary term and how that term is then displayed are managed by third-party Javascripts. The initial Javascripts that have been created provide auto-completion, displaying a list of choices that match what the user has typed so far, but other interfaces, such as displaying a tree of options for a hierarchical vocabulary, are possible. Similarly, existing scripts do relatively simple things for displaying a term - showing the term's name in the appropriate language and providing a link to an external URL with more information, but more sophisticated displays are possible. -Scripts supporting use of vocabularies from services supporting the SKOMOS protocol (see https://skosmos.org) and retrieving ORCIDs (from https://orcid.org) are available https://github.com/gdcc/dataverse-external-vocab-support. (Custom scripts can also be used and community members are encouraged to share new scripts through the dataverse-external-vocab-support repository.) +Scripts supporting use of vocabularies from services supporting the SKOMOS protocol (see https://skosmos.org), retrieving ORCIDs (from https://orcid.org), services based on Ontoportal product (see https://ontoportal.org/), and using ROR (https://ror.org/) are available https://github.com/gdcc/dataverse-external-vocab-support. (Custom scripts can also be used and community members are encouraged to share new scripts through the dataverse-external-vocab-support repository.) -Configuration involves specifying which fields are to be mapped, whether free-text entries are allowed, which vocabulary(ies) should be used, what languages those vocabulary(ies) are available in, and several service protocol and service instance specific parameters. +Configuration involves specifying which fields are to be mapped, to which Solr field they should be indexed, whether free-text entries are allowed, which vocabulary(ies) should be used, what languages those vocabulary(ies) are available in, and several service protocol and service instance specific parameters, including the ability to send HTTP headers on calls to the service. These are all defined in the :ref:`:CVocConf <:CVocConf>` setting as a JSON array. Details about the required elements as well as example JSON arrays are available at https://github.com/gdcc/dataverse-external-vocab-support, along with an example metadata block that can be used for testing. The scripts required can be hosted locally or retrieved dynamically from https://gdcc.github.io/ (similar to how dataverse-previewers work). diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java index b1717431e41..43648fa3b6d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java @@ -330,7 +330,7 @@ public Map getCVocConf(boolean byTermUriField){ logger.warning("Ignoring External Vocabulary setting for non-existent child field: " + managedFields.getString(s)); } else { - logger.info("Found: " + dft.getName()); + logger.fine("Found: " + dft.getName()); } } } @@ -372,10 +372,10 @@ public void registerExternalVocabValues(DatasetField df) { /** * Retrieves indexable strings from a cached externalvocabularyvalue entry filtered through retrieval-filtering configuration. *

- * This method externalvocabularyvalue entries have been filtered and contains a single JsonObject. - * Is handled : Strings, Array of Objects with "lang" and ("value" or "content") keys, Object with Strings as value or Object with Array of Strings as value. - * The string, or the "value/content"s for each language are added to the set. - * This method can retrieve string values to be indexed in term-uri-field (parameter defined in CVOC configuration) or in "indexIn" field (optional parameter of retrieval-filtering defined in CVOC configuration). + * This method assumes externalvocabularyvalue entries have been filtered and that they contain a single JsonObject. + * Cases Handled : A String, an Array of Strings, an Array of Objects with "value" or "content" keys, an Object with one or more entries that have String values or Array values with a set of String values. + * The string(s), or the "value/content"s for each language are added to the set. + * Retrieved string values are indexed in the term-uri-field (parameter defined in CVOC configuration) by default, or in the field specified by an optional "indexIn" parameter in the retrieval-filtering defined in the CVOC configuration. *

* Any parsing error results in no entries (there can be unfiltered entries with * unknown structure - getting some strings from such an entry could give fairly From 8496c10868a83ff8074444e59941f1a43c47a701 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 10 May 2024 10:23:11 -0400 Subject: [PATCH 025/145] rewrite ideas/features --- doc/sphinx-guides/source/contributor/intro.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/sphinx-guides/source/contributor/intro.md b/doc/sphinx-guides/source/contributor/intro.md index 08de9c642a2..951f2ec49bb 100644 --- a/doc/sphinx-guides/source/contributor/intro.md +++ b/doc/sphinx-guides/source/contributor/intro.md @@ -13,13 +13,14 @@ We aren't just looking for developers. There are many ways to contribute to Data ## Ideas/Feature Requests -Your idea or feature request might already be captured in the Dataverse [issue tracker] on GitHub but if not, the best way to bring it to the community's attention is by posting on the [dataverse-community Google Group][] or bringing it up on a [Community Call][]. You're also welcome to make some noise in [chat.dataverse.org][] or cram your idea into 280 characters and mention [@dataverseorg][] on Twitter. To discuss your idea privately, please email it to +1. Please check if your idea or feature request is already captured in our [issue tracker][] or [roadmap][]. +1. Bring your idea to the community by posting on our [Google Group][] or [chat.dataverse.org][]. +1. To discuss privately, email us at support@dataverse.org. -There's a chance your idea is already on our roadmap, which is available at - -[chat.dataverse.org]: http://chat.dataverse.org [issue tracker]: https://github.com/IQSS/dataverse/issues -[@dataverseorg]: https://twitter.com/dataverseorg +[roadmap]: https://www.iq.harvard.edu/roadmap-dataverse-project +[chat.dataverse.org]: http://chat.dataverse.org +[Google Group]: https://groups.google.com/group/dataverse-community ## Usability Testing From 6382517eddd22421164009b292df760ea7f1fd20 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 10 May 2024 10:29:57 -0400 Subject: [PATCH 026/145] link to usability testing form, move to bottom --- doc/sphinx-guides/source/contributor/intro.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/contributor/intro.md b/doc/sphinx-guides/source/contributor/intro.md index 951f2ec49bb..4d0cc86a7ea 100644 --- a/doc/sphinx-guides/source/contributor/intro.md +++ b/doc/sphinx-guides/source/contributor/intro.md @@ -22,10 +22,6 @@ We aren't just looking for developers. There are many ways to contribute to Data [chat.dataverse.org]: http://chat.dataverse.org [Google Group]: https://groups.google.com/group/dataverse-community -## Usability Testing - -Please email us at if you are interested in participating in usability testing. - ## Bug Reports/Issues An issue is a bug (a feature is no longer behaving the way it should) or a feature (something new to Dataverse that helps users complete tasks). You can browse the Dataverse [issue tracker] on GitHub by open or closed issues or by milestones. @@ -64,6 +60,12 @@ After making your pull request, your goal should be to help it advance through o Thanks for your contribution! +## Usability Testing + +Please email us at or fill in our [feedback form][] if you are interested in participating in usability testing. + +[feedback form]: https://goo.gl/forms/p7uu3GfiWYSlJrsi1 + [dataverse-community Google Group]: https://groups.google.com/group/dataverse-community [Community Call]: https://dataverse.org/community-calls [dataverse-dev Google Group]: https://groups.google.com/group/dataverse-dev From 9747685a9c44aeb85917066c43e319ca661f473c Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 10 May 2024 10:49:00 -0400 Subject: [PATCH 027/145] love your ideas, add support link --- doc/sphinx-guides/source/contributor/intro.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/contributor/intro.md b/doc/sphinx-guides/source/contributor/intro.md index 4d0cc86a7ea..5ce0afedff6 100644 --- a/doc/sphinx-guides/source/contributor/intro.md +++ b/doc/sphinx-guides/source/contributor/intro.md @@ -13,9 +13,11 @@ We aren't just looking for developers. There are many ways to contribute to Data ## Ideas/Feature Requests +We would love to hear your ideas!💡 + 1. Please check if your idea or feature request is already captured in our [issue tracker][] or [roadmap][]. 1. Bring your idea to the community by posting on our [Google Group][] or [chat.dataverse.org][]. -1. To discuss privately, email us at support@dataverse.org. +1. To discuss privately, email us at . [issue tracker]: https://github.com/IQSS/dataverse/issues [roadmap]: https://www.iq.harvard.edu/roadmap-dataverse-project From 5b813568b0207896f81d699850b38b9cbf042058 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 10 May 2024 11:04:00 -0400 Subject: [PATCH 028/145] simplify bug report section --- doc/sphinx-guides/source/contributor/intro.md | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/doc/sphinx-guides/source/contributor/intro.md b/doc/sphinx-guides/source/contributor/intro.md index 5ce0afedff6..75fd93e9914 100644 --- a/doc/sphinx-guides/source/contributor/intro.md +++ b/doc/sphinx-guides/source/contributor/intro.md @@ -24,25 +24,13 @@ We would love to hear your ideas!💡 [chat.dataverse.org]: http://chat.dataverse.org [Google Group]: https://groups.google.com/group/dataverse-community -## Bug Reports/Issues +## Bug Reports -An issue is a bug (a feature is no longer behaving the way it should) or a feature (something new to Dataverse that helps users complete tasks). You can browse the Dataverse [issue tracker] on GitHub by open or closed issues or by milestones. +Before submitting an issue, please search for existing issues in our [issue tracker][]. If there is an existing open issue that matches the issue you want to report, please add a comment to it or give it a 👍. -Before submitting an issue, please search the existing issues by using the search bar at the top of the page. If there is an existing open issue that matches the issue you want to report, please add a comment to it. +If there is no pre-existing issue or it has been closed, please open a new issue (unless it is a security issue which should be reported privately to ). -If there is no pre-existing issue or it has been closed, please click on the "New Issue" button, log in, and write in what the issue is (unless it is a security issue which should be reported privately to security@dataverse.org). - -If you do not receive a reply to your new issue or comment in a timely manner, please email with a link to the issue. - -### Writing an Issue - -For the subject of an issue, please start it by writing the feature or functionality it relates to, i.e. "Create Account:..." or "Dataset Page:...". In the body of the issue, please outline the issue you are reporting with as much detail as possible. In order for the Dataverse development team to best respond to the issue, we need as much information about the issue as you can provide. Include steps to reproduce bugs. Indicate which version you're using, which is shown at the bottom of the page. We love screenshots! - -### Issue Attachments - -You can attach certain files (images, screenshots, logs, etc.) by dragging and dropping, selecting them, or pasting from the clipboard. Files must be one of GitHub's [supported attachment formats] such as png, gif, jpg, txt, pdf, zip, etc. (Pro tip: A file ending in .log can be renamed to .txt so you can upload it.) If there's no easy way to attach your file, please include a URL that points to the file in question. - -[supported attachment formats]: https://help.github.com/articles/file-attachments-on-issues-and-pull-requests/ +If you do not receive a reply to your new issue or comment in a timely manner, please ping us at [chat.dataverse.org][]. ## Documentation From 4eede606ebabddc71705a76915fff0cc774f790b Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 10 May 2024 11:27:53 -0400 Subject: [PATCH 029/145] link to "reporting security issues" section --- doc/sphinx-guides/source/contributor/intro.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/contributor/intro.md b/doc/sphinx-guides/source/contributor/intro.md index 75fd93e9914..5348f4b307f 100644 --- a/doc/sphinx-guides/source/contributor/intro.md +++ b/doc/sphinx-guides/source/contributor/intro.md @@ -28,7 +28,7 @@ We would love to hear your ideas!💡 Before submitting an issue, please search for existing issues in our [issue tracker][]. If there is an existing open issue that matches the issue you want to report, please add a comment to it or give it a 👍. -If there is no pre-existing issue or it has been closed, please open a new issue (unless it is a security issue which should be reported privately to ). +If there is no pre-existing issue or it has been closed, please open a new issue (unless it is a security issue which should be reported privately to as discussed under {ref}`reporting-security-issues` in the Installation Guide). If you do not receive a reply to your new issue or comment in a timely manner, please ping us at [chat.dataverse.org][]. From d441361fbca99190260eca70a9aa082f5d9d2a80 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 14 May 2024 14:15:17 -0400 Subject: [PATCH 030/145] add default-provider --- scripts/installer/as-setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/installer/as-setup.sh b/scripts/installer/as-setup.sh index 34deddf51a3..8007b86a392 100755 --- a/scripts/installer/as-setup.sh +++ b/scripts/installer/as-setup.sh @@ -111,6 +111,7 @@ function preliminary_setup() ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.pid.fake.label=Fake DOI Provider" ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.pid.fake.authority=10.5072" ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.pid.fake.shoulder=FK2/" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.pid.default-provider=fake" # jvm-options use colons as separators, escape as literal #DOI_DATACITERESTAPIURL_ESC=`echo $DOI_DATACITERESTAPIURL | sed -e 's/:/\\\:/'` #./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.pid.testDC.datacite.rest-api-url=$DOI_DATACITERESTAPIURL_ESC" From 1da716f817def0d1d26f1a8c090c2e04fc11a6aa Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 14 May 2024 14:32:15 -0400 Subject: [PATCH 031/145] add check for default pid --- .../harvard/iq/dataverse/settings/ConfigCheckService.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java b/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java index 29a9d8956a3..daeff7b6cb9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.settings; import edu.harvard.iq.dataverse.MailServiceBean; +import edu.harvard.iq.dataverse.pidproviders.PidProviderFactoryBean; import edu.harvard.iq.dataverse.pidproviders.PidUtil; import edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key; import edu.harvard.iq.dataverse.util.FileUtil; @@ -32,6 +33,8 @@ public class ConfigCheckService { MailSessionProducer mailSessionProducer; @Inject MailServiceBean mailService; + @Inject + PidProviderFactoryBean pidProviderFactoryBean; public static class ConfigurationError extends RuntimeException { public ConfigurationError(String message) { @@ -132,6 +135,6 @@ public void checkSystemMailSetup() { * @return True if all checks successful, false otherwise. */ private boolean checkPidProviders() { - return PidUtil.getManagedProviderIds().size() > 0; + return (PidUtil.getManagedProviderIds().size() > 0) && (pidProviderFactoryBean.getDefaultPidGenerator()!=null); } } From 6cf286d5606965a694d375e5cdb35509b230b5ed Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 14 May 2024 14:37:40 -0400 Subject: [PATCH 032/145] add logging to distiguish failure cases --- .../iq/dataverse/settings/ConfigCheckService.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java b/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java index daeff7b6cb9..96222f40daf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java @@ -135,6 +135,16 @@ public void checkSystemMailSetup() { * @return True if all checks successful, false otherwise. */ private boolean checkPidProviders() { - return (PidUtil.getManagedProviderIds().size() > 0) && (pidProviderFactoryBean.getDefaultPidGenerator()!=null); + // Check if at least one PidProvider capable of editing/minting PIDs is configured. + boolean valid=true; + if(!(PidUtil.getManagedProviderIds().size() > 0)) { + valid = false; + logger.warning("No PID providers configured"); + } + if (pidProviderFactoryBean.getDefaultPidGenerator()==null){ + valid=false; + logger.warning("No default PID provider configured"); + } + return valid; } } From d0674f08979632d8d512f517a4d4f7e669179f4a Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 20 May 2024 10:42:07 -0400 Subject: [PATCH 033/145] This commit adds mechanisms for giving the admin some extra options in limiting the search-related bells and whistles luxuries - such as the facets and the object counts for the types left unchecked in the object types facets - more granularly. The default behavior is left intact in all cases. #10570 --- .../edu/harvard/iq/dataverse/DatasetPage.java | 21 ++++++++ .../search/SearchIncludeFragment.java | 52 ++++++++++++++++--- .../settings/SettingsServiceBean.java | 3 ++ src/main/webapp/search-include-fragment.xhtml | 9 ++-- 4 files changed, 76 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index d9cb10026a3..69ce66caa64 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -8,6 +8,7 @@ import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser; +import edu.harvard.iq.dataverse.authorization.users.GuestUser; import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.dataaccess.StorageIO; @@ -138,6 +139,7 @@ import jakarta.faces.event.AjaxBehaviorEvent; import jakarta.servlet.ServletOutputStream; import jakarta.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletRequest; import org.apache.commons.text.StringEscapeUtils; import org.apache.commons.lang3.mutable.MutableBoolean; @@ -787,6 +789,25 @@ public boolean isIndexedVersion() { return isIndexedVersion = false; } + // plus we have mechanisms for disabling the facets selectively, just for + // the guests, or anonymous users: + if (session.getUser() instanceof GuestUser) { + if (settingsWrapper.isTrueForKey(SettingsServiceBean.Key.DisableSolrFacetsForGuestUsers, false)) { + return isIndexedVersion = false; + } + + // An even lower grade of user than Guest is a truly anonymous user - + // a guest user who came without the session cookie: + Map cookies = FacesContext.getCurrentInstance().getExternalContext().getRequestCookieMap(); + if (!(cookies != null && cookies.containsKey("JSESSIONID"))) { + if (settingsWrapper.isTrueForKey(SettingsServiceBean.Key.DisableSolrFacetsForAnonymousUsers, false)) { + return isIndexedVersion = false; + } + } + + } + + // The version is SUPPOSED to be indexed if it's the latest published version, or a // draft. So if none of the above is true, we can return false right away. if (!(workingVersion.isDraft() || isThisLatestReleasedVersion())) { diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java index 9be6c34aa8b..1e2e4764869 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java @@ -21,6 +21,7 @@ import edu.harvard.iq.dataverse.SettingsWrapper; import edu.harvard.iq.dataverse.ThumbnailServiceWrapper; import edu.harvard.iq.dataverse.WidgetWrapper; +import edu.harvard.iq.dataverse.authorization.users.GuestUser; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; @@ -395,7 +396,7 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused } } - if (!wasSolrErrorEncountered() && selectedTypesList.size() < 3 && !isSolrTemporarilyUnavailable() && !isFacetsDisabled()) { + if (!wasSolrErrorEncountered() && selectedTypesList.size() < 3 && !isSolrTemporarilyUnavailable() && !isFacetsDisabled() && !isUncheckedTypesFacetDisabled()) { // If some types are NOT currently selected, we will need to // run a second search to obtain the numbers of the unselected types: @@ -1086,20 +1087,59 @@ public void setSolrTemporarilyUnavailable(boolean solrIsTemporarilyUnavailable) this.solrIsTemporarilyUnavailable = solrIsTemporarilyUnavailable; } + Boolean solrFacetsDisabled = null; /** * Indicates that the fragment should not be requesting facets in Solr * searches and rendering them on the page. * @return true if disabled; false by default */ public boolean isFacetsDisabled() { - // The method is used in rendered="..." logic. So we are using - // SettingsWrapper to make sure we are not looking it up repeatedly - // (settings are not expensive to look up, but - // still). + if (this.solrFacetsDisabled != null) { + return this.solrFacetsDisabled; + } + + if (settingsWrapper.isTrueForKey(SettingsServiceBean.Key.DisableSolrFacets, false)) { + return this.solrFacetsDisabled = true; + } + + // We also have mechanisms for disabling the facets selectively, just for + // the guests, or anonymous users: + if (session.getUser() instanceof GuestUser) { + if (settingsWrapper.isTrueForKey(SettingsServiceBean.Key.DisableSolrFacetsForGuestUsers, false)) { + return this.solrFacetsDisabled = true; + } + + // An even lower grade of user than Guest is a truly anonymous user - + // a guest user who came without the session cookie: + Map cookies = FacesContext.getCurrentInstance().getExternalContext().getRequestCookieMap(); + if (!(cookies != null && cookies.containsKey("JSESSIONID"))) { + if (settingsWrapper.isTrueForKey(SettingsServiceBean.Key.DisableSolrFacetsForAnonymousUsers, false)) { + return this.solrFacetsDisabled = true; + } + } + } - return settingsWrapper.isTrueForKey(SettingsServiceBean.Key.DisableSolrFacets, false); + return this.solrFacetsDisabled = false; } + Boolean disableSecondPassSearch = null; + + /** + * Indicates that we do not need to run the second search query to populate + * the counts for *unchecked* type facets. + * @return true if disabled; false by default + */ + public boolean isUncheckedTypesFacetDisabled() { + if (this.disableSecondPassSearch != null) { + return this.disableSecondPassSearch; + } + if (settingsWrapper.isTrueForKey(SettingsServiceBean.Key.DisableUncheckedTypesFacet, false)) { + return this.disableSecondPassSearch = true; + } + return this.disableSecondPassSearch = false; + } + + public boolean isRootDv() { return rootDv; } diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 35d70498c3f..c1d3c6bee74 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -663,6 +663,9 @@ Whether Harvesting (OAI) service is enabled * and dataset pages instantly */ DisableSolrFacets, + DisableSolrFacetsForGuestUsers, + DisableSolrFacetsForAnonymousUsers, + DisableUncheckedTypesFacet, /** * When ingesting tabular data files, store the generated tab-delimited * files *with* the variable names line up top. diff --git a/src/main/webapp/search-include-fragment.xhtml b/src/main/webapp/search-include-fragment.xhtml index 1b1b8594928..505fe681363 100644 --- a/src/main/webapp/search-include-fragment.xhtml +++ b/src/main/webapp/search-include-fragment.xhtml @@ -132,9 +132,10 @@ - + +

@@ -164,9 +165,10 @@ - + + @@ -196,9 +198,10 @@ - + + From d338caec0310d1466fc3dbcf83d672af8b9c6e6c Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 9 May 2024 15:57:11 -0400 Subject: [PATCH 034/145] only delete possible orphans --- .../iq/dataverse/search/IndexServiceBean.java | 61 +++++++++++++------ 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index e61b93a741f..6b7a74cdeef 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -486,9 +486,40 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr String solrIdDeaccessioned = determineDeaccessionedDatasetId(dataset); StringBuilder debug = new StringBuilder(); debug.append("\ndebug:\n"); + DatasetVersion latestVersion = dataset.getLatestVersion(); + String latestVersionStateString = latestVersion.getVersionState().name(); + DatasetVersion.VersionState latestVersionState = latestVersion.getVersionState(); + DatasetVersion releasedVersion = dataset.getReleasedVersion(); + boolean atLeastOnePublishedVersion = false; + if (releasedVersion != null) { + atLeastOnePublishedVersion = true; + } else { + atLeastOnePublishedVersion = false; + } + List solrIdsOfFilesToDelete = null; + + try { + solrIdsOfFilesToDelete = findFilesOfParentDataset(dataset.getId()); + List fileMetadatas = latestVersion.getFileMetadatas(); + + for (FileMetadata fileMetadata : fileMetadatas) { + String solrIdOfPublishedFile = solrDocIdentifierFile + fileMetadata.getDataFile().getId(); + solrIdsOfFilesToDelete.remove(solrIdOfPublishedFile); + } + if (releasedVersion != null && !releasedVersion.equals(latestVersion)) { + fileMetadatas = releasedVersion.getFileMetadatas(); + for (FileMetadata fileMetadata : fileMetadatas) { + String solrIdOfPublishedFile = solrDocIdentifierFile + fileMetadata.getDataFile().getId(); + solrIdsOfFilesToDelete.remove(solrIdOfPublishedFile); + } + } + } catch (SearchException | NullPointerException ex) { + logger.fine("could not run search of files to delete: " + ex); + } int numPublishedVersions = 0; List versions = dataset.getVersions(); - List solrIdsOfFilesToDelete = new ArrayList<>(); + //List solrIdsOfFilesToDelete = new ArrayList<>(); + //Debugging loop for (DatasetVersion datasetVersion : versions) { Long versionDatabaseId = datasetVersion.getId(); String versionTitle = datasetVersion.getTitle(); @@ -500,10 +531,10 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr debug.append("version found with database id " + versionDatabaseId + "\n"); debug.append("- title: " + versionTitle + "\n"); debug.append("- semanticVersion-VersionState: " + semanticVersion + "-" + versionState + "\n"); - List fileMetadatas = datasetVersion.getFileMetadatas(); List fileInfo = new ArrayList<>(); + List fileMetadatas = datasetVersion.getFileMetadatas(); + for (FileMetadata fileMetadata : fileMetadatas) { - String solrIdOfPublishedFile = solrDocIdentifierFile + fileMetadata.getDataFile().getId(); /** * It sounds weird but the first thing we'll do is preemptively * delete the Solr documents of all published files. Don't @@ -515,10 +546,9 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr * searchable. See also * https://github.com/IQSS/dataverse/issues/762 */ - solrIdsOfFilesToDelete.add(solrIdOfPublishedFile); fileInfo.add(fileMetadata.getDataFile().getId() + ":" + fileMetadata.getLabel()); } - try { +// try { /** * Preemptively delete *all* Solr documents for files associated * with the dataset based on a Solr query. @@ -539,11 +569,11 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr * @todo We should also delete the corresponding Solr * "permission" documents for the files. */ - List allFilesForDataset = findFilesOfParentDataset(dataset.getId()); - solrIdsOfFilesToDelete.addAll(allFilesForDataset); - } catch (SearchException | NullPointerException ex) { - logger.fine("could not run search of files to delete: " + ex); - } + //List allFilesForDataset = findFilesOfParentDataset(dataset.getId()); + //solrIdsOfFilesToDelete.addAll(allFilesForDataset); +// } catch (SearchException | NullPointerException ex) { +// logger.fine("could not run search of files to delete: " + ex); +// } int numFiles = 0; if (fileMetadatas != null) { numFiles = fileMetadatas.size(); @@ -555,16 +585,7 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr IndexResponse resultOfAttemptToPremptivelyDeletePublishedFiles = solrIndexService.deleteMultipleSolrIds(solrIdsOfFilesToDelete); debug.append("result of attempt to premptively deleted published files before reindexing: " + resultOfAttemptToPremptivelyDeletePublishedFiles + "\n"); } - DatasetVersion latestVersion = dataset.getLatestVersion(); - String latestVersionStateString = latestVersion.getVersionState().name(); - DatasetVersion.VersionState latestVersionState = latestVersion.getVersionState(); - DatasetVersion releasedVersion = dataset.getReleasedVersion(); - boolean atLeastOnePublishedVersion = false; - if (releasedVersion != null) { - atLeastOnePublishedVersion = true; - } else { - atLeastOnePublishedVersion = false; - } + Map desiredCards = new LinkedHashMap<>(); /** * @todo refactor all of this below and have a single method that takes From 5684020068e15904365b20030d09080ace72dff2 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 9 May 2024 16:28:40 -0400 Subject: [PATCH 035/145] more redundant deletes --- .../iq/dataverse/search/IndexServiceBean.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 6b7a74cdeef..71bd7fd9bb1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -1935,9 +1935,9 @@ private String removeDeaccessioned(Dataset dataset) { StringBuilder result = new StringBuilder(); String deleteDeaccessionedResult = removeSolrDocFromIndex(determineDeaccessionedDatasetId(dataset)); result.append(deleteDeaccessionedResult); - List docIds = findSolrDocIdsForFilesToDelete(dataset, IndexableDataset.DatasetState.DEACCESSIONED); - String deleteFilesResult = removeMultipleSolrDocs(docIds); - result.append(deleteFilesResult); +// List docIds = findSolrDocIdsForFilesToDelete(dataset, IndexableDataset.DatasetState.DEACCESSIONED); +// String deleteFilesResult = removeMultipleSolrDocs(docIds); +// result.append(deleteFilesResult); return result.toString(); } @@ -1945,9 +1945,9 @@ private String removePublished(Dataset dataset) { StringBuilder result = new StringBuilder(); String deletePublishedResult = removeSolrDocFromIndex(determinePublishedDatasetSolrDocId(dataset)); result.append(deletePublishedResult); - List docIds = findSolrDocIdsForFilesToDelete(dataset, IndexableDataset.DatasetState.PUBLISHED); - String deleteFilesResult = removeMultipleSolrDocs(docIds); - result.append(deleteFilesResult); +// List docIds = findSolrDocIdsForFilesToDelete(dataset, IndexableDataset.DatasetState.PUBLISHED); +// String deleteFilesResult = removeMultipleSolrDocs(docIds); +// result.append(deleteFilesResult); return result.toString(); } From 67dc2b06e4274abb7a0c491e4fc59049d450e45b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 22 May 2024 15:05:20 -0400 Subject: [PATCH 036/145] add suffix to checks, shuffle logging --- .../iq/dataverse/search/IndexServiceBean.java | 119 +++++++++--------- 1 file changed, 62 insertions(+), 57 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 71bd7fd9bb1..18d6d77a686 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -37,6 +37,7 @@ import java.util.concurrent.Future; import java.util.concurrent.Semaphore; import java.util.function.Function; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; import jakarta.annotation.PostConstruct; @@ -500,15 +501,18 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr try { solrIdsOfFilesToDelete = findFilesOfParentDataset(dataset.getId()); - List fileMetadatas = latestVersion.getFileMetadatas(); - - for (FileMetadata fileMetadata : fileMetadatas) { - String solrIdOfPublishedFile = solrDocIdentifierFile + fileMetadata.getDataFile().getId(); + logger.fine("Existing file docs: " + String.join(", ", solrIdsOfFilesToDelete)); + if(latestVersion.isDeaccessioned() && !atLeastOnePublishedVersion) { + List latestFileMetadatas = latestVersion.getFileMetadatas(); + String suffix = (new IndexableDataset(latestVersion)).getDatasetState().getSuffix(); + for (FileMetadata fileMetadata : latestFileMetadatas) { + String solrIdOfPublishedFile = solrDocIdentifierFile + fileMetadata.getDataFile().getId() + suffix; solrIdsOfFilesToDelete.remove(solrIdOfPublishedFile); } + } if (releasedVersion != null && !releasedVersion.equals(latestVersion)) { - fileMetadatas = releasedVersion.getFileMetadatas(); - for (FileMetadata fileMetadata : fileMetadatas) { + List releasedFileMetadatas = releasedVersion.getFileMetadatas(); + for (FileMetadata fileMetadata : releasedFileMetadatas) { String solrIdOfPublishedFile = solrDocIdentifierFile + fileMetadata.getDataFile().getId(); solrIdsOfFilesToDelete.remove(solrIdOfPublishedFile); } @@ -516,74 +520,75 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr } catch (SearchException | NullPointerException ex) { logger.fine("could not run search of files to delete: " + ex); } + logger.fine("File docs to delete: " + String.join(", ", solrIdsOfFilesToDelete)); int numPublishedVersions = 0; List versions = dataset.getVersions(); //List solrIdsOfFilesToDelete = new ArrayList<>(); - //Debugging loop - for (DatasetVersion datasetVersion : versions) { - Long versionDatabaseId = datasetVersion.getId(); - String versionTitle = datasetVersion.getTitle(); - String semanticVersion = datasetVersion.getSemanticVersion(); - DatasetVersion.VersionState versionState = datasetVersion.getVersionState(); - if (versionState.equals(DatasetVersion.VersionState.RELEASED)) { - numPublishedVersions += 1; - } - debug.append("version found with database id " + versionDatabaseId + "\n"); - debug.append("- title: " + versionTitle + "\n"); - debug.append("- semanticVersion-VersionState: " + semanticVersion + "-" + versionState + "\n"); - List fileInfo = new ArrayList<>(); - List fileMetadatas = datasetVersion.getFileMetadatas(); + if (logger.isLoggable(Level.FINE)) { + for (DatasetVersion datasetVersion : versions) { + Long versionDatabaseId = datasetVersion.getId(); + String versionTitle = datasetVersion.getTitle(); + String semanticVersion = datasetVersion.getSemanticVersion(); + DatasetVersion.VersionState versionState = datasetVersion.getVersionState(); + if (versionState.equals(DatasetVersion.VersionState.RELEASED)) { + numPublishedVersions += 1; + } + debug.append("version found with database id " + versionDatabaseId + "\n"); + debug.append("- title: " + versionTitle + "\n"); + debug.append("- semanticVersion-VersionState: " + semanticVersion + "-" + versionState + "\n"); + List fileInfo = new ArrayList<>(); + List fileMetadatas = datasetVersion.getFileMetadatas(); - for (FileMetadata fileMetadata : fileMetadatas) { - /** - * It sounds weird but the first thing we'll do is preemptively - * delete the Solr documents of all published files. Don't - * worry, published files will be re-indexed later along with - * the dataset. We do this so users can delete files from - * published versions of datasets and then re-publish a new - * version without fear that their old published files (now - * deleted from the latest published version) will be - * searchable. See also - * https://github.com/IQSS/dataverse/issues/762 - */ - fileInfo.add(fileMetadata.getDataFile().getId() + ":" + fileMetadata.getLabel()); - } + for (FileMetadata fileMetadata : fileMetadatas) { + /** + * It sounds weird but the first thing we'll do is preemptively delete the Solr + * documents of all published files. Don't worry, published files will be + * re-indexed later along with the dataset. We do this so users can delete files + * from published versions of datasets and then re-publish a new version without + * fear that their old published files (now deleted from the latest published + * version) will be searchable. See also + * https://github.com/IQSS/dataverse/issues/762 + */ + fileInfo.add(fileMetadata.getDataFile().getId() + ":" + fileMetadata.getLabel()); + } // try { /** - * Preemptively delete *all* Solr documents for files associated - * with the dataset based on a Solr query. + * Preemptively delete *all* Solr documents for files associated with the + * dataset based on a Solr query. * - * We must query Solr for this information because the file has - * been deleted from the database ( perhaps when Solr was down, - * as reported in https://github.com/IQSS/dataverse/issues/2086 - * ) so the database doesn't even know about the file. It's an - * orphan. + * We must query Solr for this information because the file has been deleted + * from the database ( perhaps when Solr was down, as reported in + * https://github.com/IQSS/dataverse/issues/2086 ) so the database doesn't even + * know about the file. It's an orphan. * - * @todo This Solr query should make the iteration above based - * on the database unnecessary because it the Solr query should - * find all files for the dataset. We can probably remove the - * iteration above after an "index all" has been performed. - * Without an "index all" we won't be able to find files based - * on parentId because that field wasn't searchable in 4.0. + * @todo This Solr query should make the iteration above based on the database + * unnecessary because it the Solr query should find all files for the + * dataset. We can probably remove the iteration above after an "index + * all" has been performed. Without an "index all" we won't be able to + * find files based on parentId because that field wasn't searchable in + * 4.0. * - * @todo We should also delete the corresponding Solr - * "permission" documents for the files. + * @todo We should also delete the corresponding Solr "permission" documents for + * the files. */ - //List allFilesForDataset = findFilesOfParentDataset(dataset.getId()); - //solrIdsOfFilesToDelete.addAll(allFilesForDataset); + // List allFilesForDataset = findFilesOfParentDataset(dataset.getId()); + // solrIdsOfFilesToDelete.addAll(allFilesForDataset); // } catch (SearchException | NullPointerException ex) { // logger.fine("could not run search of files to delete: " + ex); // } - int numFiles = 0; - if (fileMetadatas != null) { - numFiles = fileMetadatas.size(); + int numFiles = 0; + if (fileMetadatas != null) { + numFiles = fileMetadatas.size(); + } + debug.append("- files: " + numFiles + " " + fileInfo.toString() + "\n"); } - debug.append("- files: " + numFiles + " " + fileInfo.toString() + "\n"); } debug.append("numPublishedVersions: " + numPublishedVersions + "\n"); if (doNormalSolrDocCleanUp) { - IndexResponse resultOfAttemptToPremptivelyDeletePublishedFiles = solrIndexService.deleteMultipleSolrIds(solrIdsOfFilesToDelete); - debug.append("result of attempt to premptively deleted published files before reindexing: " + resultOfAttemptToPremptivelyDeletePublishedFiles + "\n"); + if(!solrIdsOfFilesToDelete.isEmpty()) { + IndexResponse resultOfAttemptToPremptivelyDeletePublishedFiles = solrIndexService.deleteMultipleSolrIds(solrIdsOfFilesToDelete); + debug.append("result of attempt to premptively deleted published files before reindexing: " + resultOfAttemptToPremptivelyDeletePublishedFiles + "\n"); + } } Map desiredCards = new LinkedHashMap<>(); From 2ad902c65eccfa8d955996965a135b14cef4cc87 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 22 May 2024 16:26:43 -0400 Subject: [PATCH 037/145] fix delete logic --- .../iq/dataverse/search/IndexServiceBean.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 18d6d77a686..437287aa755 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -502,13 +502,15 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr try { solrIdsOfFilesToDelete = findFilesOfParentDataset(dataset.getId()); logger.fine("Existing file docs: " + String.join(", ", solrIdsOfFilesToDelete)); - if(latestVersion.isDeaccessioned() && !atLeastOnePublishedVersion) { - List latestFileMetadatas = latestVersion.getFileMetadatas(); - String suffix = (new IndexableDataset(latestVersion)).getDatasetState().getSuffix(); - for (FileMetadata fileMetadata : latestFileMetadatas) { - String solrIdOfPublishedFile = solrDocIdentifierFile + fileMetadata.getDataFile().getId() + suffix; - solrIdsOfFilesToDelete.remove(solrIdOfPublishedFile); - } + //We keep the latest version's docs unless it is deaccessioned and there is no published/released version + //So skip the loop removing those docs from the delete list in that case + if ((!latestVersion.isDeaccessioned() || atLeastOnePublishedVersion)) { + List latestFileMetadatas = latestVersion.getFileMetadatas(); + String suffix = (new IndexableDataset(latestVersion)).getDatasetState().getSuffix(); + for (FileMetadata fileMetadata : latestFileMetadatas) { + String solrIdOfPublishedFile = solrDocIdentifierFile + fileMetadata.getDataFile().getId() + suffix; + solrIdsOfFilesToDelete.remove(solrIdOfPublishedFile); + } } if (releasedVersion != null && !releasedVersion.equals(latestVersion)) { List releasedFileMetadatas = releasedVersion.getFileMetadatas(); From 34b03fed73933c6687069f680ad2eac23c36164b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 22 May 2024 16:27:47 -0400 Subject: [PATCH 038/145] drafts already deleted --- .../iq/dataverse/search/IndexServiceBean.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 437287aa755..10c9b9bbe07 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -673,11 +673,11 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr desiredCards.put(DatasetVersion.VersionState.DRAFT, false); if (doNormalSolrDocCleanUp) { - List solrDocIdsForDraftFilesToDelete = findSolrDocIdsForDraftFilesToDelete(dataset); + //List solrDocIdsForDraftFilesToDelete = findSolrDocIdsForDraftFilesToDelete(dataset); String deleteDraftDatasetVersionResult = removeSolrDocFromIndex(solrIdDraftDataset); - String deleteDraftFilesResults = deleteDraftFiles(solrDocIdsForDraftFilesToDelete); - results.append("Attempting to delete traces of drafts. Result: ") - .append(deleteDraftDatasetVersionResult).append(deleteDraftFilesResults).append("\n"); + //String deleteDraftFilesResults = deleteDraftFiles(solrDocIdsForDraftFilesToDelete); + //results.append("Attempting to delete traces of drafts. Result: ") + // .append(deleteDraftDatasetVersionResult).append(deleteDraftFilesResults).append("\n"); } /** @@ -721,11 +721,11 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr desiredCards.put(DatasetVersion.VersionState.DRAFT, false); if (doNormalSolrDocCleanUp) { - List solrDocIdsForDraftFilesToDelete = findSolrDocIdsForDraftFilesToDelete(dataset); + //List solrDocIdsForDraftFilesToDelete = findSolrDocIdsForDraftFilesToDelete(dataset); String deleteDraftDatasetVersionResult = removeSolrDocFromIndex(solrIdDraftDataset); - String deleteDraftFilesResults = deleteDraftFiles(solrDocIdsForDraftFilesToDelete); - results.append("The latest version is published. Attempting to delete drafts. Result: ") - .append(deleteDraftDatasetVersionResult).append(deleteDraftFilesResults).append("\n"); + //String deleteDraftFilesResults = deleteDraftFiles(solrDocIdsForDraftFilesToDelete); + //results.append("The latest version is published. Attempting to delete drafts. Result: ") + // .append(deleteDraftDatasetVersionResult).append(deleteDraftFilesResults).append("\n"); } desiredCards.put(DatasetVersion.VersionState.DEACCESSIONED, false); @@ -822,13 +822,13 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr } } - private String deleteDraftFiles(List solrDocIdsForDraftFilesToDelete) { +/* private String deleteDraftFiles(List solrDocIdsForDraftFilesToDelete) { String deleteDraftFilesResults = ""; IndexResponse indexResponse = solrIndexService.deleteMultipleSolrIds(solrDocIdsForDraftFilesToDelete); deleteDraftFilesResults = indexResponse.toString(); return deleteDraftFilesResults; } - +*/ private IndexResponse indexDatasetPermissions(Dataset dataset) { boolean disabledForDebugging = false; if (disabledForDebugging) { From 90bfcf4234b56d8aef5f18c7a19df86483c2cb8b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 22 May 2024 16:28:49 -0400 Subject: [PATCH 039/145] don't run though file doc creation if not using it --- .../harvard/iq/dataverse/search/IndexServiceBean.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 10c9b9bbe07..985712da158 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -1224,8 +1224,8 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set Date: Thu, 23 May 2024 16:27:27 -0400 Subject: [PATCH 040/145] release note/guide entries for the new settings in the pr. #10570 --- .../10570-extra-facet-settings.md | 4 ++++ .../source/installation/config.rst | 19 ++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 doc/release-notes/10570-extra-facet-settings.md diff --git a/doc/release-notes/10570-extra-facet-settings.md b/doc/release-notes/10570-extra-facet-settings.md new file mode 100644 index 00000000000..9d68defc9a3 --- /dev/null +++ b/doc/release-notes/10570-extra-facet-settings.md @@ -0,0 +1,4 @@ +Extra settings have been added giving an instance admin more choices in +selectively limiting the availability of search facets on the Collection and Dataset pages. +See the [Disable Solr Facets](https://guides.dataverse.org/en/6.3/installation/config.html#DisableSolrFacets) sections of the Config Guide for more info. + diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 907631e6236..1d76313a92c 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3851,7 +3851,7 @@ If ``:SolrFullTextIndexing`` is set to true, the content of files of any size wi :DisableSolrFacets ++++++++++++++++++ -Setting this to ``true`` will make the collection ("dataverse") page start showing search results without the usual search facets on the left side of the page. A message will be shown in that column informing the users that facets are temporarily unavailable. Generating the facets is more resource-intensive for Solr than the main search results themselves, so applying this measure will significantly reduce the load on the search engine when its performance becomes an issue. +Setting this to ``true`` will make the collection ("dataverse") page start showing search results without the usual search facets on the left side of the page. A message will be shown in that column informing the users that facets are temporarily unavailable. Generating the facets may in some cases be more resource-intensive for Solr than the main search results themselves, so applying this measure will significantly reduce the load on the search engine when its performance becomes an issue. This setting can be used in combination with the "circuit breaker" mechanism on the Solr side (see the "Installing Solr" section of the Installation Prerequisites guide). An admin can choose to enable it, or even create an automated system for enabling it in response to Solr beginning to drop incoming requests with the HTTP code 503. @@ -3860,6 +3860,23 @@ To enable the setting:: curl -X PUT -d true "http://localhost:8080/api/admin/settings/:DisableSolrFacets" +:DisableSolrFacetsForGuestUsers ++++++++++++++++++++++++++++++++ + +Similar to the above, but will disable the facets for Guest users only. + +:DisableSolrFacetsForAnonymousUsers ++++++++++++++++++++++++++++++++++++ + +Same idea as with the 2 settings above. For the purposes of this setting, a request is considered "anonymous", if it came in without the JSESSION cookie supplied. A UI user who is browsing the holdings without logging in will have a valid JSESSION cookie, tied to a guest session. The main purpose of this setting is to hide the facets from bots, scripted crawlers and such (most of which - though not all - do not use cookies). Not letting the bots anywhere near the facets can serve a dual purpose on a busy instance experiencing problems with such abuse - some CPU cycles and resources can be saved by not having to generate the facets. And, even more importantly, it can prevent bots from attempting to crawl the facet trees, which has a potential for multiplying the service load. + +.. _:DisableUncheckedTypesFacet + +:DisableUncheckedTypesFacet ++++++++++++++++++++++++++++ + +Another option for reducing the load on solr on a busy instance. Rather than disabling all the search facets, this setting affects only one - the facet on the upper left of the collection page, where users can select the type of objects to search - Collections ("Dataverses"), Datasets and/or Files. With this option set to true, the numbers of results will only be shown for the types actually selected (i.e. only for the search results currently shown to the user). This minor feature - being able to tell the user how many files (for example) they *would* find, *if* they chose to search for files, by clicking the Files facet - essentially doubles the expense of running the search. That may still be negligible on an instance with lighter holdings, but can make a significant difference for a large and heavily used archive. + .. _:SignUpUrl: :SignUpUrl From 47f126469d6513e8164b7993761f2981c979e170 Mon Sep 17 00:00:00 2001 From: landreev Date: Fri, 24 May 2024 16:02:15 -0400 Subject: [PATCH 041/145] Update doc/sphinx-guides/source/installation/config.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/installation/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 1d76313a92c..d58199f2b21 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3863,7 +3863,7 @@ To enable the setting:: :DisableSolrFacetsForGuestUsers +++++++++++++++++++++++++++++++ -Similar to the above, but will disable the facets for Guest users only. +Similar to the above, but will disable the facets for Guest (unauthenticated) users only. :DisableSolrFacetsForAnonymousUsers +++++++++++++++++++++++++++++++++++ From a7839b50f8875a4b3e5d700c9d12a627a65f0e2a Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 3 Jun 2024 10:48:10 -0400 Subject: [PATCH 042/145] add permission doc deletes, check/delete per-version perm docs via api --- .../iq/dataverse/DataFileServiceBean.java | 8 +++ .../iq/dataverse/search/IndexServiceBean.java | 49 ++++++++++++++++++- 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java index 41ea6ae39f0..7f38107af6b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse; +import edu.harvard.iq.dataverse.DatasetVersion.VersionState; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; @@ -759,6 +760,13 @@ public List findAll() { return em.createQuery("select object(o) from DataFile as o order by o.id", DataFile.class).getResultList(); } + public List findVersionStates(Long fileId) { + Query query = em.createQuery( + "select distinct dv.versionState from DatasetVersion dv where dv.id in (select fm.datasetVersion.id from FileMetadata fm where fm.dataFile.id=:fileId)"); + query.setParameter("fileId", fileId); + return query.getResultList(); + } + public DataFile save(DataFile dataFile) { if (dataFile.isMergeable()) { diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 985712da158..bab7b196f2a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.search; import edu.harvard.iq.dataverse.*; +import edu.harvard.iq.dataverse.DatasetVersion.VersionState; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean; import edu.harvard.iq.dataverse.batch.util.LoggingUtil; @@ -503,7 +504,7 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr solrIdsOfFilesToDelete = findFilesOfParentDataset(dataset.getId()); logger.fine("Existing file docs: " + String.join(", ", solrIdsOfFilesToDelete)); //We keep the latest version's docs unless it is deaccessioned and there is no published/released version - //So skip the loop removing those docs from the delete list in that case + //So skip the loop removing those docs from the delete list except in that case if ((!latestVersion.isDeaccessioned() || atLeastOnePublishedVersion)) { List latestFileMetadatas = latestVersion.getFileMetadatas(); String suffix = (new IndexableDataset(latestVersion)).getDatasetState().getSuffix(); @@ -588,6 +589,10 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr debug.append("numPublishedVersions: " + numPublishedVersions + "\n"); if (doNormalSolrDocCleanUp) { if(!solrIdsOfFilesToDelete.isEmpty()) { + for(String file: solrIdsOfFilesToDelete) { + //Also remove associated permission docs + solrIdsOfFilesToDelete.add(file+"_permission"); + } IndexResponse resultOfAttemptToPremptivelyDeletePublishedFiles = solrIndexService.deleteMultipleSolrIds(solrIdsOfFilesToDelete); debug.append("result of attempt to premptively deleted published files before reindexing: " + resultOfAttemptToPremptivelyDeletePublishedFiles + "\n"); } @@ -2088,8 +2093,48 @@ public List findPermissionsInSolrOnly() throws SearchException { SolrDocumentList list = rsp.getResults(); for (SolrDocument doc: list) { long id = Long.parseLong((String) doc.getFieldValue(SearchFields.DEFINITION_POINT_DVOBJECT_ID)); + String docId = (String)doc.getFieldValue(SearchFields.ID); if(!dvObjectService.checkExists(id)) { - permissionInSolrOnly.add((String)doc.getFieldValue(SearchFields.ID)); + permissionInSolrOnly.add(docId); + } else { + DvObject obj = dvObjectService.findDvObject(id); + if (obj instanceof Dataset d) { + DatasetVersion dv = d.getLatestVersion(); + if (docId.endsWith("draft_permission")) { + if (!dv.isDraft()) { + permissionInSolrOnly.add(docId); + } + } else if (docId.endsWith("deaccessioned_permission")) { + if (!dv.isDeaccessioned()) { + permissionInSolrOnly.add(docId); + } + } else { + if (d.getReleasedVersion() != null) { + permissionInSolrOnly.add(docId); + } + } + } else if (obj instanceof DataFile f) { + List states = dataFileService.findVersionStates(f.getId()); + Set strings = states.stream().map(VersionState::toString).collect(Collectors.toSet()); + logger.info("for " + docId + " states: " + String.join(", ", strings)); + if (docId.endsWith("draft_permission")) { + if (!states.contains(VersionState.DRAFT)) { + permissionInSolrOnly.add(docId); + } + } else if (docId.endsWith("deaccessioned_permission")) { + if (!states.contains(VersionState.DEACCESSIONED) && states.size() == 1) { + permissionInSolrOnly.add(docId); + } + } else { + if (!states.contains(VersionState.RELEASED)) { + permissionInSolrOnly.add(docId); + } else if (!dataFileService.findMostRecentVersionFileIsIn(f).getDatasetVersion() + .equals(f.getOwner().getReleasedVersion())) { + permissionInSolrOnly.add(docId); + } + + } + } } } if (cursorMark.equals(nextCursorMark)) { From 7d17b9a496f52ae7adb979471dcf6c782bb7c400 Mon Sep 17 00:00:00 2001 From: Victoria Lubitch Date: Mon, 3 Jun 2024 14:21:00 -0400 Subject: [PATCH 043/145] Dataverse with WSL --- .../source/developers/windows.rst | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst index 53578fe980c..91e3d783177 100755 --- a/doc/sphinx-guides/source/developers/windows.rst +++ b/doc/sphinx-guides/source/developers/windows.rst @@ -16,3 +16,91 @@ See the `post + +You will be asked to create a linux user. +After installation of Linux check that you have internet connection: + +.. code-block:: bash + + ping www.google.com + +If you do not have internet connection try add in ``/etc/wsl.conf`` + +.. code-block:: bash + + [network] + generateResolvConf = false + +Also in /etc/resolv.conf add + +.. code-block:: bash + + nameserver 1.1.1.1 + +Now you can install all the tools one usually uses in Linux. For example, it is good idea to run update: + +.. code-block:: bash + + sudo apt update + sudo apt full-upgrade -y + +Install Dataverse +~~~~~~~~~~~~~~~~~ + +Now you can install Dataverse in WSL following the instructions for :doc:`classic-dev-env` +At the end check that you have ``-Ddataverse.pid.default-provider=fake`` in jvm-options. + +Now you can access dataverse in your windows browser + + - http://localhost:8080 + - username: dataverseAdmin + - password: admin + +IDE for Dataverse in Windows +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Files in WSL are accessible from Windows for editing using ``\\wsl.localhost`` or ``\\wsl$`` path. Windows files are accessible in linux in ``/mnt/c/`` directory. Therefore one can use one's favorite editor or IDE to edit dataverse project files. Then one can build using ``mvn`` in WSL and deploy manually in WSL using ``asadmin``. + +It is still though possible to use full strength of IDE, the following instructions are for Intelij users. + +- Install Intelij in Windows. + +You can open the project through ``\\wsl.localhost`` and navigate to dataverse project. +You can try to build the project in Intelij. You may get a message ``Cannot establish network connection from WSL to Windows host (could be blocked by firewall).`` In that case you can try +to disable WSL Hyperviser from firewall. +After that you should be able to build the project in Intelij. +It seems that at present it is impossible to deploy the glassfish application in Intelij. You can try to add Glassfish plugin through Settings->Plugins and in Run->Edit Configurations configure Application Server from WSL ``/usr/localhost/payara6`` with URL http://localhost:8080 and Server Domain as domian one, but it may fail since Intelij confuses the Windows and Linux paths. + +To use the full strength of Intelij with build, deployment and debugging, one will need to use Intelij ``Remote development``. Close all the projects in InteliJ and go to ``Remote development->WSL`` and press ``New Project``. In WSL instance choose your linux distribution and press ``Next``. In ``Prpject Directory`` navigate to WSL dataverse project.Then press ``Download IDE and Connect``. This will install InteliJ in WSL in ``~/.cache/JetBrains/``. Now in InteliJ you should see your project opened in a new InteliJ window. After adding Glassfish plugin and editing configuration you should be able to build the project and run the project. + +PgAdmin in Windows for Dataverse +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can access dataverse database from Windows. Install pgAdmin https://www.pgadmin.org/download/pgadmin-4-windows/ In pgAdmin register server using 127.0.0.1 with port 5432, database dvndb and dvnapp as username with secret password. Now you will be able to access and update dataverse database. + From 8eeaa31829a2e980cc13387f50a691319d9fb37d Mon Sep 17 00:00:00 2001 From: Victoria Lubitch <43550154+lubitchv@users.noreply.github.com> Date: Mon, 3 Jun 2024 14:44:29 -0400 Subject: [PATCH 044/145] Update doc/sphinx-guides/source/developers/windows.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/developers/windows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst index 91e3d783177..d72ec0e50d2 100755 --- a/doc/sphinx-guides/source/developers/windows.rst +++ b/doc/sphinx-guides/source/developers/windows.rst @@ -38,7 +38,7 @@ See the list of possible distributions: wsl --list --online -Choose the distribution you would like. Then run the following command. Notice that this installation of dataverse was tried with ubuntu distribution. +Choose the distribution you would like. Then run the following command. These instructions were tested with Ubuntu. .. code-block:: powershell From 88d9854cd0328349c0acf0b96c767b04100d7ec7 Mon Sep 17 00:00:00 2001 From: Victoria Lubitch <43550154+lubitchv@users.noreply.github.com> Date: Mon, 3 Jun 2024 14:44:53 -0400 Subject: [PATCH 045/145] Update doc/sphinx-guides/source/developers/windows.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/developers/windows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst index d72ec0e50d2..6a2fc9155fc 100755 --- a/doc/sphinx-guides/source/developers/windows.rst +++ b/doc/sphinx-guides/source/developers/windows.rst @@ -45,7 +45,7 @@ Choose the distribution you would like. Then run the following command. These in wsl --install -d You will be asked to create a linux user. -After installation of Linux check that you have internet connection: +After the installation of Linux is complete, check that you have an internet connection: .. code-block:: bash From 9b5b29cf0eac6fc21f666d05c74f8406e1ddbdbf Mon Sep 17 00:00:00 2001 From: Victoria Lubitch <43550154+lubitchv@users.noreply.github.com> Date: Mon, 3 Jun 2024 14:45:14 -0400 Subject: [PATCH 046/145] Update doc/sphinx-guides/source/developers/windows.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/developers/windows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst index 6a2fc9155fc..8986cc2e875 100755 --- a/doc/sphinx-guides/source/developers/windows.rst +++ b/doc/sphinx-guides/source/developers/windows.rst @@ -51,7 +51,7 @@ After the installation of Linux is complete, check that you have an internet con ping www.google.com -If you do not have internet connection try add in ``/etc/wsl.conf`` +If you do not have an internet connection try adding it in ``/etc/wsl.conf`` .. code-block:: bash From 87edbcba66f8c293e2074cfbd758b338da0ad057 Mon Sep 17 00:00:00 2001 From: Victoria Lubitch <43550154+lubitchv@users.noreply.github.com> Date: Mon, 3 Jun 2024 14:45:28 -0400 Subject: [PATCH 047/145] Update doc/sphinx-guides/source/developers/windows.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/developers/windows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst index 8986cc2e875..f689b172cd6 100755 --- a/doc/sphinx-guides/source/developers/windows.rst +++ b/doc/sphinx-guides/source/developers/windows.rst @@ -58,7 +58,7 @@ If you do not have an internet connection try adding it in ``/etc/wsl.conf`` [network] generateResolvConf = false -Also in /etc/resolv.conf add +Also in ``/etc/resolv.conf`` add .. code-block:: bash From 74254affb19eba566699675fae4d677ce6417ef2 Mon Sep 17 00:00:00 2001 From: Victoria Lubitch <43550154+lubitchv@users.noreply.github.com> Date: Mon, 3 Jun 2024 14:45:45 -0400 Subject: [PATCH 048/145] Update doc/sphinx-guides/source/developers/windows.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/developers/windows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst index f689b172cd6..ccba42274b8 100755 --- a/doc/sphinx-guides/source/developers/windows.rst +++ b/doc/sphinx-guides/source/developers/windows.rst @@ -64,7 +64,7 @@ Also in ``/etc/resolv.conf`` add nameserver 1.1.1.1 -Now you can install all the tools one usually uses in Linux. For example, it is good idea to run update: +Now you can install all the tools one usually uses in Linux. For example, it is good idea to run an update: .. code-block:: bash From 93c452d6bec71a402e73b4670db68cc0f95918fc Mon Sep 17 00:00:00 2001 From: Victoria Lubitch <43550154+lubitchv@users.noreply.github.com> Date: Mon, 3 Jun 2024 14:46:18 -0400 Subject: [PATCH 049/145] Update doc/sphinx-guides/source/developers/windows.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/developers/windows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst index ccba42274b8..dda34a22719 100755 --- a/doc/sphinx-guides/source/developers/windows.rst +++ b/doc/sphinx-guides/source/developers/windows.rst @@ -102,5 +102,5 @@ To use the full strength of Intelij with build, deployment and debugging, one wi PgAdmin in Windows for Dataverse ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -You can access dataverse database from Windows. Install pgAdmin https://www.pgadmin.org/download/pgadmin-4-windows/ In pgAdmin register server using 127.0.0.1 with port 5432, database dvndb and dvnapp as username with secret password. Now you will be able to access and update dataverse database. +You can access The Dataverse database from Windows. Install pgAdmin https://www.pgadmin.org/download/pgadmin-4-windows/ In pgAdmin register a server using 127.0.0.1 with port 5432, database dvndb and dvnapp as username with secret password. Now you will be able to access and update Dataverse database. From fc57ddffd12c75798fd83f2a99da20128da0821b Mon Sep 17 00:00:00 2001 From: Victoria Lubitch <43550154+lubitchv@users.noreply.github.com> Date: Mon, 3 Jun 2024 14:46:46 -0400 Subject: [PATCH 050/145] Update doc/sphinx-guides/source/developers/windows.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/developers/windows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst index dda34a22719..81b518333f9 100755 --- a/doc/sphinx-guides/source/developers/windows.rst +++ b/doc/sphinx-guides/source/developers/windows.rst @@ -30,7 +30,7 @@ If you have Docker already installed, you should already have WSL installed, oth wsl --install -If you already had WSL installed you can install specific linux distribution: +If you already had WSL installed you can install a specific Linux distribution: See the list of possible distributions: From fc25adf11a78f405045fdf64aee37f044bde4973 Mon Sep 17 00:00:00 2001 From: Victoria Lubitch <43550154+lubitchv@users.noreply.github.com> Date: Mon, 3 Jun 2024 14:46:57 -0400 Subject: [PATCH 051/145] Update doc/sphinx-guides/source/developers/windows.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/developers/windows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst index 81b518333f9..4ff309bbc9f 100755 --- a/doc/sphinx-guides/source/developers/windows.rst +++ b/doc/sphinx-guides/source/developers/windows.rst @@ -44,7 +44,7 @@ Choose the distribution you would like. Then run the following command. These in wsl --install -d -You will be asked to create a linux user. +You will be asked to create a Linux user. After the installation of Linux is complete, check that you have an internet connection: .. code-block:: bash From 841ffd06ddc475f5168549171b97c1b1288673cd Mon Sep 17 00:00:00 2001 From: Victoria Lubitch <43550154+lubitchv@users.noreply.github.com> Date: Mon, 3 Jun 2024 14:47:13 -0400 Subject: [PATCH 052/145] Update doc/sphinx-guides/source/developers/windows.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/developers/windows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst index 4ff309bbc9f..fdc6641eab1 100755 --- a/doc/sphinx-guides/source/developers/windows.rst +++ b/doc/sphinx-guides/source/developers/windows.rst @@ -94,7 +94,7 @@ It is still though possible to use full strength of IDE, the following instructi You can open the project through ``\\wsl.localhost`` and navigate to dataverse project. You can try to build the project in Intelij. You may get a message ``Cannot establish network connection from WSL to Windows host (could be blocked by firewall).`` In that case you can try to disable WSL Hyperviser from firewall. -After that you should be able to build the project in Intelij. +After that you should be able to build the project in Intellij. It seems that at present it is impossible to deploy the glassfish application in Intelij. You can try to add Glassfish plugin through Settings->Plugins and in Run->Edit Configurations configure Application Server from WSL ``/usr/localhost/payara6`` with URL http://localhost:8080 and Server Domain as domian one, but it may fail since Intelij confuses the Windows and Linux paths. To use the full strength of Intelij with build, deployment and debugging, one will need to use Intelij ``Remote development``. Close all the projects in InteliJ and go to ``Remote development->WSL`` and press ``New Project``. In WSL instance choose your linux distribution and press ``Next``. In ``Prpject Directory`` navigate to WSL dataverse project.Then press ``Download IDE and Connect``. This will install InteliJ in WSL in ``~/.cache/JetBrains/``. Now in InteliJ you should see your project opened in a new InteliJ window. After adding Glassfish plugin and editing configuration you should be able to build the project and run the project. From 185c31cac6aee4869772736bda0ea9879be61a42 Mon Sep 17 00:00:00 2001 From: Victoria Lubitch <43550154+lubitchv@users.noreply.github.com> Date: Mon, 3 Jun 2024 14:47:26 -0400 Subject: [PATCH 053/145] Update doc/sphinx-guides/source/developers/windows.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/developers/windows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst index fdc6641eab1..5d74072f264 100755 --- a/doc/sphinx-guides/source/developers/windows.rst +++ b/doc/sphinx-guides/source/developers/windows.rst @@ -92,7 +92,7 @@ It is still though possible to use full strength of IDE, the following instructi - Install Intelij in Windows. You can open the project through ``\\wsl.localhost`` and navigate to dataverse project. -You can try to build the project in Intelij. You may get a message ``Cannot establish network connection from WSL to Windows host (could be blocked by firewall).`` In that case you can try +You can try to build the project in Intellij. You may get a message ``Cannot establish network connection from WSL to Windows host (could be blocked by firewall).`` In that case you can try to disable WSL Hyperviser from firewall. After that you should be able to build the project in Intellij. It seems that at present it is impossible to deploy the glassfish application in Intelij. You can try to add Glassfish plugin through Settings->Plugins and in Run->Edit Configurations configure Application Server from WSL ``/usr/localhost/payara6`` with URL http://localhost:8080 and Server Domain as domian one, but it may fail since Intelij confuses the Windows and Linux paths. From f9fb940228106221b08f4cf60139cdd829171a6f Mon Sep 17 00:00:00 2001 From: Victoria Lubitch <43550154+lubitchv@users.noreply.github.com> Date: Mon, 3 Jun 2024 14:48:08 -0400 Subject: [PATCH 054/145] Update doc/sphinx-guides/source/developers/windows.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/developers/windows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst index 5d74072f264..e877f4612e1 100755 --- a/doc/sphinx-guides/source/developers/windows.rst +++ b/doc/sphinx-guides/source/developers/windows.rst @@ -77,7 +77,7 @@ Install Dataverse Now you can install Dataverse in WSL following the instructions for :doc:`classic-dev-env` At the end check that you have ``-Ddataverse.pid.default-provider=fake`` in jvm-options. -Now you can access dataverse in your windows browser +Now you can access Dataverse in your Windows browser (Edge, Chrome, etc.): - http://localhost:8080 - username: dataverseAdmin From d336f45154425cd0e49526807b3d087e268233d5 Mon Sep 17 00:00:00 2001 From: Victoria Lubitch <43550154+lubitchv@users.noreply.github.com> Date: Mon, 3 Jun 2024 14:53:15 -0400 Subject: [PATCH 055/145] Update doc/sphinx-guides/source/developers/windows.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/developers/windows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst index e877f4612e1..16063272966 100755 --- a/doc/sphinx-guides/source/developers/windows.rst +++ b/doc/sphinx-guides/source/developers/windows.rst @@ -85,7 +85,7 @@ Now you can access Dataverse in your Windows browser (Edge, Chrome, etc.): IDE for Dataverse in Windows ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Files in WSL are accessible from Windows for editing using ``\\wsl.localhost`` or ``\\wsl$`` path. Windows files are accessible in linux in ``/mnt/c/`` directory. Therefore one can use one's favorite editor or IDE to edit dataverse project files. Then one can build using ``mvn`` in WSL and deploy manually in WSL using ``asadmin``. +Files in WSL are accessible from Windows for editing using ``\\wsl.localhost`` or ``\\wsl$`` path. Windows files are accessible under Linux in the ``/mnt/c/`` directory. Therefore one can use one's favorite editor or IDE to edit Dataverse project files. Then one can build using ``mvn`` in WSL and deploy manually in WSL using ``asadmin``. It is still though possible to use full strength of IDE, the following instructions are for Intelij users. From 4e4af12890c00b1b484d1d98d7428c16c32faa1c Mon Sep 17 00:00:00 2001 From: Victoria Lubitch <43550154+lubitchv@users.noreply.github.com> Date: Mon, 3 Jun 2024 14:53:32 -0400 Subject: [PATCH 056/145] Update doc/sphinx-guides/source/developers/windows.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/developers/windows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst index 16063272966..ec92adf7099 100755 --- a/doc/sphinx-guides/source/developers/windows.rst +++ b/doc/sphinx-guides/source/developers/windows.rst @@ -87,7 +87,7 @@ IDE for Dataverse in Windows ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Files in WSL are accessible from Windows for editing using ``\\wsl.localhost`` or ``\\wsl$`` path. Windows files are accessible under Linux in the ``/mnt/c/`` directory. Therefore one can use one's favorite editor or IDE to edit Dataverse project files. Then one can build using ``mvn`` in WSL and deploy manually in WSL using ``asadmin``. -It is still though possible to use full strength of IDE, the following instructions are for Intelij users. +It is still though possible to use a full-strength IDE. The following instructions are for Intellij users. - Install Intelij in Windows. From 6a34b97b7e67e163553f483ab12a55e1379d4b92 Mon Sep 17 00:00:00 2001 From: Victoria Lubitch <43550154+lubitchv@users.noreply.github.com> Date: Mon, 3 Jun 2024 14:53:48 -0400 Subject: [PATCH 057/145] Update doc/sphinx-guides/source/developers/windows.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/developers/windows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst index ec92adf7099..b16e86b5c2d 100755 --- a/doc/sphinx-guides/source/developers/windows.rst +++ b/doc/sphinx-guides/source/developers/windows.rst @@ -91,7 +91,7 @@ It is still though possible to use a full-strength IDE. The following instructio - Install Intelij in Windows. -You can open the project through ``\\wsl.localhost`` and navigate to dataverse project. +You can open the project through ``\\wsl.localhost`` and navigate to Dataverse project. You can try to build the project in Intellij. You may get a message ``Cannot establish network connection from WSL to Windows host (could be blocked by firewall).`` In that case you can try to disable WSL Hyperviser from firewall. After that you should be able to build the project in Intellij. From 3b9e691c476f9cc50ca75065c80f1a08526ff386 Mon Sep 17 00:00:00 2001 From: Victoria Lubitch <43550154+lubitchv@users.noreply.github.com> Date: Mon, 3 Jun 2024 14:54:05 -0400 Subject: [PATCH 058/145] Update doc/sphinx-guides/source/developers/windows.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/developers/windows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst index b16e86b5c2d..09060b801ad 100755 --- a/doc/sphinx-guides/source/developers/windows.rst +++ b/doc/sphinx-guides/source/developers/windows.rst @@ -95,7 +95,7 @@ You can open the project through ``\\wsl.localhost`` and navigate to Dataverse p You can try to build the project in Intellij. You may get a message ``Cannot establish network connection from WSL to Windows host (could be blocked by firewall).`` In that case you can try to disable WSL Hyperviser from firewall. After that you should be able to build the project in Intellij. -It seems that at present it is impossible to deploy the glassfish application in Intelij. You can try to add Glassfish plugin through Settings->Plugins and in Run->Edit Configurations configure Application Server from WSL ``/usr/localhost/payara6`` with URL http://localhost:8080 and Server Domain as domian one, but it may fail since Intelij confuses the Windows and Linux paths. +It seems that at present it is impossible to deploy the Glassfish application in Intellij. You can try to add Glassfish plugin through Settings->Plugins and in Run->Edit Configurations configure Application Server from WSL ``/usr/localhost/payara6`` with URL http://localhost:8080 and Server Domain as domain1, but it may fail since Intellij confuses the Windows and Linux paths. To use the full strength of Intelij with build, deployment and debugging, one will need to use Intelij ``Remote development``. Close all the projects in InteliJ and go to ``Remote development->WSL`` and press ``New Project``. In WSL instance choose your linux distribution and press ``Next``. In ``Prpject Directory`` navigate to WSL dataverse project.Then press ``Download IDE and Connect``. This will install InteliJ in WSL in ``~/.cache/JetBrains/``. Now in InteliJ you should see your project opened in a new InteliJ window. After adding Glassfish plugin and editing configuration you should be able to build the project and run the project. From 2e497a65aa609eff3934a914951b18da217dac9d Mon Sep 17 00:00:00 2001 From: Victoria Lubitch <43550154+lubitchv@users.noreply.github.com> Date: Mon, 3 Jun 2024 14:54:23 -0400 Subject: [PATCH 059/145] Update doc/sphinx-guides/source/developers/windows.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/developers/windows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst index 09060b801ad..54a30e95aef 100755 --- a/doc/sphinx-guides/source/developers/windows.rst +++ b/doc/sphinx-guides/source/developers/windows.rst @@ -97,7 +97,7 @@ to disable WSL Hyperviser from firewall. After that you should be able to build the project in Intellij. It seems that at present it is impossible to deploy the Glassfish application in Intellij. You can try to add Glassfish plugin through Settings->Plugins and in Run->Edit Configurations configure Application Server from WSL ``/usr/localhost/payara6`` with URL http://localhost:8080 and Server Domain as domain1, but it may fail since Intellij confuses the Windows and Linux paths. -To use the full strength of Intelij with build, deployment and debugging, one will need to use Intelij ``Remote development``. Close all the projects in InteliJ and go to ``Remote development->WSL`` and press ``New Project``. In WSL instance choose your linux distribution and press ``Next``. In ``Prpject Directory`` navigate to WSL dataverse project.Then press ``Download IDE and Connect``. This will install InteliJ in WSL in ``~/.cache/JetBrains/``. Now in InteliJ you should see your project opened in a new InteliJ window. After adding Glassfish plugin and editing configuration you should be able to build the project and run the project. +To use the full strength of Intelij with build, deployment and debugging, one will need to use Intelij ``Remote development``. Close all the projects in Intellij and go to ``Remote development->WSL`` and press ``New Project``. In WSL instance choose your Linux distribution and press ``Next``. In ``Prpject Directory`` navigate to WSL Dataverse project. Then press ``Download IDE and Connect``. This will install Intellij in WSL in ``~/.cache/JetBrains/``. Now in Intellij you should see your project opened in a new Intellij window. After adding Glassfish plugin and editing configuration you should be able to build the project and run the project. PgAdmin in Windows for Dataverse ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 40503758427f6c2bae790f7ead45844af20293e9 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 5 Jun 2024 13:46:40 -0400 Subject: [PATCH 060/145] adding fix to file name for download guestbook responses --- .../edu/harvard/iq/dataverse/GuestbookResponsesPage.java | 5 +++-- .../java/edu/harvard/iq/dataverse/ManageGuestbooksPage.java | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponsesPage.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponsesPage.java index c53df93def8..93ba8028fa8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponsesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponsesPage.java @@ -101,8 +101,9 @@ public String init() { private String getFileName(){ // The fix below replaces any spaces in the name of the dataverse with underscores; // without it, the filename was chopped off (by the browser??), and the user - // was getting the file name "Foo", instead of "Foo and Bar in Social Sciences.csv". -- L.A. - return dataverse.getName().replace(' ', '_') + "_" + guestbook.getId() + "_GuestbookReponses.csv"; + // was getting the file name "Foo", instead of "Foo and Bar in Social Sciences.csv". -- L.A. + // Also removing some chars that have been reported to cause issues with certain browsers + return dataverse.getName().replace(' ', '_').replaceAll("[\\\\/:*?\"<>|,;]", "") + "_" + guestbook.getId() + "_GuestbookResponses.csv"; } public void streamResponsesByDataverseAndGuestbook(){ diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageGuestbooksPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageGuestbooksPage.java index cc89cfd9d56..94c36a40794 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManageGuestbooksPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManageGuestbooksPage.java @@ -220,7 +220,8 @@ private String getFileName(){ // The fix below replaces any spaces in the name of the dataverse with underscores; // without it, the filename was chopped off (by the browser??), and the user // was getting the file name "Foo", instead of "Foo and Bar in Social Sciences.csv". -- L.A. - return dataverse.getName().replace(' ', '_') + "_GuestbookReponses.csv"; + // Also removing some chars that have been reported to cause issues with certain browsers + return dataverse.getName().replace(' ', '_').replaceAll("[\\\\/:*?\"<>|,;]", "") + "_GuestbookResponses.csv"; } public void deleteGuestbook() { From 88877a44f5ac0e52168c0b86841fae5bd9d09ca6 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 5 Jun 2024 16:25:03 -0400 Subject: [PATCH 061/145] move like method to FileUtil static method and added tests --- .../harvard/iq/dataverse/GuestbookResponsesPage.java | 3 ++- .../edu/harvard/iq/dataverse/ManageGuestbooksPage.java | 3 ++- .../java/edu/harvard/iq/dataverse/util/FileUtil.java | 10 +++++++++- .../edu/harvard/iq/dataverse/util/FileUtilTest.java | 7 +++++++ 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponsesPage.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponsesPage.java index 93ba8028fa8..4276eb02882 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponsesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponsesPage.java @@ -8,6 +8,7 @@ import edu.harvard.iq.dataverse.engine.command.impl.UpdateDataverseCommand; import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import java.util.List; import java.util.logging.Logger; @@ -103,7 +104,7 @@ private String getFileName(){ // without it, the filename was chopped off (by the browser??), and the user // was getting the file name "Foo", instead of "Foo and Bar in Social Sciences.csv". -- L.A. // Also removing some chars that have been reported to cause issues with certain browsers - return dataverse.getName().replace(' ', '_').replaceAll("[\\\\/:*?\"<>|,;]", "") + "_" + guestbook.getId() + "_GuestbookResponses.csv"; + return FileUtil.sanitizeFileName(dataverse.getName() + "_" + guestbook.getId() + "_GuestbookResponses.csv"); } public void streamResponsesByDataverseAndGuestbook(){ diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageGuestbooksPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageGuestbooksPage.java index 94c36a40794..d1cc515fd01 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManageGuestbooksPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManageGuestbooksPage.java @@ -5,6 +5,7 @@ import edu.harvard.iq.dataverse.engine.command.impl.UpdateDataverseCommand; import edu.harvard.iq.dataverse.engine.command.impl.UpdateDataverseGuestbookRootCommand; import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.JsfHelper; import static edu.harvard.iq.dataverse.util.JsfHelper.JH; import java.util.LinkedList; @@ -221,7 +222,7 @@ private String getFileName(){ // without it, the filename was chopped off (by the browser??), and the user // was getting the file name "Foo", instead of "Foo and Bar in Social Sciences.csv". -- L.A. // Also removing some chars that have been reported to cause issues with certain browsers - return dataverse.getName().replace(' ', '_').replaceAll("[\\\\/:*?\"<>|,;]", "") + "_GuestbookResponses.csv"; + return FileUtil.sanitizeFileName(dataverse.getName() + "_GuestbookResponses.csv"); } public void deleteGuestbook() { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 6c427672e6d..a0c32d5c8ce 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -1816,5 +1816,13 @@ public static String getStorageDriver(DataFile dataFile) { String storageIdentifier = dataFile.getStorageIdentifier(); return storageIdentifier.substring(0, storageIdentifier.indexOf(DataAccess.SEPARATOR)); } - + + /** + * Replace spaces with "_" and remove invalid chars + * @param fileNameIn - Name before sanitization NOTE: not full path since this method removes '/' and '\' + * @return filename without spaces or invalid chars + */ + public static String sanitizeFileName(String fileNameIn) { + return fileNameIn == null ? null : fileNameIn.replace(' ', '_').replaceAll("[\\\\/:*?\"<>|,;]", ""); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java index ce8698c95eb..46359d7b02c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java @@ -434,4 +434,11 @@ public void testDetermineFileTypeROCrate() { assertEquals("Code", FileUtil.getIndexableFacetFileType(dockerDataFile)); } + @Test + public void testSanitizeFileName() { + assertEquals(null, FileUtil.sanitizeFileName(null)); + assertEquals("with_space", FileUtil.sanitizeFileName("with space")); + assertEquals("withcomma", FileUtil.sanitizeFileName("with,comma")); + assertEquals("with.txt", FileUtil.sanitizeFileName("with,\\?:;,.txt")); + } } From 18c5b9b11936027e1149a6029f8b261a81947303 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 7 Jun 2024 10:20:14 -0400 Subject: [PATCH 062/145] Update intro.md --- doc/sphinx-guides/source/contributor/intro.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/contributor/intro.md b/doc/sphinx-guides/source/contributor/intro.md index 5348f4b307f..47111d0a663 100644 --- a/doc/sphinx-guides/source/contributor/intro.md +++ b/doc/sphinx-guides/source/contributor/intro.md @@ -2,9 +2,9 @@ Please note: this is a copy/paste from but we intend to split the content on this intro page into sub-pages. -Thank you for your interest in contributing to Dataverse! We are open to contributions from everyone. You don't need permission to participate. Just jump in. If you have questions, please reach out using one or more of the channels described below. +Thank you for your interest in contributing to Dataverse! We are open to contributions from everyone. We aren't just looking for developers. There are many ways to contribute to Dataverse. We welcome contributions of ideas, bug reports, usability research/feedback, documentation, code, and more! -We aren't just looking for developers. There are many ways to contribute to Dataverse. We welcome contributions of ideas, bug reports, usability research/feedback, documentation, code, and more! +If you have questions, please reach out using one or more of the channels described below. ```{contents} Contents: :local: From 32be795562b821449bdff0076ccba2860eb23e42 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 7 Jun 2024 11:56:59 -0400 Subject: [PATCH 063/145] remove intro page and put its content in index and code pages #10531 This is a bit of an experiment discussed with @vaidap and @DieuwertjeBloemen where we are trying to put more content on the index page rather than a table of contents. Because we have a subpage (code.md) we still need to put it in a toctree (or it will be an orphan), so I hid it the best I could within the flow of the text. --- doc/sphinx-guides/source/contributor/code.md | 26 ++++++++ doc/sphinx-guides/source/contributor/index.md | 55 +++++++++++++++- doc/sphinx-guides/source/contributor/intro.md | 63 ------------------- .../source/developers/documentation.rst | 9 +++ 4 files changed, 89 insertions(+), 64 deletions(-) create mode 100644 doc/sphinx-guides/source/contributor/code.md delete mode 100644 doc/sphinx-guides/source/contributor/intro.md diff --git a/doc/sphinx-guides/source/contributor/code.md b/doc/sphinx-guides/source/contributor/code.md new file mode 100644 index 00000000000..941ce1fd6bc --- /dev/null +++ b/doc/sphinx-guides/source/contributor/code.md @@ -0,0 +1,26 @@ +# Contributing Code + +```{contents} Contents: +:local: +:depth: 3 +``` + +## Welcome! + +We love code contributions. Developers are not limited to the main Dataverse code in this git repo. You can help with API {doc}`/api/client-libraries` in your favorite language or create a new library. You can help work on configuration management code that's mentioned in the {doc}`/installation/index`. See {doc}`/api/external-tools` for how you can create your own tool that is available from within an installation of Dataverse. + +If you are interested in working on the main Dataverse code, great! Before you start coding, please reach out to us either on the [dataverse-community Google Group][], the [dataverse-dev Google Group][], [chat.dataverse.org][], or via to make sure the effort is well coordinated and we avoid merge conflicts. We maintain a list of [community contributors][] and [dev efforts][] the community is working on so please let us know if you'd like to be added or removed from either list. + +## Making a Pull Request + +Please read {doc}`/developers/version-control` in the Developer Guide to understand how we use the "git flow" model of development and how we will encourage you to create a GitHub issue (if it doesn't exist already) to associate with your pull request. That page also includes tips on making a pull request. + +After making your pull request, your goal should be to help it advance through our kanban board at . If no one has moved your pull request to the code review column in a timely manner, please reach out. Note that once a pull request is created for an issue, we'll remove the issue from the board so that we only track one card (the pull request). + +Thanks for your contribution! + +[dataverse-community Google Group]: https://groups.google.com/group/dataverse-community +[dataverse-dev Google Group]: https://groups.google.com/group/dataverse-dev +[chat.dataverse.org]: http://chat.dataverse.org +[community contributors]: https://docs.google.com/spreadsheets/d/1o9DD-MQ0WkrYaEFTD5rF_NtyL8aUISgURsAXSL7Budk/edit?usp=sharing +[dev efforts]: https://github.com/orgs/IQSS/projects/34/views/6 diff --git a/doc/sphinx-guides/source/contributor/index.md b/doc/sphinx-guides/source/contributor/index.md index 8f49785780b..7a965b994e1 100644 --- a/doc/sphinx-guides/source/contributor/index.md +++ b/doc/sphinx-guides/source/contributor/index.md @@ -1,5 +1,58 @@ # Contributor Guide +Thank you for your interest in contributing to Dataverse! We are open to contributions from everyone. We welcome contributions of ideas, bug reports, usability research/feedback, documentation, code, and more! + +```{contents} Contents: +:local: +:depth: 3 +``` + +## Ideas and Feature Requests + +We would love to hear your ideas!💡 + +1. Please check if your idea or feature request is already captured in our [issue tracker][] or [roadmap][]. +1. Bring your idea to the community by posting on our [Google Group][] or [chat.dataverse.org][]. +1. To discuss privately, email us at . + +[issue tracker]: https://github.com/IQSS/dataverse/issues +[roadmap]: https://www.iq.harvard.edu/roadmap-dataverse-project +[chat.dataverse.org]: http://chat.dataverse.org +[Google Group]: https://groups.google.com/group/dataverse-community + +## Bug Reports + +Before submitting an issue, please search for existing issues in our [issue tracker][]. If there is an existing open issue that matches the issue you want to report, please add a comment to it or give it a 👍 (thumbs up). + +If there is no pre-existing issue or it has been closed, please open a new issue (unless it is a security issue which should be reported privately to as discussed under {ref}`reporting-security-issues` in the Installation Guide). + +If you do not receive a reply to your new issue or comment in a timely manner, please ping us at [chat.dataverse.org][]. + +## Documentation + +The source for the documentation is in the GitHub repo under the "[doc][]" folder. If you find a typo or inaccuracy or something to clarify, please send us a pull request! For more on the tools used to write docs, please see the {doc}`/developers/documentation` section of the Developer Guide. + +Please observe the following when writing documentation: + +- Use American English spelling. +- Use examples when possible. +- Break up longer paragraphs. +- Use "double quotes" instead of 'single quotes'. +- Favor "and" (data and code) over slashes (data/code). + +[doc]: https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source + +## Code and Pull Requests + +Dataverse is open source and we love code contributions. Developers are not limited to the main Dataverse code in this git repo. We have projects in C, C++, Go, Java, Javascript, Julia, PHP, Python, R, Ruby, TypeScript and more. To get started, please see the following pages: + ```{toctree} -intro.md +:maxdepth: 1 +code.md ``` + +## Usability Testing + +Please email us at or fill in our [feedback form][] if you are interested in participating in usability testing. + +[feedback form]: https://goo.gl/forms/p7uu3GfiWYSlJrsi1 diff --git a/doc/sphinx-guides/source/contributor/intro.md b/doc/sphinx-guides/source/contributor/intro.md deleted file mode 100644 index 47111d0a663..00000000000 --- a/doc/sphinx-guides/source/contributor/intro.md +++ /dev/null @@ -1,63 +0,0 @@ -# Introduction - -Please note: this is a copy/paste from but we intend to split the content on this intro page into sub-pages. - -Thank you for your interest in contributing to Dataverse! We are open to contributions from everyone. We aren't just looking for developers. There are many ways to contribute to Dataverse. We welcome contributions of ideas, bug reports, usability research/feedback, documentation, code, and more! - -If you have questions, please reach out using one or more of the channels described below. - -```{contents} Contents: -:local: -:depth: 3 -``` - -## Ideas/Feature Requests - -We would love to hear your ideas!💡 - -1. Please check if your idea or feature request is already captured in our [issue tracker][] or [roadmap][]. -1. Bring your idea to the community by posting on our [Google Group][] or [chat.dataverse.org][]. -1. To discuss privately, email us at . - -[issue tracker]: https://github.com/IQSS/dataverse/issues -[roadmap]: https://www.iq.harvard.edu/roadmap-dataverse-project -[chat.dataverse.org]: http://chat.dataverse.org -[Google Group]: https://groups.google.com/group/dataverse-community - -## Bug Reports - -Before submitting an issue, please search for existing issues in our [issue tracker][]. If there is an existing open issue that matches the issue you want to report, please add a comment to it or give it a 👍. - -If there is no pre-existing issue or it has been closed, please open a new issue (unless it is a security issue which should be reported privately to as discussed under {ref}`reporting-security-issues` in the Installation Guide). - -If you do not receive a reply to your new issue or comment in a timely manner, please ping us at [chat.dataverse.org][]. - -## Documentation - -The source for the documentation is in the GitHub repo under the "[doc][]" folder. If you find a typo or inaccuracy or something to clarify, please send us a pull request! For more on the tools used to write docs, please see the {doc}`/developers/documentation` section of the Developer Guide. - -[doc]: https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source - -## Code/Pull Requests - -We love code contributions. Developers are not limited to the main Dataverse code in this git repo. You can help with API client libraries in your favorite language that are mentioned in the {doc}`/api/index` or create a new library. You can help work on configuration management code that's mentioned in the {doc}`/installation/index` . The Installation Guide also covers a relatively new concept called "external tools" that allows developers to create their own tools that are available from within an installation of Dataverse. - -If you are interested in working on the main Dataverse code, great! Before you start coding, please reach out to us either on the [dataverse-community Google Group][], the [dataverse-dev Google Group][], [chat.dataverse.org][], or via to make sure the effort is well coordinated and we avoid merge conflicts. We maintain a list of [community contributors][] and [dev efforts][] the community is working on so please let us know if you'd like to be added or removed from either list. - -Please read {doc}`/developers/version-control` in the Developer Guide to understand how we use the "git flow" model of development and how we will encourage you to create a GitHub issue (if it doesn't exist already) to associate with your pull request. That page also includes tips on making a pull request. - -After making your pull request, your goal should be to help it advance through our kanban board at . If no one has moved your pull request to the code review column in a timely manner, please reach out. Note that once a pull request is created for an issue, we'll remove the issue from the board so that we only track one card (the pull request). - -Thanks for your contribution! - -## Usability Testing - -Please email us at or fill in our [feedback form][] if you are interested in participating in usability testing. - -[feedback form]: https://goo.gl/forms/p7uu3GfiWYSlJrsi1 - -[dataverse-community Google Group]: https://groups.google.com/group/dataverse-community -[Community Call]: https://dataverse.org/community-calls -[dataverse-dev Google Group]: https://groups.google.com/group/dataverse-dev -[community contributors]: https://docs.google.com/spreadsheets/d/1o9DD-MQ0WkrYaEFTD5rF_NtyL8aUISgURsAXSL7Budk/edit?usp=sharing -[dev efforts]: https://github.com/orgs/IQSS/projects/34/views/6 diff --git a/doc/sphinx-guides/source/developers/documentation.rst b/doc/sphinx-guides/source/developers/documentation.rst index a4b8c027445..477776d7ee2 100755 --- a/doc/sphinx-guides/source/developers/documentation.rst +++ b/doc/sphinx-guides/source/developers/documentation.rst @@ -114,6 +114,13 @@ Every non-index page should use the following code to display a table of content This code should be placed below any introductory text/images and directly above the first subheading, much like a Wikipedia page. +If you are using Markdown, you should use this form instead: :: + + ```{contents} Contents: + :local: + :depth: 3 + ``` + Images ------ @@ -125,6 +132,8 @@ While PNGs in the git repo can be linked directly via URL, Sphinx-generated imag provide higher visual quality. Especially in terms of quality of content, generated images can be extendend and improved by a textbased and reviewable commit, without needing raw data or source files and no diff around. +TODO: The above covers "how" but what about "when"? + Cross References ---------------- From f76c71066f3cbb92314f13f8275b041943abae36 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 7 Jun 2024 15:21:05 -0400 Subject: [PATCH 064/145] replace content of CONTRIBUTING.md with link to new guide #10531 --- CONTRIBUTING.md | 66 +++---------------------------------------------- 1 file changed, 3 insertions(+), 63 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1430ba951a6..4fa6e955b70 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,67 +1,7 @@ # Contributing to Dataverse -Thank you for your interest in contributing to Dataverse! We are open to contributions from everyone. You don't need permission to participate. Just jump in. If you have questions, please reach out using one or more of the channels described below. +Thank you for your interest in contributing to Dataverse! We are open to contributions from everyone. -We aren't just looking for developers. There are many ways to contribute to Dataverse. We welcome contributions of ideas, bug reports, usability research/feedback, documentation, code, and more! +Please see our [Contributor Guide][] for how you can help! -## Ideas/Feature Requests - -Your idea or feature request might already be captured in the Dataverse [issue tracker] on GitHub but if not, the best way to bring it to the community's attention is by posting on the [dataverse-community Google Group][] or bringing it up on a [Community Call][]. You're also welcome to make some noise in [chat.dataverse.org][] or cram your idea into 280 characters and mention [@dataverseorg][] on Twitter. To discuss your idea privately, please email it to support@dataverse.org - -There's a chance your idea is already on our roadmap, which is available at https://www.iq.harvard.edu/roadmap-dataverse-project - -[chat.dataverse.org]: http://chat.dataverse.org -[issue tracker]: https://github.com/IQSS/dataverse/issues -[@dataverseorg]: https://twitter.com/dataverseorg - -## Usability testing - -Please email us at support@dataverse.org if you are interested in participating in usability testing. - -## Bug Reports/Issues - -An issue is a bug (a feature is no longer behaving the way it should) or a feature (something new to Dataverse that helps users complete tasks). You can browse the Dataverse [issue tracker] on GitHub by open or closed issues or by milestones. - -Before submitting an issue, please search the existing issues by using the search bar at the top of the page. If there is an existing open issue that matches the issue you want to report, please add a comment to it. - -If there is no pre-existing issue or it has been closed, please click on the "New Issue" button, log in, and write in what the issue is (unless it is a security issue which should be reported privately to security@dataverse.org). - -If you do not receive a reply to your new issue or comment in a timely manner, please email support@dataverse.org with a link to the issue. - -### Writing an Issue - -For the subject of an issue, please start it by writing the feature or functionality it relates to, i.e. "Create Account:..." or "Dataset Page:...". In the body of the issue, please outline the issue you are reporting with as much detail as possible. In order for the Dataverse development team to best respond to the issue, we need as much information about the issue as you can provide. Include steps to reproduce bugs. Indicate which version you're using, which is shown at the bottom of the page. We love screenshots! - -### Issue Attachments - -You can attach certain files (images, screenshots, logs, etc.) by dragging and dropping, selecting them, or pasting from the clipboard. Files must be one of GitHub's [supported attachment formats] such as png, gif, jpg, txt, pdf, zip, etc. (Pro tip: A file ending in .log can be renamed to .txt so you can upload it.) If there's no easy way to attach your file, please include a URL that points to the file in question. - -[supported attachment formats]: https://help.github.com/articles/file-attachments-on-issues-and-pull-requests/ - -## Documentation - -The source for the documentation at http://guides.dataverse.org/en/latest/ is in the GitHub repo under the "[doc][]" folder. If you find a typo or inaccuracy or something to clarify, please send us a pull request! For more on the tools used to write docs, please see the [documentation][] section of the Developer Guide. - -[doc]: https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source -[documentation]: http://guides.dataverse.org/en/latest/developers/documentation.html - -## Code/Pull Requests - -We love code contributions. Developers are not limited to the main Dataverse code in this git repo. You can help with API client libraries in your favorite language that are mentioned in the [API Guide][] or create a new library. You can help work on configuration management code that's mentioned in the [Installation Guide][]. The Installation Guide also covers a relatively new concept called "external tools" that allows developers to create their own tools that are available from within an installation of Dataverse. - -[API Guide]: http://guides.dataverse.org/en/latest/api -[Installation Guide]: http://guides.dataverse.org/en/latest/installation - -If you are interested in working on the main Dataverse code, great! Before you start coding, please reach out to us either on the [dataverse-community Google Group][], the [dataverse-dev Google Group][], [chat.dataverse.org][], or via support@dataverse.org to make sure the effort is well coordinated and we avoid merge conflicts. We maintain a list of [community contributors][] and [dev efforts][] the community is working on so please let us know if you'd like to be added or removed from either list. - -Please read http://guides.dataverse.org/en/latest/developers/version-control.html to understand how we use the "git flow" model of development and how we will encourage you to create a GitHub issue (if it doesn't exist already) to associate with your pull request. That page also includes tips on making a pull request. - -After making your pull request, your goal should be to help it advance through our kanban board at https://github.com/orgs/IQSS/projects/34 . If no one has moved your pull request to the code review column in a timely manner, please reach out. Note that once a pull request is created for an issue, we'll remove the issue from the board so that we only track one card (the pull request). - -Thanks for your contribution! - -[dataverse-community Google Group]: https://groups.google.com/group/dataverse-community -[Community Call]: https://dataverse.org/community-calls -[dataverse-dev Google Group]: https://groups.google.com/group/dataverse-dev -[community contributors]: https://docs.google.com/spreadsheets/d/1o9DD-MQ0WkrYaEFTD5rF_NtyL8aUISgURsAXSL7Budk/edit?usp=sharing -[dev efforts]: https://github.com/orgs/IQSS/projects/34/views/6 +[Contributor Guide]: https://guides.dataverse.org/en/latest/contributor/index.html From 36b4ed99636f55f794c332f6fff8d829ea691a9a Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 10 Jun 2024 23:02:47 -0400 Subject: [PATCH 065/145] expand "contributing code", other tweaks #10531 --- doc/sphinx-guides/source/contributor/code.md | 54 +++++++++++++++---- doc/sphinx-guides/source/developers/intro.rst | 26 +++++---- .../source/developers/version-control.rst | 2 + 3 files changed, 61 insertions(+), 21 deletions(-) diff --git a/doc/sphinx-guides/source/contributor/code.md b/doc/sphinx-guides/source/contributor/code.md index 941ce1fd6bc..9bc2b5a2027 100644 --- a/doc/sphinx-guides/source/contributor/code.md +++ b/doc/sphinx-guides/source/contributor/code.md @@ -1,26 +1,58 @@ # Contributing Code +We love code contributions! There are lots of ways you can help. + ```{contents} Contents: :local: :depth: 3 ``` -## Welcome! +## Finding an Issue to Work On + +New contributors often wonder what issues they should work on first. + +### Many Codebases, Many Languages + +The primary codebase and issue tracker for Dataverse is . It's mostly backend code written in Java. However, there are many other codebases you can work on in a variety of languages. Here are a few that are especially active: + +- (Java) +- (React) +- (TypeScript) +- (Javascript) +- (Python) +- (Ansible) +- (Javascript) + +If nothing above sparks joy, you can find more projects to work on under {doc}`/api/client-libraries`, {doc}`/api/external-tools`, {ref}`related-projects`, and {doc}`/api/apps`. + +### Picking a Good First Issue + +Once you've decided which codebase suits you, you should try to identify an issue to work on. Some codebases use a label like "good first issue" to suggest issues for newcomers. -We love code contributions. Developers are not limited to the main Dataverse code in this git repo. You can help with API {doc}`/api/client-libraries` in your favorite language or create a new library. You can help work on configuration management code that's mentioned in the {doc}`/installation/index`. See {doc}`/api/external-tools` for how you can create your own tool that is available from within an installation of Dataverse. +For the main codebase, please see {ref}`finding-github-issues-to-work-on` which includes information on labels like "good first issue". -If you are interested in working on the main Dataverse code, great! Before you start coding, please reach out to us either on the [dataverse-community Google Group][], the [dataverse-dev Google Group][], [chat.dataverse.org][], or via to make sure the effort is well coordinated and we avoid merge conflicts. We maintain a list of [community contributors][] and [dev efforts][] the community is working on so please let us know if you'd like to be added or removed from either list. +Other codebases may use different labels. Check the README or other documentation for that codebase. + +If there is a linked pull request that is trying to close the issue, you should probably find another issue. + +If you are having trouble finding an issue or have any questions at all, please do not hesitate to reach out. See {ref}`getting-help-developers`. ## Making a Pull Request -Please read {doc}`/developers/version-control` in the Developer Guide to understand how we use the "git flow" model of development and how we will encourage you to create a GitHub issue (if it doesn't exist already) to associate with your pull request. That page also includes tips on making a pull request. +For the main codebase, please see {ref}`how-to-make-a-pull-request`. + +For other codebases, consult the README. + +## Reviewing Code + +Reviewing code is a great way to learn about a codebase. For any codebase you can browse open pull requests, of course, but for the primary codebases, you can take a look at the "Ready for Review" and "In Review" columns at https://github.com/orgs/IQSS/projects/34 + +You are welcome to review code informally or to leave an actual review. We're interested in what you think. + +## Reproducing Bugs -After making your pull request, your goal should be to help it advance through our kanban board at . If no one has moved your pull request to the code review column in a timely manner, please reach out. Note that once a pull request is created for an issue, we'll remove the issue from the board so that we only track one card (the pull request). +At times, bugs are reported that we haven't had time to confirm. You can help out by reproducing bugs and commenting on the issue the results you find. -Thanks for your contribution! +## Getting Help -[dataverse-community Google Group]: https://groups.google.com/group/dataverse-community -[dataverse-dev Google Group]: https://groups.google.com/group/dataverse-dev -[chat.dataverse.org]: http://chat.dataverse.org -[community contributors]: https://docs.google.com/spreadsheets/d/1o9DD-MQ0WkrYaEFTD5rF_NtyL8aUISgURsAXSL7Budk/edit?usp=sharing -[dev efforts]: https://github.com/orgs/IQSS/projects/34/views/6 +If you have any questions at all, please do not hesitate to reach out. See {ref}`getting-help-developers`. diff --git a/doc/sphinx-guides/source/developers/intro.rst b/doc/sphinx-guides/source/developers/intro.rst index 350968012d8..0e74dc1c36f 100755 --- a/doc/sphinx-guides/source/developers/intro.rst +++ b/doc/sphinx-guides/source/developers/intro.rst @@ -2,7 +2,7 @@ Introduction ============ -Welcome! `The Dataverse Project `_ is an `open source `_ project that loves `contributors `_! +Welcome! `The Dataverse Project `_ is an `open source `_ project that loves contributors! .. contents:: |toctitle| :local: @@ -10,7 +10,7 @@ Welcome! `The Dataverse Project `_ is an `open source `_ mailing list, `community calls `_, or support@dataverse.org. +If you have any questions at all, please reach out to other developers via https://chat.dataverse.org, the `dataverse-dev `_ mailing list, the `dataverse-community `_ mailing list, or `community calls `_. .. _core-technologies: Core Technologies ----------------- -The Dataverse Software is a `Jakarta EE `_ application that is compiled into a WAR file and deployed to an application server (app server) which is configured to work with a relational database (PostgreSQL) and a search engine (Solr). +Dataverse is a `Jakarta EE `_ application that is compiled into a WAR file and deployed to an application server (app server) which is configured to work with a relational database (PostgreSQL) and a search engine (Solr). -We make use of a variety of Jakarta EE technologies such as JPA, JAX-RS, JMS, and JSF. The front end is built using PrimeFaces and Bootstrap. +We make use of a variety of Jakarta EE technologies such as JPA, JAX-RS, JMS, and JSF. In addition, we use parts of Eclipse MicroProfile such as `MicroProfile Config `_. -In addition, we start to adopt parts of Eclipse MicroProfile, namely `MicroProfile Config `_. +The frontend is built using PrimeFaces and Bootstrap. A new frontend is being built using React at https://github.com/IQSS/dataverse-frontend Roadmap ------- -For the Dataverse Software development roadmap, please see https://www.iq.harvard.edu/roadmap-dataverse-project +For the roadmap, please see https://www.iq.harvard.edu/roadmap-dataverse-project .. _kanban-board: @@ -47,20 +47,26 @@ You can get a sense of what's currently in flight (in dev, in QA, etc.) by looki Issue Tracker ------------- -We use GitHub Issues as our issue tracker: https://github.com/IQSS/dataverse/issues +The main issue tracker is https://github.com/IQSS/dataverse/issues but note that individual projects have their own issue trackers. Related Guides -------------- -If you are a developer who wants to make use of the Dataverse Software APIs, please see the :doc:`/api/index`. If you have front-end UI questions, please see the :doc:`/style/index`. +If you are wondering about how to contribute generally, please see the :doc:`/contributor/index`. -If you are a sysadmin who likes to code, you may be interested in hacking on installation scripts mentioned in the :doc:`/installation/index`. +If you are a developer who wants to make use of the Dataverse APIs, please see the :doc:`/api/index`. + +If you have frontend UI questions, please see the :doc:`/style/index`. For the new frontend, see https://github.com/IQSS/dataverse-frontend If you are a Docker enthusiasts, please check out the :doc:`/container/index`. +.. _related-projects: + Related Projects ---------------- +Note: this list is somewhat old. Please see also the :doc:`/contributor/code` section of the Contributor Guide. + As a developer, you also may be interested in these projects related to Dataverse: - External Tools - add additional features to the Dataverse Software without modifying the core: :doc:`/api/external-tools` diff --git a/doc/sphinx-guides/source/developers/version-control.rst b/doc/sphinx-guides/source/developers/version-control.rst index 07922b56b86..8648c8ce2a0 100644 --- a/doc/sphinx-guides/source/developers/version-control.rst +++ b/doc/sphinx-guides/source/developers/version-control.rst @@ -69,6 +69,8 @@ Find or Create a GitHub Issue An issue represents a bug (unexpected behavior) or a new feature in Dataverse. We'll use the issue number in the branch we create for our pull request. +.. _finding-github-issues-to-work-on: + Finding GitHub Issues to Work On ******************************** From a5c42c1825c5474cdd6f67fecf0a9b6b6cfa6ae5 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 11 Jun 2024 07:40:59 -0400 Subject: [PATCH 066/145] add translations to contrib guide #10531 --- doc/sphinx-guides/source/contributor/index.md | 4 ++++ doc/sphinx-guides/source/installation/config.rst | 2 ++ 2 files changed, 6 insertions(+) diff --git a/doc/sphinx-guides/source/contributor/index.md b/doc/sphinx-guides/source/contributor/index.md index 7a965b994e1..819a1d8a438 100644 --- a/doc/sphinx-guides/source/contributor/index.md +++ b/doc/sphinx-guides/source/contributor/index.md @@ -42,6 +42,10 @@ Please observe the following when writing documentation: [doc]: https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source +## Translations + +If you speak multiple languages, you are very welcome to help us translate Dataverse! Please see {ref}`help-translate` for details. + ## Code and Pull Requests Dataverse is open source and we love code contributions. Developers are not limited to the main Dataverse code in this git repo. We have projects in C, C++, Go, Java, Javascript, Julia, PHP, Python, R, Ruby, TypeScript and more. To get started, please see the following pages: diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 8fb9460892b..6d0da625a5a 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1741,6 +1741,8 @@ Now that you have a "languages.zip" file, you can load it into your Dataverse in Click on the languages using the drop down in the header to try them out. +.. _help-translate: + How to Help Translate the Dataverse Software Into Your Language +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ From be05cadfe307b164aee085e846662210b003fe08 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 11 Jun 2024 08:22:53 -0400 Subject: [PATCH 067/145] add support, talks, working groups #10531 --- doc/sphinx-guides/source/contributor/index.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/sphinx-guides/source/contributor/index.md b/doc/sphinx-guides/source/contributor/index.md index 819a1d8a438..78491578c5f 100644 --- a/doc/sphinx-guides/source/contributor/index.md +++ b/doc/sphinx-guides/source/contributor/index.md @@ -60,3 +60,15 @@ code.md Please email us at or fill in our [feedback form][] if you are interested in participating in usability testing. [feedback form]: https://goo.gl/forms/p7uu3GfiWYSlJrsi1 + +## Answering Questions + +People come along with questions on the [mailing list](https://groups.google.com/g/dataverse-community) and in [chat](https://chat.dataverse.org) all the time. You are very welcome to help out by answering these questions to the best of your ability. + +## Giving Talks + +If you give a recorded talk about Dataverse, we are happy to add it to [DataverseTV](https://dataverse.org/dataversetv) + +## Working Groups + +Most working groups are wide open to participation. For the current list of groups, please see . From 13c814dd706df7f3e25ca73350c581187f052e6e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 11 Jun 2024 12:37:44 -0400 Subject: [PATCH 068/145] add issue triage, other tweaks #10531 --- doc/sphinx-guides/source/contributor/index.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/contributor/index.md b/doc/sphinx-guides/source/contributor/index.md index 78491578c5f..5168b259d25 100644 --- a/doc/sphinx-guides/source/contributor/index.md +++ b/doc/sphinx-guides/source/contributor/index.md @@ -63,11 +63,21 @@ Please email us at or fill in our [feedback form][] if y ## Answering Questions -People come along with questions on the [mailing list](https://groups.google.com/g/dataverse-community) and in [chat](https://chat.dataverse.org) all the time. You are very welcome to help out by answering these questions to the best of your ability. +People come along with questions on the [mailing list](https://groups.google.com/g/dataverse-community) and in [chat][] all the time. You are very welcome to help out by answering these questions to the best of your ability. + +[chat]: https://chat.dataverse.org + +## Issue Triage + +New issues are coming in all the time, especially for the main issue tracker at . + +You can help by leaving comments that mention related issues or that answer questions. ## Giving Talks -If you give a recorded talk about Dataverse, we are happy to add it to [DataverseTV](https://dataverse.org/dataversetv) +If you give a recorded talk about Dataverse, we are happy to add it to [DataverseTV](https://dataverse.org/dataversetv). You can just leave a comment on the [spreadsheet](https://docs.google.com/spreadsheets/d/1uVk_57Ek_A49sLZ5OKdI6QASKloWNzykni3kcYNzpxA/edit#gid=0) or make some noise in [chat][]. + +For non-recorded talks, we are happy to upload your slides to . Please email . ## Working Groups From 7b9319eea4f26ad7d2b6986cfdc1976b7f433e9a Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 12 Jun 2024 11:49:38 -0400 Subject: [PATCH 069/145] ignore shapefiles if they are under a hidden directory in the zip file --- .../iq/dataverse/util/ShapefileHandler.java | 42 +++++++++++------- .../util/shapefile/ShapefileHandlerTest.java | 8 +++- src/test/resources/hiddenShapefiles.zip | Bin 0 -> 53764 bytes 3 files changed, 33 insertions(+), 17 deletions(-) create mode 100644 src/test/resources/hiddenShapefiles.zip diff --git a/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java b/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java index 9786fda4217..0c77e33712b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java @@ -15,6 +15,7 @@ import java.util.*; import java.nio.file.Files; +import java.nio.file.Paths; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; import java.util.logging.Level; import java.util.logging.Logger; @@ -695,33 +696,42 @@ private boolean examineZipfile(FileInputStream zip_file_stream){ this.filesListInDir.clear(); this.filesizeHash.clear(); this.fileGroups.clear(); - - try{ + + try{ ZipInputStream zipStream = new ZipInputStream(zip_file_stream); ZipEntry entry; - + List hiddenDirectories = new ArrayList<>(); while((entry = zipStream.getNextEntry())!=null){ + String zentryFileName = entry.getName(); + boolean isDirectory = entry.isDirectory(); + + Boolean skip = isDirectory || this.isFileToSkip(zentryFileName); + + // check if path is hidden + if (isDirectory && Files.isHidden(Paths.get(zentryFileName))) { + hiddenDirectories.add(zentryFileName); + logger.info("Ignoring files under hidden directory: " + zentryFileName); + } else { + // check if the path was already found to be hidden + for (String hidden : hiddenDirectories) { + if (zentryFileName.startsWith(hidden)) { + skip = true; + break; + } + } + } - String zentryFileName = entry.getName(); - //msg("zip entry: " + entry.getName()); - // Skip files or folders starting with __ - if (this.isFileToSkip(zentryFileName)){ - continue; - } - - if (entry.isDirectory()) { - //String dirpath = outputFolder + "/" + zentryFileName; - //createDirectory(dirpath); - continue; + if (skip) { + continue; } - + String unzipFileName = this.getFileBasename(zentryFileName); if (unzipFileName==null){ logger.warning("Zip Entry Basename is an empty string: " + zentryFileName); continue; } String unzipFolderName = this.getFolderName(zentryFileName); - + String unzipFilePath = unzipFileName; if (unzipFolderName != null) { unzipFilePath = unzipFolderName + "/" + unzipFileName; diff --git a/src/test/java/edu/harvard/iq/dataverse/util/shapefile/ShapefileHandlerTest.java b/src/test/java/edu/harvard/iq/dataverse/util/shapefile/ShapefileHandlerTest.java index f0e538616b2..3c5b4797b0a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/shapefile/ShapefileHandlerTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/shapefile/ShapefileHandlerTest.java @@ -282,8 +282,14 @@ public void testZippedShapefileWithExtraFiles() throws IOException{ msg("Passed!"); } - + @Test + public void testHiddenFiles() { + // test with shapefiles in hidden directory + ShapefileHandler shp_handler = new ShapefileHandler("src/test/resources/hiddenShapefiles.zip"); + shp_handler.DEBUG= true; + assertFalse(shp_handler.containsShapefile()); + } diff --git a/src/test/resources/hiddenShapefiles.zip b/src/test/resources/hiddenShapefiles.zip new file mode 100644 index 0000000000000000000000000000000000000000..64ed1721eb20f9c5bcaff7e205ec866a967b90d2 GIT binary patch literal 53764 zcmcG#cUTkQn=Y&(qNt!Kpi~tF6%py3sEDX2ReFi^PAEcvkVHj6sUiqcBO)TbcSuBf z2Z4khAV4SyAp{7d9)G)gojtqfoZY?7x8Gdz%r$?^d*AmwbI&{PyfcrHKF3k+L;qNe zpQ4}t)6TzajvW#?6yWFO7AR}>{KO#+PoXNudzBupK_Mp(9X?7ueCW{8^M8*t`k(FR z&H63>*Yv;A!+cNZrwKahv-%YVkgReQMN|Hl6nE92j35&b`6$={K`BXdVZM&Zss z!|wfO5?i0so_A*6{CMiG&VR(s{|xrFw?p7d2PY?&z(5!0e@ep>2G&vge^0}|N;%E_ zuPOQWDZTuk@p*eV{ui_T@1Yg{E6^dT%0OjB8Svlm@$hv6E6D#B*-EVb?X^|^e`Hg< zedqSSa0)bexC;G^|EmYK{vD^z|0hl#2fvpfkoUisIq5HP(SJl9WAkrM_{o0{d7!(4 zpUb~!_>A;$P5m4HhXnlN#yWoJh!D-_;@#Zz(?<>+ddqR>kja0mhCmPJz}tcDA^+s+ zY8g==;D2|ukEzyEQ-||f3DE*aY8-Pp4@+OtEIG0*^_sipN$VSZhv&In)b?edzE`)5j_8;K>u9b`;)IIa{h1YpT-_>xu*N{dKn>Mv#+h)qWxBA zLDiLx{c}ba9T>;_d=%!6#+UKA9R?Sz&tkuCv|H?!S)0uENc~w}JA;yXoOxw4F#&3z z<-&OHz79B&pR`ErRI~^(X;uo{UN}XO5jUczJYuiczO)DJsK2g~w9gsXXYzPG{F6Ne!j=MbKCcX+|0 zhS{l%NiXKs(Wul@1k>m~@l{n_ojomsP1ENzW>07Ipccsa4+cAWXKiCYH!Aj)u(XFI z04qyiQlHvT=ls<~yxr5=TI0!TgiXvTnwgtCdNyjwPd#;XdrO1|p(+*CLe^?+`ZaP5 z8Za^K7o45yJl81oO7GMQMs6!hs_P9_s3*i2v|qr@<0|^OZSZiLJE}z*S8|ct9(n2dQrLPU)qTetu=KH~TtSB6|M#sc067LH}#c*3I$3mIb%h?1I{dqz)R3@pzTq6lFKGb)65_8O8Qfhtd z3{%?o>2#ma$lmSLpm+~%AmX+DkP(T1>xJ&h|Y?LkuSeF{Zc~(i8AfnQ*Rt z{LCaznDCB@=7LdY5obL00*jkaO}tBwIAyVR9LXm#uAK?B8Q3R->=Y`uQr@%(PnXZd z>7{3?A$&Dl(xvNGLS*o_i;wYSOf?jAg=HN?sxZ=W^*~>=5c^!r z>B$Cfw|xyXKA84y4=?@*BAD4b6Y3A);le!6WA51X1A2EVu)Z{~GL$sLAm_@g%F-8Z zOUs8&(@?WXUL?qTNNgNnS(OHU1tgtixG7Gf;TM8czVPgE?M+hm^iG#$4Qw*1~CFo0*Ll~7g3)a+~;T9t(0EeD^4pH_v5YT$e{5xekH z@jnur8D*ytns-nZJ*q&SyD+Etd)gWC5y;u9-SRrJ#TG8%?PhI+c%2KJ1_>Ub3y2`5 zvsg73u;OmP<~t7TKctKTltMb-bk-?)&CvRItHETK-OCFB6Rti$HT6+UXuTk%+EEOA zG`s<*JR>FEn_sGfDF`r9D-baobJ1KDWp+M1C5|O?MXN3>Vda! z=J`Z?4w`rRJSivK7n;LO3$4t`_wSgW9w>u9{~Y9_5>B``3~|Kc@I|fD1Flpz3pY_O z0t_7d%+D*St>juaS+j}6G!}$Gf}NV<;?h8p0frc{v2v)j+s=xs#W?8$@HwggX2vX~ zx$I7_W6b70f7152<)O}}_0Dix*7plgzY;W_Rhh6+KIncM*O^fL02nD(3doIHE1Q*l zGUd|Alf8M|9z>NL0gMkFVe&K0nHb6f}}4m?{5d0`(v60Js{9@^x*$DFq8x-iWl+;?yWAbtwThs@rvQLU}wg|PSY zhV{Mes(wgB=_EE)1s$Uh7sN=JXPra9De^$eSX7X7|NT;l);WgP8dkjN=f>lJ3(V2_ z(amZH-S8p^)R;Ax(e1)+w*AeiJ6Rhbj;2?dnq<>o<{{H6m=+&1;w|!aG!$C^?8?Iz zedMsi6d80#A=ylB$YKhO%7oZe-+8DV^o%8Ju|nU}LQ77g;z4dJ+pegDW*#}z6`UZc z!>#cc3X7!sfKf8f6ZZndowRysLcEW%v|BD8 zvT)_Vjxps|$?}Y3plCzX=PgrmAD1wE zzo7g*#9?Mre!rnnnQ@xNg%jtQmB~H{S7xz)b?2l|6g?tp1g`&Vm1VRKd;{x9eKT9jwd@xb z{6=&&w+g+;i2qR*0OJ$f4iCdVA$evzqc%QwF6ptmtrm6d?^DwIH&Z*?nVR~Z-h#TQ zAy`s!8v3zEcM}>Wd#hgMN%&WCM5mw-+&%moIi^!Zi0vMJK3U`A0l2!myHnhORb;*# zOIYmf$A>%Njh4)E$5^wUdYOWr+H)N6WpDFzr-5bUxd!o+)Fy>k%`nB&5dQ3kbDEZx zJdAUp0d>d2Y$+kDF<{R~1eXhlqp~EqY!#;Sb_KCtd+5f1+U{AgnF2s1LM0fWJqWh8F!?ix|xut2*-_l}K&eBbd#FJ>TXizmWcUQcG z*Ms?JL*E+yhn9>kt1kT-#8skyeoUsi^l}hyNsRmKgN>Wk;E~2yv`=+u?g3&?r@|s` zzXfr%#N79r=phg$1p^Hlol>*dPjlfXDG?U7M-YOJU9-&jj;wBI*9NziJ0_VNYk`v9 z`|Sfg+YEavzMb3CoyV`Lj4gpMh^(sxO`EFh z6{aK39!QotSPLNplG=_pSr+@lcbkcZdj%6c@D!Ea>eHRd9f;jd;5+d|@%I^4TE9B? z5S9BI$NCayUu>j=zoCil>aW+08RR!d-3~B0Ez;_=|0puLPJjtgD;YGY>Uw+Wxs(B~W(+rxCGrvNdRHBikjIosQlTpVXqAK?`R0 zHyF5_8=r4pqa~dWw1x)-+r8U_vh@e0I5(v5L!ZWgS@sIU)7ee#8XNLcVhu@_izcg^ zqNoMz=3zR9^`vgviagUyY`7%<+8j#?-TGY_LFdEHJZoD(kgaJBw3Nd}+fxoqKaf3jEnL z4Pf2Pyu8oL(AD{Xdc$3e33XA~PGUZJ8k|ltfa<)Qg>6lBY}-Eev8BuoDGgQ!rJHb3 z(XYmbev1E5uS2)i7?cLREUEGmwcps`3<@uxY-qfC13t1He;0ev-A?0kmJL_ShaYbu zR4T9wrf79b3NwY{dLka{JTI72{I4+m$|R{9*;*|o333l zC$!IyTKl>cjyk~}qCBy!mh7oW7XMHl+jcO2+DnY7N9ui+74=ZnK1Kg;)kMOS|9Pr?4Slpgtn_3 zEFFLIPcSFtZBmguT3?yu^8#MbaWx+!-{c;e;`#zChF2FelWNoVbJW_&yTY73E`o6q z&jgQjxhwg<7tcYBs0H&f&N1VaGwk@8rp)I|nPJcf`wb{V3X%=f zy;s*i!xHd&F07peDuxAf2KWzs*WQeEL4HN)(Q^RuK%$d7>If^72l@rYM>~T($&=7J z7piB<4#slmNf)!ULN#{MBfg3s)~e?rlpT%IRzIKH z2Wtxdn4Q)-{AWscGxi$(Yo+O)|Mab>GBb4T+(sHQY;H$$J#?H>9K=_WyUK`rf;u@G zbX=x%&k_l$j2B{esXL36pw5rAe7|jOq>|KVGO>GN)s(0$NAlkbEF3?F{#~ST*tU~? zzsN^~izk|#vFS_W%313R>*Ot*Vy@`!2TD-5nh*~MPY-i}x=zMoj}&ja6P74aYj#&c z*a}nDGFQOVkBrCl(BL0FGqi_OEJ1=TZ?G&~1UeaXZhujBf~%o!wdEm6%i3*N{Vn=N zb$m_`Pf3J()`N}f)@~!}v1pI#c${*>aRU2AFx7FZZ2z=WHa8%y0bdwC+cJ0XZeVk9 zbNgmxU>&Q?@w3o75@R#F13Irn(ZiMY?d(#ynzUq6kOQlLT*C!=RV!}>{7VoAPL!gt zVRiLbG%BE>i{A#O_Idos=Ic4$Kr0eQFz?J@_yUIjwV4y%nl}GpJ}z`p5*O~e zP5ZsGD7m!3Qx_K9Nfl!2!@gdEUpn9%0DOlK=OUO1Ly?H_EIqer3{G+yt=2EKmafIQx&q8AQ4k_m8`PS9GuDLj-8~DC3c`c;3Oy0XL61Ys!>0;t%aJ zx~`g)w<}kI@PNt`zYcFZNRfq{C4j1SYb$6bT8k0;!G6`d!jyB9Ob_#Qm6 zS7&%yjBk2|7m)Ki^Mz2J;YH-K&MqNjGq+>ouV77yTj<4>68 zTg<+l7JpsIAKfhn=TQ9Y=A+2<=4Un=8=K*&L89IjThB+^ZoB};9~gefRUEwezD>~} zN!0+S8ONbyFs5pPdm5*+%gkMAP~t5O4H(Gv%C@6yrRQVzZ#VC1uaw1Voro6eBV=tF z_L)FTZ!ojkTKdYL*#Ua@KdCXNmjh(8Auz@9UUO(!=GWe{2B9cWar=pN2>38%R&x05wA%nDoG ze2Z6rETFC5gin>KK$?^H0*Q=xWm`mG@@5{Kn?nD*CN5L1X>w=}*a2II)!t-kC$5Zu z_i9>VEi=aV13nv!N=uUEq;Tv@MMWkF~7z1f#!Lcm05k;f%R|O zo4VMJ^UWEFVQwMrkg3$V{X)JCFsoC$DBN0#?Q(COR8r>FAYA6+6Qb6&G;>E5|5LB+ z>GV(8AOVy(mARiLIqWr$&7x;0kS+&<4l1uXQ7h~5uhHFW4~rza)n8_D1_2As@sA~b zAN(G4y~LD%tRf7p48^7;#!-(4%gvdk?ut4bOy%wHsY_@!5SV^I|j&lv1xINK2ciUY5gic*f|D<*^TB$KsYrW|aRHXoK zIFz3aw1ZEZJgKP`bAj+NqpDKSTLCrb2k~(f#~mSyuSC8o)02hHw1>EYwn$8i3#?u! zoE>ks9bfi3v()Z1Lj3bEj2o{#cPL)+OP1$r?S~rLllj``S7TO_)-%>~*NfJx)|;}q zv}D7wcbeBi$0&t(s|~a@2HjYP?x^uC-JN_xD6pa0W$OTt+tr~B zk%1;rlIPAZM7M?rs(BgbQyg-Hj8N$b=6nq`gq3%SpzopA1~feEpHR%xG27se7=Ky@9R8a6krg4G1ALs_ z&E7w~A0+uo{}kXjU{1hq?4Y=Ru9Nii%_gdc!G58|x=hHYo|sUJ4!Oq2+U?^10+dof|^BIR(iFp-56tID(=EL*aMBSe9iiL)wtP8*2zu+lwSiceCSl zYq$bW>3lK55qoXQ`s~4Xtd+&`ED}}!Wclgr)`{)?8Bn*-kVkVq`Dgb=3CR>Y=<64OI^S#k>|4%75cLP_Mtd{6HP%D0MNN!$VM8h=vX@{Z26wgXpo zBqyn!BZ#|(X*K?|0IfJg27a-J1kPRZ>C;Pa zfBcK=-xTNx(CxPw&6VbGu&eh1H=V@_x4dd}D|>R3&5RU2|XADJ-9_GZ3M8 z@kMYntp}RmOiS_RREg;_-7I{a#3>yDR~JA{%P-B&zu8n?8OZWdUFTy|2)_~`97?sA zdFJ0B;wx33y$Dld>Fr-$PDlp!E~B2nidTj_VI{N^yAtbeW4z0Z6n~@!_kl%w|~^9EqX?8)bCLSc($}UsJDKVR{!FS#KT1#<}&=J##HRGLdu03>(?%rINte5ud6=8SnH`^7xn3a9o1;}S!bCz5P2Mhy0 zIfL;bWR2+7i>if>pr7$?-(#o=;ESaU(fZ8XRlKIfO<;F<{P*3av6>5ouP%4Pq<)5Y z|0rF&5}k2fEGOjM;>Bq2b+Ksrv<3k{suReGtw z;zoy^$d?KIY=3fp%^m-A4)U2isyC32)ee|IP$f$ia7pKgP3xd-d^nh86&z3Xr1KQ{ z`KlDY$0lSiiujp;CSe#GUoCE6X?Wu{QBB+aAZ&s1nPg$FA1b@tk*s2Bf3;y`$-X#Z z)f!h)c!i5sqRUViVS`x*9FE^0Jq(We%4@+f+4MRHH&g@pVx1wn>9$?Y>%Cd#91dtk zSe;xDop1oBj6kvrOi8OrZ+5|#78!hlgM_#ln%= zYt-TE@x}g5w@Z`jO>rh21LrMF|JwIPM;?Ejxvh;qNYs#MvUhozyZP|ytyvAhfJ2I; zu&>K@HCcTSNh?_Ez(m%P=yvU!hos>lxTAUQbyj< z`R6*sJxOfFvtF*L{S0HVtO%_@yN_B!i`f!xcjnp#;tkUrf@l0t+gtX=z{U7_+%5Eo zO!$zRm|jS>2=7(R#O+7D3LeO)l?Zp_`;{bj2TfChCG7{M7 z_~$IVidKxO*E$egFkjKffJ80Uk$4`zEOMk=TFaBy!k?Mp;Q)NXqOfFT=`jg<*NfCo z_UFn24cWZRnf;qSSu|xIi&xY*n0)xe98soj|859IM7F58WQpi$U$(+r*-NYB9fuZf zIt7Uf)(A!aooi@o3(=l=mBl-~tqn%3NtLsLbmP4=2ls{2wGE^4_qr}Y*l)XdKKW|` z)(-k@-wpY(`cUq-h&_x<6lN}u?6N<`jbPhfE^!(FrLr}S@-KD0y|geOon z_MA35T)GWyn#0=Z&1_vp9UyjlNo^W|jU=Qv!e=7HFD0nVIb2*O)68N)1p>cJ-Jzuli;o0Lk%~?O=4;8wEp83Bzb^q(fbnd0=HGl+p&F=QpdmzB z26Z4CE4x-Hms_=TX+=7%@il%(XdrC5hi)pWcX~L=V&U4HxDi0}g(e9VC^`gee2F5V z{K~Qlnm{NIH8=6*U)C4eXI0;27JNFnYM5ugaOY$9W+Wzc3UE-%diVP#>Qvu0@xIX+;2K}><$%*6 z7nwAl2^`HU6u8ywV7W_R>P^*wJ%ieA_U}crm;&QJQAC#|V91=;(A0ZUVNx3QtCU)} z!@zqQpR~uVyowYgU?y6dxm)F4mM~khHMv120%u#iWLugf3U)MtU5I@bQAXv{*R(fp z)sbYI_f5S`52ngFLTg#q@*T`KfAT2es#w!*8ign@+;F`pi0)hZ=Y8=qb}(nK=$H8leKd+pzAmc z`f@NTCIO@MYV+(%-W*8mjM!4na`Mna18oPxdF4raFh6rQJ;N|OxMod|8C8Ndf-7KM zULL{4J*=>Q*o(VuUm{IJY$^j9nPb>yM}Pq#FLWoD7-CY!Udgx%UN&aY?F>e|cGU9vpj zCu#4$m0q+v)~7ao4)}rjc&?gUceII||5~O=K(mb2GQm$e?yrOyPMtlsT0}n0$-29F zoc!Zf-&h=SIo8wUAZXg@*Xg$wZ|>=4EgV~1`4QFOci8^g&Y3KZJ%o3g{deezjjfh0 z$aN}GZ>X1lss8A)(AyZJ()$gkmerF_8kat3IRAJ2Ya8pQ)Vq8glS?bLpn3OtF8ZU0 z3bC`>m$?s42Pk)FoHV#o=nN@R^NSyGc2p)=4~`uWf6*3gj*(}&g~^Vn zNFl0s<=|}0^7I#PC`DjV>WwiPvpf+R$K)3_BHXRu><|mctn&NQYbBjul35q9s5WS; z7SwZX#P-~P?uae_fXib^OW(nR_Co)lshFs8#0g+IbsU~ZC2nS0oh0AFg)~WQFn}gE zEtKN$ZF7dMKA;nZ-x~_Ge(^Dc-7#=JwJ-WjF!8qgLL;&r+~@U;itkk%@3?us&!||Z z;>>b$^0_{L*F}pl(odjClMuT{_19-KRxdGk)^G%ypsJ7+uKtN^KOvP%|0(WPJj=|z zj+!2H=3NM-9&@Y3X(e+hXZvLh1ojrVuxEh=-0 zS;MOq!|zmoSKezYIU*5g@#Y`|qB?yR@sNUx2kQO09cWT&D(g{qZ~H8(%xDWmtG4R_7c+{O)liW+O=20tPuuPk{C zX0Pnr0OP`rP2`ELBv3Ck9jvg$$~E}wENw$>sYThVA?udjrPLyn;tQ2Gc=z-(o)yXeWin*uIOU611k5Bsh4XcyV`M6-?~ts7UT_Tj&UVFI$*eP=p)yFBE%j zsK)bj(0bJ>wi+}N9@7+G&MGdie@4T+CS**HB8du_sC1a=JHG$q|HJZe4+LPF zS+HS^1d&@M`r#6H9+GPGp7D0KX0)w>^%p;vFJMDh0*ijhXYWmaZxDHaR+*+;T(+;0`UpBCG1qi=1QS5R zu{7WBZN?P<*BsXS!WxQq(k}Lqwo`ce_g~!?`?#s|sjQ;xFf)Kzx8?0b5ga&(I}XYu zYFq_B4UAPR9j@2GeX^bT<{#IoB_#NGCOM|49Z`TXh5uyzY5Mh2WzVCtF8r~p`26P5 z!FQhJ&FcX(i}FJXpezehR54NCfO$njWfYwY?`rO<`=af2S&szMg|r&2m)+yp02n~i zP<{GYyMFlAGRSF5iG^X)=-Ow4VeRAda}frF{BM^t)lMXcdTb1By3RSXwnA_#(%O0l zm~JwtRyH)LDZTAOWP(yd zo-W^5xH)+Ipou_FCfhDVhZv+M2<(X_9-z5v4N2k9Pa?Bho;$*_71*2UZ|&Z17-h$< zwD{^<^@_W|SDBFnyC}ek4N5br$1mSi66joS&O1m8nLW)W`dz@;zO@^-ZM0=oO!(JO z)Bu-)CBs1q!Jx*;s2-aeWt}0+3N*}u@Es5$NF2}CEZmh2-h{+p!-%aszzD(*wVmAy zjfR7ocN7E*Nz&A*7MyAG*ova6lk>w=PP^p(3@W?dMr}t3+Fl=_!EKu7UK=(HbpDgkA*<*93kisCQI}JEZn0&NzQN~4UmF7-} zRbSZoAar{FQ^H5PpuLS= zotuUQnm@KPdn0ZdrsIr1yYY)Dca3r<6lsFuxdgH;@%Ta$I_Z4XDT5aYsVhu@dYMVM zG_qhne=)aL{{|?pIc3|vH|7TDM|1kNZ*SiXP`+A!I<8o4*q%3FBEq1X#oa{NR9dWm zaja5UZVoM{wsnugCclKhQ`gKz%`&zh$4ZGjv|Xve^8io`BTCh* ze4ya7#ldG~9|l}Zcf^%1-i1p~gKlBmP-4&sk(oA@4W8<0(H=880S9`6YIEmL9+eH2 z?%w6T$tQLF_SgE?i`>x~*Kg<4zgrZFhPZEjrOE9Eu5*nW7K)o`tiyXmRa?#4QLL37 zO~B0Jf{r~NBJS=#YPsact!fpXv|Q`Mp=S!U$3?q+Hz}5jdp&xg_LWxe#Ty+O{Jdt( z&&gvVb2meZ7i*#;Je!Nj)t%Eq30}>m0GP(b$>lMNC<&i|i}}{` z50(i56MdTxvV8XYqtBvC*A5bm(`v^^8^}v_ppVdZn;HR(;_bTMq?_y_hF?5D0A}0; zvAdBe>;4)P3nO-=D?BZK)yh4XACg1$_!NpB!2Sy7RfYs>9BB4D+KSwAb0)n+o+F7c z{QjKR;8S`KKXrlHz|`2Oejg%3^pfu(5LJ7VOuYv;V1`&I+G0l$rnG+W8^-j$4b&RJadi8+Ma@S311*goYsVp1v1`=%(HX zxYF{V->0wQ%X4|fIKL|atEYehFq4Bm5!4O0Lp!mrRyf_~Uajz72W5FzZqo;Vz7o8b zGM)A*z>m3mE$*OSep7%Zj$L@ZCUQx#7h8_~qE;zG`?QO0|4e%}B-~&qACl4V9zto# zU?r4Q`sy>46EJfyX_gdi8LSg8_=i+o5|VO&Xk1DQ>$+giR71&-#9hKd((n--^r66U zP0N9#an11;LF0~C{Uy1jUD}3}bEiOwoaQ_LJ?9^(-(Q7&lo8dk1s;aItRUYL9s1sa zo51U;jQ6bGt{>ZxdlfKlVQR4+%g*F(_hZsryuB`WChaS{?==NSb+m$0rMp=iwIm*1 z*>1A+4xi@BGt0C6s6lC+gVfc>xW{oSN@C+7W;m-ju#y-$#ESWzc5zo{J%22Lv%~{1 z_nIJi2G!VGO9Di)j&Yq*g6s$f479G|d4z$63|~JrTmqu1;iA8vYw4c`{cJgeEM>HK zjCIM@hbry8_Nn6YyBWS&@%N%}hKnjqI0X9%KsK`{Onr}M%Cl?sRH=^z%_CH5t?8R@ zpjBCJ()vq!eGxwwVJ}=j7TsDj||G4K~UW?3<0++@}5mi9-@ea?j z5LNnLd!Plg$}0MX46MM_Z9v#MJPeVgXY~%c1;eYnW{7`e#!Hs9l}dF5WJ|8Ou)-86 z3kG)p#*y1w>!oQ8hWNU7(sO1N2}#Y-4bK{)j?_tk&91`>7(ccVz2|P0eQl1|=Igb& zUY64wwSA>G<9Zoixb0{`n3@A zy?#+TI4s$lWm~Vv1GVBriZHh8eoV#`@gYE5LQi5iRV8t6@n-kJ2LZiZRvKyH^wu2< zyx#Cs5guvg8_*bOK0jb%3@>DvGb?BtelwN*mO+{&do;xMns)nO z_W^LqUU_mbwEvSr;5D~nrfes?Wq{lKQfy{+&Zt6g*fGPQ)#c5iBsw90I@<#cX`pW} z!PG#V_qqX7lh@P$g?Rm4M*c>~p+Nswhczc2H%8ShsR9l34jvLXu`sjnt#Tu4|pp(Et7 z{RAN|NZzR)KrE(l47%YFzQzXG;Du~-&vuaCNqcZSsHnbT;t*;%6|HOJtVPa_HCGEMKiO&@`ovftLy0`i*(<&65d^#74<=jrIdyQGO4GMwt zk7X-u|I7h<5)#{9#7C%|k<$BJb$-aYwr<2AYrGyI{c!xx57^a0>(4b&@51=T{&Y&@ zlP=DtC}${VDI>IBYe#6m(~i>qpdF(fr=6gks6ATn&Y|R#r-!9io3%*3IaiHl@vei+ z881Gos_yiMXw5%$9cney&h&)25;8*|YcistO9_D(u(&-ISaz?%XlMLH3%ty6-1&z) zsBX}x=UKkDmO$HG`%PH}B762sLVkUy++0@+4wl!k*K|u+57+?zQiHI8c!Km3eRM1(PdQ#gff|->K8m61Eaa?!3$jp^HKmz)SBG?uG$AIPmd^ zaWxONQeIKd@?N!n6n-o7d;N87clkE(o1sj>_9rO7 z2N~+5OM#o$aiXsExLEL4a=ap*M`=!Yc>EG8-Y#{LJzWHa`?+d;ZI`%W++)Hgwn%t-|O0Twq6xv+ulS9n$ z`9}QalS|C?`A+=q<6_%dAA#TZd$V||3~UxDc(7m)7qtsm`n-)V0Prd+>*uFD(u&OUIh_fDgOsmUp}r zeN=rs{fdn#Wu%c1sq&$CsT9{9YorO~G{DAwk9lBx0d_O$0JfF}Zr`kxo@f5rq9CjXk>H#u z`})5keun6iX61k+HRp!dA;Gkn>w;%D&x#W3Xm4a>gh=2j*}5+SA;#(p&`DeK&H|3H zv+HMqV%58C&j*?rJUirh%+mB%ItTh{x$A4C)5W1Lo*ng)%uJ7t*dtOf0!8p9sQy;! zr7FzG-;c-irhqZ9&7X?J6M+h_-1q+TO)})^-vCDX!xpOjg6K$wq?6P;V2J29Tv<@m z-R(keNHe-TU4ha&7(-cUtoPmSDA2uE5^dUdL^CRcmwahamnZwJ!f~iy{jS`IUYP=- zBzx;epB}Rwv9c)A7k0t{y4A=s99yjxsrSUWp~0xKYzQ-o0Us?^tTQMKyfq5Z0dPv+~u$L zWUK4$4;iTT&{0i=t*5ODTtinitG1r%KpsGJArB#XkVlZmkSBltOZ_@$RQ40x8ueq; zCf@@lmO1f7c7_a!JnvF?>j{;v7 zkRQMNm~mNK%Vpb6N?ncFJQf61u&)zvFT>pMwC#L`VJ$?6&lOJ17 zBpRDICp=Ro@aPYkGk zNmh=#UwSgk=_;^lxO^$&pe*alBMQP2oqFR(cb$UrzR>8sw&Bgm*=zX{N~|u&Q~L_3 zEh%H)-@hLOai@tT88+Cq-e*%moLQ>uf%E+*IZ`R{g9LmC9koWvQ+K=ulwg z=pBeUqmUna=UqRaib(Tp`+f6x_>e!Gvg{msXXA^j=M75YeL=ohwg>BNRRczj`7%q* z%D!LBUAxk2Ec<@;po9WXT*WXiIx_yvR#hh9W&e*mwmihX9HZ=!6g`(&_GEFYWh!r# zqi#%l&CVBhw9uv?Sl(*2H!akFo^UL%lJVtpB1}2i>1ocHXA5-t1lYLn8r5PnCMoOn zV6(w_e`t==h{Dm6*i; ztN5qmbT-)MJl}3x9T1%)-m`;v#@nxvEMy| zt=@e>1;=ymaj&tJAGyENs^`RWYp+Y}-4U!xB5z)nJ$+*gfBlbK>m6O&wGlH8n-hR@ zzX^X5^&g3cLq}K0IQWZMt;@)vPi(%8!8Y$?CVEA0`TSJFSH{Pc>nE9vgV~7(tEZ$i z#H;)<$fJTZqwCk3F)c^-zJ8_3(0ngBU^#bS??5NQWL>^}Tj$W0nAcG08f><{aF^h4 z*|o*GSNgNtoy6WE?Bl%2TRWijU8cujo=n1({?Y>XN1qR8OhWDcY<*h=uy?Pa?P9Hp z5QhW*q$rH^30%I`b4p-)H__Sr8ej1p*EIXKY@1eBcicL$V;)6!wT{tvd`|nu8!JSI z22b3MvHR{;yH2i{K$vn+R7kqSSwa0fUNYzSTUuY=AKEr_1-_XG_ikP&vJpR<@$E}i z%KpNc-1qlMq8bucTj9F-GxDEMy6rzMxZ;3fH$0C5=nlUL8Ap}j589t5%El(~=aY!` zwxL1<47!&G}220}p<0NAqM3Sdh1hu~El^ueqM?AGo+&dcN_x zw|Wa<6z(P+xa@qRe~uel6LNM;;F;R885~tUEc?*-6fj##s;`~kJT8#GJb1ov<#Cg+ z)$tA2r*_*#nU-LFSFoK;`4mz4UAt&qR0onX*aGo#+xlB=aVG&uvs3?cCSxz{^Xmc$ zqo_*mSo`L^1?OvW^w=78I2aBFvu!Y~9(o{_uyZc<=e!zY*V^t-9D`owXv zq~s7(PW#IX%nS1^|EO?%sdw37AoW3W+~(z_`A247 zrYC)F^}b)y@@Z8-Qog)bRwb%ANm&|0yDFPlgmF?H4m8J{+Y0LY?JF4NDqwB0Rqi_& zxESvuC#@;{I1GFI-IHU`e)$D&-2~RMZ`Q@$9^YTZJ&xEsCi6UZy)uMqR?B9=ZmuI9 zPdw$R%1HGQk#mqAK_+b6_Km3Wc%ZEHh4G+0L(_HOVuW_r#*Op&S)#&+H(ZM@*qNWY z;#qveRmLaMn;&b`O?Ul3(C8j_X!;%euJ&j^W%L&)B3tDhXDI%CF|j**=G@z}p?W9& z2W{^e)l}Doi`r2@P^yX$0qN2e1cWGvNbe#wfE4LUl@em3OYc%b5s;2lX$ei~1nHp# zkP-+GAcUG^-@Na4zI)HPXN>djjAx9DnYEJawdQ>0oNMo_Qggh;|3ScpQQ<(oDZP^m zF^gz^ekXS7lazyte8+qRpp&b*m7XX-_YzBd89KG#-od~jd3s>=H1J53R_HR;Om)0^ z_?HmLypG*+)lSx?egMY7-EpV{X)vv$clv=j23V3x)yH&s${^*B`*?zm1Lp37_j zqk9HpmaM&4v((^RJxQa7Mb$2zAGrt|qt$tI-G7z!_l74|>zVV{9qFQB4!J2z-PS)Q zKSS#zagKofDNheyL*Y!R;(L<9XcyYFBIKnXhlBqn`%dB7lbD2^0;0tm^`otl82n*~SwY9cx$=@{!>? zK8gd=NWm{`B#qqZbThBX+8=tahv08nFU6zwB^4GW4;}QmGY4P2TUW-S>L1M1)07De zIlM>z@o~Aj^(*}~?gQ(5=K`a}$5$K^I@||UoHa0MUkjc0Q=3!2>L^b{=!$pDdLEDu z7VP}2eanXM%Uam>wX!4@$ax>a=mBe(2*WjBi7wZOjIaGyU;5dPnDZ+3FlOt{{xbg* zp?Je)V$?rik?1j8BxWABLMxXnu0!^VNCFGEd_gmVDOP!Q>CX!rH2oBoeIO#Z$Jy^X zCbcD8-kMnMtFeaM50;rlUwS($ENCDcua+4dg08KV6lUL8mGs!Akww?3rkpCclZ9NvLFzsqhw|Q|-Y)uWc?>M1eV{BadUa?0M_z_jrsWTvR5Gca+ za_}jSwuc&F>agC(XGU0a$+cM`&_ET#&u;tOD{)5e-hhgQ@rpFd>}8ll_hvqc_xzYI zf5K6Y-p^|R=aLoGeyu88d|S*+BFJI-=G6U3lP4^HfbI6pOR0Kbbst*A+a{3i_dbm+w(mx}B7*AKYc94og9;k- zzMFbt3%-kiY3yc!D^G>q%7_fOPlNZWCoh~ZaS%qxd_MucDRSDJ-&?cdeR=+qp9w#N zKW!xft(I@9Xe7*K2YG64Z1BV8=%PV!r`6-mG8tc)5wD`r;ezf(#NpJbF^TAcOQz4) zy|;teRq`Cr)-s#@E#19rEM~XA3(Ew43}HaJ*O%ND*nD%XCSy+bR9sG#yUxe`5&s(* z2oj4vn2D%C+aCNz<#20aorw4evzE~8He(-ktrbaION9Z1$JLkI`D(cwZm)2@ZA~}h zRRUTR!WEQ0qt6HZ8Le5I|F~2l7Tz9Q{_U61yTJ$_##b-ar^` z`#^3^CXWl$?pk!wuX}`>ws^kGbc*4W$hluB^_}0j-iXdSMO5j0w zph5&XObR7-{FqZpq~u2AFUpK&SKN8Gu>5q#B_<(n)6)KTlYwj}AP@JLv@H4IY>~;c zig|>0!#!%VyiCUfS8DVNO2(M3@XO1SOwH*XG8?n?*Uo1%y?skt!%Aw=_sGTcn(p9K zmA>gxcj3*#DOz#hz644kT=lpR_WR< zu+{8Va--M= zz=iN}*(*TdGUTw$_1pUNUQyqIBZMbEb1~)>LwNm0RAQl-De>rSE zeD~FEd?nv!Ai(AT-M>|#mdjAr@?~u?R6gncxccI*(vs_ZT2_q}zFV#bs_`IM+oYWV z-KS8@@#g~Wlj)ztSN?1Zi3z$1qS}2gUuzqDs7~Utfj|b74%S4YaRqU6jx}PR*w_rN ziA!swtL!H_N^s=7>riB0En~#UG(QTN`oc9sP+68)#094*a`D={W!n4p7(NW z)BBT5jtXID*j4>(HVKu&H2!E$mEiE}Z+6l7&yNDmI7BNw_wdrU%}X|P34ia2NRU& zKOp1Iq*?n}QR1D!D0wkPttrp5gB&g=a-*gx>zRiIMjtEwix1cop>qP6?+VUGU`xfV zCuXdPOY)52$FM8P)IYWdoC9pDvT-~Df!Ax!PDg5gc)-#G^o)@nW^PT#t#?+gbfLv; zUL4HcNBFHsAf4`?CSU!r)_||O((*K$zP)Vv3t#(6svyg5t=6WFB3(h&39~o$-j6W7 zflDT+n%<2 zuc4rBo=4(?<(l+Iv?pE@*N->(9b%8agb6YjVof%GiRo9NSL}12sQN#(nC)Xl8Pv?4m-(8!k(*UoBzET?VP?W_5ExyKjSx-#TdNNk(}Bl-@GP|#A)6AOc#a$ z)3Ez29NBgxKX%P4r_XE;_TbYw4r#HfYGGk$KW_KLgW?!Fy!Aw1^KK;9togyi(SU?4 zM&!@f3U%%Sju3QNuzD;bD~rJ-(nR_Kf(v-?dOgXi!!J)#;kKha*SDUhpXRoxfA*YO zHWW9HW=^TlJ1DZ`4OX`^Oeo(}7O|etG%G*o7K*TbC22l(q$gf@g|xZuX(+3YWs0iF#< z(-U@DIrvy=iMu1Ww(2XgYGhaQ9`C>&T+(tVc%}lV>3(`^^Uc$|hrY?zY-;WfOmqdt zxD|i0*it(E-4Kk`{Mq~^De?+f^Y^B6MSDEwF!N4L(xJKC(jg~L>Z9MACX?wN{nitw zvM?*0LmIJK949v){CsoqiUU!$yk6mg^oa+0(FHW+v6@|@Sqi>}Q~Kpv*!`!N@r9(m zx@|l7$-zCco=afDxgXoo12GJ<)rYf;UXQWWW{1z%nz&YVZbEr%DRK^f#j?I!W&7Az zV2q?~=aiKxCrFLm-F|qnM&RN-=E^Yx?ECiWza`f)XdKJ;U)eEvd=r;wHlBCCLpT>A zmD=K_5&EviqDo#G;(^wX3Z3ou zN4REx_Oj;2ft~H+ppMB=$qcbPQ-C5GQcpCxss4#GGKtQ|IV0i_k($N{X+3H>1btV@ z)dtwwxn2~L<+df%Hy2qQn$3SZ-)d*@Ndx+(2Qa?%uzz(&M)f$O#kJoQ`w|=9 z^a*(x+~2WT$IFH^dLf~IxDR?E-VP%#KQn+XT?Ph!Z;F;Wi;C{}^wzPHSgjeyzP~z% zeQrB0hfbIN%(*AYk^Z2@;9SrMTF`LuT;P1rhCoU6 z*O!N!4nHRT4&snAM%Ijk{iDnx&X$f|M7Jvi@!Coko6JkoT*>jIGFQJtVaFn)2MLmB zZZNUYOnt<>OR}D$3vGFYOO=QMuGrMXrYW+_2=Wudl{K0)2XsGyNi#e^mRda4kWQks z=8v&&isICwPq3xrz?=zYOcTG!8??vlRzo}X`~9QdS%?JooFqn=grD6wozHaTc6c!B z>Astt7gzM+cfWR31vz5#FJUDj)9f=k_T=KwL^4ZB$wUO2mW(gi`djm<@|vIi7j(jt zBGJQKjAW_0QVQ`RlKEPV;T4C#c?Nk-HitZ$(j|)`Reln zVR$vbY{nqAvmqqBp3*z2gNoh$D2T>a0{!t=2~=K->VP5+y4C(5`7gWtv(=k%a>9D}~;?ylq zD%Zj3b0Pi_7xga7alnckC~(NN`6S1`LnygiTwR0AF|D3iezv9%HWDNA#Fc?_K1hRp zb*fQhC~pzkb$ce<^ug#%nOKkGuHS0m(~t1->}!oN)#0;~pG^heDRJzfEG0LSd#c2K zVb&sk9(_bV`1%)z>S!`2X@*UzWPz6h-n6{`a_le=>}WZ|;d}uK$?WsK>Kn84gV95s zXTq(0bJZ?=sN9(4mRu@pQD=5RrR=5h0luXzkycDXXfhfuXtWcG{zZ}D-N#y?wcTy! zUJM|Ef|6M7Fe@HJ`X<1s_^Go~4gT^6MtF<3|b#o>QDO^IoDY8OcRQ&$I+j@_* za%9FFlhV`NIZG7dy)zl)llc)(oX{c?v;)3F*{{eqd4+i()noF$|mCNEi9 z?6&63ymXZMYnM?Q|FY+<-CyDJd!hsy2R=Xc3n3o>%4;9TyB~m*-&$XJtAG32_-2@v z9pG$fv(`s%v4jQIC_ECD8+n|$CE(CdZ~1P7zNU7r^YUMoDUVO^uOBKn7m-Mb=B$Pq z-HJqI!q?QS;m>5xUPUap>dTahti$Of8)&s5cUYr91DKU$+f($E3LLkxUEOMToitgQ zW=bP|H6})7e}X>v?O;-Fr95>g>%f>kHTdR3jZin`&o~K#S7=R(ZoTt}4X=_*3XP3z zYWFkXG;{AaxrP`GL(^dbEo%e>jcf;K-9P)7_r<8G;jWWS!4CNF3kNi9!Hm<{(yg7_ zjZ1eY>Q%3ua<-Mh(|7vivC&7HDS@9=wo?A=)>O&zZc2osYm(FJ+cnd}QFzO@9An#6 z_OIbbS2_?XX>J>EbVj@ew^}RufdlV7Zw1UE&IN3X>`c?9U#PaVq&`z0H(F_k12HwQ zXT|CXIJaGPa046rl=DW`8y9V+8dB7~g@yoUC)7(W(OF_FzW3osr)FUSA zEu9B*V*q`PLs4L?N0TadvGfnGi-?bdnSe%ne0n$*mk;$)hC_3a@OZ+o`~bjV?*==`|e=yT1rQD4vWW% z%@L#>_8M^m27KD|S)34MScEinD|j;bKSJy5bi1Tm1Y0{!j;9n?ff03%plK}tpA4nmi$Bn0r__R;-?8< zdMHCqw>V4LlXuWPSVF})spPJ++w!b9wJOwzPXQHn+n)VzDUstqZ|8Kq!#R+?NQ(eT{M@}%M_Ob}kFtpQo%+#3AV4MKRSZn?#3e79^nv6fb z3oU+PPy6F%wDgn*E8-O5uCXYv9X$bLw-Eo98Q%$6ostspXro7awvgCtv)Ak#tqTIx==rJ3Ft=7JTKLmQi^}X@C;@$ynW@{HXF}7`4X7 z^DU#ixPH%aaXNhmx$zZ89fib zoj3d{+k%&7M?J9o!_?$b7y32XBcowrdpLtdn0ZIa=%eDBIn&(yTRNK`CV*R-m6x4D zr;?xqgCXW^TZg4fm0gd9g0rYi7N3n>NTxZZ{M9v^1=2$hPPD%^TK|$5dZ{g&Tno9> z)I_Us1}+ zMRMJh%o6JSuP$R;<99d?kP+|liO9-V128{iq25c_(zASbwk2p{wXch!Bd6`~31%yg z3}I+6=$U-q?QwHCA*{URB%ULjOy$t23QEVA}^l=(4 zNXYq0R@B^o8X({Zw7rZ(A8dDu@AV0JAB6Tg3w-~faZAYOS@*-We`s3X$emmPT-})A zGOvq?9S&UGWOW|o*?df;NP*Qy=|wUAuR2j*p8&&mZKqwWRI6$!4aeBtL6A+TYwY#! zYXOW0ZfWLF!+#opKbDg}_$}e*m+kOHVX``Ihb!Ju;e{sxFY%gOx&Otd;BBOq-E5L{ z*I=;P{WSjafV6NrGXUrF_Nn`W{R>g_Z1Q}I583Y8CQbkq$?N+-W12FZ2J;tsZa{d- z^nTf2{!L9BtRbp@#hIFjO9CV(6D@0$y0O}vJV99#z;6%H)r8cXVIAgTDEl4ofJSC^ zN2>NR#0<#T*$>)$RMj~!li_Zuo2P=@;?f~c*2+cnQ z{i0IC<=q4>;0dE)S(ST&qRI(>W&ECbJNlC}mGH6JX-yIXI1 zl&a0wF*{_@^#$DT*pU0&V#4P@Lk838Q(qn>Sj4WHXJj&MtdOIpH}gPX6V4x5x+hi{dth8 z|ARVBj}){)PgO9s`^@Z*72L{iagS@K1w>@n-E)NTcE4{3xM;;E(lBN-d)Ha`Iy;rM zr-5ysSHx-80}w8A=DSFQz9jXBdiREppZ#W0ZknN`Z`9I1eteis`15q`8xc9sRDYWW z`{2TPUhs|g_(!)7{Ux8i4!BuMIV3h4D&!0C%hx4$2Xc@mCx!&i?wf3_)H<{`V_NvSN~tgavEo z`2_EsLFtG-b)`B)LiQ>*^ylADX>^Ui6p}Hnwm4~wdpm6c@D0zN8)0}L^K{jRP9X-| zd+Xwb_yD1e;e9U zOjTCod@2cZ>hwak<~(z`!BDi@G(m1d;rAQ1~ncQ=`8!0 z0W-@FjWy{#iOj^Y*Pw$_o$bu86Baz7)PypmI!stY98ev} zyjAukZD;T+519>L73kkGS`Yra$#!=yI(ueSU3$xPuH-UmjFO?PqpR;>z1+X7}# zvi(`gk@Z&#H{bmo@gZar`(qQS43e2Uz1k5VWT)t6B=)h~m>Dk9ZyWbAbpl{Z!_-Sm zspdZQ3K10e*Vc)ahOzxwhm+DoorkR_i-Xu&R+o$T5q!S zu`e(}&nOOzh(Y1YB2O7ra4@1#{mc7PdV;?-t55#kpu5@^h8g1V0+j>W^$369O4sh~ z^us_3S=W+-8r*dr8JRcyIO7SQKFf)Fi@g#|Hf3IZmPt*e^JR3k7H1kg#=WA$;ujc4 z+514rcW@lkjLoIh@BQcrH==9f^Wf&;h7xflC?i~vYQ_+iiBXRU!yYZ-8Lu^Qwx$X2 z=?%WeM|Y_Xx8wIDh~$setahB44M9xj^%CA5LGvmLiOA$ziVh^{wC7uG8^aDiqq;vj zbj~?OSSaIQFsF#=8K6KPu;)QwA=`Ib9GB<=9TO}jf41z{-{=2a5TAOb;%+`E-I?mJG!Ow zxT2kE%31ilxK&Qa)5`7+8G#m?-0Ve8-A%gi04kZlAmk0eFe&0-+3r`lEeu?tD6sSX zNb#Q`lEY-*W~QZ9Za^SP(Z&2_Px(Q(iTTC1|3rlK=DmvR;jdN)?kC>zwK9p3el9S7 z{pO1_Xw_iP(hq?#w%-|L#Xs@9!J2K<%hTzW84M~s=dPnI`Xsi1Q+^^$3!S`z&dzA+&3KqL(3rk&X4D=EQzm=h;0~0>TiN#q5;7s2FQ`Jq~dzY#s@N@LkfZ*o5 zxZkMPoG}-JJFIft4;QNbDD-c(7AeDP8TdIQn{%RI!_f(t-IA`d5Yr^o@rI{SdtnAr3Y2IOG$Q{nbo^3&zVXDh zyIgr`KE~D;H@jZ|;x0lY4lTmZ`HxJxc({}0mc*!uVEi#hr7>Fwt%)}#IfUy^!z0mE z6R>_PHGP-+ggI}@>BV6 zFlBnM-agRg*Zh#@>e3c#tY@e==!aN2VC=O9DW$K04JH+r$8VxB@@upZzD0#v%H-)S zR#Ez6E&CKmB6oAqxKR1qJNpt;BQ_gVXc8BkXx-MGIoRG$C%6IXW6#5PsPP{6Fk>9g zafxW#`pt@OAkIR^5FhljyLi-|+8;&Rg7LGWEF%e+LB)|iW$GSdh3KEx{HQFZV_Dst z0_o(!1@14=dcEvvwhrq}WBO;5-R#L4@uQGs!*Z$iN5WR(#p03Xv40~^zO2gKpd}%1 zVBs?%;}Tmyo*lukCJviSwLnDgOk--3P?BgClf%@&L?vh<2B~sHO{=XSHa+Zl;=}rP z`NLhO{ zD8s+h!CrHfc9wnUiIrbZj1NFhKB*k?8i%b!{4<=Rtef9mHn4ftfK2~3OIPjlh~kh+z<@Q|cly*u*{s7%7D5CeI#SZ-}wD6Gr|vQ?9d4Y;82 z9d)WMPYqGxOq(LcdD|9aUCI&4Fw{J6uuB)!C80srQ7Z>}&-vYpJ;e4UjO_;>(Sew~ z*vbJ`V>Ib=fLd;;lt5)>H?sE&_dmleH8sDRAg*t5+6`*{7X~ue#Ae0VP2L25a%_(Zw_a&6Edm^tq@Xh+UVf_6Eb^GxOSPl2aBkKLq zAq4Nc&JiN!ZnN75wc$4Y1XbZhJf*b0N3UGHdbQuf5Ib=ll$Cjpl$DAVpsLwsSJ8#T zk~aP5vOL`fJoZJ;RUa~cP{IUo5?CATW{nh@{bBt;Wk5WWv>SXY5xYgTqA;=zuT9c2 z%B>$FuOdLqmJrTL2g=R+<=VV}SG+-}=- z!n-&7I%NB=raNY*JO1-}vvE+PI^iz5{I2>?Q^H*la>BRz%tSlBUiE67ghWw+UiI=! z*W%>6>eLT8@|mg+^{E@U9kx(f7(sYE1cOwn)UXdI@tjY$n4UK*Hu8g%FCY?&4mi`m z37Mu@rr`T9-__28lfR>cNgxB}8L(7;ys|YJ%asxAy1j57pVh+vf-9gLYnnN>wzUHq zlycR4jCI#dP7*1m2Ol4cPm~gK<|}pABM}84T*HYAIwF!kpxCO&56gQ}hA6>rtexa< zC(YGByjqn#CR7#6#IYPFAA-xlIEejWXu#vjKNq2T)V0_r_3l`|6LU0_d3wBm|5B?V zXnbw|XT>^ufEggDAXgA#RosA840s9|V{WRzV8u`T&~&`5PbG0zm@`@4v{mUjZ-u2gqNrD1Z$RrI?c%Qj-2F)h7XI{~O*J z9>*Sg5HPDq#UfI}f22WJ1|S2tQJASnG+hBN{tHv%e+!r?3M9WH{0?Rj*$E2+h9dX= zhXkNbEspVfm=$C{EENy~EU8QN{QfDJ6Ux|w{|92ifYAR0MM@y)@CYv|2>cJp?H}os zba;#>VAhZ*7#vUnpp;XWzJH~gfN3tt@qYs&&kC#!cn_QfKH2|QI#2QXe?j0|0k8cF zi~%K##CQza124mRfH;5~Q2D1oHkm42q~cfR9`$g9{|VqFSQqg6#W862Ka%l(r8Gbi zctz}(zWPiK)tXDL*;^~PAj@2CtjN-g)w zi@pp8imrI}G2N-T@SSu{L+3$$)37EVE~vyAp_)C;7AwPfl(-M!rCdY zNIqR5kYNCYd~!^Iosds~{yWG z0&2#xyKJB)Yv!{dgxN_#|A6j{f4!6@nnD^ssd1SwIjn|g5qkY=6Nm(b%EPrunYELE zU~~U+$8kBenJjT1s1W{S({$ukzIHZvM*4-0(>IF%I35L(gPtE)~|~3*#o!E-hf>*vQP`iFf%% zzAT4Xrl~XvqiH^fnKcCd+N(iO_l!YKn}1xYkOk}4F?dYfCq{U4-&23-R^W9tN>WV+ z>Nuh1u}gDMT&{(|gZE#Sqo3Er+%Hi?4vqcJUvcbYLUmi23t57+G))3_D6NMcr9(O~bxokBS|j0f`B?rBBRIhY zC-tcZyezIzPqu7Sa;m@ks&Z!1(Y>{@tnr}HRoZ8`Hq^_wQ6ekZ8b+8tJW0JbV=mQ| z8?vAn#R@DvdDKpeLT!mwDSdn*18L(rN zT&ZmYsV(9B)GGWisn{8|BIYwZt7Ko<*uS5l^Jbq^f7Dn>OfTrKFG_K07>g>8?Pw47 z{j;91`xThHwJp|d>NUN0t^K5j&uZ6FMEtbali3sSZ-k9P0uE9P$IqH~Q*aty?2U1^ zenAwGuu5H70O>XU^>uXN43cZNCw;l(!4Co-vcP)Or>luUW1m9%!Y_#Bv406=;Dz2U zfe+r$Fl1p*HvrzDAi+4|vDXle zLnfvsLGUX;+56H4cB9%uynOy1U=}kXsz{THi08w_3Q14>A*_%IM2b2!uUE#21Sv($z{dSqd6MxxF(~FE?n*>di!0qyzRbc! zaF2KoxaY;aF)dbb5>#GT!Q`N%`M~Q^y~8tobWqW1};@Bk{C$>)>&rrS^bQWp2X z9g4A5TKdLFr`>p#f^{fsLaN@iU6TX3_wVZZPAl7#MB%M}qlRL3P9oxO&339!TNXr*_%q+jl~?^?XwDS?e#J{>!n+my?x&JKy}v zVd5#w9`8$+4tpUibjvl;Y7Ep_efW|ab`dUY35?m5_S&gXhhP)L&75rTu4+tpkfl?b zO!bq8`|;~8LKlB>CLqu-0(wtG;+1t-wcVKospYx+S# zWEUR>PA8a}Hx>`{58-cbhSisR{L@KL*x14>IPR<2sjWPM*y!WMGw;B@-h*``clDF~ z`Wc_?x-6a(r|794)+D2`NKvo9qQ8?`1<<_Duag?x$YFP0g+l0eir%yjqIzb)^2rCy zZd^d2*TbR%Q&Mm} z1S5WbEYkBV*NwXe;}#cJu0O%+PtzVR`W`#FxrkMR8t-4$=>!KvPCyU_*GEAzUwAc6 zKFhb%sgaUrmt_&j>dvT@^aY_Pg2&vuXGeOAH_buzqC1-Mclw@9(;qE@8sqQ;I(IgN zM&PZ(kw<{>#OayB7nZ97ZBS38z3+WFxR_lPvK6d#$hkv?m9DNw%_1rPT~a?k_mxh8 zsjB&M=9{0l&jGyNlH$N)nO*uAQ>$}1^{I9=DY3%*5qCvJc$9W zD9DF}I?5lJIw~tbh#yLh9k>2WP_Vq%SI>?$i~T4JxR2zXn$8R9R8QjJl6e*G_<%$b zARWYI9RC(b{wWs3$C!EX*?^MIbO0-Y>D(>E6laX`kzRMY#Iz82(Lso!kUO2@1~`#yIFl*YqYbXk1W|h}UA~38 zJC9@k_Ge+7TnPDuK1;h_cn|0et)nG>ucImFbxfAZrSFMcMnujjo!Cbl^1^jcRook|1N$n z|NEo!0546J7>W0SW)NCgcrONl)GKAYZ}FFuPR5dp>fy?Iu}5CVYf+%>-k?;OXULbU zG(ZwPMf}s(lEs<5$BukVE4Q~i^1HnMg0dP*l1%E;T9v97rar@(9wj{Pjg@rSn|voT z(#Ev72K@`fvpjt_DU2J$EhCrxOncsL)hiLG5Q?>rNUvzgsWKM%7xs|c_bsC zWo)v^)5qRdLn#!A%nV(kLvJui48RGeGvbKJZhIS9K%IR3X~j3&4Cuz4eyjbPNkCIg z@o#nYRHF^wgzW+f^)>;?4b(a`Z=5dVgc>-Sxb4s&_>r`$JN#04*Ne?E zXL*BI3a0nem6y)3u3|pxvx-Q6QPa_<@DLD3#{O0->{-idmkfp|g$AIwRC;#>rAdE7 zyRxGk)?bx8LC|&GYSNdgve}GvfgrNxcI;-dXHJBv8HJauqJ110YrgjEh^FX83+20N z5&t9FS%~wHu5h*xJIiX`bLJyQn~;D-W~ocg&C^z5$!|oAxEA z(#w@H!DJlyN$7I+g~=^b%5PRWz{u_*zk}%*InGk>sZh8o9`&b@AYQAd4P3lz<_@0O zOI3a&L$Tps`kP?R%0)|XF_?6LDLgR#Wtt@Dl1=7gWY1*2EjUX4wDi{CNb5vPsDsf{ z`?zCG&~6y+y^yiSt}zkNPZ`WQ7Hq6(de%GhmFM;(ISWv=b1GE$$lPhjh1cczD>?@JcU?aiDx-$Fh7t@^Mkb*xgPXsUM zy`F1Q3aadcGa3c%@OKOj!SWpYazM``1QB}X0mY~eVn248=f@jT@fqNCLtQFYnauIu z&wR;YaFK^d&#=GpkWuIQ!l5$XizagpQBR=V=!zStf9(pnFp8wq90a*0Z zMNXJ;1t?JTheTm|wQD=UR~L$cRB~9<3py;4K&6>}+h?w@kO(yeFELOEUZ zr@HfyX0J(@&FUXDedfybOvusMbnr#UD3D}->&AsAE(hNckD;V$kUz^J&EhT2tj0{h z=-uLRwWyyzc(1af+5BdbtxD7mnAG+QY*;Wny|Jr!^X3`wV@1PUllOvnUGi6^$QPP1 zk2=D}V%>wyIp|Z%U){U6hSL=s^AA}(I4Lu5lc;~|LCa!8LEsNjSWFVshlvohAI(fI zXBgKI^W}AqeI}-Xuv-twTC07RTdo_0? z`eFUM*+*9fq?+*+?F&L-@&47bYQIJ8rf7swmys*x@Z!uTN-Av?FEWZcIW<<71Pb6<>q|_4dVrk4qxzr?y22X%x{MQi;vP_;w?!=;b=oa^~4V2|L(+|1khHv^rNQ zF)YTvGUi)mvJX2+e4ADIM2@{e!*{D!eoeDyr=9)NQ8e7*$P*Y_eNk?>Q!3YGY6l0Pgi+@0~MK2#fjlvnBgH{K%!b#L%^P5X}; zJmOPadGH4%HL%keI!=f+T#$m8RFkk#G}JM27nKX<%A#C+6_b2@>UgBFD>S2nm!hx) zvf4o26f>Y`md*z#9n%GAO`hf2yl~_^;&F#jATVR`_~T>bQmRZeIi%T0W|>LJm6sZE;mr8|= zPJ_J@a^L$x=3 zU`h%6rR3h2I64r?DF|g523pj*2*Pp3;q>l1;jjehM=@63cSeqrA$}wors6Xgv&XIQ$}|fn-m=Tqv9LQu zPW<2Al34@T3tl^&!{)CUf?M~cnS+0Jva+!=Uq{x0lj)rP6_$CRL`}tz6DnG_j$GsI5TDT7`A8l;m@;Y z%d&g`Z{2ku=(Dm-5;z;FcGm_PYeOL2}aBE-~gqZ0hq7)!4d2&1_!g=<7 zQ^-lJ=U?5DkA2WLPWf=k$`hR&SMU4nAn8Z@jAri|uN$%Jljb~4Fce5s;mbU~yzliJ zy|55dY|4@rNAReE;4StY9zJ|Hd=oc=OkQbqmZwQl(h17te~jY$!%be2{o*!l6weDB zPe2k&h>?lKvTFj(Z&-b+zP@DwSAhR44)*UAe}~s`G zlOOEt^ikw_v(quzi#hiI)#44iPsCS-4?r)9VzZH_oG^AW`%_Q<$g1BujJmuS{< z`C1+ubc5{D*6u9I*S4T0bN{uCjUNx-i<8~IfWG%3^V$Lm&+_qmV{T!UZA`s|%k$=w zYnQIz6bh`!$?{-2(K3s#N(datdpUt<{Vcr!tY^%Z#|p!{Y=M3^uGL_Vo@?9oxXVv%ca*%3sTO7oPx_cBN@U zQxV?laRzjdLRmkkb)JLH!=m)@WB*Rv)wfwo%3Y}bXv|@vu@t0rLp#=oObodd}ua#LGTkpvjUcA`XD! zN=x_6_dUP*R4r=zEf_E~^FkGlurt7ATi@WxlT%kIItnQ3kf-(u26klrfqu&Q!}52e z)#6DtNNToQq~KN7iI6d9XDDZ`hgdL{_GVGTw3D(#|G1Il^^D__l=Xz?36A(+iT%Dy z9l_<~8K=datSZ$cAjb0$FFq=7^hLKDLx#y_sfF$PN}l`qSnX`6J^me{-Q5qyTpUt$ zv+udbv%BD}J#dEY*Lz88;``jMuhgEGvjEv&W>-?fm@hEO^_Fr4v$pVhCx|2@fbalk zkIhcC3Oc3Z&>dq6THIpAl zxmopW9>32)+=OpueYM)B6wIC|3cugaz8hHsKR&~uU16I`+1R15uRalM(0l0aggOIG zcFs{k5^qg2Ruu!$|Px0{Kp$#i=q}u%(#`)0NBN;-==1?Gg||UPh3v! za6V(UBgU5#`-JkLF7wrtCZ(|~#5}T-lRHy+GNl!Wx+#kF;D~hoi(v|QDu1^dvhU%{ z=s^=n5`|{6Dq52UwI#(gup6pn!m6Bnk>Bl9R+CBN-&; zoRj1{0|J5q0-}-`$vK1MBsq@coO2E{2$Pu0*|YoaIlFuJ-{;=_;rU>ghp+1Gs_LHp z>h0>*dMr1@u52<*aI@<4K;2BxxriPUxBF0zDdLzL)VPl{u_q6HI8ioV#KlJ@V>U51 z@ks^n^25Wl>1S!5$un1)+tVBXKO(0k%dO{;?9Z@r7)CGn4^}Z4YtW3i*(-f$z6yI=!0BU86llVDsQCn46EI~JtX1$Wq* z#vQ+1R#VJYY6gh}k&eD@;ef^Y_SK&&PzQJa@a{wDthhmQL{PKl*K1=C#-5;fB|ODR zk9ECU4(x`Tm2E0>$ZCizgukgc#*csQns79!h3LX~1E88DREOc;x?pQ(?rViAh?B3} zC2dD^x{#;YKBUYAYNOK1+V1_UD(*%?9(j;s1aE$ zX1Kod(TSLt*L#7ntsr=dt3xM)OJrPG^&{x#!&wzxC6TYr0^683r5DOI^5`tOSDtIQ z9wv>}q$8pdQY)YO8GOaBRuE~$BMoe23aHEjg0@V+ZiFNi7Uw}b0=cOb;q8q288@?{ z9NIDCPjFzi%j|Z|7b_1`ir9oH9DFl?Ow4X(=Y!%+CYnldOWRi~Y}-!xQtxbwrXcpT zz=JS_7e9J@f}9Yw))7^uCLcz*-29=DAMO*p&6-@lR^X0I;d~W+p|KGeiGrl+zI0>X zw3}~&5q&7Nk5f(+Ed2qeZn+4H!5(Q|V@JoFyO5o}YGVpw!aCzM`#i4{5d<5c?Dj3Y z{VQiRQyi#?krreR5e(=i^2)p7g7%#XY=2H83_~*5%Il^{Y(uwp+o{^bV0&`#F{OHm z^mi_nLihUnpTGu&&!qeY^>fS+j#BJfUp}W}qgGrvdoGb(vD}kVRN1gDbzac3tZ9iOZw0e*A772ff_*Wgy9%$T}h0J%NOo9py@P%Pg^*Z@!q zAljmtF7%2v6n$A+eo^$rIAb%l3R?nT$1EO1jJUTF0KyF*9Ay8wAa2O8crX-1wch+I z=9UpI_|0vv%+06C$jc4%%)~qs^*ZTDyG`nB1d)yVpgO(_{l;W_kQ&8=-dykvBXDPB zu+tb(j~AxqY^fpd8{CW~0d=FKS-$peqp+%i2?aK^zt|yPVNZp<^}5s94u!n5fVq04 zbzQi@pz$(1>-Mq;riK$_&={0;gL-%G+)jcB3VL3njj|cTsMo$Z?+^$AuDBtfwewnK zu`WWFkpA2?*#t^eW|~@$;g?g_Ucukq0QK6Dp7#}pGe3orjHrWXQ@Lz)Ag$4m73Mz5 zH_Sy9ni%gjRPaI^cfWMt01SMDD37haEaIQh#tr*&B~jLMhXG9qANyj)UDa!;#dM@J z$~8dB2A-H_$E`u($F9hxurZjwKptGbwro5Vt`TIU@XwsN^nR4O2JO`748BTqj__pC z%q1%7y|`HeJDM-e`01MVT2+xPfI0_6046(cl}*DkVb=_zz8I|d0?h_wt*e#9_W2%6 zaBB2U^n^&iBZ6{0e8OwTTB&Z(y9D6$z4|$3y_9Fy_F;6|s-Mkf$V|nzgFnd`yHDoY z&J5jdpWT61E#J%9aQ$`gBn;M=0r$`aq{h}*-jq=TE0&<3JLb4`d$P|rQFhF-JSWBv z$Db4og4l(lMsb4-b0ONF0>NCuCKg++-dBeR(6*O9P^qR4x^rs7o5`cvLfq5m19b}= z0yxW0KfHYiXDf2qP8_jQV{Ow%J5G#iVIw11nno$D^uVLL_}ke)m@>?h`=NA*0O)*W z1$^Pt7SOmqW1sMa!6oe?XxpuUMJ+j8*s~@BrFRTomX%w31^S3MK894M)_kYFIXeI{ z=AwSe`#K2BVE^dcnRJF6hbj0j!PyqrCgKBK>4;+{$os>iuO~iq+;Q!S;7IxoMRosh z@Ima#J*MdSMV78Y28>Lw4__-{7f|mMgPiAXJwt8M04Anhamwb56tuH+_F4Bv@D}$S zYF*#MiO=tO(?*Lu=oSpDhG$1%por5Q-FEKdsU3y6MPFLSoDu83i=>iv`T!5Up#|xd zgPiADoAe?9FGA?973^X?LcT;`V}vhbE7%qi513vELy0VagE=+h?>=eHK%CPGxNi0> zkViDdh^+VSB%<`@?vLy$tc(r^`UlS6P6y@75BN#wms9L}PMct!M{U!im^L5~)a%)R z9)2sW0;nFgc$F$Nmw*%w4%$-i*pY+Z<)uuoy-8Jm*$Y?#%sIz)7=3G#d*fsRXS}brCpzLZd|nyMfs(kw%`%z4|B*$@HRtPTeQDm zB!l^d;twr-FnOt>9{S^bd7=kH-)go{`rlL7^9kK6`BJKklYXe#{!*>HCL84pT39oU z3eQi6qVk?Z-%p=(K-G-+uIJv9f{@Jtpn+xxe4P0}2Jhnwvwf$oV_I&*AL}`1C5VLT zpbs5L#7r~jmhh8SFu!RqY$^ge)uj6T40@pe1>@h2I@uW-NJMg9mSn&+$pl^6G_<@1 zMls*UNN|Y1+dtcnyXsspy}~OS4q2l5j0p-}WmUG@?A&8eh#QT!zZHI@XW6nH2bTk! zpvMSy&j#kp&DMe8O3VeguBXNp^0ADLF9ElJ*O@E#K)3H_VK1sBw%vEgj>9J3x!*3>siB>{br(Y3zNzO~a-n5b zxOx>U5f58E?Fu{V#=|Y@MJvFMPn6$$^%I7tADunKd56e&djhj$7T*?3SBR<_y=+HWh+qp5(!37mdBU*T%=rfE{ecORetq_5h*ZqRIoTJn2z5R_>r+tm}6 zQ`HJ^_6+!T(Nt7nCC1ehs{V>{6=6W$8F+b<``l|Bu2rZYy4LpLBi`$U>ipa>JCM2K zZ%;eV-vQbWOM!d$%bB$A^S!NosWNvmhQ@zc9tiZSLDJ#K=ru#yAAOF`P=BPrASbc7 z+m8d2zwR8wo`K$VvZ3*?br^QP^5^E7#cP_|Nxb<~4E^N(LKDPqCF+WKf2}ppBTE^$ z9XT{((AJO@GqDyaRId>7-f^~_8RId-&-oAl z^+BkCy7oyYlq+)pjr#S=pGlTz-Z}TvBc{h1KU3J(LGo=#cAj5^Gco~w$8kR=$Roc$ zK>Jkjd4|OYcpG;_!QA-WTVQc7CM*e`Aog}iAMr6i`v^IP(nRk?UlCf_GXbakidMiU zHQTshjMO21+h9JZt#rN`!`GkVVAu+e-==7r#%pcV)35j!w81aRIOib}Svgy2$quE95B}#s^|Ic=OO!aEu>JKrAQBS6QYAe!?Y_Nn=1>@?P4j*qT?S*cM#`4R zqg4s)12+Kgw?ss1g@9K)(_kM=MqdRgV4>T?_N5#G6cFWhovVp&$6ZI{DUE4oL2J=A zSzN_ayBgbS&4!VTtjM#R_?^=Opk&}H?uc)UgQj}riX6i>p z?w49AM|#2ODf*2$$(v}Q>w9w_6^F7>CIe{l6^pavSjdvo6;qeq_R71nGw2LDEJ9S) z`JU<~Xp@;MnDX^a0f4xuPGRR6xt8|sNxtUpDHy&MlA}HZhQ%$65E#4%7ERK?48+&sGZ0SDvo^axhe6X)@gw_O z)2X8<>TS0-xQTI%_@@I+2h1Y@67dd(=;aHdm~t70WSp=tP>gW4Sp?BQQ2Q%0h1>2( zb&rJsZW470)pk6)ftJc+Xx@9S*}PKo3i7>@EIPL!CqujqXUG^={G^KSYBctVwf_oZTbwx z0fw6|VhYh&fF22(ZzruVtEtK#vT1-&X*)D|`}Z>eL6XsR$|NTZg(sT#mO+cCTvY+k z2U+15*>q-=`JzJzcgERKV|>43(U{C9Bc$g+g=e27Sw?eWGMf<5Vh79G404+%p9zw;d}Z_=gtt^1?TIi zCx$Y}tNI~XIg|ek>r4qikB!3Fw&r3!1uSFzFmqn;Jp}GQZN~%lW4^kFOqvh}r)QFS zXtLPoRzT)wefG_~y8SVwL5xs2;~J)wQz0b~olC>FW+^Ps@z7KivH*`v{*IJx$l zV$e_m=D`ZHW5A$bv%IUH)Iug4bI7HlILLCpZbyb7gjYUpe+WCqzYI{bf0+N0^jXk4 z7b0|cjT`9kQzl^*y=pFvq~4lVw>_2}e5uTr6$UxFo)dPqJcxtzb(~(jAEXwB6>};1 zf|v-COm}0Ct7(HG+B7W$d3=7HmR)pph-SIQz^uA2kwE@!?0(oq?Q1p6LoEr01J2}S zg3g`MMGL($MUendMIh!^(960rDsh~|btni2fy*9|W6cYEF_f7vmqj~z!s|W}0Vw*y zOZRfGgFcwWlmSB&&(?lgwq!J4bS{T-xydwWqy$i*6*o`e+zsl8v5Sngu1^?epo3M+ zI=NXXt5FJT_R-em!uVbmns%CDAJL0uT&o&`aip%ONh5hdB(96SZAj}kX>o%Y<6#c> zp^Gb@Ppj}R>|7p#?#8~EQcx+8+Q>_Xyo6d%Y3E-?>=zYCWK-d>w;J@F0++pKRy6xP zg$B*fwDLyMBVx{1MWB%WMUAHd0j1@x48 z*Ui(BJVkbcbdT?zT}w>V5Q;@E89ap6bD`c>-ur>&MXF#XH1jTcAoSf>U~5NT;2{o;@Z-%WUylJyoCQk z2%o+xVD3O%?ww7tYXB99#646k7baJ z7v!fB_H}8CbM|(?hkUF*>xW1)kj+JQ*8E>%${w6>u~W85zmP<0x-A0F_+q94Qbz`o zbVyX zyqnjNaGHLcF}83h-%bO1DBH>e7mAsSMZ$18&^qT*vKg8~6mpm5;3UsEC(2ByLI=NBv{?+P!uOJkWe}K4)+` zA;1gNz>>_|&frT-zE~}{ClAl)FxcYSN3nu70OG;(4MHST4+=9H$WPc^_?a)vtp7d{|XRR0*bfw zvRxV<;5kph41c@Fk4&NVe+L%22A6zu2#rI`^fQ+4q-|%YWGsLe-nZ}GqD~C&<6v!z zU!yFS;?e|igm}*pA%@StF(nI1|2#8t9FD7Y2hUw_EFG&SHM8d_KpCj)Js>-F`H#ZJ z9^aL?#8ALqVxJ{P1c3q5BpVD3xjgHvtw|^KJ2Ess?y>^wn%lCIYZ%qVB z&1i`ipx-ov0^SJ-O4en&qpHdB!R%liyj3;k!-V%4BMc?4nOX>a){bcALl4(hGTKDL zIF3e798L4(!a`{L)32v35Y+FoXZyc1r0q=kuH&72??U(m?3wjXW;e8@m87DspS3bw z`g+D$Dgi+Z(2)4R1d~ZNkZc6}2L!+QIlM=E6y5)GXOi#X;T1P`Bf&%=!Mo82Xv!Wq zV2uGbx0eFxp5FYW2>zbUO!=`sF8KPo^b$>{7ev<4{aw|tZO{%#=$oo4CM6EdIFxvN z5O0tp52qCRVREHZo`RfF5f6aLir_r;S9&hgB z!2>RY9!$^cMH8C4A~v+TmOCcO%tqqqB4~rRqi5=v@D%Y(EJ}LY(!6k?Ad|?g%kCa> z;|&ql1y?{{sV5^f)D`+*-jDbRuheuV1Vzy*qSvmi#K0PsR*SDf@uy(9J{$ zV5MI?UNbNfWD2nPmTTukvbNC#(@*I0_*nqc-_$hYefrjk~e8?L_$}(cIvDnV7jv-@> zO(l7)2yXv?W^Z(xz8%jL3^cezIkGE{Ar~9=e_Y((y)zc+3IA-SJAe+!ec7xLM!18i zH^ST$gn!ffdVYL+0EQ5rX{I9*;7U!dMM3>^QN!%3-Qd>tia5CXa%PsF z0!C|$O9xMWpr>-Ue{0?H75}68oU4oR_KD<_&O(*#n^YCp)_Hr*(>vlaPOq6O9buZq zLVOm9r$GrYzogUV z)o9>>Pu1>ixq8XK1$ObyP87yiL``ti@<`yYw%3%L_4_o%3uOa?4?=E}Gy7?unH8X8 z>*>s9cXHL8$I>6e&zK>;zAKoGU%R`zAQ0jt;i^eOM5Haq7o{r7g+bjMQG)R}J3jB4 zYmPhN#LWb!gHRWM!yQb(q~G=0(JB`EV zI)&vEQg_{t4R_ekDF{G=A8ObjwIH8ywI;!H;Qk{~IujPEzH-NgW4#i0(Eesb=71O8$dZC? zShn)vb3s%}dZN#xuR8|3)rWZ9B^T!w!&%mP{nD&j-jdHJU--pmKf3hHwz9t>^)ZCJ zO;1TzLlo@n1xZ*IRO{}Fyq2#wy&Bq3bKEF{6}WYao|E)6+eT)j_%*ipCvy}si4!6w z6teon+()OpYM~D^%HppX_H0 zy{IqQ?e;7S`sAaFCABl!5jmmKl3bpN^BOe@8pY3$HVtG5r;}pNQ|NwZ$$F)`Bzvv> ze$FBJ>ICS{hm))n*YDqMfrE71^E^b_L~CRvwO5i9HV`iKvs3{l)<3HiL3x4;JyW3o(50uVmpZ z*nKJcoHB%8b~4Lcu1GKWY68d_{lwE42ElW98S!CLYRl?5#m&zAQ)-(`R-26_-3Kh` z$z-GzNbmN>gT6x*gWZ(#mb>q){3Mt@uBz*@(_Cx!%hA!tyQjV7n>v>8r^9it-FXeM#F;WyxrMOjPSdKmY@@%qWu3-n%f z_v7asYB>ozH5ThnweMdei%o-UC4Wrc?MSf`&| zR;^*OCa;EH+Exd*r;HH66au|#cCBrH3M@OlJ>UGebopv-fbAZv>p@Fy0)XQ!Yfktl zKd}>rKBb^F^4dvdc1qtqlg>VNW6CbhkNhNIE^Cl*M0s$Xu-?rfO>TuN>9JS|BrB%- zMVtN|xgQP)!hSV{0`{6?kpt(iZPj8_P!a39u9Ipmtv_Y5W9Xi=#oV*ou_OWEM#RMU zVi{NS@q$=b+LlUjs!&3hyX;<0_s087uC)8+><>&d$_ESgIGb_QaDE=%mJE^~WIbyE zJtF)?iAiDeDrQt$i@9~Gk<%jIk2RCRhE(06&YaO1T|IaumR9C=tnjGOGKnI<;mCoyqIhu}I= zoXEvug_wG&_LR=B1ix{JTi>IN9}}`!T79a1645WKt}^-5<_&969S$I3^pEn1{Pi2OiiL4jLjnSWzEUg!tjrV-HmuD z-LAN901~BfT8+lE{KVh?D0MDKYidjS2*MgGF*Y7Am=f(8zBcjq+@9+owLrYi$-59)d{ zyoy5s|9*Y58qjUAfP+YjtiFl zJ=APSqX(Lwx@MW-_Le|sTJW58NLhlpk(JZ*#3|OM9HYc%OIQmWSA4Ad~o_zcIo6%$H6c(HcBh-qX1#Hss zQ#Al#?=OJsi?N^32~vk%@5wJdr(8{(1JC&NW54gT^evEO0MkYcU<#kBctoSTYdha5 zP1kL^N9A;^v*EI2{xSn64M{z69txDpns5_*R7RL)maWrg{BDg{8piBDz?<2*iXl65 zX&0%b+-&`+>@?yi?D2s6lkMX!|9N?kc4*F#BwnKX$M6PrAGWp>VA zoVD`o*3~3_?UNXJ@nUSOi8;fA2NLyBdJwdJb7K#k?(Wv}JHsaNeVudA<;1BGd0HL< zbzw(eKWkb(;TAL1f@pVB9^0%Wqt8FvnWYT2o7gVt*h&Pi^034zc+>4~lY%lonnGNH z0<)6HS;B&8Hq}@C6EilISsKUs9QZp-Vw!UuI=aZtJZjp&U}s{-ynmZ@!F_^mI7)Cwn!_wpu?7bD^qrzv88S< zX4B|5UqLaEKw@^DN7=+)BK1af1d8W!cgw%@{gm=SP?uV7q+f)x0lbMRbGX1P-}ClR zm(TGLA6bT>aFXb$#p9acXUR4%F+0>K<|M2D*3Hs87tTRORO@BUTT(17#K0-$k*Vnd74it^2M|uX?=k@-_Nm>D>LzJ+sj{Jn`h}Bo?G1WtHd7m|L|N_z{$u zz*v+r86ZwtF^#sA7=@gSL`SR5b?I~Y<3y*refa`qz8yBM7Gav>nJ_m{?kJW)@!I|6 zy0R%mn(NtQpL{wuLnIr#g(Nr!o~UFkFpA64W_A&|{ca4uRRcvR9JOZ6l0PjW;}n-V zV$q6Cct7(!K%H`Ej#yFDf8aoKS399B$y<}Pmp!$qJ$DY6=ydkDh5m74WPf*D3GfW& zeVu1EoI2v(*<-ppSxwSb%k+_@<7)m{BjnZ6tBpkiQ;*p^z0gL%)|0-bktEM)>H5(4 zCw{Yo3s-otg~sp1&FlPZ8Raltb{JA26dJPW;w}*)Xu3*zIz&s88($ zy~SgcHNwtOFAuGg{iau98a<6br9bg)7(AT-_r>$&~&Mj&!P&eulnPTt)U28;A89tHD2T@k_cdl4Zavs8(jcIkWUF4-LBO(;&hGRZ>|Vt`SOzay_|IR@C96FU^~Q#TC7$S?h6u5(5412C zsV|>1=?gv=d5z{m$YxvU;i_OpBR3yXO z9>jQtzHfh#xjnSQ1KMxBZ2Uk!=+*qo`O5+?&i6*F^cdx#uN@^BICoV0-r>%(vzT%M zREbubTKyHqitzfhkvVuVLGnfkIwN@<&`=Ic7x+s*w~2I3)|lHTzwwIci77AV<*G{w z=dr!-q)lIft9qK=)_Jt=O$|+)veqQBGJj?vSs6ykoTMBd&A42sT(G2|k-~~EthK}1 zXATKF)XcBc0;zD*Rn~jFtoSme3q+>?1G+_s_Y9)BDUI!dqBanK#$}$xcphJRJYsPAo3*%F$um zWD%E8zUomjQYQiBqILb#(0Nrk>B@SSj`zfTd>~Lc=euY8(Zr*0kK-KI;)$vwMQYYh z@B5Q>%HxyfLaa>{?Z4{IIc_{1nrn)T0d($bro^)Fzi;*&OWdqmm}YR$+PaU!e^l%r zPLJZY1iosF@)5fyyfbXoC3(QY^*UzoMF@OsBTU2X+YavuH~n#8b|qB5fhdU6uOobR zgL$@1w*WS~r8Ac}`n)D6%tVsHTw%*rIX)__;Z<|(-au};kC|RPRCQE$rS4|GdS-d< z(PG8&8sj=?Oolt=zJa_zXiJfMY~S~A;z-uje3nD3$@SVs(no#={!+=_E;Q#t&bJ7; z$$Za$;s4n;-#-3eb-M9O_H8UIo&RcHNZZEK%fZ!-)5*p8-#I5XVaW9dH?iEzDS7|W zTQhfh`LLUVg-SS|lS?q-vB%NybSa!s^>?hXryGKae5GxHp{-tIJSvk8U8zy&pUZ%->%?h^+UF z9r!to#X}H>Cu_urljcU>3+x5CS%If(pL;ijli9qG>mMM7Miu#_mHuT@^7Rs^dy#!1 z1PQA_S8h3WQ?3yKG|&7>O6lL?;n^AAFycx`Z0pINI-($r+x>pGJd)0jhc8#=tZQZV zd=n{~M$@c_<2Xzg(j*rq7J3T{ma#piT7@ZGrRBL;SG^Y;tvo8ou{{cK>fe;#XMfP7 z6aCpT<@~eqfUuR~*WtT;57irpi{hnidDAB5m~8aB(|K9^WNj8@978ce~vZ_*-Bw5Hx zZc&Ct_zSY`sgQ-Y4qslolRCDBJT%cRdjCv--&QFxRqoj)R={JTb$^vqy;pT64UR=8 zVQb3UT>jR{U@R;r^+k~oLM&Pj_D7}RR)~nVx)a^i5{svEzPYfu{f|*cG8wFt&QTql zz;@mki#sw@lDpk&{bzZJ+dvWK6@2EdD@!FsjuRcf2t9bt1(Owfb&lff_r9gWW5J_d z1LK9!jG4GVMCTU*rUiV&hCepss6dDQxV7VpyOG*$p+gG1hTghNpjgH<*Jwd<~1?$h33*Sy$r3P}=fO(1XoDdLpBkRy0E>@I_xO%cPJK?G-OoN02P(9jQaM zmtAN3wq{|BQ5;Xw81Br=vNlqGWw)>UDX)NE`(Dk1ue9Ks4!^>IeW>7=qHB6B6}<$N zM9SLzh`pP4Fl4+^C>o_km@~TF6m-pc&oa`&v2VZD0?ifYTt@B?V8|b`6^pf9q%UOj zAr23I+a)`3vGyTjllwYntyWP z-=pBS+a0l@T&izw|1An`9P<0^e-4G;e$Ho0Z?Ig=0K9TjX< z?zQMcBOOJ@3nq#v_wu9x2o6QQ4V zpjYvtd*#iWEjSWjA%MYu45W(Q@TNT_yfZ{t;d8M6?m}7e6dY0D>YT;CwAgg-UKP3e z-wjv(VM=>WAp9!)rscm~M1O0!ubGd9hbO0pwdY?D)JCExkRn=Y9uDj*Etk|?Xqtz{#B-i=N1oK1;CZG=7a zgS&J$NwhK*m40%_?HG;w(=FYLla`Fwa~cvtmHv3;lB)2=FB+P_o$+j@ZwdB4#BNDy ztyvNfV6fo5J^M{J+W+px-x@Qiv)(rU01K;%5DSayZ;kQtb@1}C@i4P;wy|)vvHss$ z)v6OML>hzCYt6>jlE%||{v{|PckwINYd6ZG%}0u*sPK+wCiLbP1dcWFh5WkcNDQ28UT@%n z_ukn#-J|%qPfO3Ne)0Db5%-c~XDyC+uTEs$8Ef=6*H@D=0f{bqcRJ?7J_qs_BDIju zAnM))0oU{2K1u3f@J5@Z_d7Bzt-}pF2VI}_^!5J21a7=@dh%hwmHrgUYJlq=5x%KHzOpL9%+sC-mwEyURc8on($;r6v|P@fO0UcNr69nbu9 z(U%NwFu5X4li)t7wi#nM==`NwDD_x?sXL%E^EA-AgSs-V-Ls&w^7NMO)0G(ydSffv z0AAwK5EdmaFH_3Cy}XYUf}rL!>bdsi$s8m7Gkx$o%l5CO^xziJe!kCNX_Z7p8g*>~ z+xZ>V`k_}E2Yaa32JgS)tV(6YUG2BZo0?vZhS${12lpnxlv zWE{|QXluVqZ$6*GJSejsZ5v{$H@ zv{>d}8zLF=h4=$KkI1L<$_?HhDDoG(ibq$55#vl%{ug^!4X3|&$Jx8LXMhLG-`6(!W>{Wx*_hNiIWjLxUd z#M*5~XPLT8fNQPs?K=YsE%J2jx3yY|4Zx8=@>>g~k?^R6^FX74E(NI8w z!h+%td)*{CYXFw4OPf5s>?MHp4wloj+WOJPcZ(?w8WBrR+}=M=JfZ1h{iavM%3G?H zz>mpKNNuJx>9w$(mMEftSc}Z6qI7HNjRUYa=ibS?P`uCfDZsx9mdPFa{j zwU|c-2g`>0>61Ju4(&|mFA)_?ES|kt-S$)4mxsKm(=}xB5uz^;F}Esd5iU0NsEpO@ zZk?p2Y8W44qV&_3(pB{xY|zzbm+eBS<`AN?6o6N3ZcGD9=H$J6>0&st zSmFBK06#3gMni6f8|8fKcs;xb?8+R4Pun2?-x%#toPRR8%S98_v73(~zM}l>NNUe6 z49OM!B%gGzbx;ze$F1r|!cINaViM3w&dsLlAe1j(RKa-h)1FbRUeUeF8O>E%I)p@f zZY~Reb!98;zc{r5{5O&!ab}5dLH1R4u`MCDnAJR)rtXqSPM_);dibTJ&aW)alhyR7 zbcl*Ln)RPE@BYH~QCbabpppHaeF|Lwz*IC>_6)DR+`phUMJn%ZQQ|Yr%gnh3wOoBR z7iqmi@kI0k-Oy($Y4F6-FBigoJLkIyCObjfEXfYs_pWDJ!}{#;Tl~I0!$H1FM9mW6 zZDIn^ywJpR8Eqryw~LM=+qYB?Ro|%&E%8NrpFQ;WXriZ5oV?AAYf@9j*&4m~Qiz8) z5*~x@l~u)J;UQ*JzG0W5B7Rz=UnFHRt2=MYsnf5gqcg;JZ%Q!ES{}XEm^?7GAAdqr z5F+KwJr*t!eHt?DyyBOGorrG3;+@h7d{R47F!~YsBWBuf&L&nI{duX>`R%YD)=_?F zYD>n;LuZ1I``+rY?_WI;wu5w)RD=grK~*>IeDt8#(IGL4zZ|Vd=yFjh904xi-*xSY zW^wV6Q0zzQ7Q4>lx7~lhLjkIA`WWEm5M*8Dg+vXfq4S!BQ$&RdgiU8tb#fvG@#Pp4 zCDw^!RWB)+WB>);4muxG*U-xXlV(NFyi}}N13&72eke_9RSwHxw+~NT$cu|kK?U%@ ztt}HYr`B<`3_m5#w9gaQ=)?w&efhYa7?GECzeDu$jayT|i?0hY$ookB@G@UVqe}V9 zMF=Se={FgoXSfX#3vcE?ZsA<41G$1j$eop`&FTG4SEEwhT0x#4QCnvX)V{HUHoN}s zR|I}${qzqvDac(Fxx=4upVAgHTOAo2+~$w+yQ;2mJ~W%&#^1} zpu|@7HebIp+U@Q*=WykLu#AgFd$h$Yjd{vA&V%0S)Q`W+`qA`0^z3FLn@0PLkJ&?T zlBW!LwWx2^OO|1WUD?;%Y$|$ipPeJ^JBfkEgcF`L#F;}<&B-a?nqb!Dr^9gF_D=sx zHW18iF{Z_QE|t7J{N%@}`{FL$uEFHglX)p{0%^lr29~ag5<}xSl_arA{3HTt$pOT0 zqv;Moo@(gXqKyhWG<)bxDB;(GSDifnUDf zm47&nB$_2ONDgG7CERypS`YC2B&qrY_nu@r)|?j^rrTH{k;H%3&|qV&8cUkmsenE5 zA}XD;TI)sEp+}w*v9&36A&$-0s+@*Z9M2wIn~Xwue}Mgo@#pW7>-hpbp_1vu*n_cMz_r#w@TD8EyOR;5F7stQn{G=oUpDRrg#^XC?Ly z^N1;pQs5JpY_6mErXFuHg9gNw2qURJUHyOq-qXD?TU`_cK}!f|@SdCic+zIezn@Pn7doi_eQW+k9HC1KE-px2b+b7I;>M-n zF!PD(Tgn4W2MF^N2((D;FMKU33!jm{I+4{b9SGdV?E9-G_hS@V9txNm?OIg^68CB> zsX4rZ)4ok|n-sk|+w?-*)lj~5n;!f3M2)7(wD#{AgTLktZXVKq|02dhOa1oBufP;^5={UB$0|54}?xq1~JQJpSfH2dv+&=YLgk6O353p&S33 z3Qp&nK((;4vhnn^vHpvxlu#@G&Q$+s_Q!u}lGcsd-_8CaAc{9X#03G#FJ-drv>NpoL5-v3B*nZci9LF7-GX>U6C|JuNRp_!kPoAa;wDVqOn z%$)v%ewjb%kN^Lr-^JpMmz$gOUpqzOzXuX+^k+jT{Rzl}n`1=&zZ>FzfOy(lys`N! zo-b|xFNXOoOMVaQ_9KJ_wPa4}gBufs-)MjRKX`tN^o0K^p1&->^|Ze^%;aY9sO6wD z`y3Y3e*T}~`ZF2*JFee9eBsin^E_l8Ja%|&8q*a z(0|F-{O{8Lc~1Yk^xva;q~|7f{yxLM5%=2+|DV9#y&3wkuqbc7OK%Jz-S^vH{|}hJ BF?s+1 literal 0 HcmV?d00001 From 2eea8e6a46cf22273ea2f272ac87c4517ed444e1 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 12 Jun 2024 15:42:04 -0400 Subject: [PATCH 070/145] added doc and updated zip file --- ...pefiles-under-hidden-directories-in-zip.md | 5 +++++ src/test/resources/hiddenShapefiles.zip | Bin 53764 -> 54468 bytes 2 files changed, 5 insertions(+) create mode 100644 doc/release-notes/8945-ignore-shapefiles-under-hidden-directories-in-zip.md diff --git a/doc/release-notes/8945-ignore-shapefiles-under-hidden-directories-in-zip.md b/doc/release-notes/8945-ignore-shapefiles-under-hidden-directories-in-zip.md new file mode 100644 index 00000000000..145ae5f6d55 --- /dev/null +++ b/doc/release-notes/8945-ignore-shapefiles-under-hidden-directories-in-zip.md @@ -0,0 +1,5 @@ +### Shapefile Handling will now ignore files under a hidden directory within the zip file + +Directories that are hidden will be ignored when determining if a zip file contains Shapefile files. + +For more information, see #8945. \ No newline at end of file diff --git a/src/test/resources/hiddenShapefiles.zip b/src/test/resources/hiddenShapefiles.zip index 64ed1721eb20f9c5bcaff7e205ec866a967b90d2..c733a2083ae601f27a236284bacd877542e36c78 100644 GIT binary patch delta 1530 zcmZwFUuaTM90%}oy-rAm90$hGIxpM9tPPhqT~=cz*;oscE!UImGPbadw#A@_U}QuT zL3DQ@2!e!sX%MU(wlF7MLYvPFGtP>mS3 z0i}}Dn7}MDgzX@sl5|u}yRlo$XK@b<7o>4K2*c8rYgy;U0WtT|MP3Ms1wZ}83t_P^ zLT}109DxNTH^yYWPuHvxVowd{Ja_^=qvpe`e<0k|tTM{V)8FN0l@RNgNj04y$Q*Gy zZvM=x)$d%8(G{5LiTR6%Ot+) z=ch@@BkojtZM!gf@2lKuH>{<)Vbq*GPFK<<{%y5_&ZoVEp#Mz{e9Mr$O$UfwLw-QQ zcaNUPxQM&PBo30Xi~O%~9KNF6^hL%=dNkd#kS$0Z)R=XWK@IEGfo^7A^L1T<_# zWL~P_Es;o_R8HyFgHB5B9w2)LfrRLAzK=|3Iz1WyOwHd1 zWcaE;M(NTv?DS+tU?F;V#{=eCjRU6Sg47Xjp=`%le{uC&_P+*H&4=hv!9$eV5vuIV z8f1kB#L{reMC@Ag&sZV$%CLcv$NS`<6icC-|v0%92(sebw6b`yI!HLlBkIUYF3ITZB(cyiDFAl(+-+a;3rL~HuRn&6_&oJByL0f zqLGS&xUZ6w4ZYxqejtj{HsqCzR2nYpDw(nn+pyF|4Sl2sMV)4S2O7ESr7dSL@V1)x1Jv%SNbNJYx2p<~Z?n*_C2zgWB886bahw Date: Wed, 12 Jun 2024 17:09:08 -0400 Subject: [PATCH 071/145] add sample data, artwork, other --- doc/sphinx-guides/source/contributor/index.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/sphinx-guides/source/contributor/index.md b/doc/sphinx-guides/source/contributor/index.md index 5168b259d25..83a2f59003c 100644 --- a/doc/sphinx-guides/source/contributor/index.md +++ b/doc/sphinx-guides/source/contributor/index.md @@ -67,6 +67,10 @@ People come along with questions on the [mailing list](https://groups.google.com [chat]: https://chat.dataverse.org +## Sample Data + +Consider contributing to , a git repo of realistic-looking data that is used for testing. + ## Issue Triage New issues are coming in all the time, especially for the main issue tracker at . @@ -82,3 +86,17 @@ For non-recorded talks, we are happy to upload your slides to . + +## Artwork + +Surely we can put artistic talent to use. See for a story about drawing animals chatting about Dataverse. + +As [annnounced](https://groups.google.com/g/dataverse-community/c/pM39_9O5Rug/m/CK-gJqZFBgAJ), we have a [sticker template](https://dataverse.org/sites/projects.iq.harvard.edu/files/dataverseorg/files/dataverse_community_stickers_template.zip) you can use. + +See [Illustrations from The Turing Way](https://zenodo.org/doi/10.5281/zenodo.3332807) for how that community has created artwork. Perhaps we can do the same. + +## Other Contributions + +We consulted but no list is comprehensive. + +What else should we list here? How to YOU want to contribute to Dataverse? 🎉 From 0b7cb9f354574064441b087732c12e5cfe0a9495 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 12 Jun 2024 17:25:55 -0400 Subject: [PATCH 072/145] move documentation into own sub page of contrib guide #10531 --- .../source/contributor/documentation.md | 28 +++++++++++++++++++ doc/sphinx-guides/source/contributor/index.md | 15 ++++------ .../source/developers/documentation.rst | 2 ++ 3 files changed, 35 insertions(+), 10 deletions(-) create mode 100644 doc/sphinx-guides/source/contributor/documentation.md diff --git a/doc/sphinx-guides/source/contributor/documentation.md b/doc/sphinx-guides/source/contributor/documentation.md new file mode 100644 index 00000000000..404f26982d1 --- /dev/null +++ b/doc/sphinx-guides/source/contributor/documentation.md @@ -0,0 +1,28 @@ +# Writing Documentation + +Thank you for your interest in contributing documentation to Dataverse! Good documentation is absolutely critical to the success of a software project. + +```{contents} Contents: +:local: +:depth: 3 +``` + +## Where the Source Files Live + +The source for the documentation is in the main "dataverse" GitHub repo under the "[doc][]" folder. + +[doc]: https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source + +## Tools + +The {doc}`/developers/documentation` section of the Developer Guide has, for a long time, been the primary place to find information about the tools we use to write docs. We may move some of that content here, but until then, please visit that page. + +## Writing Style Guidelines + +Please observe the following when writing documentation: + +- Use American English spelling. +- Use examples when possible. +- Break up longer paragraphs. +- Use "double quotes" instead of 'single quotes'. +- Favor "and" (data and code) over slashes (data/code). diff --git a/doc/sphinx-guides/source/contributor/index.md b/doc/sphinx-guides/source/contributor/index.md index 83a2f59003c..c13f6765a38 100644 --- a/doc/sphinx-guides/source/contributor/index.md +++ b/doc/sphinx-guides/source/contributor/index.md @@ -30,17 +30,12 @@ If you do not receive a reply to your new issue or comment in a timely manner, p ## Documentation -The source for the documentation is in the GitHub repo under the "[doc][]" folder. If you find a typo or inaccuracy or something to clarify, please send us a pull request! For more on the tools used to write docs, please see the {doc}`/developers/documentation` section of the Developer Guide. +Documentation is such a large topic (and important!) that we have a dedicate section on it: -Please observe the following when writing documentation: - -- Use American English spelling. -- Use examples when possible. -- Break up longer paragraphs. -- Use "double quotes" instead of 'single quotes'. -- Favor "and" (data and code) over slashes (data/code). - -[doc]: https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source +```{toctree} +:maxdepth: 1 +documentation.md +``` ## Translations diff --git a/doc/sphinx-guides/source/developers/documentation.rst b/doc/sphinx-guides/source/developers/documentation.rst index 477776d7ee2..f0ee2dcabf1 100755 --- a/doc/sphinx-guides/source/developers/documentation.rst +++ b/doc/sphinx-guides/source/developers/documentation.rst @@ -2,6 +2,8 @@ Writing Documentation ===================== +Note: in the Contributor Guide, there is a section called :doc:`/contributor/documentation`. + .. contents:: |toctitle| :local: From da5b10b40ab3b364218c914ca9586eae888ce809 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 12 Jun 2024 17:29:44 -0400 Subject: [PATCH 073/145] typo - released logic backwards --- .../java/edu/harvard/iq/dataverse/search/IndexServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index d2aa52ce395..98bcdf385fc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -2099,7 +2099,7 @@ public List findPermissionsInSolrOnly() throws SearchException { permissionInSolrOnly.add(docId); } } else { - if (d.getReleasedVersion() != null) { + if (d.getReleasedVersion() == null) { permissionInSolrOnly.add(docId); } } From 04580a3f4403d1a7dbf98e54cae411ff9ff5f9eb Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 12 Jun 2024 17:30:10 -0400 Subject: [PATCH 074/145] add dataset doc cleanup, fix looping error --- .../iq/dataverse/search/IndexServiceBean.java | 51 +++++++++++++------ 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 98bcdf385fc..01a6039b91b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -492,14 +492,12 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr boolean atLeastOnePublishedVersion = false; if (releasedVersion != null) { atLeastOnePublishedVersion = true; - } else { - atLeastOnePublishedVersion = false; } - List solrIdsOfFilesToDelete = null; + List solrIdsOfDocsToDelete = null; try { - solrIdsOfFilesToDelete = findFilesOfParentDataset(dataset.getId()); - logger.fine("Existing file docs: " + String.join(", ", solrIdsOfFilesToDelete)); + solrIdsOfDocsToDelete = findFilesOfParentDataset(dataset.getId()); + logger.fine("Existing file docs: " + String.join(", ", solrIdsOfDocsToDelete)); //We keep the latest version's docs unless it is deaccessioned and there is no published/released version //So skip the loop removing those docs from the delete list except in that case if ((!latestVersion.isDeaccessioned() || atLeastOnePublishedVersion)) { @@ -507,20 +505,36 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr String suffix = (new IndexableDataset(latestVersion)).getDatasetState().getSuffix(); for (FileMetadata fileMetadata : latestFileMetadatas) { String solrIdOfPublishedFile = solrDocIdentifierFile + fileMetadata.getDataFile().getId() + suffix; - solrIdsOfFilesToDelete.remove(solrIdOfPublishedFile); + solrIdsOfDocsToDelete.remove(solrIdOfPublishedFile); } } if (releasedVersion != null && !releasedVersion.equals(latestVersion)) { List releasedFileMetadatas = releasedVersion.getFileMetadatas(); for (FileMetadata fileMetadata : releasedFileMetadatas) { String solrIdOfPublishedFile = solrDocIdentifierFile + fileMetadata.getDataFile().getId(); - solrIdsOfFilesToDelete.remove(solrIdOfPublishedFile); + solrIdsOfDocsToDelete.remove(solrIdOfPublishedFile); } } + //Clear any unused dataset docs + if (!latestVersion.isDraft()) { + // The latest version is released, so should delete any draft docs for the + // dataset + solrIdsOfDocsToDelete.add(solrDocIdentifierDataset + dataset.getId() + draftSuffix); + } + if (!atLeastOnePublishedVersion) { + // There's no released version, so should delete any normal state docs for the + // dataset + solrIdsOfDocsToDelete.add(solrDocIdentifierDataset + dataset.getId()); + } + if (atLeastOnePublishedVersion || !latestVersion.isDeaccessioned()) { + // There's a released version or a draft, so should delete any deaccessioned + // state docs for the dataset + solrIdsOfDocsToDelete.add(solrDocIdentifierDataset + dataset.getId() + deaccessionedSuffix); + } } catch (SearchException | NullPointerException ex) { logger.fine("could not run search of files to delete: " + ex); } - logger.fine("File docs to delete: " + String.join(", ", solrIdsOfFilesToDelete)); + logger.fine("Solr docs to delete: " + String.join(", ", solrIdsOfDocsToDelete)); int numPublishedVersions = 0; List versions = dataset.getVersions(); //List solrIdsOfFilesToDelete = new ArrayList<>(); @@ -585,12 +599,17 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr } debug.append("numPublishedVersions: " + numPublishedVersions + "\n"); if (doNormalSolrDocCleanUp) { - if(!solrIdsOfFilesToDelete.isEmpty()) { - for(String file: solrIdsOfFilesToDelete) { + + if(!solrIdsOfDocsToDelete.isEmpty()) { + List solrIdsOfPermissionDocsToDelete = new ArrayList<>(); + for(String file: solrIdsOfDocsToDelete) { //Also remove associated permission docs - solrIdsOfFilesToDelete.add(file+"_permission"); + solrIdsOfPermissionDocsToDelete.add(file + discoverabilityPermissionSuffix); } - IndexResponse resultOfAttemptToPremptivelyDeletePublishedFiles = solrIndexService.deleteMultipleSolrIds(solrIdsOfFilesToDelete); + solrIdsOfDocsToDelete.addAll(solrIdsOfPermissionDocsToDelete); + logger.fine("Solr docs and perm docs to delete: " + String.join(", ", solrIdsOfDocsToDelete)); + + IndexResponse resultOfAttemptToPremptivelyDeletePublishedFiles = solrIndexService.deleteMultipleSolrIds(solrIdsOfDocsToDelete); debug.append("result of attempt to premptively deleted published files before reindexing: " + resultOfAttemptToPremptivelyDeletePublishedFiles + "\n"); } } @@ -2118,9 +2137,11 @@ public List findPermissionsInSolrOnly() throws SearchException { } else { if (!states.contains(VersionState.RELEASED)) { permissionInSolrOnly.add(docId); - } else if (!dataFileService.findMostRecentVersionFileIsIn(f).getDatasetVersion() - .equals(f.getOwner().getReleasedVersion())) { - permissionInSolrOnly.add(docId); + } else { + if(dataFileService.findFileMetadataByDatasetVersionIdAndDataFileId(f.getOwner().getReleasedVersion().getId(), f.getId()) == null) { + logger.info("Adding doc " + docId + " to list of permissions in Solr only"); + permissionInSolrOnly.add(docId); + } } } From c16fdd5396fab0f7890d221965a632889046467f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 12 Jun 2024 17:44:48 -0400 Subject: [PATCH 075/145] Fix to version check This is only used in determining the most recent version a dataset is in on the file page, e.g. for https://demo.dataverse.org/file.xhtml ?persistentId=doi:10.70122/FK2/FO0MPQ/KNG6PA&version=3.0 I confirmed that demo shows version 1 in this example whereas it should show version 2 (which this commit fixes). --- .../java/edu/harvard/iq/dataverse/DataFileServiceBean.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java index 7f38107af6b..21f925f8981 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java @@ -384,7 +384,8 @@ public FileMetadata findMostRecentVersionFileIsIn(DataFile file) { if (fileMetadatas == null || fileMetadatas.isEmpty()) { return null; } else { - return fileMetadatas.get(0); + // This assumes the order of filemetadatas is from first to most recent, which is true as of v6.3 + return fileMetadatas.get(fileMetadatas.size() - 1); } } From 1b3e312572ef922572fc4df20e121169de2b929e Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 12 Jun 2024 19:36:44 -0400 Subject: [PATCH 076/145] Update src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java Co-authored-by: Philip Durbin --- .../java/edu/harvard/iq/dataverse/util/ShapefileHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java b/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java index 0c77e33712b..f1440cc3c02 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java @@ -710,7 +710,7 @@ private boolean examineZipfile(FileInputStream zip_file_stream){ // check if path is hidden if (isDirectory && Files.isHidden(Paths.get(zentryFileName))) { hiddenDirectories.add(zentryFileName); - logger.info("Ignoring files under hidden directory: " + zentryFileName); + logger.fine("Ignoring files under hidden directory: " + zentryFileName); } else { // check if the path was already found to be hidden for (String hidden : hiddenDirectories) { From fb461c77d9a0929f992cede57e40bc1ebe683465 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 13 Jun 2024 09:32:08 -0400 Subject: [PATCH 077/145] shorted to just "code" #10531 --- doc/sphinx-guides/source/contributor/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/contributor/index.md b/doc/sphinx-guides/source/contributor/index.md index c13f6765a38..e603d6151b7 100644 --- a/doc/sphinx-guides/source/contributor/index.md +++ b/doc/sphinx-guides/source/contributor/index.md @@ -41,7 +41,7 @@ documentation.md If you speak multiple languages, you are very welcome to help us translate Dataverse! Please see {ref}`help-translate` for details. -## Code and Pull Requests +## Code Dataverse is open source and we love code contributions. Developers are not limited to the main Dataverse code in this git repo. We have projects in C, C++, Go, Java, Javascript, Julia, PHP, Python, R, Ruby, TypeScript and more. To get started, please see the following pages: From b0808424de5ed85fe5e6ecd7d2e788ae027c0ab0 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 13 Jun 2024 09:46:24 -0400 Subject: [PATCH 078/145] link to new "writing docs" under contrib guide #10531 --- doc/sphinx-guides/source/admin/integrations.rst | 2 +- doc/sphinx-guides/source/api/faq.rst | 4 ++-- doc/sphinx-guides/source/installation/config.rst | 2 +- doc/sphinx-guides/source/installation/intro.rst | 2 +- doc/sphinx-guides/source/qa/testing-infrastructure.md | 2 +- doc/sphinx-guides/source/versions.rst | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/sphinx-guides/source/admin/integrations.rst b/doc/sphinx-guides/source/admin/integrations.rst index e48d7a60798..7a9b9c92f77 100644 --- a/doc/sphinx-guides/source/admin/integrations.rst +++ b/doc/sphinx-guides/source/admin/integrations.rst @@ -290,4 +290,4 @@ If you have an idea for an integration, please ask on the `dataverse-community < Many integrations take the form of "external tools". See the :doc:`external-tools` section for details. External tool makers should check out the :doc:`/api/external-tools` section of the API Guide. -Please help us keep this page up to date making a pull request! To get started, see the :doc:`/developers/documentation` section of the Developer Guide. +Please help us keep this page up to date making a pull request! To get started, see the :doc:`/contributor/documentation` section of the Contributor Guide. diff --git a/doc/sphinx-guides/source/api/faq.rst b/doc/sphinx-guides/source/api/faq.rst index 439783779c3..c335b30a0e0 100644 --- a/doc/sphinx-guides/source/api/faq.rst +++ b/doc/sphinx-guides/source/api/faq.rst @@ -82,12 +82,12 @@ The following tasks cannot currently be automated via API because no API exists If you would like APIs for any of the features above, please open a GitHub issue at https://github.com/IQSS/dataverse/issues -You are also welcome to open an issue to add to the list above. Or you are welcome to make a pull request. Please see the :doc:`/developers/documentation` section of the Developer Guide for instructions. +You are also welcome to open an issue to add to the list above. Or you are welcome to make a pull request. Please see the :doc:`/contributor/documentation` section of the Contributor Guide for instructions. Why Are the Return Values (HTTP Status Codes) Not Documented? ------------------------------------------------------------- -They should be. Please consider making a pull request to help. The :doc:`/developers/documentation` section of the Developer Guide should help you get started. :ref:`create-dataverse-api` has an example you can follow or you can come up with a better way. +They should be. Please consider making a pull request to help. The :doc:`/contributor/documentation` section of the Contriburor Guide should help you get started. :ref:`create-dataverse-api` has an example you can follow or you can come up with a better way. Also, please note that we are starting to experiment with putting response codes in our OpenAPI document. See :ref:`openapi`. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 6d0da625a5a..e2f7048b6ff 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3723,7 +3723,7 @@ Note: by default, the URL is composed from the settings ``:GuidesBaseUrl`` and ` :GuidesBaseUrl ++++++++++++++ -Set ``:GuidesBaseUrl`` to override the default value "https://guides.dataverse.org". If you are interested in writing your own version of the guides, you may find the :doc:`/developers/documentation` section of the Developer Guide helpful. +Set ``:GuidesBaseUrl`` to override the default value "https://guides.dataverse.org". If you are interested in writing your own version of the guides, you may find the :doc:`/contributor/documentation` section of the Contributor Guide helpful. ``curl -X PUT -d http://dataverse.example.edu http://localhost:8080/api/admin/settings/:GuidesBaseUrl`` diff --git a/doc/sphinx-guides/source/installation/intro.rst b/doc/sphinx-guides/source/installation/intro.rst index 6d77a1209b2..1c239863e98 100644 --- a/doc/sphinx-guides/source/installation/intro.rst +++ b/doc/sphinx-guides/source/installation/intro.rst @@ -53,6 +53,6 @@ If you've encountered a problem installing Dataverse and are ready to ask for he Improving this Guide -------------------- -If you spot a typo in this guide or would like to suggest an improvement, please find the appropriate file in https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source/installation and send a pull request as explained in the :doc:`/developers/documentation` section of the Developer Guide. You are also welcome to simply open an issue at https://github.com/IQSS/dataverse/issues to describe the problem with this guide. +If you spot a typo in this guide or would like to suggest an improvement, please find the appropriate file in https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source/installation and send a pull request as explained in the :doc:`/contributor/documentation` section of the Contributor Guide. You are also welcome to simply open an issue at https://github.com/IQSS/dataverse/issues to describe the problem with this guide. Next is the :doc:`prep` section. diff --git a/doc/sphinx-guides/source/qa/testing-infrastructure.md b/doc/sphinx-guides/source/qa/testing-infrastructure.md index 804e4c0afe6..6ec26c6da49 100644 --- a/doc/sphinx-guides/source/qa/testing-infrastructure.md +++ b/doc/sphinx-guides/source/qa/testing-infrastructure.md @@ -39,7 +39,7 @@ To build and test a PR, we use a job called `IQSS_Dataverse_Internal` on . This Read the Docs preview is also mentioned under also {doc}`/developers/documentation`. +Note that changes to guides can also be previewed on Read the Docs. In the pull request, look for a link like . This Read the Docs preview is also mentioned under also {doc}`/contributor/documentation`. ## Other Servers diff --git a/doc/sphinx-guides/source/versions.rst b/doc/sphinx-guides/source/versions.rst index d76f9a889cb..850702d823e 100755 --- a/doc/sphinx-guides/source/versions.rst +++ b/doc/sphinx-guides/source/versions.rst @@ -6,7 +6,7 @@ Dataverse Software Documentation Versions This list provides a way to refer to the documentation for previous and future versions of the Dataverse Software. In order to learn more about the updates delivered from one version to another, visit the `Releases `__ page in our GitHub repo. -- pre-release `HTML (not final!) `__ and `PDF (experimental!) `__ built from the :doc:`develop ` branch :doc:`(how to contribute!) ` +- pre-release `HTML (not final!) `__ and `PDF (experimental!) `__ built from the :doc:`develop ` branch :doc:`(how to contribute!) ` - 6.2 - `6.1 `__ - `6.0 `__ From 55ce252b047a793c900a4844e7922eca83daf83e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 13 Jun 2024 10:25:32 -0400 Subject: [PATCH 079/145] move toc #10531 --- .../source/contributor/documentation.md | 16 ++++++++++++++++ .../source/developers/documentation.rst | 17 ----------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/doc/sphinx-guides/source/contributor/documentation.md b/doc/sphinx-guides/source/contributor/documentation.md index 404f26982d1..a5f5bc2d93e 100644 --- a/doc/sphinx-guides/source/contributor/documentation.md +++ b/doc/sphinx-guides/source/contributor/documentation.md @@ -26,3 +26,19 @@ Please observe the following when writing documentation: - Break up longer paragraphs. - Use "double quotes" instead of 'single quotes'. - Favor "and" (data and code) over slashes (data/code). + +## Table of Contents + +Every non-index page should use the following code to display a table of contents of internal sub-headings. This code should be placed below any introductory text and directly above the first subheading, much like a Wikipedia page. + +If the page is written in reStructuredText (.rst), use this form: + + .. contents:: |toctitle| + :local: + +If the page is written in Markdown (.md), use this form: + + ```{contents} Contents: + :local: + :depth: 3 + ``` diff --git a/doc/sphinx-guides/source/developers/documentation.rst b/doc/sphinx-guides/source/developers/documentation.rst index f0ee2dcabf1..c90b46a4ce9 100755 --- a/doc/sphinx-guides/source/developers/documentation.rst +++ b/doc/sphinx-guides/source/developers/documentation.rst @@ -106,23 +106,6 @@ You can click on the files in the ``html`` folder to preview the changes. Now you can make a commit with the changes to your own fork in GitHub and submit a pull request. See :ref:`how-to-make-a-pull-request`. -Table of Contents ------------------ - -Every non-index page should use the following code to display a table of contents of internal sub-headings: :: - - .. contents:: |toctitle| - :local: - -This code should be placed below any introductory text/images and directly above the first subheading, much like a Wikipedia page. - -If you are using Markdown, you should use this form instead: :: - - ```{contents} Contents: - :local: - :depth: 3 - ``` - Images ------ From c243a9ca325fa35ccd4147d8df6c931efa7c58fd Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 13 Jun 2024 10:30:33 -0400 Subject: [PATCH 080/145] move images and cross references #10531 --- .../source/contributor/documentation.md | 12 ++++++++++++ .../source/developers/documentation.rst | 18 ------------------ 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/doc/sphinx-guides/source/contributor/documentation.md b/doc/sphinx-guides/source/contributor/documentation.md index a5f5bc2d93e..a659e4467dc 100644 --- a/doc/sphinx-guides/source/contributor/documentation.md +++ b/doc/sphinx-guides/source/contributor/documentation.md @@ -42,3 +42,15 @@ If the page is written in Markdown (.md), use this form: :local: :depth: 3 ``` + +## Images + +A good documentation is just like a website enhanced and upgraded by adding high quality and self-explanatory images. Often images depict a lot of written text in a simple manner. Within our Sphinx docs, you can add them in two ways: a) add a PNG image directly and include or b) use inline description languages like GraphViz (current only option). + +While PNGs in the git repo can be linked directly via URL, Sphinx-generated images do not need a manual step and might provide higher visual quality. Especially in terms of quality of content, generated images can be extendend and improved by a textbased and reviewable commit, without needing raw data or source files and no diff around. + +TODO: The above covers "how" but what about "when"? + +## Cross References + +When adding ReStructured Text (.rst) [cross references](https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#ref-role), use the hyphen character (`-`) as the word separator for the cross reference label. For example, `my-reference-label` would be the preferred label for a cross reference as opposed to, for example, `my_reference_label`. diff --git a/doc/sphinx-guides/source/developers/documentation.rst b/doc/sphinx-guides/source/developers/documentation.rst index c90b46a4ce9..8ff8d1477d8 100755 --- a/doc/sphinx-guides/source/developers/documentation.rst +++ b/doc/sphinx-guides/source/developers/documentation.rst @@ -106,24 +106,6 @@ You can click on the files in the ``html`` folder to preview the changes. Now you can make a commit with the changes to your own fork in GitHub and submit a pull request. See :ref:`how-to-make-a-pull-request`. -Images ------- - -A good documentation is just like a website enhanced and upgraded by adding high quality and self-explanatory images. -Often images depict a lot of written text in a simple manner. Within our Sphinx docs, you can add them in two ways: a) add a -PNG image directly and include or b) use inline description languages like GraphViz (current only option). - -While PNGs in the git repo can be linked directly via URL, Sphinx-generated images do not need a manual step and might -provide higher visual quality. Especially in terms of quality of content, generated images can be extendend and improved -by a textbased and reviewable commit, without needing raw data or source files and no diff around. - -TODO: The above covers "how" but what about "when"? - -Cross References ----------------- - -**NOTE:** When adding ReStructured Text (RST) `cross references `_, use the hyphen character (``-``) as the word separator for the cross reference label. For example, ``my-reference-label`` would be the preferred label for a cross reference as opposed to, for example, ``my_reference_label``. - Versions -------- From a970df20a5a8146d7ce5bb2b5679f9787d16343b Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 13 Jun 2024 10:51:53 -0400 Subject: [PATCH 081/145] move writing docs page from dev to contrib guide #10531 --- .../source/contributor/documentation.md | 115 ++++++++++++++- .../source/developers/documentation.rst | 137 ------------------ doc/sphinx-guides/source/developers/index.rst | 1 - 3 files changed, 110 insertions(+), 143 deletions(-) delete mode 100755 doc/sphinx-guides/source/developers/documentation.rst diff --git a/doc/sphinx-guides/source/contributor/documentation.md b/doc/sphinx-guides/source/contributor/documentation.md index a659e4467dc..e0b83560870 100644 --- a/doc/sphinx-guides/source/contributor/documentation.md +++ b/doc/sphinx-guides/source/contributor/documentation.md @@ -7,15 +7,120 @@ Thank you for your interest in contributing documentation to Dataverse! Good doc :depth: 3 ``` -## Where the Source Files Live +## Quick Fix -The source for the documentation is in the main "dataverse" GitHub repo under the "[doc][]" folder. +If you find a typo or a small error in the documentation you can fix it using GitHub's online web editor. Generally speaking, we will be following -[doc]: https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source +- Navigate to where you will see folders for each of the guides: [admin][], [api][], [container][], etc. +- Find the file you want to edit under one of the folders above. +- Click the pencil icon in the upper-right corner. If this is your first contribution to the Dataverse Project, the hover text over the pencil icon will say "Fork this project and edit this file". +- Make changes to the file and preview them. +- In the **Commit changes** box, enter a description of the changes you have made and click **Propose file change**. +- Under the **Write** tab, delete the long welcome message and write a few words about what you fixed. +- Click **Create Pull Request**. -## Tools +That's it! Thank you for your contribution! Your pull request will be added manually to the main Dataverse Project board at and will go through code review and QA before it is merged into the "develop" branch. Along the way, developers might suggest changes or make them on your behalf. Once your pull request has been merged you will be listed as a contributor at ! 🎉 -The {doc}`/developers/documentation` section of the Developer Guide has, for a long time, been the primary place to find information about the tools we use to write docs. We may move some of that content here, but until then, please visit that page. +Please see for an example of a quick fix that was merged (the "Files changed" tab shows how a typo was fixed). + +Preview your documentation changes which will be built automatically as part of your pull request in Github. It will show up as a check entitled: `docs/readthedocs.org:dataverse-guide — Read the Docs build succeeded!`. For example, this PR built to . + +If you would like to read more about the Dataverse Project's use of GitHub, please see the {doc}`/developers/version-control` section of the Developer Guide. For bug fixes and features we request that you create an issue before making a pull request but this is not at all necessary for quick fixes to the documentation. + +[admin]: https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source/admin +[api]: https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source/api +[container]: https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source/container + +## Building the Guides with Sphinx + +The Dataverse guides are written using Sphinx (). We recommend installing Sphinx on your computer or using a Sphinx Docker container to build the guides locally so you can get an accurate preview of your changes. + +In case you decide to use a Sphinx Docker container to build the guides, you can skip the next two installation sections, but you will need to have Docker installed. + +### Installing Sphinx + +First, make a fork of https://github.com/IQSS/dataverse and clone your fork locally. Then change to the ``doc/sphinx-guides`` directory. + +``cd doc/sphinx-guides`` + +Create a Python virtual environment, activate it, then install dependencies: + +``python3 -m venv venv`` + +``source venv/bin/activate`` + +``pip install -r requirements.txt`` + +### Installing GraphViz + +In some parts of the documentation, graphs are rendered as images using the Sphinx GraphViz extension. + +Building the guides requires the ``dot`` executable from GraphViz. + +This requires having `GraphViz `_ installed and either having ``dot`` on the path or +`adding options to the make call `_. + +### Editing and Building the Guides + +To edit the existing documentation: + +- Create a branch (see :ref:`how-to-make-a-pull-request`). +- In ``doc/sphinx-guides/source`` you will find the .rst files that correspond to https://guides.dataverse.org. +- Using your preferred text editor, open and edit the necessary files, or create new ones. + +Once you are done, you can preview the changes by building the guides locally. As explained, you can build the guides with Sphinx locally installed, or with a Docker container. + +#### Building the Guides with Sphinx Locally Installed + +Open a terminal, change directories to `doc/sphinx-guides`, activate (or reactivate) your Python virtual environment, and build the guides. + +`cd doc/sphinx-guides` + +`source venv/bin/activate` + +`make clean` + +`make html` + +#### Building the Guides with a Sphinx Docker Container + +If you want to build the guides using a Docker container, execute the following command in the repository root: + +`docker run -it --rm -v $(pwd):/docs sphinxdoc/sphinx:3.5.4 bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make html"` + +#### Previewing the Guides + +After Sphinx is done processing the files you should notice that the `html` folder in `doc/sphinx-guides/build` directory has been updated. You can click on the files in the `html` folder to preview the changes. + +Now you can make a commit with the changes to your own fork in GitHub and submit a pull request. See {ref}`how-to-make-a-pull-request`. + +## Versions + +For installations hosting their own copies of the guides, note that as each version of the Dataverse Software is released, there is an updated version of the guides released with it. Google and other search engines index all versions, which may confuse users who discover your guides in the search results as to which version they should be looking at. When learning about your installation from the search results, it is best to be viewing the *latest* version. + +In order to make it clear to the crawlers that we only want the latest version discoverable in their search results, we suggest adding this to your `robots.txt` file: + + User-agent: * + Allow: /en/latest/ + Disallow: /en/ + +## PDF Version of the Guides + +The HTML version of the guides is the official one. Any other formats are maintained on a best effort basis. + +If you would like to build a PDF version of the guides and have Docker installed, please try the command below from the root of the git repo: + +`docker run -it --rm -v $(pwd):/docs sphinxdoc/sphinx-latexpdf:3.5.4 bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make latexpdf LATEXMKOPTS=\"-interaction=nonstopmode\"; cd ../.. && ls -1 doc/sphinx-guides/build/latex/Dataverse.pdf"` + +A few notes about the command above: + +- Hopefully the PDF was created at `doc/sphinx-guides/build/latex/Dataverse.pdf`. +- For now, we are using "nonstopmode" but this masks some errors. +- See requirements.txt for a note regarding the version of Sphinx we are using. + +Also, as of this writing we have enabled PDF builds from the "develop" branch. You download the PDF from + +If you would like to help improve the PDF version of the guides, please get in touch! Please see {ref}`getting-help-developers` for ways to contact the developer community. ## Writing Style Guidelines diff --git a/doc/sphinx-guides/source/developers/documentation.rst b/doc/sphinx-guides/source/developers/documentation.rst deleted file mode 100755 index 8ff8d1477d8..00000000000 --- a/doc/sphinx-guides/source/developers/documentation.rst +++ /dev/null @@ -1,137 +0,0 @@ -===================== -Writing Documentation -===================== - -Note: in the Contributor Guide, there is a section called :doc:`/contributor/documentation`. - -.. contents:: |toctitle| - :local: - -Quick Fix ------------ - -If you find a typo or a small error in the documentation you can fix it using GitHub's online web editor. Generally speaking, we will be following https://docs.github.com/en/repositories/working-with-files/managing-files/editing-files#editing-files-in-another-users-repository - -- Navigate to https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source where you will see folders for each of the guides: `admin`_, `api`_, `developers`_, `installation`_, `style`_, `user`_. -- Find the file you want to edit under one of the folders above. -- Click the pencil icon in the upper-right corner. If this is your first contribution to the Dataverse Project, the hover text over the pencil icon will say "Fork this project and edit this file". -- Make changes to the file and preview them. -- In the **Commit changes** box, enter a description of the changes you have made and click **Propose file change**. -- Under the **Write** tab, delete the long welcome message and write a few words about what you fixed. -- Click **Create Pull Request**. - -That's it! Thank you for your contribution! Your pull request will be added manually to the main Dataverse Project board at https://github.com/orgs/IQSS/projects/34 and will go through code review and QA before it is merged into the "develop" branch. Along the way, developers might suggest changes or make them on your behalf. Once your pull request has been merged you will be listed as a contributor at https://github.com/IQSS/dataverse/graphs/contributors - -Please see https://github.com/IQSS/dataverse/pull/5857 for an example of a quick fix that was merged (the "Files changed" tab shows how a typo was fixed). - -Preview your documentation changes which will be built automatically as part of your pull request in Github. It will show up as a check entitled: `docs/readthedocs.org:dataverse-guide — Read the Docs build succeeded!`. For example, this PR built to https://dataverse-guide--9249.org.readthedocs.build/en/9249/. - -If you would like to read more about the Dataverse Project's use of GitHub, please see the :doc:`version-control` section. For bug fixes and features we request that you create an issue before making a pull request but this is not at all necessary for quick fixes to the documentation. - -.. _admin: https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source/admin -.. _api: https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source/api -.. _developers: https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source/developers -.. _installation: https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source/installation -.. _style: https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source/style -.. _user: https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source/user - -Building the Guides with Sphinx -------------------------------- - -The Dataverse guides are written using Sphinx (https://sphinx-doc.org). We recommend installing Sphinx on your localhost or using a Sphinx Docker container to build the guides locally so you can get an accurate preview of your changes. - -In case you decide to use a Sphinx Docker container to build the guides, you can skip the next two installation sections, but you will need to have Docker installed. - -Installing Sphinx -~~~~~~~~~~~~~~~~~ - -First, make a fork of https://github.com/IQSS/dataverse and clone your fork locally. Then change to the ``doc/sphinx-guides`` directory. - -``cd doc/sphinx-guides`` - -Create a Python virtual environment, activate it, then install dependencies: - -``python3 -m venv venv`` - -``source venv/bin/activate`` - -``pip install -r requirements.txt`` - -Installing GraphViz -~~~~~~~~~~~~~~~~~~~ - -In some parts of the documentation, graphs are rendered as images using the Sphinx GraphViz extension. - -Building the guides requires the ``dot`` executable from GraphViz. - -This requires having `GraphViz `_ installed and either having ``dot`` on the path or -`adding options to the make call `_. - -Editing and Building the Guides -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To edit the existing documentation: - -- Create a branch (see :ref:`how-to-make-a-pull-request`). -- In ``doc/sphinx-guides/source`` you will find the .rst files that correspond to https://guides.dataverse.org. -- Using your preferred text editor, open and edit the necessary files, or create new ones. - -Once you are done, you can preview the changes by building the guides locally. As explained, you can build the guides with Sphinx locally installed, or with a Docker container. - -Building the Guides with Sphinx Locally Installed -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Open a terminal, change directories to ``doc/sphinx-guides``, activate (or reactivate) your Python virtual environment, and build the guides. - -``cd doc/sphinx-guides`` - -``source venv/bin/activate`` - -``make clean`` - -``make html`` - -Building the Guides with a Sphinx Docker Container -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If you want to build the guides using a Docker container, execute the following command in the repository root: - -``docker run -it --rm -v $(pwd):/docs sphinxdoc/sphinx:3.5.4 bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make html"`` - -Previewing the Guides -^^^^^^^^^^^^^^^^^^^^^ - -After Sphinx is done processing the files you should notice that the ``html`` folder in ``doc/sphinx-guides/build`` directory has been updated. -You can click on the files in the ``html`` folder to preview the changes. - -Now you can make a commit with the changes to your own fork in GitHub and submit a pull request. See :ref:`how-to-make-a-pull-request`. - -Versions --------- - -For installations hosting their own copies of the guides, note that as each version of the Dataverse Software is released, there is an updated version of the guides released with it. Google and other search engines index all versions, which may confuse users who discover your guides in the search results as to which version they should be looking at. When learning about your installation from the search results, it is best to be viewing the *latest* version. - -In order to make it clear to the crawlers that we only want the latest version discoverable in their search results, we suggest adding this to your ``robots.txt`` file:: - - User-agent: * - Allow: /en/latest/ - Disallow: /en/ - -PDF Version of the Guides -------------------------- - -The HTML version of the guides is the official one. Any other formats are maintained on a best effort basis. - -If you would like to build a PDF version of the guides and have Docker installed, please try the command below from the root of the git repo: - -``docker run -it --rm -v $(pwd):/docs sphinxdoc/sphinx-latexpdf:3.5.4 bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make latexpdf LATEXMKOPTS=\"-interaction=nonstopmode\"; cd ../.. && ls -1 doc/sphinx-guides/build/latex/Dataverse.pdf"`` - -A few notes about the command above: - -- Hopefully the PDF was created at ``doc/sphinx-guides/build/latex/Dataverse.pdf``. -- For now, we are using "nonstopmode" but this masks some errors. -- See requirements.txt for a note regarding the version of Sphinx we are using. - -Also, as of this writing we have enabled PDF builds from the "develop" branch. You download the PDF from http://preview.guides.gdcc.io/_/downloads/en/develop/pdf/ - -If you would like to help improve the PDF version of the guides, please get in touch! Please see :ref:`getting-help-developers` for ways to contact the developer community. diff --git a/doc/sphinx-guides/source/developers/index.rst b/doc/sphinx-guides/source/developers/index.rst index 25fea138736..6d01e13d78e 100755 --- a/doc/sphinx-guides/source/developers/index.rst +++ b/doc/sphinx-guides/source/developers/index.rst @@ -18,7 +18,6 @@ Developer Guide version-control sql-upgrade-scripts testing - documentation api-design security performance From f2871102a9b5a5c4d8dc5d97938f9b876f3ca636 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 13 Jun 2024 10:53:17 -0400 Subject: [PATCH 082/145] reorg into writing guidelines #10531 --- doc/sphinx-guides/source/contributor/documentation.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/contributor/documentation.md b/doc/sphinx-guides/source/contributor/documentation.md index e0b83560870..37d171a4279 100644 --- a/doc/sphinx-guides/source/contributor/documentation.md +++ b/doc/sphinx-guides/source/contributor/documentation.md @@ -122,7 +122,9 @@ Also, as of this writing we have enabled PDF builds from the "develop" branch. Y If you would like to help improve the PDF version of the guides, please get in touch! Please see {ref}`getting-help-developers` for ways to contact the developer community. -## Writing Style Guidelines +## Writing Guidelines + +### Writing Style Guidelines Please observe the following when writing documentation: @@ -132,7 +134,7 @@ Please observe the following when writing documentation: - Use "double quotes" instead of 'single quotes'. - Favor "and" (data and code) over slashes (data/code). -## Table of Contents +### Table of Contents Every non-index page should use the following code to display a table of contents of internal sub-headings. This code should be placed below any introductory text and directly above the first subheading, much like a Wikipedia page. @@ -148,7 +150,7 @@ If the page is written in Markdown (.md), use this form: :depth: 3 ``` -## Images +### Images A good documentation is just like a website enhanced and upgraded by adding high quality and self-explanatory images. Often images depict a lot of written text in a simple manner. Within our Sphinx docs, you can add them in two ways: a) add a PNG image directly and include or b) use inline description languages like GraphViz (current only option). @@ -156,6 +158,6 @@ While PNGs in the git repo can be linked directly via URL, Sphinx-generated imag TODO: The above covers "how" but what about "when"? -## Cross References +### Cross References When adding ReStructured Text (.rst) [cross references](https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#ref-role), use the hyphen character (`-`) as the word separator for the cross reference label. For example, `my-reference-label` would be the preferred label for a cross reference as opposed to, for example, `my_reference_label`. From 15c02842f083761bac1a56f2cd96c4f95c4f19a2 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 13 Jun 2024 11:56:27 -0400 Subject: [PATCH 083/145] add intro for writing docs #10531 --- .../source/contributor/documentation.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/contributor/documentation.md b/doc/sphinx-guides/source/contributor/documentation.md index 37d171a4279..3df81dc461c 100644 --- a/doc/sphinx-guides/source/contributor/documentation.md +++ b/doc/sphinx-guides/source/contributor/documentation.md @@ -7,6 +7,23 @@ Thank you for your interest in contributing documentation to Dataverse! Good doc :depth: 3 ``` +## Overview + +The Dataverse guides are written using [Sphinx](https://sphinx-doc.org). + +The source files are stored under [doc/sphinx-guides](https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides) in the main "dataverse" repo on GitHub. + +Historically, guides have been written in the default Sphinx format, [reStructuredText][] (.rst), but newer guides such as the {doc}`/contributor/index` are written in [Markdown][] (.md). + +[reStructuredText]: https://en.wikipedia.org/wiki/ReStructuredText +[Markdown]: https://en.wikipedia.org/wiki/Markdown + +Below we'll present a technique for making quick edits to the guides using GitHub's web editor ("quick fix"). We'll also describe how to install Sphinx locally for more significant edits. + +Finally, we'll provide some guidelines on writing content. + +We could use some help on writing this very page and helping newcomers get started. Please don't be shy about suggesting improvements! You can open an issue at , post to , write to the [mailing list](https://groups.google.com/g/dataverse-community), or suggest a change with a pull request. + ## Quick Fix If you find a typo or a small error in the documentation you can fix it using GitHub's online web editor. Generally speaking, we will be following @@ -33,7 +50,7 @@ If you would like to read more about the Dataverse Project's use of GitHub, plea ## Building the Guides with Sphinx -The Dataverse guides are written using Sphinx (). We recommend installing Sphinx on your computer or using a Sphinx Docker container to build the guides locally so you can get an accurate preview of your changes. +While the "quick fix" technique shown above should work fine for minor changes, especially for larger changes, we recommend installing Sphinx on your computer or using a Sphinx Docker container to build the guides locally so you can get an accurate preview of your changes. In case you decide to use a Sphinx Docker container to build the guides, you can skip the next two installation sections, but you will need to have Docker installed. From d92cb686fe9387c36d6557db541aaf76b88f8139 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 13 Jun 2024 12:23:47 -0400 Subject: [PATCH 084/145] rewrite "versions" content into "hosting own guides" #10531 --- .../source/contributor/documentation.md | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/doc/sphinx-guides/source/contributor/documentation.md b/doc/sphinx-guides/source/contributor/documentation.md index 3df81dc461c..8c1bce3f31c 100644 --- a/doc/sphinx-guides/source/contributor/documentation.md +++ b/doc/sphinx-guides/source/contributor/documentation.md @@ -111,16 +111,6 @@ After Sphinx is done processing the files you should notice that the `html` fold Now you can make a commit with the changes to your own fork in GitHub and submit a pull request. See {ref}`how-to-make-a-pull-request`. -## Versions - -For installations hosting their own copies of the guides, note that as each version of the Dataverse Software is released, there is an updated version of the guides released with it. Google and other search engines index all versions, which may confuse users who discover your guides in the search results as to which version they should be looking at. When learning about your installation from the search results, it is best to be viewing the *latest* version. - -In order to make it clear to the crawlers that we only want the latest version discoverable in their search results, we suggest adding this to your `robots.txt` file: - - User-agent: * - Allow: /en/latest/ - Disallow: /en/ - ## PDF Version of the Guides The HTML version of the guides is the official one. Any other formats are maintained on a best effort basis. @@ -178,3 +168,17 @@ TODO: The above covers "how" but what about "when"? ### Cross References When adding ReStructured Text (.rst) [cross references](https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#ref-role), use the hyphen character (`-`) as the word separator for the cross reference label. For example, `my-reference-label` would be the preferred label for a cross reference as opposed to, for example, `my_reference_label`. + +## Hosting Your Own Version of the Guides + +Some installations of Dataverse maintain their own versions of the guides and use settings like {ref}`:NavbarGuidesUrl` or {ref}`:GuidesBaseUrl` to point their users to them. + +### Having Google Index the Latest Version + +As each version of the Dataverse software is released, there is an updated version of the guides released with it. Google and other search engines index all versions, which may confuse users who discover your guides in the search results as to which version they should be looking at. When learning about your installation from the search results, it is best to be viewing the *latest* version. + +In order to make it clear to the crawlers that we only want the latest version discoverable in their search results, we suggest adding this to your `robots.txt` file: + + User-agent: * + Allow: /en/latest/ + Disallow: /en/ From 6b3d06ceaed8013f9c70741d4714d6750906d686 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 13 Jun 2024 12:25:40 -0400 Subject: [PATCH 085/145] move PDF content down (less important) #10531 --- .../source/contributor/documentation.md | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/doc/sphinx-guides/source/contributor/documentation.md b/doc/sphinx-guides/source/contributor/documentation.md index 8c1bce3f31c..0c4955d3e5e 100644 --- a/doc/sphinx-guides/source/contributor/documentation.md +++ b/doc/sphinx-guides/source/contributor/documentation.md @@ -111,24 +111,6 @@ After Sphinx is done processing the files you should notice that the `html` fold Now you can make a commit with the changes to your own fork in GitHub and submit a pull request. See {ref}`how-to-make-a-pull-request`. -## PDF Version of the Guides - -The HTML version of the guides is the official one. Any other formats are maintained on a best effort basis. - -If you would like to build a PDF version of the guides and have Docker installed, please try the command below from the root of the git repo: - -`docker run -it --rm -v $(pwd):/docs sphinxdoc/sphinx-latexpdf:3.5.4 bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make latexpdf LATEXMKOPTS=\"-interaction=nonstopmode\"; cd ../.. && ls -1 doc/sphinx-guides/build/latex/Dataverse.pdf"` - -A few notes about the command above: - -- Hopefully the PDF was created at `doc/sphinx-guides/build/latex/Dataverse.pdf`. -- For now, we are using "nonstopmode" but this masks some errors. -- See requirements.txt for a note regarding the version of Sphinx we are using. - -Also, as of this writing we have enabled PDF builds from the "develop" branch. You download the PDF from - -If you would like to help improve the PDF version of the guides, please get in touch! Please see {ref}`getting-help-developers` for ways to contact the developer community. - ## Writing Guidelines ### Writing Style Guidelines @@ -169,6 +151,25 @@ TODO: The above covers "how" but what about "when"? When adding ReStructured Text (.rst) [cross references](https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#ref-role), use the hyphen character (`-`) as the word separator for the cross reference label. For example, `my-reference-label` would be the preferred label for a cross reference as opposed to, for example, `my_reference_label`. +## PDF Version of the Guides + +The HTML version of the guides is the official one. Any other formats are maintained on a best effort basis. + +If you would like to build a PDF version of the guides and have Docker installed, please try the command below from the root of the git repo: + +`docker run -it --rm -v $(pwd):/docs sphinxdoc/sphinx-latexpdf:3.5.4 bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make latexpdf LATEXMKOPTS=\"-interaction=nonstopmode\"; cd ../.. && ls -1 doc/sphinx-guides/build/latex/Dataverse.pdf"` + +A few notes about the command above: + +- Hopefully the PDF was created at `doc/sphinx-guides/build/latex/Dataverse.pdf`. +- For now, we are using "nonstopmode" but this masks some errors. +- See requirements.txt for a note regarding the version of Sphinx we are using. + +Also, as of this writing we have enabled PDF builds from the "develop" branch. You download the PDF from + +If you would like to help improve the PDF version of the guides, please get in touch! Please see {ref}`getting-help-developers` for ways to contact the developer community. + + ## Hosting Your Own Version of the Guides Some installations of Dataverse maintain their own versions of the guides and use settings like {ref}`:NavbarGuidesUrl` or {ref}`:GuidesBaseUrl` to point their users to them. From 147866b92725bc38422ee89c44ededa158ea1a1e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 13 Jun 2024 12:33:09 -0400 Subject: [PATCH 086/145] add release note #10531 --- doc/release-notes/10531-contrib.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/release-notes/10531-contrib.md diff --git a/doc/release-notes/10531-contrib.md b/doc/release-notes/10531-contrib.md new file mode 100644 index 00000000000..6cfbe988992 --- /dev/null +++ b/doc/release-notes/10531-contrib.md @@ -0,0 +1 @@ +A new [Contributor Guide](https://dataverse-guide--10532.org.readthedocs.build/en/10532/contributor/index.html) has been added by the UX Working Group (#10531 and #10532). From 5c5e9f7fafb9b3a46dd3dc1c3988ee0018d947d6 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 13 Jun 2024 12:59:50 -0400 Subject: [PATCH 087/145] tweaks #10531 --- doc/sphinx-guides/source/contributor/index.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/doc/sphinx-guides/source/contributor/index.md b/doc/sphinx-guides/source/contributor/index.md index e603d6151b7..7a9555fdd34 100644 --- a/doc/sphinx-guides/source/contributor/index.md +++ b/doc/sphinx-guides/source/contributor/index.md @@ -1,6 +1,6 @@ # Contributor Guide -Thank you for your interest in contributing to Dataverse! We are open to contributions from everyone. We welcome contributions of ideas, bug reports, usability research/feedback, documentation, code, and more! +Thank you for your interest in contributing to Dataverse! We are open to contributions from everyone. We welcome contributions of ideas, bug reports, documentation, code, and more! ```{contents} Contents: :local: @@ -68,9 +68,11 @@ Consider contributing to , a git ## Issue Triage -New issues are coming in all the time, especially for the main issue tracker at . +New issues come in all the time, especially for the main issue tracker at . -You can help by leaving comments that mention related issues or that answer questions. +You can help by leaving comments. You can mention related issues or answer questions. + +If you are interested in adding issue labels or related curation, please get in touch! ## Giving Talks @@ -82,13 +84,15 @@ For non-recorded talks, we are happy to upload your slides to . +You're welcome to start your own working group, of course. We can help you get the word out. + ## Artwork -Surely we can put artistic talent to use. See for a story about drawing animals chatting about Dataverse. +Surely we can put artistic talent to use. A contributor [drew cartoon animals chatting about Dataverse](https://github.com/IQSS/chat.dataverse.org/issues/18), for example. As [annnounced](https://groups.google.com/g/dataverse-community/c/pM39_9O5Rug/m/CK-gJqZFBgAJ), we have a [sticker template](https://dataverse.org/sites/projects.iq.harvard.edu/files/dataverseorg/files/dataverse_community_stickers_template.zip) you can use. -See [Illustrations from The Turing Way](https://zenodo.org/doi/10.5281/zenodo.3332807) for how that community has created artwork. Perhaps we can do the same. +See [Illustrations from The Turing Way](https://zenodo.org/doi/10.5281/zenodo.3332807) for how that community has created artwork. Perhaps we can create a similar collection. ## Other Contributions From 7f56478d2f61161bb797cecdbc52669b67992cbe Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 13 Jun 2024 14:02:22 -0400 Subject: [PATCH 088/145] minor simplification --- .../java/edu/harvard/iq/dataverse/search/IndexServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 01a6039b91b..bd4aa27ba68 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -486,8 +486,8 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr StringBuilder debug = new StringBuilder(); debug.append("\ndebug:\n"); DatasetVersion latestVersion = dataset.getLatestVersion(); - String latestVersionStateString = latestVersion.getVersionState().name(); DatasetVersion.VersionState latestVersionState = latestVersion.getVersionState(); + String latestVersionStateString = latestVersionState.name(); DatasetVersion releasedVersion = dataset.getReleasedVersion(); boolean atLeastOnePublishedVersion = false; if (releasedVersion != null) { From 57b7ed92428bf24079f9fe969c4e0fedbb2cfa4e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 13 Jun 2024 14:40:04 -0400 Subject: [PATCH 089/145] cleanup --- .../iq/dataverse/search/IndexServiceBean.java | 260 ++++++------------ 1 file changed, 88 insertions(+), 172 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index bd4aa27ba68..2afb5d26082 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -13,6 +13,7 @@ import edu.harvard.iq.dataverse.datavariable.VariableMetadataUtil; import edu.harvard.iq.dataverse.datavariable.VariableServiceBean; import edu.harvard.iq.dataverse.harvest.client.HarvestingClient; +import edu.harvard.iq.dataverse.search.IndexableDataset.DatasetState; import edu.harvard.iq.dataverse.settings.FeatureFlags; import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; @@ -476,12 +477,8 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr * @todo should we use solrDocIdentifierDataset or * IndexableObject.IndexableTypes.DATASET.getName() + "_" ? */ - // String solrIdPublished = solrDocIdentifierDataset + dataset.getId(); String solrIdPublished = determinePublishedDatasetSolrDocId(dataset); String solrIdDraftDataset = IndexableObject.IndexableTypes.DATASET.getName() + "_" + dataset.getId() + IndexableDataset.DatasetState.WORKING_COPY.getSuffix(); - // String solrIdDeaccessioned = IndexableObject.IndexableTypes.DATASET.getName() - // + "_" + dataset.getId() + - // IndexableDataset.DatasetState.DEACCESSIONED.getSuffix(); String solrIdDeaccessioned = determineDeaccessionedDatasetId(dataset); StringBuilder debug = new StringBuilder(); debug.append("\ndebug:\n"); @@ -494,112 +491,53 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr atLeastOnePublishedVersion = true; } List solrIdsOfDocsToDelete = null; - - try { - solrIdsOfDocsToDelete = findFilesOfParentDataset(dataset.getId()); - logger.fine("Existing file docs: " + String.join(", ", solrIdsOfDocsToDelete)); - //We keep the latest version's docs unless it is deaccessioned and there is no published/released version - //So skip the loop removing those docs from the delete list except in that case - if ((!latestVersion.isDeaccessioned() || atLeastOnePublishedVersion)) { - List latestFileMetadatas = latestVersion.getFileMetadatas(); - String suffix = (new IndexableDataset(latestVersion)).getDatasetState().getSuffix(); - for (FileMetadata fileMetadata : latestFileMetadatas) { - String solrIdOfPublishedFile = solrDocIdentifierFile + fileMetadata.getDataFile().getId() + suffix; - solrIdsOfDocsToDelete.remove(solrIdOfPublishedFile); + if (logger.isLoggable(Level.FINE)) { + writeDebugInfo(debug, dataset); + } + if (doNormalSolrDocCleanUp) { + try { + solrIdsOfDocsToDelete = findFilesOfParentDataset(dataset.getId()); + logger.fine("Existing file docs: " + String.join(", ", solrIdsOfDocsToDelete)); + // We keep the latest version's docs unless it is deaccessioned and there is no + // published/released version + // So skip the loop removing those docs from the delete list except in that case + if ((!latestVersion.isDeaccessioned() || atLeastOnePublishedVersion)) { + List latestFileMetadatas = latestVersion.getFileMetadatas(); + String suffix = (new IndexableDataset(latestVersion)).getDatasetState().getSuffix(); + for (FileMetadata fileMetadata : latestFileMetadatas) { + String solrIdOfPublishedFile = solrDocIdentifierFile + fileMetadata.getDataFile().getId() + + suffix; + solrIdsOfDocsToDelete.remove(solrIdOfPublishedFile); + } } - } - if (releasedVersion != null && !releasedVersion.equals(latestVersion)) { - List releasedFileMetadatas = releasedVersion.getFileMetadatas(); - for (FileMetadata fileMetadata : releasedFileMetadatas) { - String solrIdOfPublishedFile = solrDocIdentifierFile + fileMetadata.getDataFile().getId(); - solrIdsOfDocsToDelete.remove(solrIdOfPublishedFile); + if (releasedVersion != null && !releasedVersion.equals(latestVersion)) { + List releasedFileMetadatas = releasedVersion.getFileMetadatas(); + for (FileMetadata fileMetadata : releasedFileMetadatas) { + String solrIdOfPublishedFile = solrDocIdentifierFile + fileMetadata.getDataFile().getId(); + solrIdsOfDocsToDelete.remove(solrIdOfPublishedFile); + } } - } - //Clear any unused dataset docs - if (!latestVersion.isDraft()) { - // The latest version is released, so should delete any draft docs for the - // dataset - solrIdsOfDocsToDelete.add(solrDocIdentifierDataset + dataset.getId() + draftSuffix); - } - if (!atLeastOnePublishedVersion) { - // There's no released version, so should delete any normal state docs for the - // dataset - solrIdsOfDocsToDelete.add(solrDocIdentifierDataset + dataset.getId()); - } - if (atLeastOnePublishedVersion || !latestVersion.isDeaccessioned()) { - // There's a released version or a draft, so should delete any deaccessioned - // state docs for the dataset - solrIdsOfDocsToDelete.add(solrDocIdentifierDataset + dataset.getId() + deaccessionedSuffix); - } - } catch (SearchException | NullPointerException ex) { - logger.fine("could not run search of files to delete: " + ex); - } - logger.fine("Solr docs to delete: " + String.join(", ", solrIdsOfDocsToDelete)); - int numPublishedVersions = 0; - List versions = dataset.getVersions(); - //List solrIdsOfFilesToDelete = new ArrayList<>(); - if (logger.isLoggable(Level.FINE)) { - for (DatasetVersion datasetVersion : versions) { - Long versionDatabaseId = datasetVersion.getId(); - String versionTitle = datasetVersion.getTitle(); - String semanticVersion = datasetVersion.getSemanticVersion(); - DatasetVersion.VersionState versionState = datasetVersion.getVersionState(); - if (versionState.equals(DatasetVersion.VersionState.RELEASED)) { - numPublishedVersions += 1; + // Clear any unused dataset docs + if (!latestVersion.isDraft()) { + // The latest version is released, so should delete any draft docs for the + // dataset + solrIdsOfDocsToDelete.add(solrIdDraftDataset); } - debug.append("version found with database id " + versionDatabaseId + "\n"); - debug.append("- title: " + versionTitle + "\n"); - debug.append("- semanticVersion-VersionState: " + semanticVersion + "-" + versionState + "\n"); - List fileInfo = new ArrayList<>(); - List fileMetadatas = datasetVersion.getFileMetadatas(); - - for (FileMetadata fileMetadata : fileMetadatas) { - /** - * It sounds weird but the first thing we'll do is preemptively delete the Solr - * documents of all published files. Don't worry, published files will be - * re-indexed later along with the dataset. We do this so users can delete files - * from published versions of datasets and then re-publish a new version without - * fear that their old published files (now deleted from the latest published - * version) will be searchable. See also - * https://github.com/IQSS/dataverse/issues/762 - */ - fileInfo.add(fileMetadata.getDataFile().getId() + ":" + fileMetadata.getLabel()); + if (!atLeastOnePublishedVersion) { + // There's no released version, so should delete any normal state docs for the + // dataset + solrIdsOfDocsToDelete.add(solrIdPublished); } -// try { - /** - * Preemptively delete *all* Solr documents for files associated with the - * dataset based on a Solr query. - * - * We must query Solr for this information because the file has been deleted - * from the database ( perhaps when Solr was down, as reported in - * https://github.com/IQSS/dataverse/issues/2086 ) so the database doesn't even - * know about the file. It's an orphan. - * - * @todo This Solr query should make the iteration above based on the database - * unnecessary because it the Solr query should find all files for the - * dataset. We can probably remove the iteration above after an "index - * all" has been performed. Without an "index all" we won't be able to - * find files based on parentId because that field wasn't searchable in - * 4.0. - * - * @todo We should also delete the corresponding Solr "permission" documents for - * the files. - */ - // List allFilesForDataset = findFilesOfParentDataset(dataset.getId()); - // solrIdsOfFilesToDelete.addAll(allFilesForDataset); -// } catch (SearchException | NullPointerException ex) { -// logger.fine("could not run search of files to delete: " + ex); -// } - int numFiles = 0; - if (fileMetadatas != null) { - numFiles = fileMetadatas.size(); + if (atLeastOnePublishedVersion || !latestVersion.isDeaccessioned()) { + // There's a released version or a draft, so should delete any deaccessioned + // state docs for the dataset + solrIdsOfDocsToDelete.add(solrIdDeaccessioned); } - debug.append("- files: " + numFiles + " " + fileInfo.toString() + "\n"); + } catch (SearchException | NullPointerException ex) { + logger.fine("could not run search of files to delete: " + ex); } - } - debug.append("numPublishedVersions: " + numPublishedVersions + "\n"); - if (doNormalSolrDocCleanUp) { - + logger.fine("Solr docs to delete: " + String.join(", ", solrIdsOfDocsToDelete)); + if(!solrIdsOfDocsToDelete.isEmpty()) { List solrIdsOfPermissionDocsToDelete = new ArrayList<>(); for(String file: solrIdsOfDocsToDelete) { @@ -636,19 +574,7 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr .append(indexDraftResult).append("\n"); desiredCards.put(DatasetVersion.VersionState.DEACCESSIONED, false); - if (doNormalSolrDocCleanUp) { - String deleteDeaccessionedResult = removeDeaccessioned(dataset); - results.append("Draft exists, no need for deaccessioned version. Deletion attempted for ") - .append(solrIdDeaccessioned).append(" (and files). Result: ") - .append(deleteDeaccessionedResult).append("\n"); - } - desiredCards.put(DatasetVersion.VersionState.RELEASED, false); - if (doNormalSolrDocCleanUp) { - String deletePublishedResults = removePublished(dataset); - results.append("No published version. Attempting to delete traces of published version from index. Result: ") - .append(deletePublishedResults).append("\n"); - } /** * Desired state for existence of cards: {DRAFT=true, @@ -687,19 +613,7 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr results.append("No draft version. Attempting to index as deaccessioned. Result: ").append(indexDeaccessionedVersionResult).append("\n"); desiredCards.put(DatasetVersion.VersionState.RELEASED, false); - if (doNormalSolrDocCleanUp) { - String deletePublishedResults = removePublished(dataset); - results.append("No published version. Attempting to delete traces of published version from index. Result: ").append(deletePublishedResults).append("\n"); - } - desiredCards.put(DatasetVersion.VersionState.DRAFT, false); - if (doNormalSolrDocCleanUp) { - //List solrDocIdsForDraftFilesToDelete = findSolrDocIdsForDraftFilesToDelete(dataset); - String deleteDraftDatasetVersionResult = removeSolrDocFromIndex(solrIdDraftDataset); - //String deleteDraftFilesResults = deleteDraftFiles(solrDocIdsForDraftFilesToDelete); - //results.append("Attempting to delete traces of drafts. Result: ") - // .append(deleteDraftDatasetVersionResult).append(deleteDraftFilesResults).append("\n"); - } /** * Desired state for existence of cards: {DEACCESSIONED=true, @@ -741,20 +655,7 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr results.append("Attempted to index " + solrIdPublished).append(". Result: ").append(indexReleasedVersionResult).append("\n"); desiredCards.put(DatasetVersion.VersionState.DRAFT, false); - if (doNormalSolrDocCleanUp) { - //List solrDocIdsForDraftFilesToDelete = findSolrDocIdsForDraftFilesToDelete(dataset); - String deleteDraftDatasetVersionResult = removeSolrDocFromIndex(solrIdDraftDataset); - //String deleteDraftFilesResults = deleteDraftFiles(solrDocIdsForDraftFilesToDelete); - //results.append("The latest version is published. Attempting to delete drafts. Result: ") - // .append(deleteDraftDatasetVersionResult).append(deleteDraftFilesResults).append("\n"); - } - desiredCards.put(DatasetVersion.VersionState.DEACCESSIONED, false); - if (doNormalSolrDocCleanUp) { - String deleteDeaccessionedResult = removeDeaccessioned(dataset); - results.append("No need for deaccessioned version. Deletion attempted for ") - .append(solrIdDeaccessioned).append(". Result: ").append(deleteDeaccessionedResult); - } /** * Desired state for existence of cards: {RELEASED=true, @@ -801,11 +702,6 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr .append(solrIdDraftDataset).append(" (limited visibility). Result: ").append(indexDraftResult).append("\n"); desiredCards.put(DatasetVersion.VersionState.DEACCESSIONED, false); - if (doNormalSolrDocCleanUp) { - String deleteDeaccessionedResult = removeDeaccessioned(dataset); - results.append("No need for deaccessioned version. Deletion attempted for ") - .append(solrIdDeaccessioned).append(". Result: ").append(deleteDeaccessionedResult); - } /** * Desired state for existence of cards: {DRAFT=true, @@ -843,7 +739,45 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr } } -/* private String deleteDraftFiles(List solrDocIdsForDraftFilesToDelete) { + private void writeDebugInfo(StringBuilder debug, Dataset dataset) { + List versions = dataset.getVersions(); + int numPublishedVersions = 0; + for (DatasetVersion datasetVersion : versions) { + Long versionDatabaseId = datasetVersion.getId(); + String versionTitle = datasetVersion.getTitle(); + String semanticVersion = datasetVersion.getSemanticVersion(); + DatasetVersion.VersionState versionState = datasetVersion.getVersionState(); + if (versionState.equals(DatasetVersion.VersionState.RELEASED)) { + numPublishedVersions += 1; + } + debug.append("version found with database id " + versionDatabaseId + "\n"); + debug.append("- title: " + versionTitle + "\n"); + debug.append("- semanticVersion-VersionState: " + semanticVersion + "-" + versionState + "\n"); + List fileInfo = new ArrayList<>(); + List fileMetadatas = datasetVersion.getFileMetadatas(); + + for (FileMetadata fileMetadata : fileMetadatas) { + /** + * It sounds weird but the first thing we'll do is preemptively delete the Solr + * documents of all published files. Don't worry, published files will be + * re-indexed later along with the dataset. We do this so users can delete files + * from published versions of datasets and then re-publish a new version without + * fear that their old published files (now deleted from the latest published + * version) will be searchable. See also + * https://github.com/IQSS/dataverse/issues/762 + */ + fileInfo.add(fileMetadata.getDataFile().getId() + ":" + fileMetadata.getLabel()); + } + int numFiles = 0; + if (fileMetadatas != null) { + numFiles = fileMetadatas.size(); + } + debug.append("- files: " + numFiles + " " + fileInfo.toString() + "\n"); + } + debug.append("numPublishedVersions: " + numPublishedVersions + "\n"); + } + + /* private String deleteDraftFiles(List solrDocIdsForDraftFilesToDelete) { String deleteDraftFilesResults = ""; IndexResponse indexResponse = solrIndexService.deleteMultipleSolrIds(solrDocIdsForDraftFilesToDelete); deleteDraftFilesResults = indexResponse.toString(); @@ -925,15 +859,17 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set docIds = findSolrDocIdsForFilesToDelete(dataset, IndexableDataset.DatasetState.DEACCESSIONED); -// String deleteFilesResult = removeMultipleSolrDocs(docIds); -// result.append(deleteFilesResult); - return result.toString(); - } - - private String removePublished(Dataset dataset) { - StringBuilder result = new StringBuilder(); - String deletePublishedResult = removeSolrDocFromIndex(determinePublishedDatasetSolrDocId(dataset)); - result.append(deletePublishedResult); -// List docIds = findSolrDocIdsForFilesToDelete(dataset, IndexableDataset.DatasetState.PUBLISHED); -// String deleteFilesResult = removeMultipleSolrDocs(docIds); -// result.append(deleteFilesResult); - return result.toString(); - } - private Dataverse findRootDataverseCached() { if (true) { /** From a52a8384ffebb2244e687aef1cbe479aa06f5b76 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 13 Jun 2024 16:03:33 -0400 Subject: [PATCH 090/145] docs --- doc/release-notes/10579-avoid-solr-deletes.md | 9 +++++++++ doc/sphinx-guides/source/developers/performance.rst | 1 + doc/sphinx-guides/source/installation/config.rst | 3 +++ 3 files changed, 13 insertions(+) create mode 100644 doc/release-notes/10579-avoid-solr-deletes.md diff --git a/doc/release-notes/10579-avoid-solr-deletes.md b/doc/release-notes/10579-avoid-solr-deletes.md new file mode 100644 index 00000000000..1062a2fb78f --- /dev/null +++ b/doc/release-notes/10579-avoid-solr-deletes.md @@ -0,0 +1,9 @@ +A features flag called "reduce-solr-deletes" has been added to improve how datafiles are indexed. When the flag is enabled, +Dataverse wil avoid pre-emptively deleting existing solr documents for the files prior to sending updated information. This +should improve performance and will allow additional optimizations going forward. + +The /api/admin/index/status and /api/admin/index/clear-orphans calls +(see https://guides.dataverse.org/en/latest/admin/solr-search-index.html#index-and-database-consistency) +will now find and remove (respectively) additional permissions related solr documents that were not being detected before. +Reducing the overall number of documents will improve solr performance and large sites may wish to periodically call the +clear-orphans API. \ No newline at end of file diff --git a/doc/sphinx-guides/source/developers/performance.rst b/doc/sphinx-guides/source/developers/performance.rst index 562fa330d75..0044899a581 100644 --- a/doc/sphinx-guides/source/developers/performance.rst +++ b/doc/sphinx-guides/source/developers/performance.rst @@ -121,6 +121,7 @@ While in the past Solr performance hasn't been much of a concern, in recent year We are tracking performance problems in `#10469 `_. In a meeting with a Solr expert on 2024-05-10 we were advised to avoid joins as much as possible. (It was acknowledged that many Solr users make use of joins because they have to, like we do, to keep some documents private.) Toward that end we have added two feature flags called ``avoid-expensive-solr-join`` and ``add-publicobject-solr-field`` as explained under :ref:`feature-flags`. It was confirmed experimentally that performing the join on all the public objects (published collections, datasets and files), i.e., the bulk of the content in the search index, was indeed very expensive, especially on a large instance the size of the IQSS prod. archive, especially under indexing load. We confirmed that it was in fact unnecessary and were able to replace it with a boolean field directly in the indexed documents, which is achieved by the two feature flags above. However, as of writing this, this mechanism should still be considered experimental. +Another flag, ``reduce-solr-deletes``, avoids deleting solr documents for files in a dataset prior to sending updates. This is expected to improve indexing performance to some extent and is a step towards avoiding unnecessary updates (i.e. when a doc would not change). Datasets with Large Numbers of Files or Versions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 8fb9460892b..f6e33a2678d 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3274,6 +3274,9 @@ please find all known feature flags below. Any of these flags can be activated u * - add-publicobject-solr-field - Adds an extra boolean field `PublicObject_b:true` for public content (published Collections, Datasets and Files). Once reindexed with these fields, we can rely on it to remove a very expensive Solr join on all such documents in Solr queries, significantly improving overall performance (by enabling the feature flag above, `avoid-expensive-solr-join`). These two flags are separate so that an instance can reindex their holdings before enabling the optimization in searches, thus avoiding having their public objects temporarily disappear from search results while the reindexing is in progress. - ``Off`` + * - reduce-solr-deletes + - Avoids deleting and recreating solr documents for dataset files when reindexing. + - ``Off`` **Note:** Feature flags can be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_FEATURE_XXX`` (e.g. ``DATAVERSE_FEATURE_API_SESSION_AUTH=1``). These environment variables can be set in your shell before starting Payara. If you are using :doc:`Docker for development `, you can set them in the `docker compose `_ file. From 1150ff4fa718eb3665eeddad420b33589d9c574d Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 13 Jun 2024 16:03:45 -0400 Subject: [PATCH 091/145] feature flag --- .../iq/dataverse/search/IndexServiceBean.java | 269 +++++++++++++----- .../iq/dataverse/settings/FeatureFlags.java | 13 + 2 files changed, 217 insertions(+), 65 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 2afb5d26082..1d1098a33d3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -482,6 +482,78 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr String solrIdDeaccessioned = determineDeaccessionedDatasetId(dataset); StringBuilder debug = new StringBuilder(); debug.append("\ndebug:\n"); + boolean reduceSolrDeletes = FeatureFlags.REDUCE_SOLR_DELETES.enabled(); + if (!reduceSolrDeletes) { + int numPublishedVersions = 0; + List versions = dataset.getVersions(); + List solrIdsOfFilesToDelete = new ArrayList<>(); + for (DatasetVersion datasetVersion : versions) { + Long versionDatabaseId = datasetVersion.getId(); + String versionTitle = datasetVersion.getTitle(); + String semanticVersion = datasetVersion.getSemanticVersion(); + DatasetVersion.VersionState versionState = datasetVersion.getVersionState(); + if (versionState.equals(DatasetVersion.VersionState.RELEASED)) { + numPublishedVersions += 1; + } + debug.append("version found with database id " + versionDatabaseId + "\n"); + debug.append("- title: " + versionTitle + "\n"); + debug.append("- semanticVersion-VersionState: " + semanticVersion + "-" + versionState + "\n"); + List fileMetadatas = datasetVersion.getFileMetadatas(); + List fileInfo = new ArrayList<>(); + for (FileMetadata fileMetadata : fileMetadatas) { + String solrIdOfPublishedFile = solrDocIdentifierFile + fileMetadata.getDataFile().getId(); + /** + * It sounds weird but the first thing we'll do is preemptively + * delete the Solr documents of all published files. Don't + * worry, published files will be re-indexed later along with + * the dataset. We do this so users can delete files from + * published versions of datasets and then re-publish a new + * version without fear that their old published files (now + * deleted from the latest published version) will be + * searchable. See also + * https://github.com/IQSS/dataverse/issues/762 + */ + solrIdsOfFilesToDelete.add(solrIdOfPublishedFile); + fileInfo.add(fileMetadata.getDataFile().getId() + ":" + fileMetadata.getLabel()); + } + try { + /** + * Preemptively delete *all* Solr documents for files associated + * with the dataset based on a Solr query. + * + * We must query Solr for this information because the file has + * been deleted from the database ( perhaps when Solr was down, + * as reported in https://github.com/IQSS/dataverse/issues/2086 + * ) so the database doesn't even know about the file. It's an + * orphan. + * + * @todo This Solr query should make the iteration above based + * on the database unnecessary because it the Solr query should + * find all files for the dataset. We can probably remove the + * iteration above after an "index all" has been performed. + * Without an "index all" we won't be able to find files based + * on parentId because that field wasn't searchable in 4.0. + * + * @todo We should also delete the corresponding Solr + * "permission" documents for the files. + */ + List allFilesForDataset = findFilesOfParentDataset(dataset.getId()); + solrIdsOfFilesToDelete.addAll(allFilesForDataset); + } catch (SearchException | NullPointerException ex) { + logger.fine("could not run search of files to delete: " + ex); + } + int numFiles = 0; + if (fileMetadatas != null) { + numFiles = fileMetadatas.size(); + } + debug.append("- files: " + numFiles + " " + fileInfo.toString() + "\n"); + } + debug.append("numPublishedVersions: " + numPublishedVersions + "\n"); + if (doNormalSolrDocCleanUp) { + IndexResponse resultOfAttemptToPremptivelyDeletePublishedFiles = solrIndexService.deleteMultipleSolrIds(solrIdsOfFilesToDelete); + debug.append("result of attempt to premptively deleted published files before reindexing: " + resultOfAttemptToPremptivelyDeletePublishedFiles + "\n"); + } + } DatasetVersion latestVersion = dataset.getLatestVersion(); DatasetVersion.VersionState latestVersionState = latestVersion.getVersionState(); String latestVersionStateString = latestVersionState.name(); @@ -490,65 +562,69 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr if (releasedVersion != null) { atLeastOnePublishedVersion = true; } - List solrIdsOfDocsToDelete = null; - if (logger.isLoggable(Level.FINE)) { - writeDebugInfo(debug, dataset); - } - if (doNormalSolrDocCleanUp) { - try { - solrIdsOfDocsToDelete = findFilesOfParentDataset(dataset.getId()); - logger.fine("Existing file docs: " + String.join(", ", solrIdsOfDocsToDelete)); - // We keep the latest version's docs unless it is deaccessioned and there is no - // published/released version - // So skip the loop removing those docs from the delete list except in that case - if ((!latestVersion.isDeaccessioned() || atLeastOnePublishedVersion)) { - List latestFileMetadatas = latestVersion.getFileMetadatas(); - String suffix = (new IndexableDataset(latestVersion)).getDatasetState().getSuffix(); - for (FileMetadata fileMetadata : latestFileMetadatas) { - String solrIdOfPublishedFile = solrDocIdentifierFile + fileMetadata.getDataFile().getId() - + suffix; - solrIdsOfDocsToDelete.remove(solrIdOfPublishedFile); + if (reduceSolrDeletes) { + List solrIdsOfDocsToDelete = null; + if (logger.isLoggable(Level.FINE)) { + writeDebugInfo(debug, dataset); + } + if (doNormalSolrDocCleanUp) { + try { + solrIdsOfDocsToDelete = findFilesOfParentDataset(dataset.getId()); + logger.fine("Existing file docs: " + String.join(", ", solrIdsOfDocsToDelete)); + // We keep the latest version's docs unless it is deaccessioned and there is no + // published/released version + // So skip the loop removing those docs from the delete list except in that case + if ((!latestVersion.isDeaccessioned() || atLeastOnePublishedVersion)) { + List latestFileMetadatas = latestVersion.getFileMetadatas(); + String suffix = (new IndexableDataset(latestVersion)).getDatasetState().getSuffix(); + for (FileMetadata fileMetadata : latestFileMetadatas) { + String solrIdOfPublishedFile = solrDocIdentifierFile + fileMetadata.getDataFile().getId() + + suffix; + solrIdsOfDocsToDelete.remove(solrIdOfPublishedFile); + } } - } - if (releasedVersion != null && !releasedVersion.equals(latestVersion)) { - List releasedFileMetadatas = releasedVersion.getFileMetadatas(); - for (FileMetadata fileMetadata : releasedFileMetadatas) { - String solrIdOfPublishedFile = solrDocIdentifierFile + fileMetadata.getDataFile().getId(); - solrIdsOfDocsToDelete.remove(solrIdOfPublishedFile); + if (releasedVersion != null && !releasedVersion.equals(latestVersion)) { + List releasedFileMetadatas = releasedVersion.getFileMetadatas(); + for (FileMetadata fileMetadata : releasedFileMetadatas) { + String solrIdOfPublishedFile = solrDocIdentifierFile + fileMetadata.getDataFile().getId(); + solrIdsOfDocsToDelete.remove(solrIdOfPublishedFile); + } } + // Clear any unused dataset docs + if (!latestVersion.isDraft()) { + // The latest version is released, so should delete any draft docs for the + // dataset + solrIdsOfDocsToDelete.add(solrIdDraftDataset); + } + if (!atLeastOnePublishedVersion) { + // There's no released version, so should delete any normal state docs for the + // dataset + solrIdsOfDocsToDelete.add(solrIdPublished); + } + if (atLeastOnePublishedVersion || !latestVersion.isDeaccessioned()) { + // There's a released version or a draft, so should delete any deaccessioned + // state docs for the dataset + solrIdsOfDocsToDelete.add(solrIdDeaccessioned); + } + } catch (SearchException | NullPointerException ex) { + logger.fine("could not run search of files to delete: " + ex); } - // Clear any unused dataset docs - if (!latestVersion.isDraft()) { - // The latest version is released, so should delete any draft docs for the - // dataset - solrIdsOfDocsToDelete.add(solrIdDraftDataset); - } - if (!atLeastOnePublishedVersion) { - // There's no released version, so should delete any normal state docs for the - // dataset - solrIdsOfDocsToDelete.add(solrIdPublished); - } - if (atLeastOnePublishedVersion || !latestVersion.isDeaccessioned()) { - // There's a released version or a draft, so should delete any deaccessioned - // state docs for the dataset - solrIdsOfDocsToDelete.add(solrIdDeaccessioned); - } - } catch (SearchException | NullPointerException ex) { - logger.fine("could not run search of files to delete: " + ex); - } - logger.fine("Solr docs to delete: " + String.join(", ", solrIdsOfDocsToDelete)); + logger.fine("Solr docs to delete: " + String.join(", ", solrIdsOfDocsToDelete)); - if(!solrIdsOfDocsToDelete.isEmpty()) { - List solrIdsOfPermissionDocsToDelete = new ArrayList<>(); - for(String file: solrIdsOfDocsToDelete) { - //Also remove associated permission docs - solrIdsOfPermissionDocsToDelete.add(file + discoverabilityPermissionSuffix); + if (!solrIdsOfDocsToDelete.isEmpty()) { + List solrIdsOfPermissionDocsToDelete = new ArrayList<>(); + for (String file : solrIdsOfDocsToDelete) { + // Also remove associated permission docs + solrIdsOfPermissionDocsToDelete.add(file + discoverabilityPermissionSuffix); + } + solrIdsOfDocsToDelete.addAll(solrIdsOfPermissionDocsToDelete); + logger.fine("Solr docs and perm docs to delete: " + String.join(", ", solrIdsOfDocsToDelete)); + + IndexResponse resultOfAttemptToPremptivelyDeletePublishedFiles = solrIndexService + .deleteMultipleSolrIds(solrIdsOfDocsToDelete); + debug.append("result of attempt to premptively deleted published files before reindexing: " + + resultOfAttemptToPremptivelyDeletePublishedFiles + "\n"); } - solrIdsOfDocsToDelete.addAll(solrIdsOfPermissionDocsToDelete); - logger.fine("Solr docs and perm docs to delete: " + String.join(", ", solrIdsOfDocsToDelete)); - - IndexResponse resultOfAttemptToPremptivelyDeletePublishedFiles = solrIndexService.deleteMultipleSolrIds(solrIdsOfDocsToDelete); - debug.append("result of attempt to premptively deleted published files before reindexing: " + resultOfAttemptToPremptivelyDeletePublishedFiles + "\n"); } } @@ -574,7 +650,19 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr .append(indexDraftResult).append("\n"); desiredCards.put(DatasetVersion.VersionState.DEACCESSIONED, false); + if (!reduceSolrDeletes && doNormalSolrDocCleanUp) { + String deleteDeaccessionedResult = removeDeaccessioned(dataset); + results.append("Draft exists, no need for deaccessioned version. Deletion attempted for ") + .append(solrIdDeaccessioned).append(" (and files). Result: ") + .append(deleteDeaccessionedResult).append("\n"); + } + desiredCards.put(DatasetVersion.VersionState.RELEASED, false); + if (!reduceSolrDeletes && doNormalSolrDocCleanUp) { + String deletePublishedResults = removePublished(dataset); + results.append("No published version. Attempting to delete traces of published version from index. Result: ") + .append(deletePublishedResults).append("\n"); + } /** * Desired state for existence of cards: {DRAFT=true, @@ -613,7 +701,19 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr results.append("No draft version. Attempting to index as deaccessioned. Result: ").append(indexDeaccessionedVersionResult).append("\n"); desiredCards.put(DatasetVersion.VersionState.RELEASED, false); + if (!reduceSolrDeletes && doNormalSolrDocCleanUp) { + String deletePublishedResults = removePublished(dataset); + results.append("No published version. Attempting to delete traces of published version from index. Result: ").append(deletePublishedResults).append("\n"); + } + desiredCards.put(DatasetVersion.VersionState.DRAFT, false); + if (!reduceSolrDeletes && doNormalSolrDocCleanUp) { + List solrDocIdsForDraftFilesToDelete = findSolrDocIdsForDraftFilesToDelete(dataset); + String deleteDraftDatasetVersionResult = removeSolrDocFromIndex(solrIdDraftDataset); + String deleteDraftFilesResults = deleteDraftFiles(solrDocIdsForDraftFilesToDelete); + results.append("Attempting to delete traces of drafts. Result: ") + .append(deleteDraftDatasetVersionResult).append(deleteDraftFilesResults).append("\n"); + } /** * Desired state for existence of cards: {DEACCESSIONED=true, @@ -655,7 +755,20 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr results.append("Attempted to index " + solrIdPublished).append(". Result: ").append(indexReleasedVersionResult).append("\n"); desiredCards.put(DatasetVersion.VersionState.DRAFT, false); + if (!reduceSolrDeletes && doNormalSolrDocCleanUp) { + List solrDocIdsForDraftFilesToDelete = findSolrDocIdsForDraftFilesToDelete(dataset); + String deleteDraftDatasetVersionResult = removeSolrDocFromIndex(solrIdDraftDataset); + String deleteDraftFilesResults = deleteDraftFiles(solrDocIdsForDraftFilesToDelete); + results.append("The latest version is published. Attempting to delete drafts. Result: ") + .append(deleteDraftDatasetVersionResult).append(deleteDraftFilesResults).append("\n"); + } + desiredCards.put(DatasetVersion.VersionState.DEACCESSIONED, false); + if (!reduceSolrDeletes && doNormalSolrDocCleanUp) { + String deleteDeaccessionedResult = removeDeaccessioned(dataset); + results.append("No need for deaccessioned version. Deletion attempted for ") + .append(solrIdDeaccessioned).append(". Result: ").append(deleteDeaccessionedResult); + } /** * Desired state for existence of cards: {RELEASED=true, @@ -702,6 +815,11 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr .append(solrIdDraftDataset).append(" (limited visibility). Result: ").append(indexDraftResult).append("\n"); desiredCards.put(DatasetVersion.VersionState.DEACCESSIONED, false); + if (!reduceSolrDeletes && doNormalSolrDocCleanUp) { + String deleteDeaccessionedResult = removeDeaccessioned(dataset); + results.append("No need for deaccessioned version. Deletion attempted for ") + .append(solrIdDeaccessioned).append(". Result: ").append(deleteDeaccessionedResult); + } /** * Desired state for existence of cards: {DRAFT=true, @@ -777,13 +895,6 @@ private void writeDebugInfo(StringBuilder debug, Dataset dataset) { debug.append("numPublishedVersions: " + numPublishedVersions + "\n"); } - /* private String deleteDraftFiles(List solrDocIdsForDraftFilesToDelete) { - String deleteDraftFilesResults = ""; - IndexResponse indexResponse = solrIndexService.deleteMultipleSolrIds(solrDocIdsForDraftFilesToDelete); - deleteDraftFilesResults = indexResponse.toString(); - return deleteDraftFilesResults; - } -*/ private IndexResponse indexDatasetPermissions(Dataset dataset) { boolean disabledForDebugging = false; if (disabledForDebugging) { @@ -866,10 +977,8 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set docIds = findSolrDocIdsForFilesToDelete(dataset, IndexableDataset.DatasetState.DEACCESSIONED); + String deleteFilesResult = removeMultipleSolrDocs(docIds); + result.append(deleteFilesResult); + return result.toString(); + } + + //Only used when FeatureFlags.REDUCE_SOLR_DELETES is disabled + private String removePublished(Dataset dataset) { + StringBuilder result = new StringBuilder(); + String deletePublishedResult = removeSolrDocFromIndex(determinePublishedDatasetSolrDocId(dataset)); + result.append(deletePublishedResult); + List docIds = findSolrDocIdsForFilesToDelete(dataset, IndexableDataset.DatasetState.PUBLISHED); + String deleteFilesResult = removeMultipleSolrDocs(docIds); + result.append(deleteFilesResult); + return result.toString(); + } + + // Only used when FeatureFlags.REDUCE_SOLR_DELETES is disabled + private String deleteDraftFiles(List solrDocIdsForDraftFilesToDelete) { + String deleteDraftFilesResults = ""; + IndexResponse indexResponse = solrIndexService.deleteMultipleSolrIds(solrDocIdsForDraftFilesToDelete); + deleteDraftFilesResults = indexResponse.toString(); + return deleteDraftFilesResults; + } + private Dataverse findRootDataverseCached() { if (true) { /** diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java index 14a7ab86f22..d523bf92e63 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java @@ -58,6 +58,19 @@ public enum FeatureFlags { * @since Dataverse 6.3 */ ADD_PUBLICOBJECT_SOLR_FIELD("add-publicobject-solr-field"), + /** + * Dataverse normally deletes all solr documents related to a dataset's files + * when the dataset is reindexed. With this flag enabled, additional logic is + * added to the reindex process to delete only the solr documents that are no + * longer needed. (Required docs will be updated rather than deleted and + * replaced.) Enabling this feature flag should make the reindex process + * faster without impacting the search results. + * + * @apiNote Raise flag by setting + * "dataverse.feature.reduce-solr-deletes" + * @since Dataverse 6.3 + */ + REDUCE_SOLR_DELETES("reduce-solr-deletes"), ; final String flag; From 058c28b21e500455d4dd51c13594b1073e316274 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 13 Jun 2024 16:06:43 -0400 Subject: [PATCH 092/145] info -> fine --- .../edu/harvard/iq/dataverse/search/IndexServiceBean.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 1d1098a33d3..567ef1ecbd8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -2180,7 +2180,7 @@ public List findPermissionsInSolrOnly() throws SearchException { } else if (obj instanceof DataFile f) { List states = dataFileService.findVersionStates(f.getId()); Set strings = states.stream().map(VersionState::toString).collect(Collectors.toSet()); - logger.info("for " + docId + " states: " + String.join(", ", strings)); + logger.fine("States for " + docId + ": " + String.join(", ", strings)); if (docId.endsWith("draft_permission")) { if (!states.contains(VersionState.DRAFT)) { permissionInSolrOnly.add(docId); @@ -2194,7 +2194,7 @@ public List findPermissionsInSolrOnly() throws SearchException { permissionInSolrOnly.add(docId); } else { if(dataFileService.findFileMetadataByDatasetVersionIdAndDataFileId(f.getOwner().getReleasedVersion().getId(), f.getId()) == null) { - logger.info("Adding doc " + docId + " to list of permissions in Solr only"); + logger.fine("Adding doc " + docId + " to list of permissions in Solr only"); permissionInSolrOnly.add(docId); } } From 1b0d3a1cc20438c3d3236b9cf190cedc5903a82b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 13 Jun 2024 17:27:03 -0400 Subject: [PATCH 093/145] note eliminating more orphan perm docs --- doc/sphinx-guides/source/developers/performance.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/performance.rst b/doc/sphinx-guides/source/developers/performance.rst index 0044899a581..6c864bec257 100644 --- a/doc/sphinx-guides/source/developers/performance.rst +++ b/doc/sphinx-guides/source/developers/performance.rst @@ -121,7 +121,7 @@ While in the past Solr performance hasn't been much of a concern, in recent year We are tracking performance problems in `#10469 `_. In a meeting with a Solr expert on 2024-05-10 we were advised to avoid joins as much as possible. (It was acknowledged that many Solr users make use of joins because they have to, like we do, to keep some documents private.) Toward that end we have added two feature flags called ``avoid-expensive-solr-join`` and ``add-publicobject-solr-field`` as explained under :ref:`feature-flags`. It was confirmed experimentally that performing the join on all the public objects (published collections, datasets and files), i.e., the bulk of the content in the search index, was indeed very expensive, especially on a large instance the size of the IQSS prod. archive, especially under indexing load. We confirmed that it was in fact unnecessary and were able to replace it with a boolean field directly in the indexed documents, which is achieved by the two feature flags above. However, as of writing this, this mechanism should still be considered experimental. -Another flag, ``reduce-solr-deletes``, avoids deleting solr documents for files in a dataset prior to sending updates. This is expected to improve indexing performance to some extent and is a step towards avoiding unnecessary updates (i.e. when a doc would not change). +Another flag, ``reduce-solr-deletes``, avoids deleting solr documents for files in a dataset prior to sending updates. It also eliminates several causes of orphan permission documents. This is expected to improve indexing performance to some extent and is a step towards avoiding unnecessary updates (i.e. when a doc would not change). Datasets with Large Numbers of Files or Versions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 00020e2e14be599d49bbd7400d37454cf91717b7 Mon Sep 17 00:00:00 2001 From: Steven Ferey Date: Fri, 14 Jun 2024 17:39:06 +0200 Subject: [PATCH 094/145] 10288 Add keywordTermURI metadata in keyword block (#10371) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * New keywordTermURI Metadata in keyword Metadata Block * update of the keywordVocabularyURI metadata to make it consistent with its name Controlled Vocabulary URL * fix description and watermark properties * 10288 adding documentation * 10288 - adaptation of the SolR schema and dataset exports * 10288 - Adjustment of typo and sql * Adaptations for Dataverse 6.2 * 10288 - rollback keywordVocabularyURL to keywordVocabularyURI * 10288 - removing obsolete SQL script * 10288 - Label modification to follow Dataverse recommendations * 10288 - Added valueURI attribute for OpenAire export * Fix NoResultException on DatasetServiceBean.findDeep (.getSingleResult():L137) --------- Co-authored-by: Ludovic DANIEL Co-authored-by: Jérôme ROUCOU --- conf/solr/9.3.0/schema.xml | 2 + ...-add-term_uri-metadata-in-keyword-block.md | 53 ++++++++ ...dataset-create-new-all-default-fields.json | 12 ++ scripts/api/data/metadatablocks/citation.tsv | 113 +++++++++--------- .../iq/dataverse/DatasetFieldConstant.java | 5 +- .../harvard/iq/dataverse/DatasetKeyword.java | 68 ----------- .../export/openaire/OpenAireExportUtil.java | 22 ++-- .../java/propertyFiles/citation.properties | 3 + .../export/OpenAireExportUtilTest.java | 2 + .../export/SchemaDotOrgExporterTest.java | 1 + .../dataverse/export/dataset-all-defaults.txt | 12 ++ .../dataset-create-new-all-ddi-fields.json | 12 ++ .../dataverse/export/ddi/dataset-finch1.json | 12 ++ src/test/resources/json/dataset-finch2.json | 12 ++ .../json/dataset-long-description.json | 12 ++ 15 files changed, 208 insertions(+), 133 deletions(-) create mode 100644 doc/release-notes/10288-add-term_uri-metadata-in-keyword-block.md delete mode 100644 src/main/java/edu/harvard/iq/dataverse/DatasetKeyword.java diff --git a/conf/solr/9.3.0/schema.xml b/conf/solr/9.3.0/schema.xml index 521e7a7db72..5dde750573d 100644 --- a/conf/solr/9.3.0/schema.xml +++ b/conf/solr/9.3.0/schema.xml @@ -326,6 +326,7 @@ + @@ -565,6 +566,7 @@ + diff --git a/doc/release-notes/10288-add-term_uri-metadata-in-keyword-block.md b/doc/release-notes/10288-add-term_uri-metadata-in-keyword-block.md new file mode 100644 index 00000000000..eb3a79dbf25 --- /dev/null +++ b/doc/release-notes/10288-add-term_uri-metadata-in-keyword-block.md @@ -0,0 +1,53 @@ +### New keywordTermURI Metadata in keyword Metadata Block + +Adding a new metadata `keywordTermURI` to the `keyword` metadata block to facilitate the integration of controlled vocabulary services, in particular by adding the possibility of saving the "term" and its associated URI. For more information, see #10288 and PR #10371. + +## Upgrade Instructions + +1\. Update the Citation metadata block + +- `wget https://github.com/IQSS/dataverse/releases/download/v6.3/citation.tsv` +- `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @citation.tsv -H "Content-type: text/tab-separated-values"` + +2\. Update your Solr `schema.xml` to include the new field. + + For details, please see https://guides.dataverse.org/en/latest/admin/metadatacustomization.html#updating-the-solr-schema + + +3\. Reindex Solr. + + Once the schema.xml is updated, Solr must be restarted and a reindex initiated. + For details, see https://guides.dataverse.org/en/latest/admin/solr-search-index.html but here is the reindex command: + + `curl http://localhost:8080/api/admin/index` + + +4\. Run ReExportAll to update dataset metadata exports. Follow the instructions in the [Metadata Export of Admin Guide](https://guides.dataverse.org/en/latest/admin/metadataexport.html#batch-exports-through-the-api). + + +## Notes for Dataverse Installation Administrators + +### Data migration to the new `keywordTermURI` field + +You can migrate your `keywordValue` data containing URIs to the new `keywordTermURI` field. +In case of data migration, view the affected data with the following database query: + +``` +SELECT value FROM datasetfieldvalue dfv +INNER JOIN datasetfield df ON df.id = dfv.datasetfield_id +WHERE df.datasetfieldtype_id = (SELECT id FROM datasetfieldtype WHERE name = 'keywordValue') +AND value ILIKE 'http%'; +``` + +If you wish to migrate your data, a database update is then necessary: + +``` +UPDATE datasetfield df +SET datasetfieldtype_id = (SELECT id FROM datasetfieldtype WHERE name = 'keywordTermURI') +FROM datasetfieldvalue dfv +WHERE dfv.datasetfield_id = df.id +AND df.datasetfieldtype_id = (SELECT id FROM datasetfieldtype WHERE name = 'keywordValue') +AND dfv.value ILIKE 'http%'; +``` + +A ['Reindex in Place'](https://guides.dataverse.org/en/latest/admin/solr-search-index.html#reindex-in-place) will be required and ReExportAll will need to be run to update the metadata exports of the dataset. Follow the directions in the [Admin Guide](http://guides.dataverse.org/en/latest/admin/metadataexport.html#batch-exports-through-the-api). \ No newline at end of file diff --git a/scripts/api/data/dataset-create-new-all-default-fields.json b/scripts/api/data/dataset-create-new-all-default-fields.json index 3bcf134bc76..cc856c6372f 100644 --- a/scripts/api/data/dataset-create-new-all-default-fields.json +++ b/scripts/api/data/dataset-create-new-all-default-fields.json @@ -231,6 +231,12 @@ "typeClass": "primitive", "value": "KeywordTerm1" }, + "keywordTermURI": { + "typeName": "keywordTermURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://keywordTermURI1.org" + }, "keywordVocabulary": { "typeName": "keywordVocabulary", "multiple": false, @@ -251,6 +257,12 @@ "typeClass": "primitive", "value": "KeywordTerm2" }, + "keywordTermURI": { + "typeName": "keywordTermURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://keywordTermURI2.org" + }, "keywordVocabulary": { "typeName": "keywordVocabulary", "multiple": false, diff --git a/scripts/api/data/metadatablocks/citation.tsv b/scripts/api/data/metadatablocks/citation.tsv index 82da5a12eaf..18354f2b1f7 100644 --- a/scripts/api/data/metadatablocks/citation.tsv +++ b/scripts/api/data/metadatablocks/citation.tsv @@ -23,62 +23,63 @@ subject Subject The area of study relevant to the Dataset text 19 TRUE TRUE TRUE TRUE TRUE TRUE citation http://purl.org/dc/terms/subject keyword Keyword A key term that describes an important aspect of the Dataset and information about any controlled vocabulary used none 20 FALSE FALSE TRUE FALSE TRUE FALSE citation keywordValue Term A key term that describes important aspects of the Dataset text 21 #VALUE TRUE FALSE FALSE TRUE TRUE FALSE keyword citation - keywordVocabulary Controlled Vocabulary Name The controlled vocabulary used for the keyword term (e.g. LCSH, MeSH) text 22 (#VALUE) FALSE FALSE FALSE FALSE TRUE FALSE keyword citation - keywordVocabularyURI Controlled Vocabulary URL The URL where one can access information about the term's controlled vocabulary https:// url 23 #VALUE FALSE FALSE FALSE FALSE TRUE FALSE keyword citation - topicClassification Topic Classification Indicates a broad, important topic or subject that the Dataset covers and information about any controlled vocabulary used none 24 FALSE FALSE TRUE FALSE FALSE FALSE citation - topicClassValue Term A topic or subject term text 25 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE topicClassification citation - topicClassVocab Controlled Vocabulary Name The controlled vocabulary used for the keyword term (e.g. LCSH, MeSH) text 26 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE topicClassification citation - topicClassVocabURI Controlled Vocabulary URL The URL where one can access information about the term's controlled vocabulary https:// url 27 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE topicClassification citation - publication Related Publication The article or report that uses the data in the Dataset. The full list of related publications will be displayed on the metadata tab none 28 FALSE FALSE TRUE FALSE TRUE FALSE citation http://purl.org/dc/terms/isReferencedBy - publicationCitation Citation The full bibliographic citation for the related publication textbox 29 #VALUE TRUE FALSE FALSE FALSE TRUE FALSE publication citation http://purl.org/dc/terms/bibliographicCitation - publicationIDType Identifier Type The type of identifier that uniquely identifies a related publication text 30 #VALUE: TRUE TRUE FALSE FALSE TRUE FALSE publication citation http://purl.org/spar/datacite/ResourceIdentifierScheme - publicationIDNumber Identifier The identifier for a related publication text 31 #VALUE TRUE FALSE FALSE FALSE TRUE FALSE publication citation http://purl.org/spar/datacite/ResourceIdentifier - publicationURL URL The URL form of the identifier entered in the Identifier field, e.g. the DOI URL if a DOI was entered in the Identifier field. Used to display what was entered in the ID Type and ID Number fields as a link. If what was entered in the Identifier field has no URL form, the URL of the publication webpage is used, e.g. a journal article webpage https:// url 32 #VALUE FALSE FALSE FALSE FALSE TRUE FALSE publication citation https://schema.org/distribution - notesText Notes Additional information about the Dataset textbox 33 FALSE FALSE FALSE FALSE TRUE FALSE citation - language Language A language that the Dataset's files is written in text 34 TRUE TRUE TRUE TRUE FALSE FALSE citation http://purl.org/dc/terms/language - producer Producer The entity, such a person or organization, managing the finances or other administrative processes involved in the creation of the Dataset none 35 FALSE FALSE TRUE FALSE FALSE FALSE citation - producerName Name The name of the entity, e.g. the person's name or the name of an organization 1) FamilyName, GivenName or 2) Organization text 36 #VALUE TRUE FALSE FALSE TRUE FALSE TRUE producer citation - producerAffiliation Affiliation The name of the entity affiliated with the producer, e.g. an organization's name Organization XYZ text 37 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE producer citation - producerAbbreviation Abbreviated Name The producer's abbreviated name (e.g. IQSS, ICPSR) text 38 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE producer citation - producerURL URL The URL of the producer's website https:// url 39 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE producer citation - producerLogoURL Logo URL The URL of the producer's logo https:// url 40
FALSE FALSE FALSE FALSE FALSE FALSE producer citation - productionDate Production Date The date when the data were produced (not distributed, published, or archived) YYYY-MM-DD date 41 TRUE FALSE FALSE TRUE FALSE FALSE citation - productionPlace Production Location The location where the data and any related materials were produced or collected text 42 TRUE FALSE TRUE TRUE FALSE FALSE citation - contributor Contributor The entity, such as a person or organization, responsible for collecting, managing, or otherwise contributing to the development of the Dataset none 43 : FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/contributor - contributorType Type Indicates the type of contribution made to the dataset text 44 #VALUE TRUE TRUE FALSE TRUE FALSE FALSE contributor citation - contributorName Name The name of the contributor, e.g. the person's name or the name of an organization 1) FamilyName, GivenName or 2) Organization text 45 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE contributor citation - grantNumber Funding Information Information about the Dataset's financial support none 46 : FALSE FALSE TRUE FALSE FALSE FALSE citation https://schema.org/sponsor - grantNumberAgency Agency The agency that provided financial support for the Dataset Organization XYZ text 47 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE grantNumber citation - grantNumberValue Identifier The grant identifier or contract identifier of the agency that provided financial support for the Dataset text 48 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE grantNumber citation - distributor Distributor The entity, such as a person or organization, designated to generate copies of the Dataset, including any editions or revisions none 49 FALSE FALSE TRUE FALSE FALSE FALSE citation - distributorName Name The name of the entity, e.g. the person's name or the name of an organization 1) FamilyName, GivenName or 2) Organization text 50 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE distributor citation - distributorAffiliation Affiliation The name of the entity affiliated with the distributor, e.g. an organization's name Organization XYZ text 51 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE distributor citation - distributorAbbreviation Abbreviated Name The distributor's abbreviated name (e.g. IQSS, ICPSR) text 52 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE distributor citation - distributorURL URL The URL of the distributor's webpage https:// url 53 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE distributor citation - distributorLogoURL Logo URL The URL of the distributor's logo image, used to show the image on the Dataset's page https:// url 54
FALSE FALSE FALSE FALSE FALSE FALSE distributor citation - distributionDate Distribution Date The date when the Dataset was made available for distribution/presentation YYYY-MM-DD date 55 TRUE FALSE FALSE TRUE FALSE FALSE citation - depositor Depositor The entity, such as a person or organization, that deposited the Dataset in the repository 1) FamilyName, GivenName or 2) Organization text 56 FALSE FALSE FALSE FALSE FALSE FALSE citation - dateOfDeposit Deposit Date The date when the Dataset was deposited into the repository YYYY-MM-DD date 57 FALSE FALSE FALSE TRUE FALSE FALSE citation http://purl.org/dc/terms/dateSubmitted - timePeriodCovered Time Period The time period that the data refer to. Also known as span. This is the time period covered by the data, not the dates of coding, collecting data, or making documents machine-readable none 58 ; FALSE FALSE TRUE FALSE FALSE FALSE citation https://schema.org/temporalCoverage - timePeriodCoveredStart Start Date The start date of the time period that the data refer to YYYY-MM-DD date 59 #NAME: #VALUE TRUE FALSE FALSE TRUE FALSE FALSE timePeriodCovered citation - timePeriodCoveredEnd End Date The end date of the time period that the data refer to YYYY-MM-DD date 60 #NAME: #VALUE TRUE FALSE FALSE TRUE FALSE FALSE timePeriodCovered citation - dateOfCollection Date of Collection The dates when the data were collected or generated none 61 ; FALSE FALSE TRUE FALSE FALSE FALSE citation - dateOfCollectionStart Start Date The date when the data collection started YYYY-MM-DD date 62 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE dateOfCollection citation - dateOfCollectionEnd End Date The date when the data collection ended YYYY-MM-DD date 63 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE dateOfCollection citation - kindOfData Data Type The type of data included in the files (e.g. survey data, clinical data, or machine-readable text) text 64 TRUE FALSE TRUE TRUE FALSE FALSE citation http://rdf-vocabulary.ddialliance.org/discovery#kindOfData - series Series Information about the dataset series to which the Dataset belong none 65 : FALSE FALSE TRUE FALSE FALSE FALSE citation - seriesName Name The name of the dataset series text 66 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE series citation - seriesInformation Information Can include 1) a history of the series and 2) a summary of features that apply to the series textbox 67 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE series citation - software Software Information about the software used to generate the Dataset none 68 , FALSE FALSE TRUE FALSE FALSE FALSE citation https://www.w3.org/TR/prov-o/#wasGeneratedBy - softwareName Name The name of software used to generate the Dataset text 69 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE software citation - softwareVersion Version The version of the software used to generate the Dataset, e.g. 4.11 text 70 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE software citation - relatedMaterial Related Material Information, such as a persistent ID or citation, about the material related to the Dataset, such as appendices or sampling information available outside of the Dataset textbox 71 FALSE FALSE TRUE FALSE FALSE FALSE citation - relatedDatasets Related Dataset Information, such as a persistent ID or citation, about a related dataset, such as previous research on the Dataset's subject textbox 72 FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/relation - otherReferences Other Reference Information, such as a persistent ID or citation, about another type of resource that provides background or supporting material to the Dataset text 73 FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/references - dataSources Data Source Information, such as a persistent ID or citation, about sources of the Dataset (e.g. a book, article, serial, or machine-readable data file) textbox 74 FALSE FALSE TRUE FALSE FALSE FALSE citation https://www.w3.org/TR/prov-o/#wasDerivedFrom - originOfSources Origin of Historical Sources For historical sources, the origin and any rules followed in establishing them as sources textbox 75 FALSE FALSE FALSE FALSE FALSE FALSE citation - characteristicOfSources Characteristic of Sources Characteristics not already noted elsewhere textbox 76 FALSE FALSE FALSE FALSE FALSE FALSE citation - accessToSources Documentation and Access to Sources 1) Methods or procedures for accessing data sources and 2) any special permissions needed for access textbox 77 FALSE FALSE FALSE FALSE FALSE FALSE citation + keywordTermURI Term URI A URI that points to the web presence of the Keyword Term https:// url 22 #VALUE FALSE FALSE FALSE FALSE TRUE FALSE keyword citation + keywordVocabulary Controlled Vocabulary Name The controlled vocabulary used for the keyword term (e.g. LCSH, MeSH) text 23 (#VALUE) FALSE FALSE FALSE FALSE TRUE FALSE keyword citation + keywordVocabularyURI Controlled Vocabulary URL The URL where one can access information about the term's controlled vocabulary https:// url 24 #VALUE FALSE FALSE FALSE FALSE TRUE FALSE keyword citation + topicClassification Topic Classification Indicates a broad, important topic or subject that the Dataset covers and information about any controlled vocabulary used none 25 FALSE FALSE TRUE FALSE FALSE FALSE citation + topicClassValue Term A topic or subject term text 26 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE topicClassification citation + topicClassVocab Controlled Vocabulary Name The controlled vocabulary used for the keyword term (e.g. LCSH, MeSH) text 27 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE topicClassification citation + topicClassVocabURI Controlled Vocabulary URL The URL where one can access information about the term's controlled vocabulary https:// url 28 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE topicClassification citation + publication Related Publication The article or report that uses the data in the Dataset. The full list of related publications will be displayed on the metadata tab none 29 FALSE FALSE TRUE FALSE TRUE FALSE citation http://purl.org/dc/terms/isReferencedBy + publicationCitation Citation The full bibliographic citation for the related publication textbox 30 #VALUE TRUE FALSE FALSE FALSE TRUE FALSE publication citation http://purl.org/dc/terms/bibliographicCitation + publicationIDType Identifier Type The type of identifier that uniquely identifies a related publication text 31 #VALUE: TRUE TRUE FALSE FALSE TRUE FALSE publication citation http://purl.org/spar/datacite/ResourceIdentifierScheme + publicationIDNumber Identifier The identifier for a related publication text 32 #VALUE TRUE FALSE FALSE FALSE TRUE FALSE publication citation http://purl.org/spar/datacite/ResourceIdentifier + publicationURL URL The URL form of the identifier entered in the Identifier field, e.g. the DOI URL if a DOI was entered in the Identifier field. Used to display what was entered in the ID Type and ID Number fields as a link. If what was entered in the Identifier field has no URL form, the URL of the publication webpage is used, e.g. a journal article webpage https:// url 33 #VALUE FALSE FALSE FALSE FALSE TRUE FALSE publication citation https://schema.org/distribution + notesText Notes Additional information about the Dataset textbox 34 FALSE FALSE FALSE FALSE TRUE FALSE citation + language Language A language that the Dataset's files is written in text 35 TRUE TRUE TRUE TRUE FALSE FALSE citation http://purl.org/dc/terms/language + producer Producer The entity, such a person or organization, managing the finances or other administrative processes involved in the creation of the Dataset none 36 FALSE FALSE TRUE FALSE FALSE FALSE citation + producerName Name The name of the entity, e.g. the person's name or the name of an organization 1) FamilyName, GivenName or 2) Organization text 37 #VALUE TRUE FALSE FALSE TRUE FALSE TRUE producer citation + producerAffiliation Affiliation The name of the entity affiliated with the producer, e.g. an organization's name Organization XYZ text 38 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE producer citation + producerAbbreviation Abbreviated Name The producer's abbreviated name (e.g. IQSS, ICPSR) text 39 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE producer citation + producerURL URL The URL of the producer's website https:// url 40 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE producer citation + producerLogoURL Logo URL The URL of the producer's logo https:// url 41
FALSE FALSE FALSE FALSE FALSE FALSE producer citation + productionDate Production Date The date when the data were produced (not distributed, published, or archived) YYYY-MM-DD date 42 TRUE FALSE FALSE TRUE FALSE FALSE citation + productionPlace Production Location The location where the data and any related materials were produced or collected text 43 TRUE FALSE TRUE TRUE FALSE FALSE citation + contributor Contributor The entity, such as a person or organization, responsible for collecting, managing, or otherwise contributing to the development of the Dataset none 44 : FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/contributor + contributorType Type Indicates the type of contribution made to the dataset text 45 #VALUE TRUE TRUE FALSE TRUE FALSE FALSE contributor citation + contributorName Name The name of the contributor, e.g. the person's name or the name of an organization 1) FamilyName, GivenName or 2) Organization text 46 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE contributor citation + grantNumber Funding Information Information about the Dataset's financial support none 47 : FALSE FALSE TRUE FALSE FALSE FALSE citation https://schema.org/sponsor + grantNumberAgency Agency The agency that provided financial support for the Dataset Organization XYZ text 48 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE grantNumber citation + grantNumberValue Identifier The grant identifier or contract identifier of the agency that provided financial support for the Dataset text 49 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE grantNumber citation + distributor Distributor The entity, such as a person or organization, designated to generate copies of the Dataset, including any editions or revisions none 50 FALSE FALSE TRUE FALSE FALSE FALSE citation + distributorName Name The name of the entity, e.g. the person's name or the name of an organization 1) FamilyName, GivenName or 2) Organization text 51 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE distributor citation + distributorAffiliation Affiliation The name of the entity affiliated with the distributor, e.g. an organization's name Organization XYZ text 52 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE distributor citation + distributorAbbreviation Abbreviated Name The distributor's abbreviated name (e.g. IQSS, ICPSR) text 53 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE distributor citation + distributorURL URL The URL of the distributor's webpage https:// url 54 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE distributor citation + distributorLogoURL Logo URL The URL of the distributor's logo image, used to show the image on the Dataset's page https:// url 55
FALSE FALSE FALSE FALSE FALSE FALSE distributor citation + distributionDate Distribution Date The date when the Dataset was made available for distribution/presentation YYYY-MM-DD date 56 TRUE FALSE FALSE TRUE FALSE FALSE citation + depositor Depositor The entity, such as a person or organization, that deposited the Dataset in the repository 1) FamilyName, GivenName or 2) Organization text 57 FALSE FALSE FALSE FALSE FALSE FALSE citation + dateOfDeposit Deposit Date The date when the Dataset was deposited into the repository YYYY-MM-DD date 58 FALSE FALSE FALSE TRUE FALSE FALSE citation http://purl.org/dc/terms/dateSubmitted + timePeriodCovered Time Period The time period that the data refer to. Also known as span. This is the time period covered by the data, not the dates of coding, collecting data, or making documents machine-readable none 59 ; FALSE FALSE TRUE FALSE FALSE FALSE citation https://schema.org/temporalCoverage + timePeriodCoveredStart Start Date The start date of the time period that the data refer to YYYY-MM-DD date 60 #NAME: #VALUE TRUE FALSE FALSE TRUE FALSE FALSE timePeriodCovered citation + timePeriodCoveredEnd End Date The end date of the time period that the data refer to YYYY-MM-DD date 61 #NAME: #VALUE TRUE FALSE FALSE TRUE FALSE FALSE timePeriodCovered citation + dateOfCollection Date of Collection The dates when the data were collected or generated none 62 ; FALSE FALSE TRUE FALSE FALSE FALSE citation + dateOfCollectionStart Start Date The date when the data collection started YYYY-MM-DD date 63 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE dateOfCollection citation + dateOfCollectionEnd End Date The date when the data collection ended YYYY-MM-DD date 64 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE dateOfCollection citation + kindOfData Data Type The type of data included in the files (e.g. survey data, clinical data, or machine-readable text) text 65 TRUE FALSE TRUE TRUE FALSE FALSE citation http://rdf-vocabulary.ddialliance.org/discovery#kindOfData + series Series Information about the dataset series to which the Dataset belong none 66 : FALSE FALSE TRUE FALSE FALSE FALSE citation + seriesName Name The name of the dataset series text 67 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE series citation + seriesInformation Information Can include 1) a history of the series and 2) a summary of features that apply to the series textbox 68 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE series citation + software Software Information about the software used to generate the Dataset none 69 , FALSE FALSE TRUE FALSE FALSE FALSE citation https://www.w3.org/TR/prov-o/#wasGeneratedBy + softwareName Name The name of software used to generate the Dataset text 70 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE software citation + softwareVersion Version The version of the software used to generate the Dataset, e.g. 4.11 text 71 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE software citation + relatedMaterial Related Material Information, such as a persistent ID or citation, about the material related to the Dataset, such as appendices or sampling information available outside of the Dataset textbox 72 FALSE FALSE TRUE FALSE FALSE FALSE citation + relatedDatasets Related Dataset Information, such as a persistent ID or citation, about a related dataset, such as previous research on the Dataset's subject textbox 73 FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/relation + otherReferences Other Reference Information, such as a persistent ID or citation, about another type of resource that provides background or supporting material to the Dataset text 74 FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/references + dataSources Data Source Information, such as a persistent ID or citation, about sources of the Dataset (e.g. a book, article, serial, or machine-readable data file) textbox 75 FALSE FALSE TRUE FALSE FALSE FALSE citation https://www.w3.org/TR/prov-o/#wasDerivedFrom + originOfSources Origin of Historical Sources For historical sources, the origin and any rules followed in establishing them as sources textbox 76 FALSE FALSE FALSE FALSE FALSE FALSE citation + characteristicOfSources Characteristic of Sources Characteristics not already noted elsewhere textbox 77 FALSE FALSE FALSE FALSE FALSE FALSE citation + accessToSources Documentation and Access to Sources 1) Methods or procedures for accessing data sources and 2) any special permissions needed for access textbox 78 FALSE FALSE FALSE FALSE FALSE FALSE citation #controlledVocabulary DatasetField Value identifier displayOrder subject Agricultural Sciences D01 0 subject Arts and Humanities D0 1 diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java index 22bad42df96..d91aa101eb5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java @@ -91,8 +91,9 @@ public class DatasetFieldConstant implements java.io.Serializable { public final static String datasetVersionValue="datasetVersionValue"; public final static String versionDate="versionDate"; public final static String keywordValue="keywordValue"; - public final static String keywordVocab="keywordVocabulary"; //SEK 6/10/2016 to match what is in the db - public final static String keywordVocabURI="keywordVocabularyURI"; //SEK 6/10/2016 to match what is in the db + public final static String keywordTermURI="keywordTermURI"; + public final static String keywordVocab="keywordVocabulary"; + public final static String keywordVocabURI="keywordVocabularyURI"; public final static String topicClassValue="topicClassValue"; public final static String topicClassVocab="topicClassVocab"; public final static String topicClassVocabURI="topicClassVocabURI"; diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetKeyword.java b/src/main/java/edu/harvard/iq/dataverse/DatasetKeyword.java deleted file mode 100644 index 747e3c068f1..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetKeyword.java +++ /dev/null @@ -1,68 +0,0 @@ -package edu.harvard.iq.dataverse; - -/** - * - * @author skraffmiller - */ - -public class DatasetKeyword { - - private int displayOrder; - public int getDisplayOrder() { - return this.displayOrder; - } - public void setDisplayOrder(int displayOrder) { - this.displayOrder = displayOrder; - } - - private DatasetField value; - public DatasetField getValue() { - return this.value; - } - public void setValue(DatasetField value) { - this.value = value; - } - - private DatasetVersion datasetVersion; - public DatasetVersion getDatasetVersion() { - return datasetVersion; - } - public void setDatasetVersion(DatasetVersion metadata) { - this.datasetVersion = metadata; - } - /* - @Version - private Long version; - public Long getVersion() { - return this.version; - } - public void setVersion(Long version) { - this.version = version; - } */ - - private DatasetField vocab; - public DatasetField getVocab() { - return this.vocab; - } - public void setVocab(DatasetField vocab) { - this.vocab = vocab; - } - - private DatasetField vocabURI; - public DatasetField getVocabURI() { - return this.vocabURI; - } - public void setVocabURI(DatasetField vocabURI) { - this.vocabURI = vocabURI; - } - - - public boolean isEmpty() { - /*return ((value==null || value.getValue().trim().equals("")) - && (vocab==null || vocab.getValue().trim().equals("")) - && (vocabURI==null || vocabURI.getValue().trim().equals("")));*/ - return false; - } - - -} diff --git a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java index 49ceabc5900..4b8822e8b66 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java @@ -437,7 +437,7 @@ public static void writeSubjectsElement(XMLStreamWriter xmlw, DatasetVersionDTO for (String subject : fieldDTO.getMultipleVocab()) { if (StringUtils.isNotBlank(subject)) { subject_check = writeOpenTag(xmlw, "subjects", subject_check); - writeSubjectElement(xmlw, null, null, subject, language); + writeSubjectElement(xmlw, null, null, null, subject, language); } } } @@ -446,7 +446,8 @@ public static void writeSubjectsElement(XMLStreamWriter xmlw, DatasetVersionDTO for (HashSet fieldDTOs : fieldDTO.getMultipleCompound()) { String subject = null; String subjectScheme = null; - String schemeURI = null; + String keywordTermURI = null; + String keywordVocabURI = null; for (Iterator iterator = fieldDTOs.iterator(); iterator.hasNext();) { FieldDTO next = iterator.next(); @@ -454,18 +455,22 @@ public static void writeSubjectsElement(XMLStreamWriter xmlw, DatasetVersionDTO subject = next.getSinglePrimitive(); } + if (DatasetFieldConstant.keywordTermURI.equals(next.getTypeName())) { + keywordTermURI = next.getSinglePrimitive(); + } + if (DatasetFieldConstant.keywordVocab.equals(next.getTypeName())) { subjectScheme = next.getSinglePrimitive(); } - + if (DatasetFieldConstant.keywordVocabURI.equals(next.getTypeName())) { - schemeURI = next.getSinglePrimitive(); + keywordVocabURI = next.getSinglePrimitive(); } } if (StringUtils.isNotBlank(subject)) { subject_check = writeOpenTag(xmlw, "subjects", subject_check); - writeSubjectElement(xmlw, subjectScheme, schemeURI, subject, language); + writeSubjectElement(xmlw, subjectScheme, keywordTermURI, keywordVocabURI, subject, language); } } } @@ -493,7 +498,7 @@ public static void writeSubjectsElement(XMLStreamWriter xmlw, DatasetVersionDTO if (StringUtils.isNotBlank(subject)) { subject_check = writeOpenTag(xmlw, "subjects", subject_check); - writeSubjectElement(xmlw, subjectScheme, schemeURI, subject, language); + writeSubjectElement(xmlw, subjectScheme, null, schemeURI, subject, language); } } } @@ -513,7 +518,7 @@ public static void writeSubjectsElement(XMLStreamWriter xmlw, DatasetVersionDTO * @param language * @throws XMLStreamException */ - private static void writeSubjectElement(XMLStreamWriter xmlw, String subjectScheme, String schemeURI, String value, String language) throws XMLStreamException { + private static void writeSubjectElement(XMLStreamWriter xmlw, String subjectScheme, String valueURI, String schemeURI, String value, String language) throws XMLStreamException { // write a subject Map subject_map = new HashMap(); @@ -524,6 +529,9 @@ private static void writeSubjectElement(XMLStreamWriter xmlw, String subjectSche if (StringUtils.isNotBlank(subjectScheme)) { subject_map.put("subjectScheme", subjectScheme); } + if (StringUtils.isNotBlank(valueURI)) { + subject_map.put("valueURI", valueURI); + } if (StringUtils.isNotBlank(schemeURI)) { subject_map.put("schemeURI", schemeURI); } diff --git a/src/main/java/propertyFiles/citation.properties b/src/main/java/propertyFiles/citation.properties index f35ede79b50..cb864eb78e9 100644 --- a/src/main/java/propertyFiles/citation.properties +++ b/src/main/java/propertyFiles/citation.properties @@ -22,6 +22,7 @@ datasetfieldtype.dsDescriptionValue.title=Text datasetfieldtype.dsDescriptionDate.title=Date datasetfieldtype.subject.title=Subject datasetfieldtype.keyword.title=Keyword +datasetfieldtype.keywordTermURI.title=Term URI datasetfieldtype.keywordValue.title=Term datasetfieldtype.keywordVocabulary.title=Controlled Vocabulary Name datasetfieldtype.keywordVocabularyURI.title=Controlled Vocabulary URL @@ -100,6 +101,7 @@ datasetfieldtype.dsDescriptionValue.description=A summary describing the purpose datasetfieldtype.dsDescriptionDate.description=The date when the description was added to the Dataset. If the Dataset contains more than one description, e.g. the data producer supplied one description and the data repository supplied another, this date is used to distinguish between the descriptions datasetfieldtype.subject.description=The area of study relevant to the Dataset datasetfieldtype.keyword.description=A key term that describes an important aspect of the Dataset and information about any controlled vocabulary used +datasetfieldtype.keywordTermURI.description=A URI that points to the web presence of the Keyword Term datasetfieldtype.keywordValue.description=A key term that describes important aspects of the Dataset datasetfieldtype.keywordVocabulary.description=The controlled vocabulary used for the keyword term (e.g. LCSH, MeSH) datasetfieldtype.keywordVocabularyURI.description=The URL where one can access information about the term's controlled vocabulary @@ -178,6 +180,7 @@ datasetfieldtype.dsDescriptionValue.watermark= datasetfieldtype.dsDescriptionDate.watermark=YYYY-MM-DD datasetfieldtype.subject.watermark= datasetfieldtype.keyword.watermark= +datasetfieldtype.keywordTermURI.watermark=https:// datasetfieldtype.keywordValue.watermark= datasetfieldtype.keywordVocabulary.watermark= datasetfieldtype.keywordVocabularyURI.watermark=https:// diff --git a/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java index 8ebdeea6243..2da15147255 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java @@ -305,8 +305,10 @@ public void testSubjectsElement() throws XMLStreamException, IOException { + "Engineering" + "Law" + "KeywordTerm1" + "KeywordTerm2" + "", stringWriter.toString()); diff --git a/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java b/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java index d600ccac53c..2139589b4c3 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java @@ -322,6 +322,7 @@ private static void mockDatasetFieldSvc() { DatasetFieldType keywordType = datasetFieldTypeSvc.add(new DatasetFieldType("keyword", DatasetFieldType.FieldType.TEXT, true)); Set keywordChildTypes = new HashSet<>(); keywordChildTypes.add(datasetFieldTypeSvc.add(new DatasetFieldType("keywordValue", DatasetFieldType.FieldType.TEXT, false))); + keywordChildTypes.add(datasetFieldTypeSvc.add(new DatasetFieldType("keywordTermURI", DatasetFieldType.FieldType.TEXT, false))); keywordChildTypes.add(datasetFieldTypeSvc.add(new DatasetFieldType("keywordVocabulary", DatasetFieldType.FieldType.TEXT, false))); keywordChildTypes.add(datasetFieldTypeSvc.add(new DatasetFieldType("keywordVocabularyURI", DatasetFieldType.FieldType.TEXT, false))); keywordType.setChildDatasetFieldTypes(keywordChildTypes); diff --git a/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt b/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt index 342d4b6fabf..431f069cb03 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt +++ b/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt @@ -271,6 +271,12 @@ "typeClass": "primitive", "value": "KeywordTerm1" }, + "keywordTermURI": { + "typeName": "keywordTermURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://keywordTermURI1.org" + }, "keywordVocabulary": { "typeName": "keywordVocabulary", "multiple": false, @@ -291,6 +297,12 @@ "typeClass": "primitive", "value": "KeywordTerm2" }, + "keywordTermURI": { + "typeName": "keywordTermURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://keywordTermURI2.org" + }, "keywordVocabulary": { "typeName": "keywordVocabulary", "multiple": false, diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json index b23741517c9..9cf04bd0e05 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json @@ -230,6 +230,12 @@ "typeClass": "primitive", "value": "KeywordTerm1" }, + "keywordTermURI": { + "typeName": "keywordTermURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://keywordTermURI1.org" + }, "keywordVocabulary": { "typeName": "keywordVocabulary", "multiple": false, @@ -250,6 +256,12 @@ "typeClass": "primitive", "value": "KeywordTerm2" }, + "keywordTermURI": { + "typeName": "keywordTermURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://keywordTermURI2.org" + }, "keywordVocabulary": { "typeName": "keywordVocabulary", "multiple": false, diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.json b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.json index f9882aed3dd..2d4ca078962 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.json +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.json @@ -238,6 +238,12 @@ "typeClass": "primitive", "value": "Keyword Value 1" }, + "keywordTermURI": { + "typeName": "keywordTermURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://keywordTermURI1.org" + }, "keywordVocabulary": { "typeName": "keywordVocabulary", "multiple": false, @@ -258,6 +264,12 @@ "typeClass": "primitive", "value": "Keyword Value Two" }, + "keywordTermURI": { + "typeName": "keywordTermURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://keywordTermURI1.org" + }, "keywordVocabulary": { "typeName": "keywordVocabulary", "multiple": false, diff --git a/src/test/resources/json/dataset-finch2.json b/src/test/resources/json/dataset-finch2.json index 4bd6f33eb42..b214eacfa3c 100644 --- a/src/test/resources/json/dataset-finch2.json +++ b/src/test/resources/json/dataset-finch2.json @@ -100,6 +100,12 @@ "typeClass": "primitive", "value": "KeywordTerm1" }, + "keywordTermURI": { + "typeName": "keywordTermURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://keywordTermURI1.org" + }, "keywordVocabulary": { "typeName": "keywordVocabulary", "multiple": false, @@ -120,6 +126,12 @@ "typeClass": "primitive", "value": "KeywordTerm2" }, + "keywordTermURI": { + "typeName": "keywordTermURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://keywordTermURI2.org" + }, "keywordVocabulary": { "typeName": "keywordVocabulary", "multiple": false, diff --git a/src/test/resources/json/dataset-long-description.json b/src/test/resources/json/dataset-long-description.json index a6e5c291322..4d5478b0f63 100644 --- a/src/test/resources/json/dataset-long-description.json +++ b/src/test/resources/json/dataset-long-description.json @@ -92,6 +92,12 @@ "typeClass": "primitive", "value": "KeywordTerm1" }, + "keywordTermURI": { + "typeName": "keywordTermURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://keywordTermURI1.org" + }, "keywordVocabulary": { "typeName": "keywordVocabulary", "multiple": false, @@ -112,6 +118,12 @@ "typeClass": "primitive", "value": "KeywordTerm2" }, + "keywordTermURI": { + "typeName": "keywordTermURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://keywordTermURI2.org" + }, "keywordVocabulary": { "typeName": "keywordVocabulary", "multiple": false, From 946d1ddd75e24d053ddd0ed0ad4e0b4b848b0fd8 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 14 Jun 2024 11:49:35 -0400 Subject: [PATCH 095/145] incorporate feedback from @philippconzett #10531 Via https://groups.google.com/g/dataverse-community/c/ejJrJmH6ojI/m/NnLSTlaMDwAJ --- doc/sphinx-guides/source/contributor/code.md | 2 +- doc/sphinx-guides/source/contributor/documentation.md | 10 +++++----- doc/sphinx-guides/source/contributor/index.md | 4 ++++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/doc/sphinx-guides/source/contributor/code.md b/doc/sphinx-guides/source/contributor/code.md index 9bc2b5a2027..2a1dec08c05 100644 --- a/doc/sphinx-guides/source/contributor/code.md +++ b/doc/sphinx-guides/source/contributor/code.md @@ -45,7 +45,7 @@ For other codebases, consult the README. ## Reviewing Code -Reviewing code is a great way to learn about a codebase. For any codebase you can browse open pull requests, of course, but for the primary codebases, you can take a look at the "Ready for Review" and "In Review" columns at https://github.com/orgs/IQSS/projects/34 +Reviewing code is a great way to learn about a codebase. For any codebase you can browse open pull requests, of course, but for the primary codebases, you can take a look at the "Ready for Review" and "In Review" columns at . You are welcome to review code informally or to leave an actual review. We're interested in what you think. diff --git a/doc/sphinx-guides/source/contributor/documentation.md b/doc/sphinx-guides/source/contributor/documentation.md index 0c4955d3e5e..67db855dc60 100644 --- a/doc/sphinx-guides/source/contributor/documentation.md +++ b/doc/sphinx-guides/source/contributor/documentation.md @@ -1,6 +1,6 @@ # Writing Documentation -Thank you for your interest in contributing documentation to Dataverse! Good documentation is absolutely critical to the success of a software project. +Thank you for your interest in contributing documentation to Dataverse! Good documentation is absolutely critical to the success of software. ```{contents} Contents: :local: @@ -26,23 +26,23 @@ We could use some help on writing this very page and helping newcomers get start ## Quick Fix -If you find a typo or a small error in the documentation you can fix it using GitHub's online web editor. Generally speaking, we will be following +If you find a typo or a small error in the documentation you can fix it using GitHub's online web editor. Generally speaking, we will be following [GitHub's guidance on editing files in another user's repository](https://docs.github.com/en/repositories/working-with-files/managing-files/editing-files#editing-files-in-another-users-repository). - Navigate to where you will see folders for each of the guides: [admin][], [api][], [container][], etc. - Find the file you want to edit under one of the folders above. -- Click the pencil icon in the upper-right corner. If this is your first contribution to the Dataverse Project, the hover text over the pencil icon will say "Fork this project and edit this file". +- Click the pencil icon in the upper-right corner. If this is your first contribution to Dataverse, the hover text over the pencil icon will say "Fork this project and edit this file". - Make changes to the file and preview them. - In the **Commit changes** box, enter a description of the changes you have made and click **Propose file change**. - Under the **Write** tab, delete the long welcome message and write a few words about what you fixed. - Click **Create Pull Request**. -That's it! Thank you for your contribution! Your pull request will be added manually to the main Dataverse Project board at and will go through code review and QA before it is merged into the "develop" branch. Along the way, developers might suggest changes or make them on your behalf. Once your pull request has been merged you will be listed as a contributor at ! 🎉 +That's it! Thank you for your contribution! Your pull request will be added manually to the main Dataverse project board at and will go through code review and QA before it is merged into the "develop" branch. Along the way, developers might suggest changes or make them on your behalf. Once your pull request has been merged you will be listed as a contributor at ! 🎉 Please see for an example of a quick fix that was merged (the "Files changed" tab shows how a typo was fixed). Preview your documentation changes which will be built automatically as part of your pull request in Github. It will show up as a check entitled: `docs/readthedocs.org:dataverse-guide — Read the Docs build succeeded!`. For example, this PR built to . -If you would like to read more about the Dataverse Project's use of GitHub, please see the {doc}`/developers/version-control` section of the Developer Guide. For bug fixes and features we request that you create an issue before making a pull request but this is not at all necessary for quick fixes to the documentation. +If you would like to read more about the Dataverse's use of GitHub, please see the {doc}`/developers/version-control` section of the Developer Guide. For bug fixes and features we request that you create an issue before making a pull request but this is not at all necessary for quick fixes to the documentation. [admin]: https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source/admin [api]: https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source/api diff --git a/doc/sphinx-guides/source/contributor/index.md b/doc/sphinx-guides/source/contributor/index.md index 7a9555fdd34..e75cc58bccd 100644 --- a/doc/sphinx-guides/source/contributor/index.md +++ b/doc/sphinx-guides/source/contributor/index.md @@ -86,6 +86,10 @@ Most working groups are wide open to participation. For the current list of grou You're welcome to start your own working group, of course. We can help you get the word out. +## GDCC + +The popularity of the Dataverse software has resulted in a continuously growing community with different needs and requirements. The Global Dataverse Community Consortium (GDCC) helps coordinate community efforts and sustain the software and community in the long term. Please consider contributing to GDCC by joining as an institutional member (). + ## Artwork Surely we can put artistic talent to use. A contributor [drew cartoon animals chatting about Dataverse](https://github.com/IQSS/chat.dataverse.org/issues/18), for example. From d113d7c358147b9caa7e4b148807dd28eb32c9d9 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 18 Jun 2024 09:02:30 -0400 Subject: [PATCH 096/145] Deprecate rsync: remove from guides, mark code as deprecated (#10620) * remove rsync docs #8985 * add deprecation notices to Java code for rsync #8985 * delete deprecated config options from guides #8985 * whoops, put :UploadMethods doc ref back #8985 * restore :PublicInstall to non-deprecated #8985 --- doc/release-notes/8985-deprecate-rsync.md | 8 + .../checksumValidationSuccess.json | 5 - doc/sphinx-guides/source/api/native-api.rst | 6 +- .../source/developers/big-data-support.rst | 178 +----------------- .../source/installation/config.rst | 24 +-- .../source/user/dataset-management.rst | 34 +--- .../source/user/find-use-data.rst | 13 -- .../data/storageSites/add-storage-site.json | 6 - .../harvard/iq/dataverse/SettingsWrapper.java | 11 +- .../harvard/iq/dataverse/api/Datasets.java | 1 + .../DataCaptureModuleException.java | 2 + .../DataCaptureModuleUtil.java | 7 + .../impl/RequestRsyncScriptCommand.java | 1 + ...RepositoryStorageAbstractionLayerPage.java | 4 + ...RepositoryStorageAbstractionLayerUtil.java | 6 + .../settings/SettingsServiceBean.java | 8 + .../iq/dataverse/util/SystemConfig.java | 14 +- 17 files changed, 61 insertions(+), 267 deletions(-) create mode 100644 doc/release-notes/8985-deprecate-rsync.md delete mode 100644 doc/sphinx-guides/source/_static/installation/files/root/big-data-support/checksumValidationSuccess.json delete mode 100644 scripts/api/data/storageSites/add-storage-site.json diff --git a/doc/release-notes/8985-deprecate-rsync.md b/doc/release-notes/8985-deprecate-rsync.md new file mode 100644 index 00000000000..44563f292fd --- /dev/null +++ b/doc/release-notes/8985-deprecate-rsync.md @@ -0,0 +1,8 @@ +Support for rsync has been deprecated. Information has been removed from the guides for rsync and related software such as Data Capture Module (DCM) and Repository Storage Abstraction Layer (RSAL). You can still find this information in [older versions](https://guides.dataverse.org/en/6.2/developers/big-data-support.html#data-capture-module-dcm) of the guides. + +The following related database settings have been deprecated as well: + +- :DataCaptureModuleUrl +- :DownloadMethods +- :LocalDataAccessPath +- :RepositoryStorageAbstractionLayerUrl diff --git a/doc/sphinx-guides/source/_static/installation/files/root/big-data-support/checksumValidationSuccess.json b/doc/sphinx-guides/source/_static/installation/files/root/big-data-support/checksumValidationSuccess.json deleted file mode 100644 index 6b609c4c65e..00000000000 --- a/doc/sphinx-guides/source/_static/installation/files/root/big-data-support/checksumValidationSuccess.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "status": "validation passed", - "uploadFolder": "OS7O8Y", - "totalSize": 72 -} diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 8c54a937353..04123321e54 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2315,7 +2315,7 @@ The fully expanded example above (without environment variables) looks like this curl "https://demo.dataverse.org/api/datasets/24/locks?type=Ingest" -Currently implemented lock types are ``Ingest``, ``Workflow``, ``InReview``, ``DcmUpload``, ``finalizePublication``, ``EditInProgress`` and ``FileValidationFailed``. +Currently implemented lock types are ``Ingest``, ``Workflow``, ``InReview``, ``DcmUpload`` (deprecated), ``finalizePublication``, ``EditInProgress`` and ``FileValidationFailed``. The API will output the list of locks, for example:: @@ -2406,7 +2406,7 @@ Use the following API to list ALL the locks on all the datasets in your installa The listing can be filtered by specific lock type **and/or** user, using the following *optional* query parameters: * ``userIdentifier`` - To list the locks owned by a specific user -* ``type`` - To list the locks of the type specified. If the supplied value does not match a known lock type, the API will return an error and a list of valid lock types. As of writing this, the implemented lock types are ``Ingest``, ``Workflow``, ``InReview``, ``DcmUpload``, ``finalizePublication``, ``EditInProgress`` and ``FileValidationFailed``. +* ``type`` - To list the locks of the type specified. If the supplied value does not match a known lock type, the API will return an error and a list of valid lock types. As of writing this, the implemented lock types are ``Ingest``, ``Workflow``, ``InReview``, ``DcmUpload`` (deprecated), ``finalizePublication``, ``EditInProgress`` and ``FileValidationFailed``. For example: @@ -3192,7 +3192,7 @@ Note: you can use the combination of cURL's ``-J`` (``--remote-header-name``) an Restrict Files ~~~~~~~~~~~~~~ -Restrict or unrestrict an existing file where ``id`` is the database id of the file or ``pid`` is the persistent id (DOI or Handle) of the file to restrict. Note that some Dataverse installations do not allow the ability to restrict files. +Restrict or unrestrict an existing file where ``id`` is the database id of the file or ``pid`` is the persistent id (DOI or Handle) of the file to restrict. Note that some Dataverse installations do not allow the ability to restrict files (see :ref:`:PublicInstall`). A curl example using an ``id`` diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index 5ea97029271..4aaed10512e 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -1,7 +1,7 @@ Big Data Support ================ -Big data support includes some highly experimental options. Eventually more of this content will move to the Installation Guide. +Big data support includes some experimental options. Eventually more of this content will move to the Installation Guide. .. contents:: |toctitle| :local: @@ -187,179 +187,3 @@ As described in that document, Globus transfers can be initiated by choosing the An overview of the control and data transfer interactions between components was presented at the 2022 Dataverse Community Meeting and can be viewed in the `Integrations and Tools Session Video `_ around the 1 hr 28 min mark. See also :ref:`Globus settings <:GlobusSettings>`. - -Data Capture Module (DCM) -------------------------- - -Please note: The DCM feature is deprecated. - -Data Capture Module (DCM) is an experimental component that allows users to upload large datasets via rsync over ssh. - -DCM was developed and tested using Glassfish but these docs have been updated with references to Payara. - -Install a DCM -~~~~~~~~~~~~~ - -Installation instructions can be found at https://github.com/sbgrid/data-capture-module/blob/master/doc/installation.md. Note that shared storage (posix or AWS S3) between your Dataverse installation and your DCM is required. You cannot use a DCM with Swift at this point in time. - -.. FIXME: Explain what ``dataverse.files.dcm-s3-bucket-name`` is for and what it has to do with ``dataverse.files.s3.bucket-name``. - -Once you have installed a DCM, you will need to configure two database settings on the Dataverse installation side. These settings are documented in the :doc:`/installation/config` section of the Installation Guide: - -- ``:DataCaptureModuleUrl`` should be set to the URL of a DCM you installed. -- ``:UploadMethods`` should include ``dcm/rsync+ssh``. - -This will allow your Dataverse installation to communicate with your DCM, so that your Dataverse installation can download rsync scripts for your users. - -Downloading rsync scripts via Your Dataverse Installation's API -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The rsync script can be downloaded from your Dataverse installation via API using an authorized API token. In the curl example below, substitute ``$PERSISTENT_ID`` with a DOI or Handle: - -``curl -H "X-Dataverse-key: $API_TOKEN" $DV_BASE_URL/api/datasets/:persistentId/dataCaptureModule/rsync?persistentId=$PERSISTENT_ID`` - -How a DCM reports checksum success or failure to your Dataverse Installation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Once the user uploads files to a DCM, that DCM will perform checksum validation and report to your Dataverse installation the results of that validation. The DCM must be configured to pass the API token of a superuser. The implementation details, which are subject to change, are below. - -The JSON that a DCM sends to your Dataverse installation on successful checksum validation looks something like the contents of :download:`checksumValidationSuccess.json <../_static/installation/files/root/big-data-support/checksumValidationSuccess.json>` below: - -.. literalinclude:: ../_static/installation/files/root/big-data-support/checksumValidationSuccess.json - :language: json - -- ``status`` - The valid strings to send are ``validation passed`` and ``validation failed``. -- ``uploadFolder`` - This is the directory on disk where your Dataverse installation should attempt to find the files that a DCM has moved into place. There should always be a ``files.sha`` file and a least one data file. ``files.sha`` is a manifest of all the data files and their checksums. The ``uploadFolder`` directory is inside the directory where data is stored for the dataset and may have the same name as the "identifier" of the persistent id (DOI or Handle). For example, you would send ``"uploadFolder": "DNXV2H"`` in the JSON file when the absolute path to this directory is ``/usr/local/payara6/glassfish/domains/domain1/files/10.5072/FK2/DNXV2H/DNXV2H``. -- ``totalSize`` - Your Dataverse installation will use this value to represent the total size in bytes of all the files in the "package" that's created. If 360 data files and one ``files.sha`` manifest file are in the ``uploadFolder``, this value is the sum of the 360 data files. - - -Here's the syntax for sending the JSON. - -``curl -H "X-Dataverse-key: $API_TOKEN" -X POST -H 'Content-type: application/json' --upload-file checksumValidationSuccess.json $DV_BASE_URL/api/datasets/:persistentId/dataCaptureModule/checksumValidation?persistentId=$PERSISTENT_ID`` - - -Steps to set up a DCM mock for Development -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -See instructions at https://github.com/sbgrid/data-capture-module/blob/master/doc/mock.md - - -Add Dataverse Installation settings to use mock (same as using DCM, noted above): - -- ``curl http://localhost:8080/api/admin/settings/:DataCaptureModuleUrl -X PUT -d "http://localhost:5000"`` -- ``curl http://localhost:8080/api/admin/settings/:UploadMethods -X PUT -d "dcm/rsync+ssh"`` - -At this point you should be able to download a placeholder rsync script. Your Dataverse installation is then waiting for news from the DCM about if checksum validation has succeeded or not. First, you have to put files in place, which is usually the job of the DCM. You should substitute "X1METO" for the "identifier" of the dataset you create. You must also use the proper path for where you store files in your dev environment. - -- ``mkdir /usr/local/payara6/glassfish/domains/domain1/files/10.5072/FK2/X1METO`` -- ``mkdir /usr/local/payara6/glassfish/domains/domain1/files/10.5072/FK2/X1METO/X1METO`` -- ``cd /usr/local/payara6/glassfish/domains/domain1/files/10.5072/FK2/X1METO/X1METO`` -- ``echo "hello" > file1.txt`` -- ``shasum file1.txt > files.sha`` - - - -Now the files are in place and you need to send JSON to your Dataverse installation with a success or failure message as described above. Make a copy of ``doc/sphinx-guides/source/_static/installation/files/root/big-data-support/checksumValidationSuccess.json`` and put the identifier in place such as "X1METO" under "uploadFolder"). Then use curl as described above to send the JSON. - -Troubleshooting -~~~~~~~~~~~~~~~ - -The following low level command should only be used when troubleshooting the "import" code a DCM uses but is documented here for completeness. - -``curl -H "X-Dataverse-key: $API_TOKEN" -X POST "$DV_BASE_URL/api/batch/jobs/import/datasets/files/$DATASET_DB_ID?uploadFolder=$UPLOAD_FOLDER&totalSize=$TOTAL_SIZE"`` - -Repository Storage Abstraction Layer (RSAL) -------------------------------------------- - -Please note: The RSAL feature is deprecated. - -Steps to set up a DCM via Docker for Development -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -See https://github.com/IQSS/dataverse/blob/develop/conf/docker-dcm/readme.md - -Using the RSAL Docker Containers -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -- Create a dataset (either with the procedure mentioned in DCM Docker Containers, or another process) -- Publish the dataset (from the client container): ``cd /mnt; ./publish_major.bash ${database_id}`` -- Run the RSAL component of the workflow (from the host): ``docker exec -it rsalsrv /opt/rsal/scn/pub.py`` -- If desired, from the client container you can download the dataset following the instructions in the dataset access section of the dataset page. - -Configuring the RSAL Mock -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Info for configuring the RSAL Mock: https://github.com/sbgrid/rsal/tree/master/mocks - -Also, to configure your Dataverse installation to use the new workflow you must do the following (see also the :doc:`workflows` section): - -1. Configure the RSAL URL: - -``curl -X PUT -d 'http://:5050' http://localhost:8080/api/admin/settings/:RepositoryStorageAbstractionLayerUrl`` - -2. Update workflow json with correct URL information: - -Edit internal-httpSR-workflow.json and replace url and rollbackUrl to be the url of your RSAL mock. - -3. Create the workflow: - -``curl http://localhost:8080/api/admin/workflows -X POST --data-binary @internal-httpSR-workflow.json -H "Content-type: application/json"`` - -4. List available workflows: - -``curl http://localhost:8080/api/admin/workflows`` - -5. Set the workflow (id) as the default workflow for the appropriate trigger: - -``curl http://localhost:8080/api/admin/workflows/default/PrePublishDataset -X PUT -d 2`` - -6. Check that the trigger has the appropriate default workflow set: - -``curl http://localhost:8080/api/admin/workflows/default/PrePublishDataset`` - -7. Add RSAL to whitelist - -8. When finished testing, unset the workflow: - -``curl -X DELETE http://localhost:8080/api/admin/workflows/default/PrePublishDataset`` - -Configuring download via rsync -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In order to see the rsync URLs, you must run this command: - -``curl -X PUT -d 'rsal/rsync' http://localhost:8080/api/admin/settings/:DownloadMethods`` - -.. TODO: Document these in the Installation Guide once they're final. - -To specify replication sites that appear in rsync URLs: - -Download :download:`add-storage-site.json <../../../../scripts/api/data/storageSites/add-storage-site.json>` and adjust it to meet your needs. The file should look something like this: - -.. literalinclude:: ../../../../scripts/api/data/storageSites/add-storage-site.json - -Then add the storage site using curl: - -``curl -H "Content-type:application/json" -X POST http://localhost:8080/api/admin/storageSites --upload-file add-storage-site.json`` - -You make a storage site the primary site by passing "true". Pass "false" to make it not the primary site. (id "1" in the example): - -``curl -X PUT -d true http://localhost:8080/api/admin/storageSites/1/primaryStorage`` - -You can delete a storage site like this (id "1" in the example): - -``curl -X DELETE http://localhost:8080/api/admin/storageSites/1`` - -You can view a single storage site like this: (id "1" in the example): - -``curl http://localhost:8080/api/admin/storageSites/1`` - -You can view all storage site like this: - -``curl http://localhost:8080/api/admin/storageSites`` - -In the GUI, this is called "Local Access". It's where you can compute on files on your cluster. - -``curl http://localhost:8080/api/admin/settings/:LocalDataAccessPath -X PUT -d "/programs/datagrid"`` - - diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 8fb9460892b..213ac827819 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -4218,20 +4218,6 @@ This is useful for specific cases where an installation's files are stored in pu ``curl -X PUT -d true http://localhost:8080/api/admin/settings/:PublicInstall`` -:DataCaptureModuleUrl -+++++++++++++++++++++ - -The URL for your Data Capture Module (DCM) installation. This component is experimental and can be downloaded from https://github.com/sbgrid/data-capture-module . - -``curl -X PUT -d 'https://dcm.example.edu' http://localhost:8080/api/admin/settings/:DataCaptureModuleUrl`` - -:RepositoryStorageAbstractionLayerUrl -+++++++++++++++++++++++++++++++++++++ - -The URL for your Repository Storage Abstraction Layer (RSAL) installation. This component is experimental and can be downloaded from https://github.com/sbgrid/rsal . - -``curl -X PUT -d 'https://rsal.example.edu' http://localhost:8080/api/admin/settings/:RepositoryStorageAbstractionLayerUrl`` - .. _:UploadMethods: :UploadMethods @@ -4241,23 +4227,15 @@ This setting controls which upload methods are available to users of your Datave - ``native/http``: Corresponds to "Upload with HTTP via your browser" and APIs that use HTTP (SWORD and native). - ``dvwebloader``: Corresponds to :ref:`folder-upload`. Note that ``dataverse.files..upload-redirect`` must be set to "true" on an S3 store for this method to show up in the UI. In addition, :ref:`:WebloaderUrl` must be set. CORS allowed on the S3 bucket. See :ref:`cors-s3-bucket`. -- ``dcm/rsync+ssh``: Corresponds to "Upload with rsync+ssh via Data Capture Module (DCM)". A lot of setup is required, as explained in the :doc:`/developers/big-data-support` section of the Developer Guide. Out of the box only ``native/http`` is enabled and will work without further configuration. To add multiple upload method, separate them using a comma like this: -``curl -X PUT -d 'native/http,dcm/rsync+ssh' http://localhost:8080/api/admin/settings/:UploadMethods`` +``curl -X PUT -d 'native/http,dvwebloader' http://localhost:8080/api/admin/settings/:UploadMethods`` You'll always want at least one upload method, so the easiest way to remove one of them is to simply ``PUT`` just the one you want, like this: ``curl -X PUT -d 'native/http' http://localhost:8080/api/admin/settings/:UploadMethods`` -:DownloadMethods -++++++++++++++++ - -This setting is experimental and related to Repository Storage Abstraction Layer (RSAL). - -``curl -X PUT -d 'rsal/rsync' http://localhost:8080/api/admin/settings/:DownloadMethods`` - :GuestbookResponsesPageDisplayLimit +++++++++++++++++++++++++++++++++++ diff --git a/doc/sphinx-guides/source/user/dataset-management.rst b/doc/sphinx-guides/source/user/dataset-management.rst index d803aae6d19..a1e214589e3 100755 --- a/doc/sphinx-guides/source/user/dataset-management.rst +++ b/doc/sphinx-guides/source/user/dataset-management.rst @@ -84,7 +84,7 @@ HTTP Upload is a common browser-based file upload tool you may be familiar with Once you have uploaded files, you will be able to edit file metadata, restrict access to files [#f1]_ , and/or add tags. Click "Save Changes" to complete the upload. If you uploaded a file by mistake, you can delete it before saving by clicking the checkbox to select the file, and then clicking the "Delete" button above the Files Table. -File upload limit size varies based on Dataverse installation. The file upload size limit can be found in the text above the HTTP upload widget. If you need to upload a very large file or a very large *number* of files, consider using rsync + SSH upload if your Dataverse installation offers it. +File upload limit size varies based on Dataverse installation. The file upload size limit can be found in the text above the HTTP upload widget. If you need to upload a very large file or a very large *number* of files, consider using DVUploader (see :ref:`DVUploader`). .. [#f1] Some Dataverse installations do not allow this feature. @@ -100,37 +100,7 @@ Folder Upload Some Dataverse installations support the ability to upload files from a local folder and subfolders. To do this, click the "Upload from Folder" button, select the folder you wish to upload, select/unselect specific files, and click "Start Uploads". More detailed instructions are available in the `DVWebloader wiki `_. -.. _rsync_upload: - -rsync + SSH Upload ------------------- - -rsync is typically used for synchronizing files and directories between two different systems, using SSH to connect rather than HTTP. Some Dataverse installations allow uploads using rsync, to facilitate large file transfers in a reliable and secure manner. - -File Upload Script -~~~~~~~~~~~~~~~~~~ - -An rsync-enabled Dataverse installation has a file upload process that differs from the traditional browser-based upload process you may be used to. In order to transfer your data to the Dataverse installation's storage, you will need to complete the following steps: - -1. Create your dataset. In rsync-enabled Dataverse installations, you cannot upload files until the dataset creation process is complete. After you hit "Save Dataset" on the Dataset Creation page, you will be taken to the page for your dataset. - -2. On the dataset page, click the "+ Upload Files" button. This will open a box with instructions and a link to the file upload script. - -3. Make sure your files are ready for upload. You will need to have one directory that you can point the upload script to. All files in this directory and in any subdirectories will be uploaded. The directory structure will be preserved, and will be reproduced when your dataset is downloaded from the Dataverse installation. Note that your data will be uploaded in the form of a data package, and each dataset can only host one such package. Be sure that all files you want to include are present before you upload. - -4. Download the rsync file upload script by clicking the "Download Script" button in the Upload Files instruction box. There are no requirements for where you save the script; put it somewhere you can find it. Downloading the upload script will put a temporary lock on your dataset to prepare it for upload. While your dataset is locked, you will not be able to delete or publish your dataset, or edit its metadata. Once you upload your files and Dataverse installation processes them, your dataset will be automatically unlocked and these disabled functions will be enabled again. If you have downloaded the script and locked your dataset, but you have then changed your mind and decided *not* to upload files, please contact Support about unlocking your dataset. - -5. To begin the upload process, you will need to run the script you downloaded. For this, you will have to go outside your browser and open a terminal (AKA command line) window on your computer. Use the terminal to navigate to the directory where you saved the upload script, and run the command that the Upload Files instruction box provides. This will begin the upload script. Please note that this upload script will expire 7 days after you downloaded it. If it expires and you still need to use it, simply download the script from the Dataverse installation again. - -**Note:** Unlike other operating systems, Windows does not come with rsync supported by default. We have not optimized this feature for Windows users, but you may be able to get it working if you install the right Unix utilities. (If you have found a way to get this feature working for you on Windows, you can contribute it to our project. Please reference our `Contributing to the Dataverse Project `_ document in the root of the source tree.) - -6. Follow the instructions provided by the upload script running in your terminal. It will direct you to enter the full path of the directory where your dataset files are located, and then it will start the upload process. Once you've initiated the upload, if you need to cancel it then you can do so by canceling the script running in your terminal window. If your upload gets interrupted, you can resume it from the same point later. - -7. Once the upload script completes its job, the Dataverse installation will begin processing your data upload and running a checksum validation. This may take some time depending on the file size of your upload. During processing, you will see a blue bar at the bottom of the dataset page that reads "Upload in progress..." - -8. Once processing is complete, you will be notified. At this point you can publish your dataset and your data will be available for download on the dataset page. - -**Note:** A dataset can only hold one data package. If you need to replace the data package in your dataset, contact Support. +.. _DVUploader: Command-line DVUploader ----------------------- diff --git a/doc/sphinx-guides/source/user/find-use-data.rst b/doc/sphinx-guides/source/user/find-use-data.rst index bea23cbcd0e..4bf45774b53 100755 --- a/doc/sphinx-guides/source/user/find-use-data.rst +++ b/doc/sphinx-guides/source/user/find-use-data.rst @@ -142,19 +142,6 @@ Downloading a Dataverse File Package via URL Dataverse File Packages are typically used to represent extremely large files or bundles containing a large number of files. Dataverse File Packages are often too large to be reliably downloaded using a web browser. When you click to download a Dataverse File Package, instead of automatically initiating the download in your web browser, the Dataverse installation displays a plaintext URL for the location of the file. To ensure a reliable, resumable download, we recommend using `GNU Wget `_ in a command line terminal or using a download manager software of your choice. If you try to simply paste the URL into your web browser then the download may overwhelm your browser, resulting in an interrupted, timed out, or otherwise failed download. -.. _rsync_download: - -Downloading a Dataverse File Package via rsync -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -rsync is typically used for synchronizing files and directories between two different systems. Some Dataverse installations allow downloads using rsync, to facilitate large file transfers in a reliable and secure manner. - -rsync-enabled Dataverse installations offer a new file download process that differs from traditional browser-based downloading. Instead of multiple files, each dataset uploaded via rsync contains a single "Dataverse File Package". When you download this package you will receive a folder that contains all files from the dataset, arranged in the exact folder structure in which they were originally uploaded. - -In a dataset containing a Dataverse File Package, the information to download and/or access is in outlined the **Data Access** listed under the Access File button. If the data is locally available to you (on a shared drive, for example) you will find the folder path to access the data locally. To download, use one of the rsync commands provided. There may be multiple commands, each corresponding to a different mirror that hosts the Dataverse File Package. Go outside your browser and open a terminal (AKA command line) window on your computer. Use the terminal to run the command that corresponds with the mirror of your choice. It’s usually best to choose the mirror that is geographically closest to you. Running this command will initiate the download process. - -After you've downloaded the Dataverse File Package, you may want to double-check that your download went perfectly. Under **Verify Data**, you'll find a command that you can run in your terminal that will initiate a checksum to ensure that the data you downloaded matches the data in the Dataverse installation precisely. This way, you can ensure the integrity of the data you're working with. - Explore Data ------------ diff --git a/scripts/api/data/storageSites/add-storage-site.json b/scripts/api/data/storageSites/add-storage-site.json deleted file mode 100644 index d13ec2f165d..00000000000 --- a/scripts/api/data/storageSites/add-storage-site.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "hostname": "dataverse.librascholar.edu", - "name": "LibraScholar, USA", - "primaryStorage": true, - "transferProtocols": "rsync,posix,globus" -} diff --git a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java index 7854f5adfd8..48196591b19 100644 --- a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java +++ b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java @@ -305,14 +305,16 @@ public boolean isPublicInstall(){ } return publicInstall; } - + + @Deprecated(forRemoval = true, since = "2024-07-07") public boolean isRsyncUpload() { if (rsyncUpload == null) { rsyncUpload = getUploadMethodAvailable(SystemConfig.FileUploadMethods.RSYNC.toString()); } return rsyncUpload; } - + + @Deprecated(forRemoval = true, since = "2024-07-07") public boolean isRsyncDownload() { if (rsyncDownload == null) { rsyncDownload = systemConfig.isRsyncDownload(); @@ -379,7 +381,8 @@ public boolean isWebloaderUpload() { } return webloaderUpload; } - + + @Deprecated(forRemoval = true, since = "2024-07-07") public boolean isRsyncOnly() { if (rsyncOnly == null) { String downloadMethods = getValueForKey(SettingsServiceBean.Key.DownloadMethods); @@ -398,7 +401,7 @@ public boolean isRsyncOnly() { } return rsyncOnly; } - + public boolean isHTTPUpload(){ if (httpUpload == null) { httpUpload = getUploadMethodAvailable(SystemConfig.FileUploadMethods.NATIVE.toString()); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index fc0afc562fc..9e9f89c8140 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -2297,6 +2297,7 @@ public Response removeDatasetLogo(@Context ContainerRequestContext crc, @PathPar } } + @Deprecated(forRemoval = true, since = "2024-07-07") @GET @AuthRequired @Path("{identifier}/dataCaptureModule/rsync") diff --git a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleException.java b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleException.java index 3329d92b7a9..474674bda73 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleException.java +++ b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleException.java @@ -1,7 +1,9 @@ package edu.harvard.iq.dataverse.datacapturemodule; +@Deprecated(forRemoval = true, since = "2024-07-07") public class DataCaptureModuleException extends Exception { + @Deprecated(forRemoval = true, since = "2024-07-07") public DataCaptureModuleException(String message, Throwable cause) { super(message, cause); } diff --git a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java index 460e4727afc..094d3976133 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java @@ -12,10 +12,12 @@ import jakarta.json.JsonObject; import jakarta.json.JsonObjectBuilder; +@Deprecated(forRemoval = true, since = "2024-07-07") public class DataCaptureModuleUtil { private static final Logger logger = Logger.getLogger(DataCaptureModuleUtil.class.getCanonicalName()); + @Deprecated(forRemoval = true, since = "2024-07-07") public static boolean rsyncSupportEnabled(String uploadMethodsSettings) { logger.fine("uploadMethodsSettings: " + uploadMethodsSettings);; if (uploadMethodsSettings==null){ @@ -28,6 +30,7 @@ public static boolean rsyncSupportEnabled(String uploadMethodsSettings) { /** * generate JSON to send to DCM */ + @Deprecated(forRemoval = true, since = "2024-07-07") public static JsonObject generateJsonForUploadRequest(AuthenticatedUser user, Dataset dataset) { JsonObjectBuilder jab = Json.createObjectBuilder(); // The general rule should be to always pass the user id and dataset identifier to the DCM. @@ -39,6 +42,7 @@ public static JsonObject generateJsonForUploadRequest(AuthenticatedUser user, Da /** * transfer script from DCM */ + @Deprecated(forRemoval = true, since = "2024-07-07") public static ScriptRequestResponse getScriptFromRequest(HttpResponse uploadRequest) { int status = uploadRequest.getStatus(); JsonNode body = uploadRequest.getBody(); @@ -54,6 +58,7 @@ public static ScriptRequestResponse getScriptFromRequest(HttpResponse return scriptRequestResponse; } + @Deprecated(forRemoval = true, since = "2024-07-07") static UploadRequestResponse makeUploadRequest(HttpResponse uploadRequest) { int status = uploadRequest.getStatus(); String body = uploadRequest.getBody(); @@ -61,6 +66,7 @@ static UploadRequestResponse makeUploadRequest(HttpResponse uploadReques return new UploadRequestResponse(uploadRequest.getStatus(), body); } + @Deprecated(forRemoval = true, since = "2024-07-07") public static String getMessageFromException(DataCaptureModuleException ex) { if (ex == null) { return "DataCaptureModuleException was null!"; @@ -76,6 +82,7 @@ public static String getMessageFromException(DataCaptureModuleException ex) { return message + " was caused by " + cause.getMessage(); } + @Deprecated(forRemoval = true, since = "2024-07-07") public static String getScriptName(DatasetVersion datasetVersion) { return "upload-" + datasetVersion.getDataset().getIdentifier().replace("/", "_") + ".bash"; } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java index a29e7fdd59c..6b7baa7d01b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java @@ -27,6 +27,7 @@ * "actiontype" in the actionlogrecord rather than "InternalError" if you throw * a CommandExecutionException. */ +@Deprecated(forRemoval = true, since = "2024-07-07") @RequiredPermissions(Permission.EditDataset) public class RequestRsyncScriptCommand extends AbstractCommand { diff --git a/src/main/java/edu/harvard/iq/dataverse/repositorystorageabstractionlayer/RepositoryStorageAbstractionLayerPage.java b/src/main/java/edu/harvard/iq/dataverse/repositorystorageabstractionlayer/RepositoryStorageAbstractionLayerPage.java index c252d2e3330..9edb536eda2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/repositorystorageabstractionlayer/RepositoryStorageAbstractionLayerPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/repositorystorageabstractionlayer/RepositoryStorageAbstractionLayerPage.java @@ -11,6 +11,7 @@ import jakarta.inject.Named; import jakarta.json.JsonArray; +@Deprecated(forRemoval = true, since = "2024-07-07") @Stateless @Named public class RepositoryStorageAbstractionLayerPage { @@ -22,17 +23,20 @@ public class RepositoryStorageAbstractionLayerPage { @EJB StorageSiteServiceBean storageSiteServiceBean; + @Deprecated(forRemoval = true, since = "2024-07-07") public String getLocalDataAccessDirectory(DatasetVersion datasetVersion) { String localDataAccessParentDir = settingsService.getValueForKey(SettingsServiceBean.Key.LocalDataAccessPath); return RepositoryStorageAbstractionLayerUtil.getLocalDataAccessDirectory(localDataAccessParentDir, datasetVersion.getDataset()); } + @Deprecated(forRemoval = true, since = "2024-07-07") public List getRsyncSites(DatasetVersion datasetVersion) { List storageSites = storageSiteServiceBean.findAll(); JsonArray storageSitesAsJson = RepositoryStorageAbstractionLayerUtil.getStorageSitesAsJson(storageSites); return RepositoryStorageAbstractionLayerUtil.getRsyncSites(datasetVersion.getDataset(), storageSitesAsJson); } + @Deprecated(forRemoval = true, since = "2024-07-07") public String getVerifyDataCommand(DatasetVersion datasetVersion) { return RepositoryStorageAbstractionLayerUtil.getVerifyDataCommand(datasetVersion.getDataset()); } diff --git a/src/main/java/edu/harvard/iq/dataverse/repositorystorageabstractionlayer/RepositoryStorageAbstractionLayerUtil.java b/src/main/java/edu/harvard/iq/dataverse/repositorystorageabstractionlayer/RepositoryStorageAbstractionLayerUtil.java index 8501fba3ce0..0d547402676 100644 --- a/src/main/java/edu/harvard/iq/dataverse/repositorystorageabstractionlayer/RepositoryStorageAbstractionLayerUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/repositorystorageabstractionlayer/RepositoryStorageAbstractionLayerUtil.java @@ -13,10 +13,12 @@ import jakarta.json.JsonArrayBuilder; import jakarta.json.JsonObject; +@Deprecated(forRemoval = true, since = "2024-07-07") public class RepositoryStorageAbstractionLayerUtil { private static final Logger logger = Logger.getLogger(RepositoryStorageAbstractionLayerUtil.class.getCanonicalName()); + @Deprecated(forRemoval = true, since = "2024-07-07") public static List getRsyncSites(Dataset dataset, JsonArray rsalSitesAsJson) { List rsalSites = new ArrayList<>(); boolean leafDirectoryOnly = false; @@ -30,6 +32,7 @@ public static List getRsyncSites(Dataset dataset, JsonArray rsalSites return rsalSites; } + @Deprecated(forRemoval = true, since = "2024-07-07") static String getLocalDataAccessDirectory(String localDataAccessParentDir, Dataset dataset) { if (localDataAccessParentDir == null) { localDataAccessParentDir = File.separator + "UNCONFIGURED ( " + SettingsServiceBean.Key.LocalDataAccessPath + " )"; @@ -38,6 +41,7 @@ static String getLocalDataAccessDirectory(String localDataAccessParentDir, Datas return localDataAccessParentDir + File.separator + getDirectoryContainingTheData(dataset, leafDirectoryOnly); } + @Deprecated(forRemoval = true, since = "2024-07-07") static String getVerifyDataCommand(Dataset dataset) { boolean leafDirectoryOnly = true; // TODO: if "files.sha" is defined somewhere, use it. @@ -51,6 +55,7 @@ static String getVerifyDataCommand(Dataset dataset) { * leafDirectoryOnly. See also * http://www.gnu.org/software/coreutils/manual/html_node/basename-invocation.html */ + @Deprecated(forRemoval = true, since = "2024-07-07") public static String getDirectoryContainingTheData(Dataset dataset, boolean leafDirectoryOnly) { /** * FIXME: What if there is more than one package in the dataset? @@ -81,6 +86,7 @@ public static String getDirectoryContainingTheData(Dataset dataset, boolean leaf * RSAL or some other "big data" component live for a list of remotes sites * to which a particular dataset is replicated to. */ + @Deprecated(forRemoval = true, since = "2024-07-07") static JsonArray getStorageSitesAsJson(List storageSites) { JsonArrayBuilder arraybuilder = Json.createArrayBuilder(); if (storageSites == null || storageSites.isEmpty()) { diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 35d70498c3f..a1975b0b975 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -55,6 +55,10 @@ public enum Key { CustomDatasetSummaryFields, /** * Defines a public installation -- all datafiles are unrestricted + * + * This was added along with CloudEnvironmentName and ComputeBaseUrl. + * See https://github.com/IQSS/dataverse/issues/3776 and + * https://github.com/IQSS/dataverse/pull/3967 */ PublicInstall, /** @@ -75,9 +79,12 @@ public enum Key { /** * For example, https://datacapture.example.org */ + @Deprecated(forRemoval = true, since = "2024-07-07") DataCaptureModuleUrl, + @Deprecated(forRemoval = true, since = "2024-07-07") RepositoryStorageAbstractionLayerUrl, UploadMethods, + @Deprecated(forRemoval = true, since = "2024-07-07") DownloadMethods, /** * If the data replicated around the world using RSAL (Repository @@ -87,6 +94,7 @@ public enum Key { * TODO: Think about if it makes sense to make this a column in the * StorageSite database table. */ + @Deprecated(forRemoval = true, since = "2024-07-07") LocalDataAccessPath, /** * The algorithm used to generate PIDs, randomString (default) or diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 3f2f36ea36a..f9801419e47 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -752,6 +752,7 @@ public enum FileUploadMethods { * DCM stands for Data Capture Module. Right now it supports upload over * rsync+ssh but DCM may support additional methods in the future. */ + @Deprecated(forRemoval = true, since = "2024-07-07") RSYNC("dcm/rsync+ssh"), /** * Traditional Dataverse file handling, which tends to involve users @@ -809,6 +810,7 @@ public enum FileDownloadMethods { * RSAL stands for Repository Storage Abstraction Layer. Downloads don't * go through Glassfish. */ + @Deprecated(forRemoval = true, since = "2024-07-07") RSYNC("rsal/rsync"), NATIVE("native/http"), GLOBUS("globus") @@ -862,6 +864,7 @@ public String toString() { */ public enum TransferProtocols { + @Deprecated(forRemoval = true, since = "2024-07-07") RSYNC("rsync"), /** * POSIX includes NFS. This is related to Key.LocalDataAccessPath in @@ -898,7 +901,8 @@ public boolean isPublicInstall(){ boolean saneDefault = false; return settingsService.isTrueForKey(SettingsServiceBean.Key.PublicInstall, saneDefault); } - + + @Deprecated(forRemoval = true, since = "2024-07-07") public boolean isRsyncUpload(){ return getMethodAvailable(SystemConfig.FileUploadMethods.RSYNC.toString(), true); } @@ -915,7 +919,8 @@ public boolean isWebloaderUpload(){ public boolean isHTTPUpload(){ return getMethodAvailable(SystemConfig.FileUploadMethods.NATIVE.toString(), true); } - + + @Deprecated(forRemoval = true, since = "2024-07-07") public boolean isRsyncOnly(){ String downloadMethods = settingsService.getValueForKey(SettingsServiceBean.Key.DownloadMethods); if(downloadMethods == null){ @@ -931,11 +936,12 @@ public boolean isRsyncOnly(){ return Arrays.asList(uploadMethods.toLowerCase().split("\\s*,\\s*")).size() == 1 && uploadMethods.toLowerCase().equals(SystemConfig.FileUploadMethods.RSYNC.toString()); } } - + + @Deprecated(forRemoval = true, since = "2024-07-07") public boolean isRsyncDownload() { return getMethodAvailable(SystemConfig.FileUploadMethods.RSYNC.toString(), false); } - + public boolean isHTTPDownload() { return getMethodAvailable(SystemConfig.FileUploadMethods.NATIVE.toString(), false); } From 853965ec6058ddb988e09ef4a8fcb38c1a151619 Mon Sep 17 00:00:00 2001 From: Juan Pablo Tosca Villanueva <142103991+jp-tosca@users.noreply.github.com> Date: Tue, 18 Jun 2024 15:19:44 -0400 Subject: [PATCH 097/145] Banner IT improvements after resolving issues on rocky. (#10565) * Rocky IX * Logging pretty print * Removed use of function with hardcoded valie * Clearing banners * Added codeblock with JSON * Banners size * Trace added * Import fix * Pull test * lOG TEST * Log test * Log test * ASD * Debug commit * Debug commit * Dbug commit * Dbug commit * Bbug commit * Dbug cmmt * dbg cmmt * dbg cmmit * Dbg cmmt * reset bannermessage * Reset Admin * Reset AdminIT * Reset admin IT * Clear console output * Last changes * Last changes for this test * Last update * Locale check * Docs and makefile to build guides * Makefile changes * Value of sphinx version loaded from requirements.txt * Last changes to makefile and docs * Correction to the docs --- .../10565-banner-test-improvements.md | 1 + doc/sphinx-guides/source/api/native-api.rst | 2 + .../source/developers/documentation.rst | 15 ++++- makefile | 14 ++++ .../harvard/iq/dataverse/BannerMessage.java | 2 +- .../dataverse/BannerMessageServiceBean.java | 2 + .../edu/harvard/iq/dataverse/api/Admin.java | 24 +++++-- .../edu/harvard/iq/dataverse/api/AdminIT.java | 66 +++++++++++-------- .../edu/harvard/iq/dataverse/api/UtilIT.java | 14 ---- 9 files changed, 90 insertions(+), 50 deletions(-) create mode 100644 doc/release-notes/10565-banner-test-improvements.md create mode 100644 makefile diff --git a/doc/release-notes/10565-banner-test-improvements.md b/doc/release-notes/10565-banner-test-improvements.md new file mode 100644 index 00000000000..d9030f2a0c3 --- /dev/null +++ b/doc/release-notes/10565-banner-test-improvements.md @@ -0,0 +1 @@ +The endpoint `api/admin/bannerMessage` has been extended so the ID is returned when created \ No newline at end of file diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 04123321e54..894f84d2aac 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -5194,6 +5194,8 @@ Delete the setting under ``name``:: Manage Banner Messages ~~~~~~~~~~~~~~~~~~~~~~ +.. warning:: Adding a banner message with a language that is not supported by the installation will result in a 500-Internal Server Error response when trying to access to the /bannerMessage. + Communications to users can be handled via banner messages that are displayed at the top of all pages within your Dataverse installation. Two types of banners can be configured: - A banner message where dismissibleByUser is set to false will be displayed to anyone viewing the application. These messages will be dismissible for a given session but will be displayed in any subsequent session until they are deleted by the Admin. This type of banner message is useful for situations such as upcoming maintenance windows and other downtime. diff --git a/doc/sphinx-guides/source/developers/documentation.rst b/doc/sphinx-guides/source/developers/documentation.rst index a4b8c027445..5ec74249f4d 100755 --- a/doc/sphinx-guides/source/developers/documentation.rst +++ b/doc/sphinx-guides/source/developers/documentation.rst @@ -92,9 +92,20 @@ Open a terminal, change directories to ``doc/sphinx-guides``, activate (or react Building the Guides with a Sphinx Docker Container ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Building with Docker and Makefile:** + +We have added a Makefile to simplify the process of building the guides using a Docker container, you can use some of the following from the repository root: + +- `make docs-html` +- `make docs-pdf` +- `make docs-epub` +- `make docs-all` + +**Building with Docker and CLI:** + If you want to build the guides using a Docker container, execute the following command in the repository root: -``docker run -it --rm -v $(pwd):/docs sphinxdoc/sphinx:3.5.4 bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make html"`` +``docker run -it --rm -v $(pwd):/docs sphinxdoc/sphinx:7.2.6 bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make html"`` Previewing the Guides ^^^^^^^^^^^^^^^^^^^^^ @@ -148,7 +159,7 @@ The HTML version of the guides is the official one. Any other formats are mainta If you would like to build a PDF version of the guides and have Docker installed, please try the command below from the root of the git repo: -``docker run -it --rm -v $(pwd):/docs sphinxdoc/sphinx-latexpdf:3.5.4 bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make latexpdf LATEXMKOPTS=\"-interaction=nonstopmode\"; cd ../.. && ls -1 doc/sphinx-guides/build/latex/Dataverse.pdf"`` +``docker run -it --rm -v $(pwd):/docs sphinxdoc/sphinx-latexpdf:7.2.6 bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make latexpdf LATEXMKOPTS=\"-interaction=nonstopmode\"; cd ../.. && ls -1 doc/sphinx-guides/build/latex/Dataverse.pdf"`` A few notes about the command above: diff --git a/makefile b/makefile new file mode 100644 index 00000000000..315ff9c508c --- /dev/null +++ b/makefile @@ -0,0 +1,14 @@ + +SPHINX_VERSION = $(shell grep "Sphinx" ./doc/sphinx-guides/requirements.txt | awk -F'==' '{print $$2}') +docs-html: + docker run -it --rm -v $$(pwd):/docs sphinxdoc/sphinx:$(SPHINX_VERSION) bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make clean && make html" + +docs-pdf: + docker run -it --rm -v $$(pwd):/docs sphinxdoc/sphinx-latexpdf:$(SPHINX_VERSION) bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make clean && make latexpdf LATEXMKOPTS=\"-interaction=nonstopmode\"; cd ../.. && ls -1 doc/sphinx-guides/build/latex/Dataverse.pdf" + +docs-epub: + docker run -it --rm -v $$(pwd):/docs sphinxdoc/sphinx:$(SPHINX_VERSION) bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make clean && make epub" + +docs-all: + docker run -it --rm -v $$(pwd):/docs sphinxdoc/sphinx:$(SPHINX_VERSION) bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make clean && make html && make epub" + docker run -it --rm -v $$(pwd):/docs sphinxdoc/sphinx-latexpdf:$(SPHINX_VERSION) bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make latexpdf LATEXMKOPTS=\"-interaction=nonstopmode\"; cd ../.. && ls -1 doc/sphinx-guides/build/latex/Dataverse.pdf" \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/BannerMessage.java b/src/main/java/edu/harvard/iq/dataverse/BannerMessage.java index 214e26965fa..003d1057972 100644 --- a/src/main/java/edu/harvard/iq/dataverse/BannerMessage.java +++ b/src/main/java/edu/harvard/iq/dataverse/BannerMessage.java @@ -46,7 +46,7 @@ public void setBannerMessageTexts(Collection bannerMessageTex public String getDisplayValue(){ - String retVal = ""; + String retVal = null; for (BannerMessageText msgTxt : this.getBannerMessageTexts()) { if (msgTxt.getLang().equals(BundleUtil.getCurrentLocale().getLanguage())) { retVal = msgTxt.getMessage(); diff --git a/src/main/java/edu/harvard/iq/dataverse/BannerMessageServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/BannerMessageServiceBean.java index 0e757998d58..3961bd064db 100644 --- a/src/main/java/edu/harvard/iq/dataverse/BannerMessageServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/BannerMessageServiceBean.java @@ -46,8 +46,10 @@ public List findAllBannerMessages() { public void save( BannerMessage message ) { em.persist(message); + em.flush(); } + public void deleteBannerMessage(Object pk) { BannerMessage message = em.find(BannerMessage.class, pk); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 154fa2350bd..46bffb4a7f6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -2357,8 +2357,13 @@ public Response addBannerMessage(JsonObject jsonObject) throws WrappedResponse { messageText.setBannerMessage(toAdd); toAdd.getBannerMessageTexts().add(messageText); } - bannerMessageService.save(toAdd); - return ok("Banner Message added successfully."); + bannerMessageService.save(toAdd); + + JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder() + .add("message", "Banner Message added successfully.") + .add("id", toAdd.getId()); + + return ok(jsonObjectBuilder); } catch (Exception e) { logger.warning("Unexpected Exception: " + e.getMessage()); @@ -2398,10 +2403,19 @@ public Response deactivateBannerMessage(@PathParam("id") Long id) throws Wrapped @Path("/bannerMessage") public Response getBannerMessages(@PathParam("id") Long id) throws WrappedResponse { - return ok(bannerMessageService.findAllBannerMessages().stream() - .map(m -> jsonObjectBuilder().add("id", m.getId()).add("displayValue", m.getDisplayValue())) - .collect(toJsonArray())); + List messagesList = bannerMessageService.findAllBannerMessages(); + + for (BannerMessage message : messagesList) { + if ("".equals(message.getDisplayValue())) { + return error(Response.Status.INTERNAL_SERVER_ERROR, "No banner messages found for this locale."); + } + } + JsonArrayBuilder messages = messagesList.stream() + .map(m -> jsonObjectBuilder().add("id", m.getId()).add("displayValue", m.getDisplayValue())) + .collect(toJsonArray()); + + return ok(messages); } @POST diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java index 44f062e8254..6d7dd2eae29 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java @@ -15,22 +15,21 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import static jakarta.ws.rs.core.Response.Status.FORBIDDEN; -import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; + + + import java.util.Map; import java.util.UUID; +import java.util.logging.Level; import java.util.logging.Logger; -import static jakarta.ws.rs.core.Response.Status.CREATED; -import static jakarta.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR; -import static jakarta.ws.rs.core.Response.Status.OK; -import static jakarta.ws.rs.core.Response.Status.UNAUTHORIZED; +import static jakarta.ws.rs.core.Response.Status.*; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.notNullValue; @@ -832,36 +831,47 @@ public void testClearThumbnailFailureFlag(){ @Test public void testBannerMessages(){ - - String pathToJsonFile = "scripts/api/data/bannerMessageError.json"; - Response addBannerMessageErrorResponse = UtilIT.addBannerMessage(pathToJsonFile); + + //We check for existing banner messages and get the number of existing messages + Response getBannerMessageResponse = UtilIT.getBannerMessages(); + getBannerMessageResponse.prettyPrint(); + getBannerMessageResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + Integer numBannerMessages = + JsonPath.from(getBannerMessageResponse.getBody().asString()).getInt("data.size()"); + + //We add a banner message with an error in the json file + String pathToJsonFile = "scripts/api/data/bannerMessageError.json"; + Response addBannerMessageErrorResponse = UtilIT.addBannerMessage(pathToJsonFile); addBannerMessageErrorResponse.prettyPrint(); - String body = addBannerMessageErrorResponse.getBody().asString(); - String status = JsonPath.from(body).getString("status"); - assertEquals("ERROR", status); + addBannerMessageErrorResponse.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("status", equalTo("ERROR")); + //We add a banner message with a correct json file pathToJsonFile = "scripts/api/data/bannerMessageTest.json"; - Response addBannerMessageResponse = UtilIT.addBannerMessage(pathToJsonFile); addBannerMessageResponse.prettyPrint(); - body = addBannerMessageResponse.getBody().asString(); - status = JsonPath.from(body).getString("status"); - assertEquals("OK", status); + addBannerMessageResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("status", equalTo("OK")) + .body("data.message", equalTo("Banner Message added successfully.")); + Long addedBanner = Long.valueOf( + JsonPath.from(addBannerMessageResponse.getBody().asString()).getLong("data.id")); - Response getBannerMessageResponse = UtilIT.getBannerMessages(); + //We get the banner messages and check that the number of messages has increased by 1 + getBannerMessageResponse = UtilIT.getBannerMessages(); getBannerMessageResponse.prettyPrint(); - body = getBannerMessageResponse.getBody().asString(); - status = JsonPath.from(body).getString("status"); - assertEquals("OK", status); - String deleteId = UtilIT.getBannerMessageIdFromResponse(getBannerMessageResponse.getBody().asString()); - - System.out.print("delete id: " + deleteId); - - Response deleteBannerMessageResponse = UtilIT.deleteBannerMessage(new Long (deleteId)); + getBannerMessageResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.size()", equalTo(numBannerMessages + 1)); + + //We delete the banner message + Response deleteBannerMessageResponse = UtilIT.deleteBannerMessage(addedBanner); deleteBannerMessageResponse.prettyPrint(); - body = deleteBannerMessageResponse.getBody().asString(); - status = JsonPath.from(body).getString("status"); - assertEquals("OK", status); + deleteBannerMessageResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("status", equalTo("OK")); } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index c107ea97b51..b9ae97649a9 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -3387,20 +3387,6 @@ static Response deleteBannerMessage(Long id) { return deleteBannerMessageResponse; } - static String getBannerMessageIdFromResponse(String getBannerMessagesResponse) { - StringReader rdr = new StringReader(getBannerMessagesResponse); - JsonObject json = Json.createReader(rdr).readObject(); - - for (JsonObject obj : json.getJsonArray("data").getValuesAs(JsonObject.class)) { - String message = obj.getString("displayValue"); - if (message.equals("Banner Message For Deletion")) { - return obj.getJsonNumber("id").toString(); - } - } - - return "0"; - } - static Response getDatasetJsonLDMetadata(Integer datasetId, String apiToken) { Response response = given() .header(API_TOKEN_HTTP_HEADER, apiToken) From c141941b0b1363176f994b224e27b70f0d735109 Mon Sep 17 00:00:00 2001 From: Juan Pablo Tosca Villanueva <142103991+jp-tosca@users.noreply.github.com> Date: Thu, 20 Jun 2024 10:33:00 -0400 Subject: [PATCH 098/145] Null banner JSON message fix (#10640) * Null banner JSON message fix * message change --- src/main/java/edu/harvard/iq/dataverse/api/Admin.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 46bffb4a7f6..d60884bad2f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -2337,6 +2337,7 @@ public Response addBannerMessage(JsonObject jsonObject) throws WrappedResponse { BannerMessage toAdd = new BannerMessage(); try { + String dismissible = jsonObject.getString("dismissibleByUser"); boolean dismissibleByUser = false; @@ -2367,7 +2368,7 @@ public Response addBannerMessage(JsonObject jsonObject) throws WrappedResponse { } catch (Exception e) { logger.warning("Unexpected Exception: " + e.getMessage()); - return error(Status.BAD_REQUEST, "Add Banner Message unexpected exception: " + e.getMessage()); + return error(Status.BAD_REQUEST, "Add Banner Message unexpected exception: invalid or missing JSON object."); } } From 83a904a5604af2699f27b43cfbf5d0c152272c31 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 20 Jun 2024 12:54:18 -0400 Subject: [PATCH 099/145] renamed the "anonymous"/"no jsession" setting, per review #10570 --- doc/sphinx-guides/source/installation/config.rst | 4 ++-- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 2 +- .../harvard/iq/dataverse/search/SearchIncludeFragment.java | 2 +- .../harvard/iq/dataverse/settings/SettingsServiceBean.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index cb8161a6f69..009e964f27e 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3871,8 +3871,8 @@ To enable the setting:: Similar to the above, but will disable the facets for Guest (unauthenticated) users only. -:DisableSolrFacetsForAnonymousUsers -+++++++++++++++++++++++++++++++++++ +:DisableSolrFacetsWithoutJsession ++++++++++++++++++++++++++++++++++ Same idea as with the 2 settings above. For the purposes of this setting, a request is considered "anonymous", if it came in without the JSESSION cookie supplied. A UI user who is browsing the holdings without logging in will have a valid JSESSION cookie, tied to a guest session. The main purpose of this setting is to hide the facets from bots, scripted crawlers and such (most of which - though not all - do not use cookies). Not letting the bots anywhere near the facets can serve a dual purpose on a busy instance experiencing problems with such abuse - some CPU cycles and resources can be saved by not having to generate the facets. And, even more importantly, it can prevent bots from attempting to crawl the facet trees, which has a potential for multiplying the service load. diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index f160b2409ee..eae4a9f2977 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -800,7 +800,7 @@ public boolean isIndexedVersion() { // a guest user who came without the session cookie: Map cookies = FacesContext.getCurrentInstance().getExternalContext().getRequestCookieMap(); if (!(cookies != null && cookies.containsKey("JSESSIONID"))) { - if (settingsWrapper.isTrueForKey(SettingsServiceBean.Key.DisableSolrFacetsForAnonymousUsers, false)) { + if (settingsWrapper.isTrueForKey(SettingsServiceBean.Key.DisableSolrFacetsWithoutJsession, false)) { return isIndexedVersion = false; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java index 542c392eb94..4f3f6e46e48 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java @@ -1113,7 +1113,7 @@ public boolean isFacetsDisabled() { // a guest user who came without the session cookie: Map cookies = FacesContext.getCurrentInstance().getExternalContext().getRequestCookieMap(); if (!(cookies != null && cookies.containsKey("JSESSIONID"))) { - if (settingsWrapper.isTrueForKey(SettingsServiceBean.Key.DisableSolrFacetsForAnonymousUsers, false)) { + if (settingsWrapper.isTrueForKey(SettingsServiceBean.Key.DisableSolrFacetsWithoutJsession, false)) { return this.solrFacetsDisabled = true; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 094268f80b7..8ed96690e84 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -672,7 +672,7 @@ Whether Harvesting (OAI) service is enabled */ DisableSolrFacets, DisableSolrFacetsForGuestUsers, - DisableSolrFacetsForAnonymousUsers, + DisableSolrFacetsWithoutJsession, DisableUncheckedTypesFacet, /** * When ingesting tabular data files, store the generated tab-delimited From a1ffe0b08bf691d6d47d3d97d723f8f3c1718ffd Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 20 Jun 2024 13:25:32 -0400 Subject: [PATCH 100/145] missing : in a setting hyperlink #10570 --- doc/sphinx-guides/source/installation/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 009e964f27e..c36a8026a6f 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3876,7 +3876,7 @@ Similar to the above, but will disable the facets for Guest (unauthenticated) us Same idea as with the 2 settings above. For the purposes of this setting, a request is considered "anonymous", if it came in without the JSESSION cookie supplied. A UI user who is browsing the holdings without logging in will have a valid JSESSION cookie, tied to a guest session. The main purpose of this setting is to hide the facets from bots, scripted crawlers and such (most of which - though not all - do not use cookies). Not letting the bots anywhere near the facets can serve a dual purpose on a busy instance experiencing problems with such abuse - some CPU cycles and resources can be saved by not having to generate the facets. And, even more importantly, it can prevent bots from attempting to crawl the facet trees, which has a potential for multiplying the service load. -.. _:DisableUncheckedTypesFacet +.. _:DisableUncheckedTypesFacet: :DisableUncheckedTypesFacet +++++++++++++++++++++++++++ From 0f0c3b67975262bc00c4b31990d8cfe11bf8a95f Mon Sep 17 00:00:00 2001 From: landreev Date: Thu, 20 Jun 2024 17:36:50 -0400 Subject: [PATCH 101/145] Upgrade Solr to 9.4.1 (#10636) * As discussed in sec.92, getting rid of the conf/solr/9.3.0 directory, in favor of keeping the config files in conf/solr/ going forward. * Removed the hard-coded solr version number from the installer makefile * removed the hard-coded 9.3.0 from the metadata customization guide. * changed to reflect 9.4.1 as the currently required version * updated the "classic" dev. install instruction, to reflect the new solr version * a release note. * #92sec move solr conf files into versioned subdirectory * Revert "#92sec move solr conf files into versioned subdirectory" This reverts commit eaa9c7c997ffe1eda48721a08a7edadd7b4affee. * getting rid of 9.3.0 in this shellspec script #10636 * updated solr.version in the parent pom file (it is used in making docker images, apparently). #10636 * removed the remaining instances of the old solr version here and there #10636 * Removed the solr insallation instructions from the "classic dev. install" guide, replaced them with "follow the instructions provided in the main Installation/prerequisites gujide". No need to duplicate these in both places. #10636. * checking in @stevenwinship's workaround for the failing shellcheck test. #10636 --------- Co-authored-by: Don Sizemore --- conf/solr/{9.3.0 => }/schema.xml | 0 conf/solr/{9.3.0 => }/solrconfig.xml | 0 conf/solr/{9.3.0 => }/update-fields.sh | 0 doc/release-notes/solr-9.4.1.md | 14 ++++++++ .../installation/files/etc/init.d/solr | 2 +- .../files/etc/systemd/solr.service | 6 ++-- .../source/admin/metadatacustomization.rst | 4 +-- .../source/developers/classic-dev-env.rst | 36 ++----------------- .../source/installation/prerequisites.rst | 16 ++++----- docker/compose/demo/compose.yml | 2 +- modules/container-configbaker/assembly.xml | 2 +- modules/dataverse-parent/pom.xml | 2 +- scripts/installer/Makefile | 8 ++--- tests/shell/spec/update_fields_spec.sh | 27 +++++++------- 14 files changed, 51 insertions(+), 68 deletions(-) rename conf/solr/{9.3.0 => }/schema.xml (100%) rename conf/solr/{9.3.0 => }/solrconfig.xml (100%) rename conf/solr/{9.3.0 => }/update-fields.sh (100%) create mode 100644 doc/release-notes/solr-9.4.1.md diff --git a/conf/solr/9.3.0/schema.xml b/conf/solr/schema.xml similarity index 100% rename from conf/solr/9.3.0/schema.xml rename to conf/solr/schema.xml diff --git a/conf/solr/9.3.0/solrconfig.xml b/conf/solr/solrconfig.xml similarity index 100% rename from conf/solr/9.3.0/solrconfig.xml rename to conf/solr/solrconfig.xml diff --git a/conf/solr/9.3.0/update-fields.sh b/conf/solr/update-fields.sh similarity index 100% rename from conf/solr/9.3.0/update-fields.sh rename to conf/solr/update-fields.sh diff --git a/doc/release-notes/solr-9.4.1.md b/doc/release-notes/solr-9.4.1.md new file mode 100644 index 00000000000..13624a272ab --- /dev/null +++ b/doc/release-notes/solr-9.4.1.md @@ -0,0 +1,14 @@ +Solr 9.4.1 is now the version recommended in our installation guides and used with automated testing. There is a known security issue in the previously recommended version 9.3.0: https://nvd.nist.gov/vuln/detail/CVE-2023-36478. While the risk of an exploit should not be significant unless the Solr instance is accessible from the outside networks (which we have always recommended against), existing Dataverse installations should consider upgrading. + +For the upgrade instructions section: + +[note that 6.3 will contain other solr-related changes, so the instructions may need to contain information merged from multiple release notes!] + +If you are upgrading Solr: + - Install solr-9.4.1 following the instructions from the Installation guide. + - Run a full reindex to populate the search catalog. + - Note that it may be possible to skip the reindexing step by simply moving the existing `.../server/solr/collection1/` under the new `solr-9.4.1` installation directory. This however has not been thoroughly tested and is not officially supported. + + + + diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr index f7dba504e70..14df734cca7 100755 --- a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr +++ b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr @@ -5,7 +5,7 @@ # chkconfig: 35 92 08 # description: Starts and stops Apache Solr -SOLR_DIR="/usr/local/solr/solr-9.3.0" +SOLR_DIR="/usr/local/solr/solr-9.4.1" SOLR_COMMAND="bin/solr" SOLR_ARGS="-m 1g" SOLR_USER=solr diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service index 2ceeb0016d6..8ccf7652a49 100644 --- a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service +++ b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service @@ -5,9 +5,9 @@ After = syslog.target network.target remote-fs.target nss-lookup.target [Service] User = solr Type = forking -WorkingDirectory = /usr/local/solr/solr-9.3.0 -ExecStart = /usr/local/solr/solr-9.3.0/bin/solr start -m 1g -ExecStop = /usr/local/solr/solr-9.3.0/bin/solr stop +WorkingDirectory = /usr/local/solr/solr-9.4.1 +ExecStart = /usr/local/solr/solr-9.4.1/bin/solr start -m 1g +ExecStop = /usr/local/solr/solr-9.4.1/bin/solr stop LimitNOFILE=65000 LimitNPROC=65000 Restart=on-failure diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index e70cf0e0897..8752f11c1e5 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -513,7 +513,7 @@ the Solr schema configuration, including any enabled metadata schemas: ``curl "http://localhost:8080/api/admin/index/solr/schema"`` -You can use :download:`update-fields.sh <../../../../conf/solr/9.3.0/update-fields.sh>` to easily add these to the +You can use :download:`update-fields.sh <../../../../conf/solr/update-fields.sh>` to easily add these to the Solr schema you installed for your Dataverse installation. The script needs a target XML file containing your Solr schema. (See the :doc:`/installation/prerequisites/` section of @@ -537,7 +537,7 @@ from some place else than your Dataverse installation). Please note that reconfigurations of your Solr index might require a re-index. Usually release notes indicate a necessary re-index, but for your custom metadata you will need to keep track on your own. -Please note also that if you are going to make a pull request updating ``conf/solr/9.3.0/schema.xml`` with fields you have +Please note also that if you are going to make a pull request updating ``conf/solr/schema.xml`` with fields you have added, you should first load all the custom metadata blocks in ``scripts/api/data/metadatablocks`` (including ones you don't care about) to create a complete list of fields. (This might change in the future.) diff --git a/doc/sphinx-guides/source/developers/classic-dev-env.rst b/doc/sphinx-guides/source/developers/classic-dev-env.rst index 82e10b727ef..0ebb9431383 100755 --- a/doc/sphinx-guides/source/developers/classic-dev-env.rst +++ b/doc/sphinx-guides/source/developers/classic-dev-env.rst @@ -136,41 +136,9 @@ On Linux, you should just install PostgreSQL using your favorite package manager Install Solr ^^^^^^^^^^^^ -`Solr `_ 9.3.0 is required. +`Solr `_ 9.4.1 is required. -To install Solr, execute the following commands: - -``sudo mkdir /usr/local/solr`` - -``sudo chown $USER /usr/local/solr`` - -``cd /usr/local/solr`` - -``curl -O https://archive.apache.org/dist/solr/solr/9.3.0/solr-9.3.0.tgz`` - -``tar xvfz solr-9.3.0.tgz`` - -``cd solr-9.3.0/server/solr`` - -``cp -r configsets/_default collection1`` - -``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/9.3.0/schema.xml`` - -``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/9.3.0/schema_dv_mdb_fields.xml`` - -``mv schema*.xml collection1/conf`` - -``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/9.3.0/solrconfig.xml`` - -``mv solrconfig.xml collection1/conf/solrconfig.xml`` - -``cd /usr/local/solr/solr-9.3.0`` - -(Please note that the extra jetty argument below is a security measure to limit connections to Solr to only your computer. For extra security, run a firewall.) - -``bin/solr start -j "-Djetty.host=127.0.0.1"`` - -``bin/solr create_core -c collection1 -d server/solr/collection1/conf`` +Follow the instructions in the "Installing Solr" section of :doc:`/installation/prerequisites` in the main Installation guide. Install Service Dependencies Using Docker Compose ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index 7f6e34789b8..571223e0aa6 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -163,7 +163,7 @@ The Dataverse software search index is powered by Solr. Supported Versions ================== -The Dataverse software has been tested with Solr version 9.3.0. Future releases in the 9.x series are likely to be compatible. Please get in touch (:ref:`support`) if you are having trouble with a newer version. +The Dataverse software has been tested with Solr version 9.4.1. Future releases in the 9.x series are likely to be compatible. Please get in touch (:ref:`support`) if you are having trouble with a newer version. Installing Solr =============== @@ -178,19 +178,19 @@ Become the ``solr`` user and then download and configure Solr:: su - solr cd /usr/local/solr - wget https://archive.apache.org/dist/solr/solr/9.3.0/solr-9.3.0.tgz - tar xvzf solr-9.3.0.tgz - cd solr-9.3.0 + wget https://archive.apache.org/dist/solr/solr/9.4.1/solr-9.4.1.tgz + tar xvzf solr-9.4.1.tgz + cd solr-9.4.1 cp -r server/solr/configsets/_default server/solr/collection1 You should already have a "dvinstall.zip" file that you downloaded from https://github.com/IQSS/dataverse/releases . Unzip it into ``/tmp``. Then copy the files into place:: - cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-9.3.0/server/solr/collection1/conf - cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-9.3.0/server/solr/collection1/conf + cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-9.4.1/server/solr/collection1/conf + cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-9.4.1/server/solr/collection1/conf Note: The Dataverse Project team has customized Solr to boost results that come from certain indexed elements inside the Dataverse installation, for example prioritizing results from Dataverse collections over Datasets. If you would like to remove this, edit your ``solrconfig.xml`` and remove the ```` element and its contents. If you have ideas about how this boosting could be improved, feel free to contact us through our Google Group https://groups.google.com/forum/#!forum/dataverse-dev . -A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-9.3.0/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400`` +A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-9.4.1/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400`` Solr will warn about needing to increase the number of file descriptors and max processes in a production environment but will still run with defaults. We have increased these values to the recommended levels by adding ulimit -n 65000 to the init script, and the following to ``/etc/security/limits.conf``:: @@ -209,7 +209,7 @@ Solr launches asynchronously and attempts to use the ``lsof`` binary to watch fo Finally, you need to tell Solr to create the core "collection1" on startup:: - echo "name=collection1" > /usr/local/solr/solr-9.3.0/server/solr/collection1/core.properties + echo "name=collection1" > /usr/local/solr/solr-9.4.1/server/solr/collection1/core.properties Dataverse collection ("dataverse") page uses Solr very heavily. On a busy instance this may cause the search engine to become the performance bottleneck, making these pages take increasingly longer to load, potentially affecting the overall performance of the application and/or causing Solr itself to crash. If this is observed on your instance, we recommend uncommenting the following lines in the ```` section of the ``solrconfig.xml`` file:: diff --git a/docker/compose/demo/compose.yml b/docker/compose/demo/compose.yml index 6c2bdcf79a4..e6ffc9f392a 100644 --- a/docker/compose/demo/compose.yml +++ b/docker/compose/demo/compose.yml @@ -103,7 +103,7 @@ services: solr: container_name: "solr" hostname: "solr" - image: solr:9.3.0 + image: solr:9.4.1 depends_on: - solr_initializer restart: on-failure diff --git a/modules/container-configbaker/assembly.xml b/modules/container-configbaker/assembly.xml index 3285eef510a..3c55e6d64b6 100644 --- a/modules/container-configbaker/assembly.xml +++ b/modules/container-configbaker/assembly.xml @@ -8,7 +8,7 @@ - conf/solr/9.3.0 + conf/solr solr diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 612902b47a4..02f601ba0f6 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -150,7 +150,7 @@ 6.2023.8 42.7.2 - 9.3.0 + 9.4.1 1.12.290 26.30.0 diff --git a/scripts/installer/Makefile b/scripts/installer/Makefile index 399bc65168a..8ea95534986 100644 --- a/scripts/installer/Makefile +++ b/scripts/installer/Makefile @@ -55,13 +55,13 @@ ${JHOVE_SCHEMA}: ../../conf/jhove/jhoveConfig.xsd ${INSTALLER_ZIP_DIR} @echo copying jhove schema file /bin/cp ../../conf/jhove/jhoveConfig.xsd ${INSTALLER_ZIP_DIR} -${SOLR_SCHEMA}: ../../conf/solr/9.3.0/schema.xml ../../conf/solr/9.3.0/update-fields.sh ${INSTALLER_ZIP_DIR} +${SOLR_SCHEMA}: ../../conf/solr/schema.xml ../../conf/solr/update-fields.sh ${INSTALLER_ZIP_DIR} @echo copying Solr schema file - /bin/cp ../../conf/solr/9.3.0/schema.xml ../../conf/solr/9.3.0/update-fields.sh ${INSTALLER_ZIP_DIR} + /bin/cp ../../conf/solr/schema.xml ../../conf/solr/update-fields.sh ${INSTALLER_ZIP_DIR} -${SOLR_CONFIG}: ../../conf/solr/9.3.0/solrconfig.xml ${INSTALLER_ZIP_DIR} +${SOLR_CONFIG}: ../../conf/solr/solrconfig.xml ${INSTALLER_ZIP_DIR} @echo copying Solr config file - /bin/cp ../../conf/solr/9.3.0/solrconfig.xml ${INSTALLER_ZIP_DIR} + /bin/cp ../../conf/solr/solrconfig.xml ${INSTALLER_ZIP_DIR} ${PYTHON_FILES}: README_python.txt install.py installConfig.py installAppServer.py installUtils.py requirements.txt default.config interactive.config ${INSTALLER_ZIP_DIR} @echo copying Python installer files diff --git a/tests/shell/spec/update_fields_spec.sh b/tests/shell/spec/update_fields_spec.sh index 48054a121b7..fa61743bf6f 100644 --- a/tests/shell/spec/update_fields_spec.sh +++ b/tests/shell/spec/update_fields_spec.sh @@ -1,16 +1,17 @@ #shellcheck shell=sh +#shellcheck disable=SC2154 update_fields() { - ../../conf/solr/9.3.0/update-fields.sh "$@" + ../../conf/solr/update-fields.sh "$@" } Describe "Update fields command" Describe "can operate on upstream data" - copyUpstreamSchema() { cp ../../conf/solr/9.3.0/schema.xml data/solr/upstream-schema.xml; } + copyUpstreamSchema() { cp ../../conf/solr/schema.xml data/solr/upstream-schema.xml; } AfterAll 'copyUpstreamSchema' - Path schema-xml="../../conf/solr/9.3.0/schema.xml" + Path schema-xml="../../conf/solr/schema.xml" It "needs upstream schema.xml" The path schema-xml should be exist End @@ -115,16 +116,16 @@ Describe "Update fields command" End Describe "succeeds because" - setup() { cp data/solr/minimal-schema.xml data/solr/minimal-schema-work.xml; } - cleanup() { rm data/solr/minimal-schema-work.xml; } - BeforeEach 'setup' - AfterEach 'cleanup' + setup1() { cp data/solr/minimal-schema.xml data/solr/minimal-schema-work.xml; } + cleanup1() { rm data/solr/minimal-schema-work.xml; } + BeforeEach 'setup1' + AfterEach 'cleanup1' deleteUpstreamSchema() { rm data/solr/upstream-schema.xml; } AfterAll 'deleteUpstreamSchema' - match_content() { - grep -q "$@" "${match_content}" + match_content1() { + grep -q "$@" "${match_content1}" } It "prints nothing when editing minimal schema" @@ -133,8 +134,8 @@ Describe "Update fields command" The status should equal 0 The output should equal "" The path data/solr/minimal-schema-work.xml should be file - The path data/solr/minimal-schema-work.xml should satisfy match_content " Date: Fri, 21 Jun 2024 08:31:29 +0200 Subject: [PATCH 102/145] build,doc: update with Payara 6.2024.6 release --- doc/release-notes/10494-payara-upgrade.md | 4 ++-- doc/sphinx-guides/source/developers/classic-dev-env.rst | 6 +++--- doc/sphinx-guides/source/installation/prerequisites.rst | 6 +++--- doc/sphinx-guides/source/qa/test-automation.md | 2 +- modules/dataverse-parent/pom.xml | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/release-notes/10494-payara-upgrade.md b/doc/release-notes/10494-payara-upgrade.md index 050faee1f3e..1e8f9aef7ba 100644 --- a/doc/release-notes/10494-payara-upgrade.md +++ b/doc/release-notes/10494-payara-upgrade.md @@ -1,6 +1,6 @@ -# Upgrade Payara to v6.2024.4 +# Upgrade Payara to v6.2024.6 -With this version of Dataverse, we encourage you to upgrade to version 6.2024.4. +With this version of Dataverse, we encourage you to upgrade to version 6.2024.6. This will address security issues accumulated since the release of 6.2023.8, which was required since Dataverse release 6.0. If you are using GDCC containers, this upgrade is included when pulling new release images. diff --git a/doc/sphinx-guides/source/developers/classic-dev-env.rst b/doc/sphinx-guides/source/developers/classic-dev-env.rst index a193c1f1b40..d305019004e 100755 --- a/doc/sphinx-guides/source/developers/classic-dev-env.rst +++ b/doc/sphinx-guides/source/developers/classic-dev-env.rst @@ -93,15 +93,15 @@ On Linux, install ``jq`` from your package manager or download a binary from htt Install Payara ~~~~~~~~~~~~~~ -Payara 6.2024.4 or higher is required. +Payara 6.2024.6 or higher is required. To install Payara, run the following commands: ``cd /usr/local`` -``sudo curl -O -L https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2024.4/payara-6.2024.4.zip`` +``sudo curl -O -L https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2024.6/payara-6.2024.6.zip`` -``sudo unzip payara-6.2024.4.zip`` +``sudo unzip payara-6.2024.6.zip`` ``sudo chown -R $USER /usr/local/payara6`` diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index eb3db50fdd6..151d44e6841 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -44,7 +44,7 @@ On RHEL/derivative you can make Java 17 the default with the ``alternatives`` co Payara ------ -Payara 6.2024.4 is recommended. Newer versions might work fine. Regular updates are recommended. +Payara 6.2024.6 is recommended. Newer versions might work fine. Regular updates are recommended. Installing Payara ================= @@ -55,8 +55,8 @@ Installing Payara - Download and install Payara (installed in ``/usr/local/payara6`` in the example commands below):: - # wget https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2024.4/payara-6.2024.4.zip - # unzip payara-6.2024.4.zip + # wget https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2024.6/payara-6.2024.6.zip + # unzip payara-6.2024.6.zip # mv payara6 /usr/local If nexus.payara.fish is ever down for maintenance, Payara distributions are also available from https://repo1.maven.org/maven2/fish/payara/distributions/payara/ diff --git a/doc/sphinx-guides/source/qa/test-automation.md b/doc/sphinx-guides/source/qa/test-automation.md index d2de33b48a5..fe0d51f9174 100644 --- a/doc/sphinx-guides/source/qa/test-automation.md +++ b/doc/sphinx-guides/source/qa/test-automation.md @@ -52,7 +52,7 @@ Go to the end of the log and then scroll up, looking for the failure. A failed A ``` TASK [dataverse : download payara zip] ***************************************** -fatal: [localhost]: FAILED! => {"changed": false, "dest": "/tmp/payara.zip", "elapsed": 10, "msg": "Request failed: ", "url": "https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2024.4/payara-6.2024.4.zip"} +fatal: [localhost]: FAILED! => {"changed": false, "dest": "/tmp/payara.zip", "elapsed": 10, "msg": "Request failed: ", "url": "https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2024.6/payara-6.2024.6.zip"} ``` In the example above, if Payara can't be downloaded, we're obviously going to have problems deploying Dataverse to it! diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index c7166e69813..e81c087056a 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -148,7 +148,7 @@ -Duser.timezone=${project.timezone} -Dfile.encoding=${project.build.sourceEncoding} -Duser.language=${project.language} -Duser.region=${project.region} - 6.2024.4 + 6.2024.6 42.7.2 9.4.1 1.12.290 From 7dd1f05e8c4b7d5509d7e6aa3f09c9caa1a57e02 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 21 Jun 2024 08:32:51 +0200 Subject: [PATCH 103/145] refactor: remove stale property for Jakarta EE API version As of Dataverse 6.0 we use Jakarta EE 10. The version of these dependencies comes from the Payara BOM and this property has no longer any use. Removing it will avoid future confusion. --- modules/dataverse-parent/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index e81c087056a..f286a44841f 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -155,7 +155,6 @@ 26.30.0 - 8.0.0 1.7.35 2.15.1 1.2 From 10b63fe53f9a1ef214c5196ec7016e62a9f1ec83 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 21 Jun 2024 09:20:04 +0200 Subject: [PATCH 104/145] doc: add upgrade instructions for Payara 6.2024.6 #10494 As requested by @qqmyers --- doc/release-notes/10494-payara-upgrade.md | 106 ++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/doc/release-notes/10494-payara-upgrade.md b/doc/release-notes/10494-payara-upgrade.md index 1e8f9aef7ba..e8ecb6756e0 100644 --- a/doc/release-notes/10494-payara-upgrade.md +++ b/doc/release-notes/10494-payara-upgrade.md @@ -3,4 +3,110 @@ With this version of Dataverse, we encourage you to upgrade to version 6.2024.6. This will address security issues accumulated since the release of 6.2023.8, which was required since Dataverse release 6.0. +## Instructions for Upgrading + If you are using GDCC containers, this upgrade is included when pulling new release images. +No manual intervention is necessary. + +We recommend you ensure you followed all update instructions from the past releases regarding Payara. +(Latest Payara update was for [v6.0](https://github.com/IQSS/dataverse/releases/tag/v6.0)) + +Upgrading requires a maintenance window and downtime. Please plan ahead, create backups of your database, etc. + +The steps below are a simple matter of reusing your existing domain directory with the new distribution. +But we also recommend that you review the Payara upgrade instructions as it could be helpful during any troubleshooting: +[Payara Release Notes](https://docs.payara.fish/community/docs/Release%20Notes/Release%20Notes%206.2024.6.html) +We assume you are already on a Dataverse 6.x installation, using a Payara 6.x release. + +```shell +export PAYARA=/usr/local/payara5 +``` + +(or `setenv PAYARA /usr/local/payara6` if you are using a `csh`-like shell) + +1\. Undeploy the previous version + +```shell + $PAYARA/bin/asadmin list-applications + $PAYARA/bin/asadmin undeploy dataverse<-version> +``` + +2\. Stop Payara + +```shell + service payara stop + rm -rf $PAYARA/glassfish/domains/domain1/generated + rm -rf $PAYARA/glassfish/domains/domain1/osgi-cache + rm -rf $PAYARA/glassfish/domains/domain1/lib/databases +``` + +3\. Move the current Payara directory out of the way + +```shell + mv $PAYARA $PAYARA.MOVED +``` + +4\. Download the new Payara version (6.2024.6), and unzip it in its place + +5\. Replace the brand new payara/glassfish/domains/domain1 with your old, preserved domain1 + +6\. Make sure that you have the following `--add-opens` options in your domain.xml. If not present, add them: + +```diff +--- payara-6.2023.8/glassfish/domains/domain1/config/domain.xml ++++ payara-6.2024.6/glassfish/domains/domain1/config/domain.xml +@@ -212,12 +212,16 @@ + --add-opens=java.naming/javax.naming.spi=ALL-UNNAMED + --add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED + --add-opens=java.logging/java.util.logging=ALL-UNNAMED ++ --add-opens=java.management/javax.management=ALL-UNNAMED ++ --add-opens=java.management/javax.management.openmbean=ALL-UNNAMED + [17|]--add-exports=java.base/sun.net.www=ALL-UNNAMED + [17|]--add-exports=java.base/sun.security.util=ALL-UNNAMED + [17|]--add-opens=java.base/java.lang.invoke=ALL-UNNAMED + [17|]--add-opens=java.desktop/java.beans=ALL-UNNAMED + [17|]--add-exports=jdk.naming.dns/com.sun.jndi.dns=ALL-UNNAMED + [17|]--add-exports=java.naming/com.sun.jndi.ldap=ALL-UNNAMED ++ [17|]--add-opens=java.base/java.io=ALL-UNNAMED ++ [21|]--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED + -Xmx512m + -XX:NewRatio=2 + -XX:+UnlockDiagnosticVMOptions +@@ -447,12 +451,16 @@ + --add-opens=java.naming/javax.naming.spi=ALL-UNNAMED + --add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED + --add-opens=java.logging/java.util.logging=ALL-UNNAMED ++ --add-opens=java.management/javax.management=ALL-UNNAMED ++ --add-opens=java.management/javax.management.openmbean=ALL-UNNAMED + [17|]--add-exports=java.base/sun.net.www=ALL-UNNAMED + [17|]--add-exports=java.base/sun.security.util=ALL-UNNAMED + [17|]--add-opens=java.base/java.lang.invoke=ALL-UNNAMED + [17|]--add-opens=java.desktop/java.beans=ALL-UNNAMED + [17|]--add-exports=jdk.naming.dns/com.sun.jndi.dns=ALL-UNNAMED + [17|]--add-exports=java.naming/com.sun.jndi.ldap=ALL-UNNAMED ++ [17|]--add-opens=java.base/java.io=ALL-UNNAMED ++ [21|]--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED + -Xmx512m + -XX:NewRatio=2 + -XX:+UnlockDiagnosticVMOptions +``` +(You can also save this as a patch file and try to apply it.) + + +7\. Start Payara + +```shell + service payara start +``` + +8\. Deploy this version. + +```shell + $PAYARA/bin/asadmin deploy dataverse-6.3.war +``` + +9\. Restart payara + +```shell + service payara stop + service payara start From b405081a62ac42dbcb94d25b2a9b7d2afb055939 Mon Sep 17 00:00:00 2001 From: luddaniel <83018819+luddaniel@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:37:28 +0200 Subject: [PATCH 105/145] CVOC: Fix NPE (#10603) * Fixed NPE * CVOC - Improved logger message and avoid http call on invalid url --- .../iq/dataverse/DatasetFieldServiceBean.java | 44 ++++++++++++++----- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java index 14d1f564ee6..34595728fa7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java @@ -6,6 +6,7 @@ import java.net.URISyntaxException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; +import java.security.InvalidParameterException; import java.sql.Timestamp; import java.text.MessageFormat; import java.time.Instant; @@ -20,8 +21,6 @@ import jakarta.ejb.EJB; import jakarta.ejb.Stateless; -import jakarta.ejb.TransactionAttribute; -import jakarta.ejb.TransactionAttributeType; import jakarta.inject.Named; import jakarta.json.Json; import jakarta.json.JsonArray; @@ -523,10 +522,19 @@ public void registerExternalTerm(JsonObject cvocEntry, String term, List Date: Fri, 21 Jun 2024 12:15:27 -0400 Subject: [PATCH 106/145] Fix DOI/ID message when deaccession a Dataset (#10610) * Fix DOI/ID message when deaccession a Dataset * Change to pull the DS from the Version * Test added and changed to use asString over toString * Change Global to PID * Refactor deaccessionDataset method to handle both datasetId and persistentId --- .../harvard/iq/dataverse/api/Datasets.java | 5 +++- .../harvard/iq/dataverse/api/DatasetsIT.java | 17 ++++++++++- .../edu/harvard/iq/dataverse/api/FilesIT.java | 1 + .../edu/harvard/iq/dataverse/api/UtilIT.java | 29 ++++++++++++++++--- 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 9e9f89c8140..8daefd02f5c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -4839,7 +4839,10 @@ public Response deaccessionDataset(@Context ContainerRequestContext crc, @PathPa } } execCommand(new DeaccessionDatasetVersionCommand(req, datasetVersion, false)); - return ok("Dataset " + datasetId + " deaccessioned for version " + versionId); + + return ok("Dataset " + + (":persistentId".equals(datasetId) ? datasetVersion.getDataset().getGlobalId().asString() : datasetId) + + " deaccessioned for version " + versionId); } catch (JsonParsingException jpe) { return error(Response.Status.BAD_REQUEST, "Error parsing Json: " + jpe.getMessage()); } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index d2d14b824bd..cb9481d3491 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -3910,6 +3910,7 @@ public void getDatasetVersionCitation() { String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.prettyPrint(); createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode()); int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); @@ -3928,7 +3929,9 @@ public void getDatasetVersionCitation() { publishDatasetResponse.then().assertThat().statusCode(OK.getStatusCode()); Response deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST_PUBLISHED, "Test deaccession reason.", null, apiToken); - deaccessionDatasetResponse.then().assertThat().statusCode(OK.getStatusCode()); + deaccessionDatasetResponse.prettyPrint(); + deaccessionDatasetResponse.then().assertThat().statusCode(OK.getStatusCode()) + .assertThat().body("data.message", containsString(String.valueOf(datasetId))); // includeDeaccessioned false Response getDatasetVersionCitationNotDeaccessioned = UtilIT.getDatasetVersionCitation(datasetId, DS_VERSION_LATEST_PUBLISHED, false, apiToken); @@ -3939,8 +3942,20 @@ public void getDatasetVersionCitation() { getDatasetVersionCitationDeaccessioned.then().assertThat() .statusCode(OK.getStatusCode()) .body("data.message", containsString("DEACCESSIONED VERSION")); + + publishDatasetResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken); + publishDatasetResponse.prettyPrint(); + publishDatasetResponse.then().assertThat().statusCode(OK.getStatusCode()); + + String persistentId = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId"); + + deaccessionDatasetResponse = UtilIT.deaccessionDataset(persistentId, DS_VERSION_LATEST_PUBLISHED, "Test deaccession reason.", null, apiToken); + deaccessionDatasetResponse.prettyPrint(); + deaccessionDatasetResponse.then().assertThat().statusCode(OK.getStatusCode()) + .assertThat().body("data.message", containsString(String.valueOf(persistentId))); } + @Test public void getVersionFiles() throws IOException, InterruptedException { Response createUser = UtilIT.createRandomUser(); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index 7150b32c2b0..e3c26284d55 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -1526,6 +1526,7 @@ public void testGetFileInfo() { .body("data.label", equalTo(newFileNameSecondUpdate)); // The following tests cover cases where the dataset version is deaccessioned + Response deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, "3.0", "Test reason", null, superUserApiToken); deaccessionDatasetResponse.then().assertThat().statusCode(OK.getStatusCode()); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index b9ae97649a9..257610dbc32 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -3845,17 +3845,38 @@ static Response getHasBeenDeleted(String dataFileId, String apiToken) { .get("/api/files/" + dataFileId + "/hasBeenDeleted"); } - static Response deaccessionDataset(Integer datasetId, String version, String deaccessionReason, String deaccessionForwardURL, String apiToken) { + static Response deaccessionDataset(int datasetId, String version, String deaccessionReason, String deaccessionForwardURL, String apiToken) { + return deaccessionDataset(String.valueOf(datasetId), version, deaccessionReason, deaccessionForwardURL, apiToken); + } + + static Response deaccessionDataset(String datasetIdOrPersistentId, String versionId, String deaccessionReason, String deaccessionForwardURL, String apiToken) { + + String idInPath = datasetIdOrPersistentId; // Assume it's a number. + String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path. + if (!NumberUtils.isCreatable(datasetIdOrPersistentId)) { + idInPath = ":persistentId"; + optionalQueryParam = "?persistentId=" + datasetIdOrPersistentId; + } + JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder(); jsonObjectBuilder.add("deaccessionReason", deaccessionReason); if (deaccessionForwardURL != null) { jsonObjectBuilder.add("deaccessionForwardURL", deaccessionForwardURL); } + String jsonString = jsonObjectBuilder.build().toString(); + StringBuilder query = new StringBuilder() + .append("/api/datasets/") + .append(idInPath) + .append("/versions/") + .append(versionId) + .append("/deaccession") + .append(optionalQueryParam); + return given() - .header(API_TOKEN_HTTP_HEADER, apiToken) - .body(jsonString) - .post("/api/datasets/" + datasetId + "/versions/" + version + "/deaccession"); + .header(API_TOKEN_HTTP_HEADER, apiToken) + .body(jsonString) + .post(query.toString()); } static Response getDownloadSize(Integer datasetId, From ee986d59dd04643b8d007c8c39d1e0cc6be21b1e Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 24 Jun 2024 15:58:13 -0400 Subject: [PATCH 107/145] A quick fix for #10611. --- .../harvard/iq/dataverse/search/IndexServiceBean.java | 8 +++++++- .../harvard/iq/dataverse/search/SearchServiceBean.java | 5 ++--- .../edu/harvard/iq/dataverse/settings/FeatureFlags.java | 9 +++++++++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 0102459ab9f..d3286d3be4b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -888,8 +888,14 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set Date: Mon, 24 Jun 2024 22:27:58 +0200 Subject: [PATCH 108/145] CVOC : Adding hidden metadata fields (Ontoportal integration) (#10503) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add hidden metadata fields for cvoc JS * adding release note * Accept review suggestion on documentation * CVOC: fix typo to have "data-cvoc-managedfields" on dataset page visualization * Add cvoc index to identity bloc metadata --------- Co-authored-by: Jérôme ROUCOU --- .../10503-cvoc-hidden-html-fields.md | 11 ++++++++ src/main/webapp/metadataFragment.xhtml | 25 ++++++++++++++----- 2 files changed, 30 insertions(+), 6 deletions(-) create mode 100644 doc/release-notes/10503-cvoc-hidden-html-fields.md diff --git a/doc/release-notes/10503-cvoc-hidden-html-fields.md b/doc/release-notes/10503-cvoc-hidden-html-fields.md new file mode 100644 index 00000000000..e3ea0463fb8 --- /dev/null +++ b/doc/release-notes/10503-cvoc-hidden-html-fields.md @@ -0,0 +1,11 @@ +## Release Highlights + +### Updates on Support for External Vocabulary Services + +#### Hidden HTML Fields + +External Controlled Vocabulary scripts, configured via [:CVocConf](https://guides.dataverse.org/en/6.3/installation/config.html#cvocconf), can now access the values of managed fields as well as the term-uri-field for use in constructing the metadata view for a dataset. + +Those values are hidden and can be found with the html attribute `data-cvoc-metadata-name`. + +For more information, see [#10503](https://github.com/IQSS/dataverse/pull/10503). diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index 24d17b27a1f..43d54f64c43 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -90,7 +90,7 @@ - + - + @@ -125,7 +125,7 @@ - + @@ -151,7 +151,8 @@ - + + - + + + + + + + + - +
From e24405cd0f3a5d50e5c46916f84108aca318fa89 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 25 Jun 2024 10:35:45 -0400 Subject: [PATCH 109/145] skip remove loops if list starts empty --- .../iq/dataverse/search/IndexServiceBean.java | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 42b740076ec..63ce35114e2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -571,23 +571,26 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr try { solrIdsOfDocsToDelete = findFilesOfParentDataset(dataset.getId()); logger.fine("Existing file docs: " + String.join(", ", solrIdsOfDocsToDelete)); - // We keep the latest version's docs unless it is deaccessioned and there is no - // published/released version - // So skip the loop removing those docs from the delete list except in that case - if ((!latestVersion.isDeaccessioned() || atLeastOnePublishedVersion)) { - List latestFileMetadatas = latestVersion.getFileMetadatas(); - String suffix = (new IndexableDataset(latestVersion)).getDatasetState().getSuffix(); - for (FileMetadata fileMetadata : latestFileMetadatas) { - String solrIdOfPublishedFile = solrDocIdentifierFile + fileMetadata.getDataFile().getId() - + suffix; - solrIdsOfDocsToDelete.remove(solrIdOfPublishedFile); + if (!solrIdsOfDocsToDelete.isEmpty()) { + // We keep the latest version's docs unless it is deaccessioned and there is no + // published/released version + // So skip the loop removing those docs from the delete list except in that case + if ((!latestVersion.isDeaccessioned() || atLeastOnePublishedVersion)) { + List latestFileMetadatas = latestVersion.getFileMetadatas(); + String suffix = (new IndexableDataset(latestVersion)).getDatasetState().getSuffix(); + for (FileMetadata fileMetadata : latestFileMetadatas) { + String solrIdOfPublishedFile = solrDocIdentifierFile + + fileMetadata.getDataFile().getId() + suffix; + solrIdsOfDocsToDelete.remove(solrIdOfPublishedFile); + } } - } - if (releasedVersion != null && !releasedVersion.equals(latestVersion)) { - List releasedFileMetadatas = releasedVersion.getFileMetadatas(); - for (FileMetadata fileMetadata : releasedFileMetadatas) { - String solrIdOfPublishedFile = solrDocIdentifierFile + fileMetadata.getDataFile().getId(); - solrIdsOfDocsToDelete.remove(solrIdOfPublishedFile); + if (releasedVersion != null && !releasedVersion.equals(latestVersion)) { + List releasedFileMetadatas = releasedVersion.getFileMetadatas(); + for (FileMetadata fileMetadata : releasedFileMetadatas) { + String solrIdOfPublishedFile = solrDocIdentifierFile + + fileMetadata.getDataFile().getId(); + solrIdsOfDocsToDelete.remove(solrIdOfPublishedFile); + } } } // Clear any unused dataset docs From db8b2d2ceef7cb69ee1f03d4b318025545a6754d Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 25 Jun 2024 11:47:00 -0400 Subject: [PATCH 110/145] Delete src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java --- .../dataverse/export/PdfCodeBookExporter.java | 81 ------------------- 1 file changed, 81 deletions(-) delete mode 100644 src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java diff --git a/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java deleted file mode 100644 index 4772c09ffd5..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java +++ /dev/null @@ -1,81 +0,0 @@ - - -package edu.harvard.iq.dataverse.export; - -import com.google.auto.service.AutoService; - -import edu.harvard.iq.dataverse.export.ddi.DdiExportUtil; -import io.gdcc.spi.export.ExportDataProvider; -import io.gdcc.spi.export.ExportException; -import io.gdcc.spi.export.Exporter; -import edu.harvard.iq.dataverse.util.BundleUtil; -import jakarta.ws.rs.core.MediaType; - -import javax.xml.stream.XMLStreamException; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Locale; -import java.util.Optional; - -@AutoService(Exporter.class) -public class PdfCodeBookExporter implements Exporter { - - @Override - public String getFormatName() { - return "pdf"; - } - - @Override - public String getDisplayName(Locale locale) { - String displayName = BundleUtil.getStringFromBundle("dataset.exportBtn.itemLabel.pdf", locale); - return Optional.ofNullable(displayName).orElse("DDI pdf codebook"); - } - - @Override - public void exportDataset(ExportDataProvider dataProvider, OutputStream outputStream) throws ExportException { - Optional ddiInputStreamOptional = dataProvider.getPrerequisiteInputStream(); - if (ddiInputStreamOptional.isPresent()) { - try (InputStream ddiInputStream = ddiInputStreamOptional.get()) { - DdiExportUtil.datasetPdfDDI(ddiInputStream, outputStream); - } catch (IOException e) { - throw new ExportException("Cannot open export_ddi cached file"); - } catch (XMLStreamException xse) { - throw new ExportException("Caught XMLStreamException performing DDI export"); - } - } else { - throw new ExportException("No prerequisite input stream found"); - } - } - - @Override - public Boolean isHarvestable() { - // No, we don't want this format to be harvested! - // For datasets with tabular data the portions of the DDIs - // become huge and expensive to parse; even as they don't contain any - // metadata useful to remote harvesters. -- L.A. 4.5 - return false; - } - - @Override - public Boolean isAvailableToUsers() { - return true; - } - - @Override - public Optional getPrerequisiteFormatName() { - //This exporter relies on being able to get the output of the ddi exporter - return Optional.of("ddi"); - } - - @Override - public String getMediaType() { - return MediaType.WILDCARD; - }; -} - - - From 88e86c17f3578c2baf86a2e836bdb0b0ddd59007 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 25 Jun 2024 11:48:26 -0400 Subject: [PATCH 111/145] Update pom.xml --- pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pom.xml b/pom.xml index efb10d8ed5e..f45e8fd9033 100644 --- a/pom.xml +++ b/pom.xml @@ -463,12 +463,6 @@ commons-compress - - - org.apache.xmlgraphics - fop - 2.8 - org.duracloud common From a95bb715947d2d610bc6a12f5f62a63af67ae244 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 25 Jun 2024 11:49:48 -0400 Subject: [PATCH 112/145] Update Datasets.java --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 1bb0d559e84..1a5eba52a62 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -206,7 +206,7 @@ public Response getDataset(@Context ContainerRequestContext crc, @PathParam("id" // WORKS on published datasets, which are open to the world. -- L.A. 4.5 @GET @Path("/export") - @Produces({"application/xml", "application/json", "application/html", "*/*" }) + @Produces({"application/xml", "application/json", "application/html", "application/ld+json", "*/*" }) public Response exportDataset(@QueryParam("persistentId") String persistentId, @QueryParam("exporter") String exporter, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) { try { From 899f1509b62d5a707473e463ec6c58e6cbce9f9c Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 25 Jun 2024 11:51:39 -0400 Subject: [PATCH 113/145] Update DdiExportUtil.java --- .../dataverse/export/ddi/DdiExportUtil.java | 53 ------------------- 1 file changed, 53 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 2d59cd0b514..9a689f7a4ed 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -27,7 +27,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.net.URL; import java.nio.file.Files; import java.nio.file.Paths; import java.time.LocalDate; @@ -63,22 +62,6 @@ import javax.xml.transform.stream.StreamResult; import java.io.InputStream; - -import java.io.OutputStream; -import javax.xml.transform.Result; -import javax.xml.transform.Source; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.sax.SAXResult; -import javax.xml.transform.stream.StreamSource; - -import org.apache.fop.apps.FOUserAgent; -import org.apache.fop.apps.Fop; -import org.apache.fop.apps.FopFactory; -import org.apache.fop.apps.MimeConstants; -import java.io.File; - - public class DdiExportUtil { private static final Logger logger = Logger.getLogger(DdiExportUtil.class.getCanonicalName()); @@ -2116,42 +2099,6 @@ private static boolean checkParentElement(XMLStreamWriter xmlw, String elementNa return true; } - public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { - try { - String sysId = DdiExportUtil.class.getClassLoader().getResource("edu/harvard/iq/dataverse/ddi-to-fo.xsl").toURI().toString(); - InputStream styleSheetInput = DdiExportUtil.class.getClassLoader().getResourceAsStream("edu/harvard/iq/dataverse/ddi-to-fo.xsl"); - - final FopFactory fopFactory = FopFactory.newInstance(new File(".").toURI()); - FOUserAgent foUserAgent = fopFactory.newFOUserAgent(); - - try { - Fop fop = fopFactory.newFop(MimeConstants.MIME_PDF, foUserAgent, outputStream); - // Setup XSLT - TransformerFactory factory = TransformerFactory.newInstance(); - Source mySrc = new StreamSource(styleSheetInput); - mySrc.setSystemId(sysId); - Transformer transformer = factory.newTransformer(mySrc); - - // Set the value of a in the stylesheet - transformer.setParameter("versionParam", "2.0"); - - // Setup input for XSLT transformation - Source src = new StreamSource(datafile); - - // Resulting SAX events (the generated FO) must be piped through to FOP - Result res = new SAXResult(fop.getDefaultHandler()); - - // Start XSLT transformation and FOP processing - transformer.transform(src, res); - - } catch (Exception e) { - logger.severe(e.getMessage()); - } - } catch (Exception e) { - logger.severe(e.getMessage()); - } - } - public static void datasetHtmlDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); From 91dcca7d99d85cb6ede7603b94cb0e810514b488 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 25 Jun 2024 11:52:20 -0400 Subject: [PATCH 114/145] Delete src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/i18n.inc.xsl --- .../resources/edu/harvard/iq/dataverse/ddi-pdf/i18n.inc.xsl | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/i18n.inc.xsl diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/i18n.inc.xsl b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/i18n.inc.xsl deleted file mode 100644 index edf876f3b04..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/i18n.inc.xsl +++ /dev/null @@ -1,5 +0,0 @@ - - - - - From a5591a5d4a78996af6771adfbaee93bb00a28389 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 25 Jun 2024 11:52:58 -0400 Subject: [PATCH 115/145] Delete src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_en.properties.xml --- .../ddi-pdf/messages_en.properties.xml | 174 ------------------ 1 file changed, 174 deletions(-) delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_en.properties.xml diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_en.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_en.properties.xml deleted file mode 100644 index d8e98dfd3c6..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_en.properties.xml +++ /dev/null @@ -1,174 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 -Valid -Frequency table not shown -Derivation -discrete -Data Collection Mode -Other Processing -Other Acknowledgment(s) -Untitled -Identification -Click here to access/export data files from Nesstar format -Value -Percentage -Weighting -Primary Investigator(s) -This document was generated using the -Sampling -Cases -Access Conditions -Source -Modalities -Rights & Disclaimer -Definition -Estimates of Sampling Error -Data Files -Geographic Coverage -April -Mean -Metadata Production -Data Collection -Missing Data -Scripts and programs -Variable(s) -Interviewer instructions -Funding Agency/ies -November -Missing -Version -Universe -Contributor(s) -Access Authority -Data Processing & Appraisal -Scope -Administrative documents -StdDev -Contact(s) -Label -Technical documents -Decimal(s) -Type -Literal question -Concepts -Range -Abstract -June -Supervision -Other Forms of Data Appraisal -References -Accessibility -Data Collection Dates -Data Editing -Questionnaires -Valid case(s) -Reports and analytical documents -Copyright -Documentation -Deviations from Sample Design -Publisher(s) -February -Dataset contains -Acknowledgment(s) -Continuous -Standard deviation -Variables Description -Producer -Production Date - -The Explorer allows you to view data files and export them to common statistical formats -Discrete -Group -July -Filename -Cases -Name -Warning: these figures indicate the number of cases found in the data file. They cannot be interpreted as summary statistics of the population of interest. -Statistical tables -December -Subjects -Processing Checks -software -Interviewer's instructions -Table of Contents -Document Information -Subgroup(s) -Keywords -group(s) -W -Weight -Files Description -Notes -Data Collection Notes -file(s) -continuous -Disclaimer -Content -variable(s) -Other Producer(s) -Producers & Sponsors -Data Cleaning Notes -Distributor(s) -Overview -Citation Requirements -September -Category -Confidentiality -Statistics -May -Undetermined -Structure -file -Pre-question -Response Rate -Width -Recoding and Derivation -Series -October -Unit of Analysis -Data Processing Notes -Kind of Data -File -Time Period(s) -File Content -Invalid -Vars -cont. -Key(s) -Question -Source of information -Imputation -Security -To open this file, you will need the free -Other resources -Data Dictionnary -Information -January -Other documents -Minimum -Scope & Coverage -Metadata Producer(s) -Show more info -Data Collector(s) -Post-question -Topics -Sampling Procedure -File Structure -Variables List -Format -Sampling Notes -Variables Group(s) -Description -Categories -Maximum -Depositor(s) -August -NW -Cover Page -Weighted -March - total - showing a subset of -Countries -question details - From 81c2abd2afeee0be516ddba1d70fd35e770be1bf Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 25 Jun 2024 11:53:15 -0400 Subject: [PATCH 116/145] Delete src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_es.properties.xml --- .../ddi-pdf/messages_es.properties.xml | 170 ------------------ 1 file changed, 170 deletions(-) delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_es.properties.xml diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_es.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_es.properties.xml deleted file mode 100644 index 9cfcdaf6e7e..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_es.properties.xml +++ /dev/null @@ -1,170 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 -Válido -No se presentan las tablas de frecuencias -Derivación -discreta -Método de Recolección -Otros relacionados al procesamiento -Otros Reconocimientos -Sin título -Identificación -Presione aquí para acceder/exportar al archivo(s) de datos -Valor -Porcentaje -Ponderando -Investigadores Principales -Este documento fue producido utilizando el -Muestreo -Casos -Condiciones de uso -Fuente -Modalidades -Derechos y Notas Legales -Definición -Estimaciones del Error Muestral -Archivo de Datos -Cobertura Geográfica -Abril -Media -Producción de los Metadatos -Recolección de Datos -Datos perdidos -Programas informáticos -Variable(s) -Manual del encuestador -Agencias Auspiciadoras -Noviembre -Valores perdidos -Versión -Universo -Contribuidor(es) -Institución Propietaria -Tratamiento y Validación de Datos -Dominio Temático -Documentos Administrativos -Desviación -Contacto(s) -Etiqueta -Documentos Técnicos -Decimal(es) -Tipo -Pregunta textual -Conceptos -Rango -Resumen -Junio -Supervisión -Otras Formas de Validación de los Datos -Referencias -Accesibilidad -Fechas de Recolección de Datos -Procesamiento de Datos -Cuestionarios -Casos válidos -Reportes y documentos analíticos -Derechos de Autor -Documentación -Modificaciones al Diseño Muestral -Editor(es) -Febrero -Contenido de la Base de Datos -Reconocimiento(s) -Contínua -Desviación estándar -Descripción de la variable -Productor -Fecha de Producción -El Explorador NESSTAR permite visualizar los archivos de datos y exportarlos a diferentes formatos estadísticos -Discreta -Grupo -Julio -Nombre del Archivo -Casos -Nombre -Cuadros estadísticos -Diciembre -Temas -Controles de Tratamiento -software -Manual del encuestador -Indice -Información acerca de la Documentación -Subgrupo(s) -Palabra Clave -grupo(s) -P -Ponderador -Descripción de los Archivos -Notas -Notas sobre la Recolección de Datos -archivo(s) -continua -Nota Legal -Contenido -variable(s) -Otros Productores -Productores y Auspiciadores -Notas acerca de la Depuración de los Datos -Distribuidor(es) -Resumen General -Forma de citar -Septiembre -Categoría -Confidencialidad -Estadística - -Mayo -Indeterminado -Estructura -archivo -Pre-pregunta -Tasa de Respuesta -Ancho -Recodificación y Derivación -Series -Octubre -Unidad de Análisis -Notas sobre el Procesamiento de Datos -Tipo de Datos -Archivo -Periodo de Referencia -Contenido del Archivo -Inválido -Vars. -cont. -Clave(s) -Pregunta -Fuente de información -Imputación -Seguridad -Para abrir este archivo se necesita el software gratuito -Otros recursos -Diccionario de Datos -Información -Enero -Otros documentos -Mínimo -Cobertura y Dominio Temático -Productor de los Metadatos -Mostrar más información -Entrevistador(es) -Pos-pregunta -Temas -Procedimiento de Muestreo -Estructura del Archivo -Lista de variables -Formato -Notas sobre el Muestreo -Grupo(s) de Variables -Descripción -Categorías -Máximo -Depositante(s) -Agosto -NP -Carátula -Ponderado -Marzo - From 61fe51f7624d153a13a73c23d82e9c8a56fb32e6 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 25 Jun 2024 11:53:26 -0400 Subject: [PATCH 117/145] Delete src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_fr.properties.xml --- .../ddi-pdf/messages_fr.properties.xml | 173 ------------------ 1 file changed, 173 deletions(-) delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_fr.properties.xml diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_fr.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_fr.properties.xml deleted file mode 100644 index 9fa4d2178b1..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_fr.properties.xml +++ /dev/null @@ -1,173 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 -Valide -Tableau de fréquences non-affiché -Mode de calcul -discrète -Méthode de collecte -Autre traitement -Autre(s) remerciement(s) -Sans titre -Identification -Cliquer ici pour accéder/exporter les fichiers de données du format Nesstar -Valeur -Pourcentage -Pondération -Enquêteur(s) principal/aux -Ce document a été généré à l'aide du -Echantillonage -Enregistrements -Conditions d'accès -Source -Modalités -Responsabilité et droits d'auteurs -Définition -Estimation des erreurs d'échantillonage -Fichiers de données -Couverture géographique -Avril -Moyenne -Production des méta-données -Collecte des données -Valeures manquantes -Programmes informatiques -Variable(s) -Instructions aux enquêteurs -Financement -Novembre -Manquant -Version -Univers -Contributeur(s) -Agence propriétaire -Traitement et évaluation des données -Domaines thématiques -Documents administratifs -Ecart type -Contact(s) -Libellé -Documents techniques -Décimale(s) -Type -Formulation de la question -Concepts -Gamme -Résumé -Juin -Supervision -Autres formes d'évaluation des données -Références -Accessibilité -Dates de collecte -Edition des données -Questionnaires -Cas valide(s) -Rapports et documents analytiques -Droits d'auteurs -Documentation -Déviations par rapport à l'échantillon initial -Editeur(s) -Février -Le jeu de données contient -Remerciement(s) -Continue -Ecart type -Description des variables -Producteur -Date de production - -L'Explorer vous permet d'accéder aux données et de les exporter vers les formats statistiques les plus courants -Discrète -Groupe -Juillet -Nom du fichier -Enreg. -Nom -Avertissement: Ces chiffres indiquent le nombre de cas identifiés dans le fichier de données. Ils ne peuvent pas être interpretés comme étant représentatifs de la population concernée. -Tableaux statistiques -Décembre -Sujets -Contrôles de traitement - -Instructions aux enquêteurs -Table des matières -Informations sur le document -Sous-groupe(s) -Mots-clé -groupe(s) -P -Pondération -Description des fichiers -Notes -Notes sur la collecte -fichier(s) -continue -Responsabilité(s) -Contenu -variable(s) -Autre(s) producteur(s) -Producteurs et sponsors -Notes sur l'apurement des données -Distributeur(s) -Aperçu -Citation -Septembre -Catégorie -Confidentialité -Statistiques -Mai -Indéterminé -Structure -fichier -Pré-question -Taux de réponse -Taille -Formulation de la question -Recodage et dérivation -Série -Octobre -Unité d'analyse -Notes sur le traitement des données -Type d'étude -Fichier -Période(s) de référence -Contenu du fichier -Non-valide -Vars -suite -Clé(s) -Question -Source d'information -Imputation -Sécurité -Pour ouvrir ce fichier, vous avez besoin du logiciel gratuit -Autres resources -Dictionnaire des variables -Information -Janvier -Autres documents -Minimum -Domaines thématiques et couverture -Producteur(s) des méta-données -Information complémentaire -Enquêteurs -Post-question -Thèmes -Procédure d'échantillonage -Structure du fichier -Liste des variables -Format -Notes sur l'échantillonage -Groupe(s) de variables -Description -Catégories -Maximum -Dépositaire(s) -Août -NP -Couverture -Pondéré -Mars -question details - From a87d1bcdee77f78fcfa287502838526cd85b54b5 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 25 Jun 2024 11:53:42 -0400 Subject: [PATCH 118/145] Delete src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ja.properties.xml --- .../ddi-pdf/messages_ja.properties.xml | 161 ------------------ 1 file changed, 161 deletions(-) delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ja.properties.xml diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ja.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ja.properties.xml deleted file mode 100644 index bc5dbb06154..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ja.properties.xml +++ /dev/null @@ -1,161 +0,0 @@ - - -Generated by Properties2Xml on Tue Feb 13 13:55:43 EST 2007 -有効な -度数表(Frequency table)は表示されません -由来 -不連続な -データ収集モード -その他の確認事項 -識別番号 -データファイルにアクセスするにはここをクリックしてください -無題 -値 -割合 -ウェイティング -第一次調査官 -この文書はToolkitを使用して作られました -サンプリング -ケース -アクセス条件 -情報源 -様相 -権利及び声明文 -定義 -サンプルエラーの見積もり -データファイル -地理的な適用範囲 -４月 -平均 -メタデータ製作 -データ収集 -損失データ -スクリプトおよびプログラム -可変的 -面接者の指示 -出資機関 -１１月 -バージョン -共通の -貢献者 -アクセス権限 -データ処理、評価 -範囲, 領域 -管理用文章 -連絡先 -ラベル -技術的な文書 -小数点 -タイプ -文字の質問 -概念 -範囲 -要約 -６月 -監督 -その他ファーマットのデータ評価 -参照 -アクセス、入手法 -データ収集日 -データ編集 -質問 -レポートおよび分析的な文書 -有効な場合 -コピーライト -書類 -サンプルデザインによる偏差 -発行者 -２月 -データセットに含まれる -確認事項 -連続的な -標準偏差 -変数の記述 -製作者 -製作日 -” Explorer”によってデータファイルを参照することも一般的に使えわれている統計データフォーマットに変換。抽出することも可能です -不連続性 -グループ -７月 -ファイルの名前 -ケース -名前 -統計表 -１２月 -主題, 内容 -工程監査 -ソフト -面接者への指示 -目録 -書類の情報 -サブグループ -キーワード - -グループ -ウェイト -ファイルの詳細 -メモ -データ収集メモ -ファイル -継続的な -声明文 -内容 -変数 -その他の製作者 -製作者とスポンサー -データクリーニングメモ -分配者 -概略 -引用する場合の必要条件 -９月 -カテゴリー -機密性、コンフィデンシャリティー -５月 -未定 -構造 -ファイル -調査前の質問 -回答比率 -幅 -記録と誘導 -シリー -１０月 -分析の単位 -データ処理メモ -データの種類 - -ファイル -期間 -ファイルの内容 -無効 -キー -情報源 -非難 -セキュリティー -このファイルを開けるには、無料で配布されているNesstar Explorer が必要です。 -その他の資料 -データ辞典 -情報 -１月 -その他の書類 -最小値 -規模及び適用範囲 -メタデータ製作者 -さらにインフォメーションを表示 -データ収集者 -調査後の質問 -サンプリングの手順 -ファイルの構造 -変数のリスト -フォーマット -サンプリングメモ -変数のグループ -詳細 -カテゴリー -最大値 -デポジター、提供者、供託者 -８月 -表紙 -ウェイトされた -３月 - From 73edb3da1228c71d9dddc5be01140290610cd74e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 25 Jun 2024 11:53:53 -0400 Subject: [PATCH 119/145] Delete src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_nn.properties.xml --- .../ddi-pdf/messages_nn.properties.xml | 174 ------------------ 1 file changed, 174 deletions(-) delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_nn.properties.xml diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_nn.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_nn.properties.xml deleted file mode 100644 index fdf14f5dfcd..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_nn.properties.xml +++ /dev/null @@ -1,174 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 -Gyldige -Frekvenstabell ikke vist -Avledning -diskret -Type datainnsamling -Annen prosessering -Andre identifikatorer og krediteringer -Uten tittel -Identifisering -Click here to access/export data files from Nesstar format -Verdi -Prosent -Vekting -Primary Investigator(s) -Dette dokumentet ble generert av -Utvalg -Enheter -Tilgangsbetingelser -Kilde -Modaliteter -Rights & Disclaimer -Definisjon -Estimert utvalgsfeil -Datafiler -Geografisk omfang -April -Mean -Metadata-produksjon -Datainnsamling -Manglende data -Script og programmer -Variable(r) -Instruksjoner til intervjueren -Sponsor/finansierende institusjon(er) -November -Missing -Versjon -Univers -Bidragsyter(e) -Tilgangskontrollør -Dataprosessering og -evaluering -Omfang -Administrative dokumenter -Standardavvik -Kontaktperson(er) -Merkelapp -Tekniske dokumenter -Desimal(er) -Type -Spørsmålstekst -Begrep(er) -Rekkevidde -Sammendrag -Juni -Supervision -Andre former for dataevaluering -Referanser -Tilgjengelighet -Datainnsamlingsdatoer -Dataredigering -Spørreskjema -Gyldige enheter -Rapporter og analysedokumenter -Copyright -Dokumentasjon -Avvik fra utvalgsdesign -Utgiver(e) -Februar -Datasettet inneholder -Krediteringer -Kontinuerlig -Standardavvik -Variabelbeskrivelse -Produsent -Produksjonsdato - -The Explorer allows you to view data files and export them to common statistical formats -Diskret -Gruppe -Juli -Filnavn -Enheter -Navn -Advarsel: disse tallene indikerer antall enheter (cases) i datafilen. De kan ikke tolkes som oppsummert statistikk for populasjonen. -Statistiske tabeller -Desember -Emner -Prosesseringssjekk -programvare -Instruksjoner til intervjueren -Innholdsfortegnelse -Dokumentinformasjon -Undergruppe(r) -Nøkkelord -gruppe(r) -W -Vekt -Filbeskrivelse -Kommentarer -Datainnsamlingskommentarer -file(r) -kontinuerlig -Fraskrivelse -Innhold -variable(r) -Andre produsenter -Produsenter og sponsorer -Kommentarer om datarensing -Distributør(er) -Oversikt -Sitatkrav -September -Kategori -Konfidensialitet -Statistikk -Mai -Uavklart -Struktur -fil -Tekst før spørsmål -Responsrate -Bredde -Omkodinger og utledninger -Serie -Oktober -Analyseenhet -Dataprosesseringskommentarer -Datatype -Fil -Tidsperiode(r) -Filinnhold -Ugyldig -Variabler -kont. -Nøkler -Spørsmål -Kilde for informasjon -Imputasjon -Sikkerhet -For å åpne denne filen trenger du følgende gratisverktøy -Andre ressurser -Dataordbok -Informasjon -Januar -Andre dokumenter -Minimum -Omfang -Metadataprodusenter -Vis mer informasjon -Datainnsamler(e) -Tekst etter spørsmål -Emner -Utvalgsprosedyre -Filstruktur -Variabelliste -Format -Utvalgskommentarer -Variabelgrupper -Beskrivelse -Kategorier -Maksimum -Utgiver(e) -August -NW -Forside -Vektet -Mars - total - viser et utvalg av -Land -spørsmålsdetaljer - From f3dd0461184d70f75daff6727f041e76e22df637 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 25 Jun 2024 11:54:04 -0400 Subject: [PATCH 120/145] Delete src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_no.properties.xml --- .../ddi-pdf/messages_no.properties.xml | 174 ------------------ 1 file changed, 174 deletions(-) delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_no.properties.xml diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_no.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_no.properties.xml deleted file mode 100644 index fdf14f5dfcd..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_no.properties.xml +++ /dev/null @@ -1,174 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 -Gyldige -Frekvenstabell ikke vist -Avledning -diskret -Type datainnsamling -Annen prosessering -Andre identifikatorer og krediteringer -Uten tittel -Identifisering -Click here to access/export data files from Nesstar format -Verdi -Prosent -Vekting -Primary Investigator(s) -Dette dokumentet ble generert av -Utvalg -Enheter -Tilgangsbetingelser -Kilde -Modaliteter -Rights & Disclaimer -Definisjon -Estimert utvalgsfeil -Datafiler -Geografisk omfang -April -Mean -Metadata-produksjon -Datainnsamling -Manglende data -Script og programmer -Variable(r) -Instruksjoner til intervjueren -Sponsor/finansierende institusjon(er) -November -Missing -Versjon -Univers -Bidragsyter(e) -Tilgangskontrollør -Dataprosessering og -evaluering -Omfang -Administrative dokumenter -Standardavvik -Kontaktperson(er) -Merkelapp -Tekniske dokumenter -Desimal(er) -Type -Spørsmålstekst -Begrep(er) -Rekkevidde -Sammendrag -Juni -Supervision -Andre former for dataevaluering -Referanser -Tilgjengelighet -Datainnsamlingsdatoer -Dataredigering -Spørreskjema -Gyldige enheter -Rapporter og analysedokumenter -Copyright -Dokumentasjon -Avvik fra utvalgsdesign -Utgiver(e) -Februar -Datasettet inneholder -Krediteringer -Kontinuerlig -Standardavvik -Variabelbeskrivelse -Produsent -Produksjonsdato - -The Explorer allows you to view data files and export them to common statistical formats -Diskret -Gruppe -Juli -Filnavn -Enheter -Navn -Advarsel: disse tallene indikerer antall enheter (cases) i datafilen. De kan ikke tolkes som oppsummert statistikk for populasjonen. -Statistiske tabeller -Desember -Emner -Prosesseringssjekk -programvare -Instruksjoner til intervjueren -Innholdsfortegnelse -Dokumentinformasjon -Undergruppe(r) -Nøkkelord -gruppe(r) -W -Vekt -Filbeskrivelse -Kommentarer -Datainnsamlingskommentarer -file(r) -kontinuerlig -Fraskrivelse -Innhold -variable(r) -Andre produsenter -Produsenter og sponsorer -Kommentarer om datarensing -Distributør(er) -Oversikt -Sitatkrav -September -Kategori -Konfidensialitet -Statistikk -Mai -Uavklart -Struktur -fil -Tekst før spørsmål -Responsrate -Bredde -Omkodinger og utledninger -Serie -Oktober -Analyseenhet -Dataprosesseringskommentarer -Datatype -Fil -Tidsperiode(r) -Filinnhold -Ugyldig -Variabler -kont. -Nøkler -Spørsmål -Kilde for informasjon -Imputasjon -Sikkerhet -For å åpne denne filen trenger du følgende gratisverktøy -Andre ressurser -Dataordbok -Informasjon -Januar -Andre dokumenter -Minimum -Omfang -Metadataprodusenter -Vis mer informasjon -Datainnsamler(e) -Tekst etter spørsmål -Emner -Utvalgsprosedyre -Filstruktur -Variabelliste -Format -Utvalgskommentarer -Variabelgrupper -Beskrivelse -Kategorier -Maksimum -Utgiver(e) -August -NW -Forside -Vektet -Mars - total - viser et utvalg av -Land -spørsmålsdetaljer - From 7a350fa948ad695cc68986f7ab9965855019f63b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 25 Jun 2024 11:54:14 -0400 Subject: [PATCH 121/145] Delete src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ru.properties.xml --- .../ddi-pdf/messages_ru.properties.xml | 169 ------------------ 1 file changed, 169 deletions(-) delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ru.properties.xml diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ru.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ru.properties.xml deleted file mode 100644 index 06fde85af5e..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ru.properties.xml +++ /dev/null @@ -1,169 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 -Валидный -Частотная таблица не выводится -Расчет -дискретная -Способ сбора данных -Прочая обработка -Другие участники -Безымянный -Индентификация -Щелкните здесь, чтобы получить доступ к файлам или экспортировать их -Значение -Процент -Взвешивание -Первичный(е) исследователь(и) -Документ был сгенерирован с использованием -Выборка -Наблюдения -Условия доступа -Источник -Модальности -Авторские права и ограничения ответственности -Определение -Оценки ошибок выборки -Файлы данных -Географический охват -Апрель -Среднее -Разработка метаданных -Сбор данных -Пропущенные данные -Скрипты и программы -Переменная(ые) -Инструкции интервьюеру -Кто финансировал -Ноябрь -Пропущенные -Версия -Генеральная совокупность -Участник(и) -Права доступа -Обработка и инспекция данных -Охват -Административные документы -СтдОткл -Контак(ы) -Метка -Технические документы -Десятичные -Тип -Текст вопроса -Концепции -Диапазон -Резюме -Июнь -Контроль -Другие формы инспекции данных -Установки -Доступность -Даты сбора данных -Редактирование данных -Вопросники -Валидное(ые) наблюдение(я) -Отчеты и аналитические документы -Авторские права -Документация -Отклонения от дизайна выборки -Издатель(и) -Февраль -Набор данных содержит -Участник(и) -Непрерывная -Стандартное отклонение -Описание переменных -Разработчик -Дата разработки -Проводник позволяет просматривать файлы данных и экспортировать их в распространенные статистические форматы -Дикретная -Группа -Июль -Имя файла -Наблюдения -Имя -Статистичсекие таблицы -Декабрь -Темы -Контроль обработки -программное обеспечение -Инструкции интервьюеру -Оглавление -Информация о документе -Подгруппа(ы) -Ключевые слова -группа(ы) -B -Вес -Описание файла -Примечания -Примечания по сбору данных -файл(ы) -непрерывная -Ограничения ответственности -Содержание -переменная(ые) -Другие разработчики -Разработчики и спонсоры -Примечания по чистке данных -Дистрибьютор(ы) -Обзор -Требования по цитированию -Сентябрь -Категория -Конфиденциальность -Статистики -Май -Неопределенный -Структура -файл -Текст, предваряющий вопрос -Доля ответов -Ширина -Перекодировка и расчеты -Серия -Октябрь -Единица анализа -Примечания по обработке данных -Тип данных -Файл -Период(ы) времени -Содержание файла -Некорректный -Переменные -непр. -Ключ(и) -Вопрос -Источник информации -Импутация -Безопасность -Чтобы открыть этот файл, необходимо иметь свободным -Прочие источники -Словарь данных -Информация -Январь -Прочие документы -Минимум -Охват и покрытие -Разработчик(и) метаданных -Показать дополнительную информацию -Кто собирал данные -Текст после вопроса -Разделы -Процедура выборки -Структура файла -Список переменных -Формат -Примечания по выборке -Группа(ы) переменных -Описание -Категории -Максимум -Депозитор(ы) -Август -HB -Титульный лист -взвешенные -Март - From ab6dd1ce828686ddadc22307dfa683e078c85992 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 25 Jun 2024 11:54:34 -0400 Subject: [PATCH 122/145] Delete src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl --- .../edu/harvard/iq/dataverse/ddi-to-fo.xsl | 4435 ----------------- 1 file changed, 4435 deletions(-) delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl deleted file mode 100644 index 26fd7c23479..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl +++ /dev/null @@ -1,4435 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - default-page - - - - - - - - - - - - - - - - - - - - - - Times - - - - - - - - - - - - - - - - - - - - - - - - - 50 - - - - - - - 0 - 0 - 0 - 1 - - - - - - - 1 - 0 - - - - - - - 0 - 1 - - - - - - ddp - toolkit - toolkit - toolkit - - - - - - - - () - - - - - - , - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1 - 1 - 1 - 1 - 0 - - - - - 1 - 1 - 1 - 1 - 0 - - - - - 1 - 1 - 1 - 1 - 1 - 0 - - - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 0 - - - - - 1 - 1 - 1 - 0 - - - - - 1 - 1 - 1 - 1 - 1 - 1 - 0 - - - - - 1 - 1 - 0 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - : - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - , - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - : - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - , - - - - - - - - - - - - - - - - - - , - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - *** - - **** - - - - - - - - - - - - - - - , - - - - - - - - : - - - - - - - - - - - - - () - , - - - - , - - - - - - - - - - - () - , - - - - - - - : - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # - - - - - - - - - - - # - - - - - - - - - - - - - - - - - : - - - - - - :  - - - - - ,  - - -  () - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - : - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - () - , - - - - - - - - - - , - - , - - - - - - - - - - () - , - - , - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - , - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - .. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - . - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -   - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # - - - - - : - - - - - - - - - - - - - - - - - - - - - [= - - - - - - ] - - - - [= - - - - - - - . - - - ] - - - - [= - - ] - - - [ - - =* - - / - - - ] - - - - - - - - - - - - - - [ - - / - - ] - - - - - - - - - - - - - - - - - - - - - - - - - - - [ - - = - - - / - - - - - - - - - - ] - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - , - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - () - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ( - ) - - - - - - - - - - - - - - - - - - - - : - - - - - - - - - - - - - - - - - - *** - *** - - - - - , - - - - - - - , - - - - - - , - - - - - - - - - - - - - , - - - - - , - - - - - , " - - " - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - , - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - , - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <h2> - - - </h2> - - - <br/> - - - - - - - - - - - - - - - - - - - -   - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Z - - - - - - - - - - - - - - - - - - - - - - - - - - - - NaN - - - - - - - - - - - From f73c5f89ddee573d2fb52e81f4f61d6186081538 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 25 Jun 2024 11:55:27 -0400 Subject: [PATCH 123/145] Update Bundle.properties --- src/main/java/propertyFiles/Bundle.properties | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 0e32ba5b30d..157f2ecaf54 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1433,7 +1433,6 @@ dataset.exportBtn.itemLabel.json=JSON dataset.exportBtn.itemLabel.oai_ore=OAI_ORE dataset.exportBtn.itemLabel.dataciteOpenAIRE=OpenAIRE dataset.exportBtn.itemLabel.html=DDI HTML Codebook -dataset.exportBtn.itemLabel.pdf=DDI PDF Codebook license.custom=Custom Dataset Terms license.custom.description=Custom terms specific to this dataset metrics.title=Metrics From b464b24296a15ffcb137d283f07965a2a2449e4b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 25 Jun 2024 15:06:48 -0400 Subject: [PATCH 124/145] drop COMMIT_WITHIN which breaks autoSoftCommit by maxTime in solrconfig --- .../harvard/iq/dataverse/search/IndexServiceBean.java | 11 +++++------ .../iq/dataverse/search/SolrIndexServiceBean.java | 6 +++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 0102459ab9f..10e6c2e6516 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -312,7 +312,7 @@ public Future indexDataverse(Dataverse dataverse, boolean processPaths) String status; try { if (dataverse.getId() != null) { - solrClientService.getSolrClient().add(docs, COMMIT_WITHIN); + solrClientService.getSolrClient().add(docs); } else { logger.info("WARNING: indexing of a dataverse with no id attempted"); } @@ -345,7 +345,6 @@ public void indexDatasetInNewTransaction(Long datasetId) { //Dataset dataset) { private static final Map INDEXING_NOW = new ConcurrentHashMap<>(); // semaphore for async indexing private static final Semaphore ASYNC_INDEX_SEMAPHORE = new Semaphore(JvmSettings.MAX_ASYNC_INDEXES.lookupOptional(Integer.class).orElse(4), true); - static final int COMMIT_WITHIN = 30000; //Same as current autoHardIndex time @Inject @Metric(name = "index_permit_wait_time", absolute = true, unit = MetricUnits.NANOSECONDS, @@ -1562,7 +1561,7 @@ private String addOrUpdateDataset(IndexableDataset indexableDataset, Set d final SolrInputDocuments docs = toSolrDocs(indexableDataset, datafilesInDraftVersion); try { - solrClientService.getSolrClient().add(docs.getDocuments(), COMMIT_WITHIN); + solrClientService.getSolrClient().add(docs.getDocuments()); } catch (SolrServerException | IOException ex) { if (ex.getCause() instanceof SolrServerException) { throw new SolrServerException(ex); @@ -1814,7 +1813,7 @@ private void updatePathForExistingSolrDocs(DvObject object) throws SolrServerExc sid.removeField(SearchFields.SUBTREE); sid.addField(SearchFields.SUBTREE, paths); - UpdateResponse addResponse = solrClientService.getSolrClient().add(sid, COMMIT_WITHIN); + UpdateResponse addResponse = solrClientService.getSolrClient().add(sid); if (object.isInstanceofDataset()) { for (DataFile df : dataset.getFiles()) { solrQuery.setQuery(SearchUtil.constructQuery(SearchFields.ENTITY_ID, df.getId().toString())); @@ -1869,7 +1868,7 @@ public String delete(Dataverse doomed) { logger.fine("deleting Solr document for dataverse " + doomed.getId()); UpdateResponse updateResponse; try { - updateResponse = solrClientService.getSolrClient().deleteById(solrDocIdentifierDataverse + doomed.getId(), COMMIT_WITHIN); + updateResponse = solrClientService.getSolrClient().deleteById(solrDocIdentifierDataverse + doomed.getId()); } catch (SolrServerException | IOException ex) { return ex.toString(); } @@ -1889,7 +1888,7 @@ public String removeSolrDocFromIndex(String doomed) { logger.fine("deleting Solr document: " + doomed); UpdateResponse updateResponse; try { - updateResponse = solrClientService.getSolrClient().deleteById(doomed, COMMIT_WITHIN); + updateResponse = solrClientService.getSolrClient().deleteById(doomed); } catch (SolrServerException | IOException ex) { return ex.toString(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java index 19235bb5a14..cfe29ea08c7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java @@ -356,7 +356,7 @@ private void persistToSolr(Collection docs) throws SolrServer /** * @todo Do something with these responses from Solr. */ - UpdateResponse addResponse = solrClientService.getSolrClient().add(docs, IndexServiceBean.COMMIT_WITHIN); + UpdateResponse addResponse = solrClientService.getSolrClient().add(docs); } public IndexResponse indexPermissionsOnSelfAndChildren(long definitionPointId) { @@ -496,7 +496,7 @@ public IndexResponse deleteMultipleSolrIds(List solrIdsToDelete) { return new IndexResponse("nothing to delete"); } try { - solrClientService.getSolrClient().deleteById(solrIdsToDelete, IndexServiceBean.COMMIT_WITHIN); + solrClientService.getSolrClient().deleteById(solrIdsToDelete); } catch (SolrServerException | IOException ex) { /** * @todo mark these for re-deletion @@ -509,7 +509,7 @@ public IndexResponse deleteMultipleSolrIds(List solrIdsToDelete) { public JsonObjectBuilder deleteAllFromSolrAndResetIndexTimes() throws SolrServerException, IOException { JsonObjectBuilder response = Json.createObjectBuilder(); logger.info("attempting to delete all Solr documents before a complete re-index"); - solrClientService.getSolrClient().deleteByQuery("*:*", IndexServiceBean.COMMIT_WITHIN); + solrClientService.getSolrClient().deleteByQuery("*:*"); int numRowsAffected = dvObjectService.clearAllIndexTimes(); response.add(numRowsClearedByClearAllIndexTimes, numRowsAffected); response.add(messageString, "Solr index and database index timestamps cleared."); From 9fc757f9fa37ef43383a0ad628fd137231249de6 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 25 Jun 2024 15:21:58 -0400 Subject: [PATCH 125/145] 10581 request access email fix (#10653) * #10581 fix subject line of request access email * #10581 add sorter * #10581 addl user info for request email * #10581 add break between custom Q responses * #10581 remove orderby --- .../iq/dataverse/GuestbookResponse.java | 40 +++++++++++++++---- .../harvard/iq/dataverse/MailServiceBean.java | 2 +- .../harvard/iq/dataverse/util/MailUtil.java | 5 ++- src/main/java/propertyFiles/Bundle.properties | 6 ++- .../iq/dataverse/util/MailUtilTest.java | 8 +++- 5 files changed, 49 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java index 9041ccf887c..1ea7d02791d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java +++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java @@ -17,6 +17,8 @@ import java.util.List; import jakarta.persistence.*; import jakarta.validation.constraints.Size; +import java.util.Collections; +import java.util.Comparator; /** * @@ -178,7 +180,7 @@ public GuestbookResponse(GuestbookResponse source){ this.setSessionId(source.getSessionId()); List customQuestionResponses = new ArrayList<>(); if (!source.getCustomQuestionResponses().isEmpty()){ - for (CustomQuestionResponse customQuestionResponse : source.getCustomQuestionResponses() ){ + for (CustomQuestionResponse customQuestionResponse : source.getCustomQuestionResponsesSorted() ){ CustomQuestionResponse customQuestionResponseAdd = new CustomQuestionResponse(); customQuestionResponseAdd.setResponse(customQuestionResponse.getResponse()); customQuestionResponseAdd.setCustomQuestion(customQuestionResponse.getCustomQuestion()); @@ -254,6 +256,18 @@ public String getResponseDate() { public List getCustomQuestionResponses() { return customQuestionResponses; } + + public List getCustomQuestionResponsesSorted(){ + + Collections.sort(customQuestionResponses, (CustomQuestionResponse cqr1, CustomQuestionResponse cqr2) -> { + int a = cqr1.getCustomQuestion().getDisplayOrder(); + int b = cqr2.getCustomQuestion().getDisplayOrder(); + return Integer.valueOf(a).compareTo(b); + }); + + + return customQuestionResponses; + } public void setCustomQuestionResponses(List customQuestionResponses) { this.customQuestionResponses = customQuestionResponses; @@ -317,7 +331,11 @@ public void setSessionId(String sessionId) { this.sessionId= sessionId; } - public String toHtmlFormattedResponse() { + public String toHtmlFormattedResponse(){ + return toHtmlFormattedResponse(null); + } + + public String toHtmlFormattedResponse(AuthenticatedUser requestor) { StringBuilder sb = new StringBuilder(); @@ -326,17 +344,25 @@ public String toHtmlFormattedResponse() { sb.append(BundleUtil.getStringFromBundle("dataset.guestbookResponse.respondent") + "
    \n
  • " + BundleUtil.getStringFromBundle("name") + ": " + getName() + "
    • \n
  • "); sb.append(" " + BundleUtil.getStringFromBundle("email") + ": " + getEmail() + "
    • \n
  • "); - sb.append( - " " + BundleUtil.getStringFromBundle("institution") + ": " + wrapNullAnswer(getInstitution()) + "
    • \n
  • "); - sb.append(" " + BundleUtil.getStringFromBundle("position") + ": " + wrapNullAnswer(getPosition()) + "
\n"); + sb.append(" " + BundleUtil.getStringFromBundle("institution") + ": " + wrapNullAnswer(getInstitution()) + "\n
  • "); + sb.append(" " + BundleUtil.getStringFromBundle("position") + ": " + wrapNullAnswer(getPosition()) + "
    • "); + + //Add requestor information to response to help dataset admin with request processing + if (requestor != null){ + sb.append("\n
  • " + BundleUtil.getStringFromBundle("dataset.guestbookResponse.requestor.id") + ": " + requestor.getId()+ "
    • "); + sb.append("\n
  • " + BundleUtil.getStringFromBundle("dataset.guestbookResponse.requestor.identifier") + ": " + requestor.getIdentifier()+ "
    • \n"); + } else { + sb.append("\n"); + } + sb.append(BundleUtil.getStringFromBundle("dataset.guestbookResponse.guestbook.additionalQuestions") + ":
      \n"); - for (CustomQuestionResponse cqr : getCustomQuestionResponses()) { + for (CustomQuestionResponse cqr : getCustomQuestionResponsesSorted()) { sb.append("
    • " + BundleUtil.getStringFromBundle("dataset.guestbookResponse.question") + ": " + cqr.getCustomQuestion().getQuestionString() + "
      " + BundleUtil.getStringFromBundle("dataset.guestbookResponse.answer") + ": " - + wrapNullAnswer(cqr.getResponse()) + "
      • \n"); + + wrapNullAnswer(cqr.getResponse()) + "\n
      "); } sb.append("
    "); return sb.toString(); diff --git a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java index 1eee9c65501..7359ef8eb33 100644 --- a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java @@ -456,7 +456,7 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio GuestbookResponse gbr = far.getGuestbookResponse(); if (gbr != null) { messageText += MessageFormat.format( - BundleUtil.getStringFromBundle("notification.email.requestFileAccess.guestbookResponse"), gbr.toHtmlFormattedResponse()); + BundleUtil.getStringFromBundle("notification.email.requestFileAccess.guestbookResponse"), gbr.toHtmlFormattedResponse(requestor)); } return messageText; case GRANTFILEACCESS: diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java index ccec3e5f09b..36c249de834 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java @@ -35,7 +35,10 @@ public static String getSubjectTextBasedOnNotification(UserNotification userNoti case CREATEDV: return BundleUtil.getStringFromBundle("notification.email.create.dataverse.subject", rootDvNameAsList); case REQUESTFILEACCESS: - return BundleUtil.getStringFromBundle("notification.email.request.file.access.subject", Arrays.asList(rootDvNameAsList.get(0), datasetDisplayName)); + String userNameFirst = userNotification.getRequestor().getFirstName(); + String userNameLast = userNotification.getRequestor().getLastName(); + String userIdentifier = userNotification.getRequestor().getIdentifier(); + return BundleUtil.getStringFromBundle("notification.email.request.file.access.subject", Arrays.asList(rootDvNameAsList.get(0), userNameFirst, userNameLast, userIdentifier, datasetDisplayName)); case REQUESTEDFILEACCESS: return BundleUtil.getStringFromBundle("notification.email.requested.file.access.subject", Arrays.asList(rootDvNameAsList.get(0), datasetDisplayName)); case GRANTFILEACCESS: diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 2996ccb509b..3e19a19d8dc 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -757,8 +757,8 @@ dashboard.card.datamove.dataset.command.error.indexingProblem=Dataset could not notification.email.create.dataverse.subject={0}: Your dataverse has been created notification.email.create.dataset.subject={0}: Dataset "{1}" has been created notification.email.dataset.created.subject={0}: Dataset "{1}" has been created -notification.email.request.file.access.subject={0}: Access has been requested for a restricted file in dataset "{1}" -notification.email.requested.file.access.subject={0}: You have requested access to a restricted file in dataset "{1}" +notification.email.request.file.access.subject={0}: {1} {2} ({3}) requested access to dataset "{4}" +notification.email.requested.file.access.subject={0}: You have requested access to a restricted file in dataset "{1}" notification.email.grant.file.access.subject={0}: You have been granted access to a restricted file notification.email.rejected.file.access.subject={0}: Your request for access to a restricted file has been rejected notification.email.submit.dataset.subject={0}: Dataset "{1}" has been submitted for review @@ -1400,6 +1400,8 @@ dataset.guestbookResponse.respondent=Respondent dataset.guestbookResponse.question=Q dataset.guestbookResponse.answer=A dataset.guestbookResponse.noResponse=(No Response) +dataset.guestbookResponse.requestor.id=authenticatedUserId +dataset.guestbookResponse.requestor.identifier=authenticatedUserIdentifier # dataset.xhtml diff --git a/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java index 0756c4650fb..f9236ab8338 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java @@ -2,6 +2,7 @@ import edu.harvard.iq.dataverse.DataverseServiceBean; import edu.harvard.iq.dataverse.UserNotification; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; @@ -82,7 +83,12 @@ public void testSubjectRevokeRole() { @Test public void testSubjectRequestFileAccess() { userNotification.setType(UserNotification.Type.REQUESTFILEACCESS); - assertEquals("LibraScholar: Access has been requested for a restricted file in dataset \"\"", MailUtil.getSubjectTextBasedOnNotification(userNotification, null)); + AuthenticatedUser requestor = new AuthenticatedUser(); + requestor.setFirstName("Tom"); + requestor.setLastName("Jones"); + requestor.setUserIdentifier("TJ-1234"); + userNotification.setRequestor(requestor); + assertEquals("LibraScholar: Tom Jones (@TJ-1234) requested access to dataset \"\"", MailUtil.getSubjectTextBasedOnNotification(userNotification, null)); } @Test From fc020be4e95c163509a055f452111aadd06eddcb Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 25 Jun 2024 15:37:03 -0400 Subject: [PATCH 126/145] missed use --- .../java/edu/harvard/iq/dataverse/search/IndexServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 10e6c2e6516..b7b2760e79b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -1826,7 +1826,7 @@ private void updatePathForExistingSolrDocs(DvObject object) throws SolrServerExc } sid.removeField(SearchFields.SUBTREE); sid.addField(SearchFields.SUBTREE, paths); - addResponse = solrClientService.getSolrClient().add(sid, COMMIT_WITHIN); + addResponse = solrClientService.getSolrClient().add(sid); } } } From 9f504e214438af9fa6fa450a88fdabc3ac994ccf Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 25 Jun 2024 19:13:32 -0400 Subject: [PATCH 127/145] optional indexing of the "metadata source" for harvested files as well - it appears that it may have been missed in the original #10464 (? - will confirm). #10611 --- .../harvard/iq/dataverse/search/IndexServiceBean.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index d3286d3be4b..1eaf012876d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -1402,7 +1402,14 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set Date: Tue, 25 Jun 2024 19:32:53 -0400 Subject: [PATCH 128/145] a release note (#10611) --- doc/release-notes/10611-harvested-origin-facet.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/release-notes/10611-harvested-origin-facet.md diff --git a/doc/release-notes/10611-harvested-origin-facet.md b/doc/release-notes/10611-harvested-origin-facet.md new file mode 100644 index 00000000000..89ab6eb7639 --- /dev/null +++ b/doc/release-notes/10611-harvested-origin-facet.md @@ -0,0 +1,10 @@ +NOTE that this release note supercedes the 10464-add-name-harvesting-client-facet.md note from the PR 10464. + +An option has been added to index the name of the Harvesting Client as the "Metadata Source" of harvested datasets and files; if enabled, the Metadata Source facet will be showing separate entries for the content harvested from different sources, instead of the current, default behavior where there is one "Harvested" facet for all such content. + + +TODO: for the v6.3 release note: +If you choose to enable the extended "Metadata Souce" facet for harvested content, set the optional feature flage (jvm option) `dataverse.feature.index-harvested-metadata-source=true` before reindexing. + +[Please note that the upgrade instruction in 6.3 will contain a suggestion to run full reindex, as part of the Solr upgrade, so the sentence above will need to be added to that section] + From bb7944928942a683718d93900775f7c1535f3609 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 25 Jun 2024 19:39:51 -0400 Subject: [PATCH 129/145] an entry in the config guide for the new feature flag. --- doc/sphinx-guides/source/installation/config.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 213ac827819..4b2542f45fd 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3274,6 +3274,9 @@ please find all known feature flags below. Any of these flags can be activated u * - add-publicobject-solr-field - Adds an extra boolean field `PublicObject_b:true` for public content (published Collections, Datasets and Files). Once reindexed with these fields, we can rely on it to remove a very expensive Solr join on all such documents in Solr queries, significantly improving overall performance (by enabling the feature flag above, `avoid-expensive-solr-join`). These two flags are separate so that an instance can reindex their holdings before enabling the optimization in searches, thus avoiding having their public objects temporarily disappear from search results while the reindexing is in progress. - ``Off`` + * - index-harvested-metadata-source + - If enabled, this will index the name of the Harvesting Client as the "Metadata Source" of harvested datasets and files; so that the Metadata Source facet on the collection page will be showing separate entries for the content harvested from different sources/via different clients, instead of the current, default behavior where there is one "Harvested" facet for all such content. Requires a reindex. + - ``Off`` **Note:** Feature flags can be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_FEATURE_XXX`` (e.g. ``DATAVERSE_FEATURE_API_SESSION_AUTH=1``). These environment variables can be set in your shell before starting Payara. If you are using :doc:`Docker for development `, you can set them in the `docker compose `_ file. From bcc50aab30e9a6685a51f5695bf5d03306038dc1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 26 Jun 2024 09:43:26 -0400 Subject: [PATCH 130/145] fixes file access request email perm check --- .../java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java index ab9e4f9be66..5370e9ac564 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java @@ -573,7 +573,7 @@ public boolean requestAccess(DataFile dataFile, GuestbookResponse gbr){ public void sendRequestFileAccessNotification(Dataset dataset, Long fileId, AuthenticatedUser requestor) { Timestamp ts = new Timestamp(new Date().getTime()); - permissionService.getUsersWithPermissionOn(Permission.ManageDatasetPermissions, dataset).stream().forEach((au) -> { + permissionService.getUsersWithPermissionOn(Permission.ManageFilePermissions, dataset).stream().forEach((au) -> { userNotificationService.sendNotification(au, ts, UserNotification.Type.REQUESTFILEACCESS, fileId, null, requestor, true); }); //send the user that requested access a notification that they requested the access From 65b2dcbdbfc6321be89035294a7caca9d3f29f87 Mon Sep 17 00:00:00 2001 From: landreev Date: Thu, 27 Jun 2024 10:56:04 -0400 Subject: [PATCH 131/145] Update 10494-payara-upgrade.md --- doc/release-notes/10494-payara-upgrade.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/10494-payara-upgrade.md b/doc/release-notes/10494-payara-upgrade.md index e8ecb6756e0..b1a6c6fda8e 100644 --- a/doc/release-notes/10494-payara-upgrade.md +++ b/doc/release-notes/10494-payara-upgrade.md @@ -19,7 +19,7 @@ But we also recommend that you review the Payara upgrade instructions as it coul We assume you are already on a Dataverse 6.x installation, using a Payara 6.x release. ```shell -export PAYARA=/usr/local/payara5 +export PAYARA=/usr/local/payara6 ``` (or `setenv PAYARA /usr/local/payara6` if you are using a `csh`-like shell) @@ -91,7 +91,13 @@ export PAYARA=/usr/local/payara5 -XX:+UnlockDiagnosticVMOptions ``` (You can also save this as a patch file and try to apply it.) - +TODO: For the combined 6.3 release note, I would consider replacing the patch format above with just the 4 specific options, for clarity etc+. (L.A.) As in: +``` + --add-opens=java.management/javax.management=ALL-UNNAMED + --add-opens=java.management/javax.management.openmbean=ALL-UNNAMED + [17|]--add-opens=java.base/java.io=ALL-UNNAMED + [21|]--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED +``` 7\. Start Payara From 7b7f2e398a1f663de199eb11cb4af3c5929d7523 Mon Sep 17 00:00:00 2001 From: landreev Date: Thu, 27 Jun 2024 10:57:10 -0400 Subject: [PATCH 132/145] Update 10494-payara-upgrade.md --- doc/release-notes/10494-payara-upgrade.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/release-notes/10494-payara-upgrade.md b/doc/release-notes/10494-payara-upgrade.md index b1a6c6fda8e..23ee0e698f7 100644 --- a/doc/release-notes/10494-payara-upgrade.md +++ b/doc/release-notes/10494-payara-upgrade.md @@ -91,7 +91,8 @@ export PAYARA=/usr/local/payara6 -XX:+UnlockDiagnosticVMOptions ``` (You can also save this as a patch file and try to apply it.) -TODO: For the combined 6.3 release note, I would consider replacing the patch format above with just the 4 specific options, for clarity etc+. (L.A.) As in: + +TODO: For the combined 6.3 release note, I would consider replacing the patch format above with just the 4 specific options, for clarity etc. (L.A.) As in: ``` --add-opens=java.management/javax.management=ALL-UNNAMED --add-opens=java.management/javax.management.openmbean=ALL-UNNAMED From 9985378dbebe0d4d76ea6b85f3501ccdef27fa12 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 28 Jun 2024 11:20:15 -0400 Subject: [PATCH 133/145] IQSS/10137-2 Add flag to remove Return To Author Reason (#10655) * drop COMMIT_WITHIN which breaks autoSoftCommit by maxTime in solrconfig * add disable-return-to-author-reason feature flag * release note, doc updates * missed use --- .../10137-2-add-disable-reason-flag.md | 6 ++++ doc/sphinx-guides/source/api/native-api.rst | 4 +-- .../source/installation/config.rst | 8 ++++-- .../harvard/iq/dataverse/api/Datasets.java | 4 ++- .../impl/ReturnDatasetToAuthorCommand.java | 3 +- .../iq/dataverse/settings/FeatureFlags.java | 10 +++++++ src/main/java/propertyFiles/Bundle.properties | 3 +- src/main/webapp/dataset.xhtml | 26 +++++++++-------- .../ReturnDatasetToAuthorCommandTest.java | 28 ++++++++++++++++++- 9 files changed, 73 insertions(+), 19 deletions(-) create mode 100644 doc/release-notes/10137-2-add-disable-reason-flag.md diff --git a/doc/release-notes/10137-2-add-disable-reason-flag.md b/doc/release-notes/10137-2-add-disable-reason-flag.md new file mode 100644 index 00000000000..ee5257466ee --- /dev/null +++ b/doc/release-notes/10137-2-add-disable-reason-flag.md @@ -0,0 +1,6 @@ +## Release Highlights + +### Feature flag to remove the required "reason" field in the "Return To Author" dialog + +A reason field, that is required to not be empty, was added in v6.2. Installations that handle author communications through email or another system may prefer to not be required to use this new field. v6.2 includes a new +disable-return-to-author-reason feature flag that can be enabled to drop the reason field from the dialog and make sending a reason optional in the api/datasets/{id}/returnToAuthor call. diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 894f84d2aac..2d0dc714132 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2231,7 +2231,7 @@ The people who need to review the dataset (often curators or journal editors) ca Return a Dataset to Author ~~~~~~~~~~~~~~~~~~~~~~~~~~ -After the curators or journal editors have reviewed a dataset that has been submitted for review (see "Submit for Review", above) they can either choose to publish the dataset (see the ``:publish`` "action" above) or return the dataset to its authors. In the web interface there is a "Return to Author" button (see :doc:`/user/dataset-management`), but the interface does not provide a way to explain **why** the dataset is being returned. There is a way to do this outside of this interface, however. Instead of clicking the "Return to Author" button in the UI, a curator can write a "reason for return" into the database via API. +After the curators or journal editors have reviewed a dataset that has been submitted for review (see "Submit for Review", above) they can either choose to publish the dataset (see the ``:publish`` "action" above) or return the dataset to its authors. In the web interface there is a "Return to Author" button (see :doc:`/user/dataset-management`). The same operation can be done via this API call. Here's how curators can send a "reason for return" to the dataset authors. First, the curator creates a JSON file that contains the reason for return: @@ -2254,7 +2254,7 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/datasets/:persistentId/returnToAuthor?persistentId=doi:10.5072/FK2/J8SJZB" -H "Content-type: application/json" -d @reason-for-return.json The review process can sometimes resemble a tennis match, with the authors submitting and resubmitting the dataset over and over until the curators are satisfied. Each time the curators send a "reason for return" via API, that reason is sent by email and is persisted into the database, stored at the dataset version level. -The reason is required, please note that you can still type a creative and meaningful comment such as "The author would like to modify his dataset", "Files are missing", "Nothing to report" or "A curation report with comments and suggestions/instructions will follow in another email" that suits your situation. +Note the reason is required, unless the `disable-return-to-author-reason` feature flag has been set (see :ref:`feature-flags`). Reason is a free text field and could be as simple as "The author would like to modify his dataset", "Files are missing", "Nothing to report" or "A curation report with comments and suggestions/instructions will follow in another email" that suits your situation. The :ref:`send-feedback` API call may be useful as a way to move the conversation to email. However, note that these emails go to contacts (versus authors) and there is no database record of the email contents. (:ref:`dataverse.mail.cc-support-on-contact-email` will send a copy of these emails to the support email address which would provide a record.) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index f6b6b5bb968..d06a396c190 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3277,9 +3277,13 @@ please find all known feature flags below. Any of these flags can be activated u * - reduce-solr-deletes - Avoids deleting and recreating solr documents for dataset files when reindexing. - ``Off`` - * - index-harvested-metadata-source - - If enabled, this will index the name of the Harvesting Client as the "Metadata Source" of harvested datasets and files; so that the Metadata Source facet on the collection page will be showing separate entries for the content harvested from different sources/via different clients, instead of the current, default behavior where there is one "Harvested" facet for all such content. Requires a reindex. + * - reduce-solr-deletes + - Avoids deleting and recreating solr documents for dataset files when reindexing. - ``Off`` + * - disable-return-to-author-reason + - Removes the reason field in the `Publish/Return To Author` dialog that was added as a required field in v6.2 and makes the reason an optional parameter in the :ref:`return-a-dataset` API call. + - ``Off`` + **Note:** Feature flags can be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_FEATURE_XXX`` (e.g. ``DATAVERSE_FEATURE_API_SESSION_AUTH=1``). These environment variables can be set in your shell before starting Payara. If you are using :doc:`Docker for development `, you can set them in the `docker compose `_ file. diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 78e551780dc..0e0ebaaf54f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -45,6 +45,7 @@ import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; import edu.harvard.iq.dataverse.search.IndexServiceBean; +import edu.harvard.iq.dataverse.settings.FeatureFlags; import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.storageuse.UploadSessionQuotaLimit; @@ -2478,7 +2479,8 @@ public Response returnToAuthor(@Context ContainerRequestContext crc, @PathParam( Dataset dataset = findDatasetOrDie(idSupplied); String reasonForReturn = null; reasonForReturn = json.getString("reasonForReturn"); - if (reasonForReturn == null || reasonForReturn.isEmpty()) { + if ((reasonForReturn == null || reasonForReturn.isEmpty()) + && !FeatureFlags.DISABLE_RETURN_TO_AUTHOR_REASON.enabled()) { return error(Response.Status.BAD_REQUEST, BundleUtil.getStringFromBundle("dataset.reject.datasetNotInReview")); } AuthenticatedUser authenticatedUser = getRequestAuthenticatedUserOrDie(crc); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ReturnDatasetToAuthorCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ReturnDatasetToAuthorCommand.java index f3b33f82524..8d8fddeda6b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ReturnDatasetToAuthorCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ReturnDatasetToAuthorCommand.java @@ -11,6 +11,7 @@ import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; +import edu.harvard.iq.dataverse.settings.FeatureFlags; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.workflows.WorkflowComment; import java.io.IOException; @@ -26,7 +27,7 @@ public class ReturnDatasetToAuthorCommand extends AbstractDatasetCommand + +

    - #{bundle['dataset.rejectMessage']} + #{bundle['dataset.rejectMessage']} #{disableReasonField ? '':bundle['dataset.rejectMessageReason']}

    - - -

    - -

    - - + + + +

    + +

    + + +
    getUsersWithPermissionOn(Permission permission, D } */ @Test + @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "false", varArgs = "disable-return-to-author-reason") void testDatasetNull() { assertThrows(IllegalArgumentException.class, () -> new ReturnDatasetToAuthorCommand(dataverseRequest, null, "")); @@ -179,6 +183,7 @@ public void testNotInReviewDataset() { } @Test + @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "false", varArgs = "disable-return-to-author-reason") public void testEmptyOrNullComment(){ dataset.getLatestVersion().setVersionState(DatasetVersion.VersionState.DRAFT); Dataset updatedDataset = null; @@ -198,6 +203,27 @@ public void testEmptyOrNullComment(){ } assertEquals(expected, actual); } + + /** Test the disable reason flag + * @throws Exception when the test is in error. + */ + @Test + @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "true", varArgs = "disable-return-to-author-reason") + public void testEmptyOrNullCommentWhenDisabled() throws Exception { + dataset.getLatestVersion().setVersionState(DatasetVersion.VersionState.DRAFT); + Dataset updatedDataset = null; + + testEngine.submit(new AddLockCommand(dataverseRequest, dataset, + new DatasetLock(DatasetLock.Reason.InReview, dataverseRequest.getAuthenticatedUser()))); + + updatedDataset = testEngine.submit(new ReturnDatasetToAuthorCommand(dataverseRequest, dataset, null)); + assertNotNull(updatedDataset); + testEngine.submit(new AddLockCommand(dataverseRequest, dataset, + new DatasetLock(DatasetLock.Reason.InReview, dataverseRequest.getAuthenticatedUser()))); + updatedDataset = testEngine.submit(new ReturnDatasetToAuthorCommand(dataverseRequest, dataset, "")); + assertNotNull(updatedDataset); + } + @Test public void testAllGood() { From 1026a3ab343cdf8f2d23500d89a227d0b61e70b2 Mon Sep 17 00:00:00 2001 From: Steven Ferey Date: Mon, 1 Jul 2024 17:07:52 +0200 Subject: [PATCH 134/145] remove duplicate properties keys (#9718) * remove duplicate properties keys * Change of label for the advanced.search.datasets.persistentId.tip key * initial value for advanced.search.datasets.persistentId.tip key * Update Bundle.properties Assuming the change is OK with the submitter so we can move this forward --------- Co-authored-by: qqmyers --- src/main/java/propertyFiles/Bundle.properties | 20 ++----------------- .../java/propertyFiles/biomedical.properties | 1 - .../java/propertyFiles/customGSD.properties | 2 -- 3 files changed, 2 insertions(+), 21 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index c0705e70b1a..0325a47f626 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -960,18 +960,16 @@ advanced.search.header.datasets=Datasets advanced.search.header.files=Files advanced.search.files.name.tip=The name given to identify the file. advanced.search.files.description.tip=A summary describing the file and its variables. -advanced.search.files.persistentId.tip=The persistent identifier for the file. advanced.search.files.persistentId=Data File Persistent ID -advanced.search.files.persistentId.tip=The unique persistent identifier for a data file, which can be a Handle or DOI in Dataverse. +advanced.search.files.persistentId.tip=The unique persistent identifier for the file. advanced.search.files.fileType=File Type advanced.search.files.fileType.tip=The file type, e.g. Comma Separated Values, Plain Text, R, etc. advanced.search.files.variableName=Variable Name advanced.search.files.variableName.tip=The name of the variable's column in the data frame. advanced.search.files.variableLabel=Variable Label advanced.search.files.variableLabel.tip=A short description of the variable. -advanced.search.datasets.persistentId.tip=The persistent identifier for the Dataset. advanced.search.datasets.persistentId=Persistent Identifier -advanced.search.datasets.persistentId.tip=The Dataset's unique persistent identifier, either a DOI or Handle +advanced.search.datasets.persistentId.tip=The persistent identifier for the Dataset. advanced.search.files.fileTags=File Tags advanced.search.files.fileTags.tip=Terms such "Documentation", "Data", or "Code" that have been applied to files. @@ -2341,20 +2339,6 @@ citationFrame.banner.closeIcon=Close this message, go to dataset citationFrame.banner.countdownMessage= This message will close in citationFrame.banner.countdownMessage.seconds=seconds -# Friendly AuthenticationProvider names -authenticationProvider.name.builtin=Dataverse -authenticationProvider.name.null=(provider is unknown) -authenticationProvider.name.github=GitHub -authenticationProvider.name.google=Google -authenticationProvider.name.orcid=ORCiD -authenticationProvider.name.orcid-sandbox=ORCiD Sandbox -authenticationProvider.name.shib=Shibboleth -ingest.csv.invalidHeader=Invalid header row. One of the cells is empty. -ingest.csv.lineMismatch=Mismatch between line counts in first and final passes!, {0} found on first pass, but {1} found on second. -ingest.csv.recordMismatch=Reading mismatch, line {0} of the Data file: {1} delimited values expected, {2} found. -ingest.csv.nullStream=Stream can't be null. -citationFrame.banner.countdownMessage.seconds=seconds - #file-edit-popup-fragment.xhtml #editFilesFragment.xhtml dataset.access.accessHeader=Restrict Access dataset.access.accessHeader.invalid.state=Define Data Access diff --git a/src/main/java/propertyFiles/biomedical.properties b/src/main/java/propertyFiles/biomedical.properties index 1bffed2ee03..7392ba823c4 100644 --- a/src/main/java/propertyFiles/biomedical.properties +++ b/src/main/java/propertyFiles/biomedical.properties @@ -96,7 +96,6 @@ controlledvocabulary.studyAssayMeasurementType.targeted_sequencing=targeted sequ controlledvocabulary.studyAssayMeasurementType.transcription_factor_binding_(chip-seq)=transcription factor binding (ChIP-Seq) controlledvocabulary.studyAssayMeasurementType.transcription_factor_binding_site_identification=transcription factor binding site identification controlledvocabulary.studyAssayMeasurementType.transcription_profiling=transcription profiling -controlledvocabulary.studyAssayMeasurementType.transcription_profiling=transcription profiling controlledvocabulary.studyAssayMeasurementType.transcription_profiling_(microarray)=transcription profiling (Microarray) controlledvocabulary.studyAssayMeasurementType.transcription_profiling_(rna-seq)=transcription profiling (RNA-Seq) controlledvocabulary.studyAssayMeasurementType.trap_translational_profiling=TRAP translational profiling diff --git a/src/main/java/propertyFiles/customGSD.properties b/src/main/java/propertyFiles/customGSD.properties index 40dc0328053..2375596fe2f 100644 --- a/src/main/java/propertyFiles/customGSD.properties +++ b/src/main/java/propertyFiles/customGSD.properties @@ -161,7 +161,6 @@ controlledvocabulary.gsdFacultyName.mcloskey,_karen=MCloskey, Karen controlledvocabulary.gsdFacultyName.mehrotra,_rahul=Mehrotra, Rahul controlledvocabulary.gsdFacultyName.menchaca,_alejandra=Menchaca, Alejandra controlledvocabulary.gsdFacultyName.menges,_achim=Menges, Achim -controlledvocabulary.gsdFacultyName.menges,_achim=Menges, Achim controlledvocabulary.gsdFacultyName.michalatos,_panagiotis=Michalatos, Panagiotis controlledvocabulary.gsdFacultyName.moe,_kiel=Moe, Kiel controlledvocabulary.gsdFacultyName.molinsky,_jennifer=Molinsky, Jennifer @@ -507,7 +506,6 @@ controlledvocabulary.gsdCourseName.06323:_brownfields_practicum=06323: Brownfiel controlledvocabulary.gsdCourseName.06333:_aquatic_ecology=06333: Aquatic Ecology controlledvocabulary.gsdCourseName.06335:_phytotechnologies=06335: Phytotechnologies controlledvocabulary.gsdCourseName.06337:_changing_natural_and_built_coastal_environments=06337: Changing Natural and Built Coastal Environments -controlledvocabulary.gsdCourseName.06337:_changing_natural_and_built_coastal_environments=06337: Changing Natural and Built Coastal Environments controlledvocabulary.gsdCourseName.06338:_introduction_to_computational_design=06338: Introduction to Computational Design controlledvocabulary.gsdCourseName.06436:_expanded_mechanisms_/_empirical_materialisms=06436: Expanded Mechanisms / Empirical Materialisms controlledvocabulary.gsdCourseName.06450:_high_performance_buildings_and_systems_integration=06450: High Performance Buildings and Systems Integration From 9bd0ccff917a94eb76a75aacffa71d8a9d42fd8b Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 1 Jul 2024 12:13:23 -0400 Subject: [PATCH 135/145] re-apply changes about sphinx and docker #10531 These changes were in 853965e as part of PR #10565 but were lost during resolution of a merge conflict. --- .../source/contributor/documentation.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/contributor/documentation.md b/doc/sphinx-guides/source/contributor/documentation.md index 67db855dc60..6a24e653152 100644 --- a/doc/sphinx-guides/source/contributor/documentation.md +++ b/doc/sphinx-guides/source/contributor/documentation.md @@ -87,7 +87,7 @@ To edit the existing documentation: Once you are done, you can preview the changes by building the guides locally. As explained, you can build the guides with Sphinx locally installed, or with a Docker container. -#### Building the Guides with Sphinx Locally Installed +#### Building the Guides with Sphinx Installed Locally Open a terminal, change directories to `doc/sphinx-guides`, activate (or reactivate) your Python virtual environment, and build the guides. @@ -99,11 +99,20 @@ Open a terminal, change directories to `doc/sphinx-guides`, activate (or reactiv `make html` -#### Building the Guides with a Sphinx Docker Container +#### Building the Guides with a Sphinx Docker Container and a Makefile + +We have added a Makefile to simplify the process of building the guides using a Docker container, you can use the following commands from the repository root: + +- `make docs-html` +- `make docs-pdf` +- `make docs-epub` +- `make docs-all` + +#### Building the Guides with a Sphinx Docker Container and CLI If you want to build the guides using a Docker container, execute the following command in the repository root: -`docker run -it --rm -v $(pwd):/docs sphinxdoc/sphinx:3.5.4 bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make html"` +`docker run -it --rm -v $(pwd):/docs sphinxdoc/sphinx:7.2.6 bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make html"` #### Previewing the Guides @@ -157,7 +166,7 @@ The HTML version of the guides is the official one. Any other formats are mainta If you would like to build a PDF version of the guides and have Docker installed, please try the command below from the root of the git repo: -`docker run -it --rm -v $(pwd):/docs sphinxdoc/sphinx-latexpdf:3.5.4 bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make latexpdf LATEXMKOPTS=\"-interaction=nonstopmode\"; cd ../.. && ls -1 doc/sphinx-guides/build/latex/Dataverse.pdf"` +`docker run -it --rm -v $(pwd):/docs sphinxdoc/sphinx-latexpdf:7.2.6 bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make latexpdf LATEXMKOPTS=\"-interaction=nonstopmode\"; cd ../.. && ls -1 doc/sphinx-guides/build/latex/Dataverse.pdf"` A few notes about the command above: From 7a64dea885c6fcaf71e654a4be9e920dc21c3bab Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 1 Jul 2024 12:15:32 -0400 Subject: [PATCH 136/145] remove TODO about how vs when #10531 --- doc/sphinx-guides/source/contributor/documentation.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/sphinx-guides/source/contributor/documentation.md b/doc/sphinx-guides/source/contributor/documentation.md index 6a24e653152..12a4266c9ff 100644 --- a/doc/sphinx-guides/source/contributor/documentation.md +++ b/doc/sphinx-guides/source/contributor/documentation.md @@ -154,8 +154,6 @@ A good documentation is just like a website enhanced and upgraded by adding high While PNGs in the git repo can be linked directly via URL, Sphinx-generated images do not need a manual step and might provide higher visual quality. Especially in terms of quality of content, generated images can be extendend and improved by a textbased and reviewable commit, without needing raw data or source files and no diff around. -TODO: The above covers "how" but what about "when"? - ### Cross References When adding ReStructured Text (.rst) [cross references](https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#ref-role), use the hyphen character (`-`) as the word separator for the cross reference label. For example, `my-reference-label` would be the preferred label for a cross reference as opposed to, for example, `my_reference_label`. From dddcf29188a5c35174f3c94ffc1c4cb1d7fc0552 Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Mon, 1 Jul 2024 13:23:46 -0400 Subject: [PATCH 137/145] bump AWS SDK to current version (#10648) Co-authored-by: Don Sizemore --- modules/dataverse-parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index f286a44841f..4a67e301f5b 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -151,7 +151,7 @@ 6.2024.6 42.7.2 9.4.1 - 1.12.290 + 1.12.748 26.30.0 From b42222fd35006c24e57eccece81d3496dcac31b8 Mon Sep 17 00:00:00 2001 From: Guillermo Portas Date: Tue, 2 Jul 2024 15:59:19 +0100 Subject: [PATCH 138/145] Fix metadata field type display condition in dataverses/{id}/metadatablocks API endpoint (#10642) * Fixed: metadata field type display condition in dataverses/{id}/metadatablocks * Changed: json object builder instantiation * Added: extended test coverage for testUpdateInputLevels and testFeatureDataverse * Added: release notes for #10637 * Fixed: JsonPrinter metadata blocks dataset field type isRequired logic * Refactor: simpler conditions in jsonPrinter * Refactor: reordered condition in jsonPrinter * Fixed: displayCondition in jsonPrinter --- .../10637-fix-dataverse-metadatablocks-api.md | 2 + .../edu/harvard/iq/dataverse/Dataverse.java | 18 ++++--- .../iq/dataverse/util/json/JsonPrinter.java | 18 +++++-- .../iq/dataverse/api/DataversesIT.java | 52 +++++++++++++------ .../edu/harvard/iq/dataverse/api/UtilIT.java | 21 ++++---- 5 files changed, 76 insertions(+), 35 deletions(-) create mode 100644 doc/release-notes/10637-fix-dataverse-metadatablocks-api.md diff --git a/doc/release-notes/10637-fix-dataverse-metadatablocks-api.md b/doc/release-notes/10637-fix-dataverse-metadatablocks-api.md new file mode 100644 index 00000000000..c8c9c4fa66f --- /dev/null +++ b/doc/release-notes/10637-fix-dataverse-metadatablocks-api.md @@ -0,0 +1,2 @@ +dataverses/{id}/metadatablocks API endpoint has been fixed, since the fields returned for each metadata block when returnDatasetTypes query parameter is set to true was not correct. + diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java index 78b1827c798..978c716e058 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java @@ -412,12 +412,18 @@ public List getDataverseFieldTypeInputLevels() { } public boolean isDatasetFieldTypeRequiredAsInputLevel(Long datasetFieldTypeId) { - for(DataverseFieldTypeInputLevel dataverseFieldTypeInputLevel : dataverseFieldTypeInputLevels) { - if (dataverseFieldTypeInputLevel.getDatasetFieldType().getId().equals(datasetFieldTypeId) && dataverseFieldTypeInputLevel.isRequired()) { - return true; - } - } - return false; + return dataverseFieldTypeInputLevels.stream() + .anyMatch(inputLevel -> inputLevel.getDatasetFieldType().getId().equals(datasetFieldTypeId) && inputLevel.isRequired()); + } + + public boolean isDatasetFieldTypeIncludedAsInputLevel(Long datasetFieldTypeId) { + return dataverseFieldTypeInputLevels.stream() + .anyMatch(inputLevel -> inputLevel.getDatasetFieldType().getId().equals(datasetFieldTypeId) && inputLevel.isInclude()); + } + + public boolean isDatasetFieldTypeInInputLevels(Long datasetFieldTypeId) { + return dataverseFieldTypeInputLevels.stream() + .anyMatch(inputLevel -> inputLevel.getDatasetFieldType().getId().equals(datasetFieldTypeId)); } public Template getDefaultTemplate() { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 95f14b79ece..c72dfc1d127 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -640,18 +640,26 @@ public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printO JsonObjectBuilder fieldsBuilder = Json.createObjectBuilder(); Set datasetFieldTypes = new TreeSet<>(metadataBlock.getDatasetFieldTypes()); + for (DatasetFieldType datasetFieldType : datasetFieldTypes) { - boolean requiredInOwnerDataverse = ownerDataverse != null && ownerDataverse.isDatasetFieldTypeRequiredAsInputLevel(datasetFieldType.getId()); - boolean displayCondition = !printOnlyDisplayedOnCreateDatasetFieldTypes || - datasetFieldType.isDisplayOnCreate() || - requiredInOwnerDataverse; + Long datasetFieldTypeId = datasetFieldType.getId(); + boolean requiredAsInputLevelInOwnerDataverse = ownerDataverse != null && ownerDataverse.isDatasetFieldTypeRequiredAsInputLevel(datasetFieldTypeId); + boolean includedAsInputLevelInOwnerDataverse = ownerDataverse != null && ownerDataverse.isDatasetFieldTypeIncludedAsInputLevel(datasetFieldTypeId); + boolean isNotInputLevelInOwnerDataverse = ownerDataverse != null && !ownerDataverse.isDatasetFieldTypeInInputLevels(datasetFieldTypeId); + + DatasetFieldType parentDatasetFieldType = datasetFieldType.getParentDatasetFieldType(); + boolean isRequired = parentDatasetFieldType == null ? datasetFieldType.isRequired() : parentDatasetFieldType.isRequired(); + + boolean displayCondition = printOnlyDisplayedOnCreateDatasetFieldTypes + ? (datasetFieldType.isDisplayOnCreate() || isRequired || requiredAsInputLevelInOwnerDataverse) + : ownerDataverse == null || includedAsInputLevelInOwnerDataverse || isNotInputLevelInOwnerDataverse; + if (displayCondition) { fieldsBuilder.add(datasetFieldType.getName(), json(datasetFieldType, ownerDataverse)); } } jsonObjectBuilder.add("fields", fieldsBuilder); - return jsonObjectBuilder; } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index b072a803aa4..79cc46cfa79 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -702,8 +702,10 @@ public void testListMetadataBlocks() { Response setMetadataBlocksResponse = UtilIT.setMetadataBlocks(dataverseAlias, Json.createArrayBuilder().add("citation").add("astrophysics"), apiToken); setMetadataBlocksResponse.then().assertThat().statusCode(OK.getStatusCode()); - String[] testInputLevelNames = {"geographicCoverage", "country"}; - Response updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInputLevelNames, apiToken); + String[] testInputLevelNames = {"geographicCoverage", "country", "city"}; + boolean[] testRequiredInputLevels = {false, true, false}; + boolean[] testIncludedInputLevels = {false, true, true}; + Response updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInputLevelNames, testRequiredInputLevels, testIncludedInputLevels, apiToken); updateDataverseInputLevelsResponse.then().assertThat().statusCode(OK.getStatusCode()); // Dataverse not found @@ -769,6 +771,21 @@ public void testListMetadataBlocks() { assertThat(expectedAllMetadataBlockDisplayNames, hasItemInArray(actualMetadataBlockDisplayName2)); assertThat(expectedAllMetadataBlockDisplayNames, hasItemInArray(actualMetadataBlockDisplayName3)); + // Check dataset fields for the updated input levels are retrieved + int geospatialMetadataBlockIndex = actualMetadataBlockDisplayName1.equals("Geospatial Metadata") ? 0 : actualMetadataBlockDisplayName2.equals("Geospatial Metadata") ? 1 : 2; + + // Since the included property of geographicCoverage is set to false, we should retrieve the total number of fields minus one + listMetadataBlocksResponse.then().assertThat() + .body(String.format("data[%d].fields.size()", geospatialMetadataBlockIndex), equalTo(10)); + + String actualMetadataField1 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.geographicCoverage.name", geospatialMetadataBlockIndex)); + String actualMetadataField2 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.country.name", geospatialMetadataBlockIndex)); + String actualMetadataField3 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.city.name", geospatialMetadataBlockIndex)); + + assertNull(actualMetadataField1); + assertNotNull(actualMetadataField2); + assertNotNull(actualMetadataField3); + // Existent dataverse and onlyDisplayedOnCreate=true and returnDatasetFieldTypes=true listMetadataBlocksResponse = UtilIT.listMetadataBlocks(dataverseAlias, true, true, apiToken); listMetadataBlocksResponse.then().assertThat().statusCode(OK.getStatusCode()); @@ -785,16 +802,18 @@ public void testListMetadataBlocks() { assertThat(expectedOnlyDisplayedOnCreateMetadataBlockDisplayNames, hasItemInArray(actualMetadataBlockDisplayName2)); // Check dataset fields for the updated input levels are retrieved - int geospatialMetadataBlockIndex = actualMetadataBlockDisplayName2.equals("Geospatial Metadata") ? 1 : 0; + geospatialMetadataBlockIndex = actualMetadataBlockDisplayName2.equals("Geospatial Metadata") ? 1 : 0; listMetadataBlocksResponse.then().assertThat() - .body(String.format("data[%d].fields.size()", geospatialMetadataBlockIndex), equalTo(2)); + .body(String.format("data[%d].fields.size()", geospatialMetadataBlockIndex), equalTo(1)); - String actualMetadataField1 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.geographicCoverage.name", geospatialMetadataBlockIndex)); - String actualMetadataField2 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.country.name", geospatialMetadataBlockIndex)); + actualMetadataField1 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.geographicCoverage.name", geospatialMetadataBlockIndex)); + actualMetadataField2 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.country.name", geospatialMetadataBlockIndex)); + actualMetadataField3 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.city.name", geospatialMetadataBlockIndex)); - assertNotNull(actualMetadataField1); + assertNull(actualMetadataField1); assertNotNull(actualMetadataField2); + assertNull(actualMetadataField3); // User has no permissions on the requested dataverse Response createSecondUserResponse = UtilIT.createRandomUser(); @@ -898,12 +917,16 @@ public void testUpdateInputLevels() { // Update valid input levels String[] testInputLevelNames = {"geographicCoverage", "country"}; - Response updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInputLevelNames, apiToken); + boolean[] testRequiredInputLevels = {true, false}; + boolean[] testIncludedInputLevels = {true, false}; + Response updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInputLevelNames, testRequiredInputLevels, testIncludedInputLevels, apiToken); + String actualInputLevelName = updateDataverseInputLevelsResponse.then().extract().path("data.inputLevels[0].datasetFieldTypeName"); + int geographicCoverageInputLevelIndex = actualInputLevelName.equals("geographicCoverage") ? 0 : 1; updateDataverseInputLevelsResponse.then().assertThat() - .body("data.inputLevels[0].required", equalTo(true)) - .body("data.inputLevels[0].include", equalTo(true)) - .body("data.inputLevels[1].required", equalTo(true)) - .body("data.inputLevels[1].include", equalTo(true)) + .body(String.format("data.inputLevels[%d].include", geographicCoverageInputLevelIndex), equalTo(true)) + .body(String.format("data.inputLevels[%d].required", geographicCoverageInputLevelIndex), equalTo(true)) + .body(String.format("data.inputLevels[%d].include", 1 - geographicCoverageInputLevelIndex), equalTo(false)) + .body(String.format("data.inputLevels[%d].required", 1 - geographicCoverageInputLevelIndex), equalTo(false)) .statusCode(OK.getStatusCode()); String actualFieldTypeName1 = updateDataverseInputLevelsResponse.then().extract().path("data.inputLevels[0].datasetFieldTypeName"); String actualFieldTypeName2 = updateDataverseInputLevelsResponse.then().extract().path("data.inputLevels[1].datasetFieldTypeName"); @@ -913,15 +936,14 @@ public void testUpdateInputLevels() { // Update input levels with an invalid field type name String[] testInvalidInputLevelNames = {"geographicCoverage", "invalid1"}; - updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInvalidInputLevelNames, apiToken); + updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInvalidInputLevelNames, testRequiredInputLevels, testIncludedInputLevels, apiToken); updateDataverseInputLevelsResponse.then().assertThat() .body("message", equalTo("Invalid dataset field type name: invalid1")) .statusCode(BAD_REQUEST.getStatusCode()); // Update invalid empty input levels testInputLevelNames = new String[]{}; - updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInputLevelNames, apiToken); - updateDataverseInputLevelsResponse.prettyPrint(); + updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInputLevelNames, testRequiredInputLevels, testIncludedInputLevels, apiToken); updateDataverseInputLevelsResponse.then().assertThat() .body("message", equalTo("Error while updating dataverse input levels: Input level list cannot be null or empty")) .statusCode(INTERNAL_SERVER_ERROR.getStatusCode()); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 257610dbc32..0216859b869 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -3960,22 +3960,25 @@ static Response requestGlobusUploadPaths(Integer datasetId, JsonObject body, Str .post("/api/datasets/" + datasetId + "/requestGlobusUploadPaths"); } - static Response updateDataverseInputLevels(String dataverseAlias, String[] inputLevelNames, String apiToken) { - JsonArrayBuilder contactArrayBuilder = Json.createArrayBuilder(); - for(String inputLevelName : inputLevelNames) { - contactArrayBuilder.add(Json.createObjectBuilder() - .add("datasetFieldTypeName", inputLevelName) - .add("required", true) - .add("include", true) - ); + public static Response updateDataverseInputLevels(String dataverseAlias, String[] inputLevelNames, boolean[] requiredInputLevels, boolean[] includedInputLevels, String apiToken) { + JsonArrayBuilder inputLevelsArrayBuilder = Json.createArrayBuilder(); + for (int i = 0; i < inputLevelNames.length; i++) { + inputLevelsArrayBuilder.add(createInputLevelObject(inputLevelNames[i], requiredInputLevels[i], includedInputLevels[i])); } return given() .header(API_TOKEN_HTTP_HEADER, apiToken) - .body(contactArrayBuilder.build().toString()) + .body(inputLevelsArrayBuilder.build().toString()) .contentType(ContentType.JSON) .put("/api/dataverses/" + dataverseAlias + "/inputLevels"); } + private static JsonObjectBuilder createInputLevelObject(String name, boolean required, boolean include) { + return Json.createObjectBuilder() + .add("datasetFieldTypeName", name) + .add("required", required) + .add("include", include); + } + public static Response getOpenAPI(String accept, String format) { Response response = given() .header("Accept", accept) From eda4def4b4c3279e5517883daa37842a8254dfbd Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Tue, 2 Jul 2024 16:29:21 -0400 Subject: [PATCH 139/145] Bump version to 6.3 --- doc/sphinx-guides/source/conf.py | 4 ++-- doc/sphinx-guides/source/versions.rst | 3 ++- modules/dataverse-parent/pom.xml | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py index 6478f15655e..c719fb05e3c 100755 --- a/doc/sphinx-guides/source/conf.py +++ b/doc/sphinx-guides/source/conf.py @@ -68,9 +68,9 @@ # built documents. # # The short X.Y version. -version = '6.2' +version = '6.3' # The full version, including alpha/beta/rc tags. -release = '6.2' +release = '6.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/sphinx-guides/source/versions.rst b/doc/sphinx-guides/source/versions.rst index 850702d823e..952eba72616 100755 --- a/doc/sphinx-guides/source/versions.rst +++ b/doc/sphinx-guides/source/versions.rst @@ -7,7 +7,8 @@ Dataverse Software Documentation Versions This list provides a way to refer to the documentation for previous and future versions of the Dataverse Software. In order to learn more about the updates delivered from one version to another, visit the `Releases `__ page in our GitHub repo. - pre-release `HTML (not final!) `__ and `PDF (experimental!) `__ built from the :doc:`develop ` branch :doc:`(how to contribute!) ` -- 6.2 +- 6.3 +- `6.2 `__ - `6.1 `__ - `6.0 `__ - `5.14 `__ diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 4a67e301f5b..62efbf62317 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -131,7 +131,7 @@ - 6.2 + 6.3 17 UTF-8 From ecb5e94489aad1592e0f48e0f4d3bab3a6ca623f Mon Sep 17 00:00:00 2001 From: Juan Pablo Tosca Villanueva <142103991+jp-tosca@users.noreply.github.com> Date: Tue, 2 Jul 2024 16:54:29 -0400 Subject: [PATCH 140/145] 6.3 Release Notes (#10662) * Initial changes * Update with 4th lvl H * Table of contet change * MD format test * External voc info update * Update 1:04 * First version for PR * a few quick changes to the upgrade instruction #10646 * Update 6.3-release-notes.md * keywordTermURI heading #10646 * removed reExportAll from the upgrade instruction; it's only needed if they choose to perform the optional step of migrating some metadata fields; and it is already mentioned in the instruction on how to do that. #10646 * simplified the jvm options part of the payara upgrade instruction #10646 * moved payara upgrade into the main upgrade instruction #10646 * typos #10646 * added the Solr upgrade instruction #10646 * cosmetic #10646 * cosmetic #10646 * Deleted the solr upgrade note. #10646 * Initial changes * #10646 add note about file access request email update * Add issue number test * add featured collection link * Test issue link * Batch of issues # * Bug fixes issues no * #10646 add issue link * normalize line endings #10646 * various improvments #10646 * reorder #10646 * add Contributor Guide to 6.3 release notes #10646 * incorporate #10637 into release note #10646 * Update 6.3-release-notes.md * Make Hidden HTML fields one more element in the ext. vocab updates * Removing snippets * small tweaks to upgrade instructions #10646 * no need to mention #10637 and #10642 as they were pre-release #10646 * s/safe/supported/ and remove dupe #10646 --------- Co-authored-by: Leonid Andreev Co-authored-by: qqmyers Co-authored-by: Stephen Kraffmiller Co-authored-by: Philip Durbin --- .../10015-RO-Crate-metadata-file.md | 10 - .../10022_upload_redirect_without_tagging.md | 5 - ...10116-incomplete-metadata-label-setting.md | 1 - .../10137-2-add-disable-reason-flag.md | 6 - .../10236-openapi-definition-endpoint.md | 8 - doc/release-notes/10242-add-feature-dv-api | 1 - ...-add-term_uri-metadata-in-keyword-block.md | 53 -- doc/release-notes/10316_cvoc_http_headers.md | 5 - .../10330-api-change-latest-version-status.md | 1 - doc/release-notes/10339-workflow.md | 3 - .../10389-metadatablocks-api-extension.md | 6 - ...pi-performance-issues-on-large-datasets.md | 4 - doc/release-notes/10425-add-MIT-License.md | 3 - .../10464-add-name-harvesting-client-facet.md | 3 - .../10466-math-challenge-403-error-page.md | 1 - .../10468-doc-datalad-integration.md | 1 - ...tadatablocks-api-extension-input-levels.md | 3 - ...dd-isreleased-to-get-dataverse-response.md | 22 - doc/release-notes/10494-payara-upgrade.md | 119 ----- .../10503-cvoc-hidden-html-fields.md | 11 - doc/release-notes/10531-contrib.md | 1 - doc/release-notes/10547-solr-updates.md | 1 - .../10554-avoid-solr-join-guest.md | 5 - doc/release-notes/10561-3dviewer.md | 1 - .../10565-banner-test-improvements.md | 1 - doc/release-notes/10568-Fix File Reingest.md | 1 - .../10570-extra-facet-settings.md | 4 - doc/release-notes/10579-avoid-solr-deletes.md | 9 - .../10611-harvested-origin-facet.md | 10 - .../10637-fix-dataverse-metadatablocks-api.md | 2 - .../5621_dataset image in header.md | 1 - doc/release-notes/6.3-release-notes.md | 470 ++++++++++++++++++ .../8243-improve-language-controlled-vocab.md | 11 - ...655-re-add-cell-counting-biomedical-tsv.md | 12 - ...8936-more-than-50000-entries-in-sitemap.md | 11 - doc/release-notes/8985-deprecate-rsync.md | 8 - ...ow-flexible-params-in-retrievaluri-cvoc.md | 14 - doc/release-notes/9276-doc-cvoc-index-in.md | 18 - doc/release-notes/9375-retention-period.md | 8 - doc/release-notes/9729-release-notes.md | 1 - doc/release-notes/9739-url-validator.md | 7 - .../9887-new-superuser-status-endpoint.md | 1 - doc/release-notes/solr-9.4.1.md | 14 - doc/sphinx-guides/source/api/native-api.rst | 2 + 44 files changed, 472 insertions(+), 407 deletions(-) delete mode 100644 doc/release-notes/10015-RO-Crate-metadata-file.md delete mode 100644 doc/release-notes/10022_upload_redirect_without_tagging.md delete mode 100644 doc/release-notes/10116-incomplete-metadata-label-setting.md delete mode 100644 doc/release-notes/10137-2-add-disable-reason-flag.md delete mode 100644 doc/release-notes/10236-openapi-definition-endpoint.md delete mode 100644 doc/release-notes/10242-add-feature-dv-api delete mode 100644 doc/release-notes/10288-add-term_uri-metadata-in-keyword-block.md delete mode 100644 doc/release-notes/10316_cvoc_http_headers.md delete mode 100644 doc/release-notes/10330-api-change-latest-version-status.md delete mode 100644 doc/release-notes/10339-workflow.md delete mode 100644 doc/release-notes/10389-metadatablocks-api-extension.md delete mode 100644 doc/release-notes/10415-fix-api-performance-issues-on-large-datasets.md delete mode 100644 doc/release-notes/10425-add-MIT-License.md delete mode 100644 doc/release-notes/10464-add-name-harvesting-client-facet.md delete mode 100644 doc/release-notes/10466-math-challenge-403-error-page.md delete mode 100644 doc/release-notes/10468-doc-datalad-integration.md delete mode 100644 doc/release-notes/10477-metadatablocks-api-extension-input-levels.md delete mode 100644 doc/release-notes/10491-add-isreleased-to-get-dataverse-response.md delete mode 100644 doc/release-notes/10494-payara-upgrade.md delete mode 100644 doc/release-notes/10503-cvoc-hidden-html-fields.md delete mode 100644 doc/release-notes/10531-contrib.md delete mode 100644 doc/release-notes/10547-solr-updates.md delete mode 100644 doc/release-notes/10554-avoid-solr-join-guest.md delete mode 100644 doc/release-notes/10561-3dviewer.md delete mode 100644 doc/release-notes/10565-banner-test-improvements.md delete mode 100644 doc/release-notes/10568-Fix File Reingest.md delete mode 100644 doc/release-notes/10570-extra-facet-settings.md delete mode 100644 doc/release-notes/10579-avoid-solr-deletes.md delete mode 100644 doc/release-notes/10611-harvested-origin-facet.md delete mode 100644 doc/release-notes/10637-fix-dataverse-metadatablocks-api.md delete mode 100644 doc/release-notes/5621_dataset image in header.md create mode 100644 doc/release-notes/6.3-release-notes.md delete mode 100644 doc/release-notes/8243-improve-language-controlled-vocab.md delete mode 100644 doc/release-notes/8655-re-add-cell-counting-biomedical-tsv.md delete mode 100644 doc/release-notes/8936-more-than-50000-entries-in-sitemap.md delete mode 100644 doc/release-notes/8985-deprecate-rsync.md delete mode 100644 doc/release-notes/9276-allow-flexible-params-in-retrievaluri-cvoc.md delete mode 100644 doc/release-notes/9276-doc-cvoc-index-in.md delete mode 100644 doc/release-notes/9375-retention-period.md delete mode 100644 doc/release-notes/9729-release-notes.md delete mode 100644 doc/release-notes/9739-url-validator.md delete mode 100644 doc/release-notes/9887-new-superuser-status-endpoint.md delete mode 100644 doc/release-notes/solr-9.4.1.md diff --git a/doc/release-notes/10015-RO-Crate-metadata-file.md b/doc/release-notes/10015-RO-Crate-metadata-file.md deleted file mode 100644 index 4b018a634f7..00000000000 --- a/doc/release-notes/10015-RO-Crate-metadata-file.md +++ /dev/null @@ -1,10 +0,0 @@ -Detection of mime-types based on a filename with extension and detection of the RO-Crate metadata files. - -From now on, filenames with extensions can be added into `MimeTypeDetectionByFileName.properties` file. Filenames added there will take precedence over simply recognizing files by extensions. For example, two new filenames are added into that file: -``` -ro-crate-metadata.json=application/ld+json; profile="http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate" -ro-crate-metadata.jsonld=application/ld+json; profile="http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate" -``` - -Therefore, files named `ro-crate-metadata.json` will be then detected as RO-Crated metadata files from now on, instead as generic `JSON` files. -For more information on the RO-Crate specifications, see https://www.researchobject.org/ro-crate diff --git a/doc/release-notes/10022_upload_redirect_without_tagging.md b/doc/release-notes/10022_upload_redirect_without_tagging.md deleted file mode 100644 index 7ff17f08f4c..00000000000 --- a/doc/release-notes/10022_upload_redirect_without_tagging.md +++ /dev/null @@ -1,5 +0,0 @@ -If your S3 store does not support tagging and gives an error if you configure direct uploads, you can disable the tagging by using the ``dataverse.files..disable-tagging`` JVM option. For more details see https://dataverse-guide--10029.org.readthedocs.build/en/10029/developers/big-data-support.html#s3-tags #10022 and #10029. - -## New config options - -- dataverse.files..disable-tagging diff --git a/doc/release-notes/10116-incomplete-metadata-label-setting.md b/doc/release-notes/10116-incomplete-metadata-label-setting.md deleted file mode 100644 index 769100c3804..00000000000 --- a/doc/release-notes/10116-incomplete-metadata-label-setting.md +++ /dev/null @@ -1 +0,0 @@ -Bug fixed for the ``incomplete metadata`` label being shown for published dataset with incomplete metadata in certain scenarios. This label will now be shown for draft versions of such datasets and published datasets that the user can edit. This label can also be made invisible for published datasets (regardless of edit rights) with the new option ``dataverse.ui.show-validity-label-when-published`` set to `false`. diff --git a/doc/release-notes/10137-2-add-disable-reason-flag.md b/doc/release-notes/10137-2-add-disable-reason-flag.md deleted file mode 100644 index ee5257466ee..00000000000 --- a/doc/release-notes/10137-2-add-disable-reason-flag.md +++ /dev/null @@ -1,6 +0,0 @@ -## Release Highlights - -### Feature flag to remove the required "reason" field in the "Return To Author" dialog - -A reason field, that is required to not be empty, was added in v6.2. Installations that handle author communications through email or another system may prefer to not be required to use this new field. v6.2 includes a new -disable-return-to-author-reason feature flag that can be enabled to drop the reason field from the dialog and make sending a reason optional in the api/datasets/{id}/returnToAuthor call. diff --git a/doc/release-notes/10236-openapi-definition-endpoint.md b/doc/release-notes/10236-openapi-definition-endpoint.md deleted file mode 100644 index 60492c29d78..00000000000 --- a/doc/release-notes/10236-openapi-definition-endpoint.md +++ /dev/null @@ -1,8 +0,0 @@ -In Dataverse 6.0 Payara was updated, which caused the url `/openapi` to stop working: - -- https://github.com/IQSS/dataverse/issues/9981 -- https://github.com/payara/Payara/issues/6369 - -When it worked in Dataverse 5.x, the `/openapi` output was generated automatically by Payara, but in this release we have switched to OpenAPI output produced by the [SmallRye OpenAPI plugin](https://github.com/smallrye/smallrye-open-api/tree/main/tools/maven-plugin). This gives us finer control over the output. - -For more information, see the section on [OpenAPI](https://dataverse-guide--10328.org.readthedocs.build/en/10328/api/getting-started.html#openapi) in the API Guide. diff --git a/doc/release-notes/10242-add-feature-dv-api b/doc/release-notes/10242-add-feature-dv-api deleted file mode 100644 index 5c786554ff9..00000000000 --- a/doc/release-notes/10242-add-feature-dv-api +++ /dev/null @@ -1 +0,0 @@ -New api endpoints have been added to allow you to add or remove featured collections from a dataverse collection. diff --git a/doc/release-notes/10288-add-term_uri-metadata-in-keyword-block.md b/doc/release-notes/10288-add-term_uri-metadata-in-keyword-block.md deleted file mode 100644 index eb3a79dbf25..00000000000 --- a/doc/release-notes/10288-add-term_uri-metadata-in-keyword-block.md +++ /dev/null @@ -1,53 +0,0 @@ -### New keywordTermURI Metadata in keyword Metadata Block - -Adding a new metadata `keywordTermURI` to the `keyword` metadata block to facilitate the integration of controlled vocabulary services, in particular by adding the possibility of saving the "term" and its associated URI. For more information, see #10288 and PR #10371. - -## Upgrade Instructions - -1\. Update the Citation metadata block - -- `wget https://github.com/IQSS/dataverse/releases/download/v6.3/citation.tsv` -- `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @citation.tsv -H "Content-type: text/tab-separated-values"` - -2\. Update your Solr `schema.xml` to include the new field. - - For details, please see https://guides.dataverse.org/en/latest/admin/metadatacustomization.html#updating-the-solr-schema - - -3\. Reindex Solr. - - Once the schema.xml is updated, Solr must be restarted and a reindex initiated. - For details, see https://guides.dataverse.org/en/latest/admin/solr-search-index.html but here is the reindex command: - - `curl http://localhost:8080/api/admin/index` - - -4\. Run ReExportAll to update dataset metadata exports. Follow the instructions in the [Metadata Export of Admin Guide](https://guides.dataverse.org/en/latest/admin/metadataexport.html#batch-exports-through-the-api). - - -## Notes for Dataverse Installation Administrators - -### Data migration to the new `keywordTermURI` field - -You can migrate your `keywordValue` data containing URIs to the new `keywordTermURI` field. -In case of data migration, view the affected data with the following database query: - -``` -SELECT value FROM datasetfieldvalue dfv -INNER JOIN datasetfield df ON df.id = dfv.datasetfield_id -WHERE df.datasetfieldtype_id = (SELECT id FROM datasetfieldtype WHERE name = 'keywordValue') -AND value ILIKE 'http%'; -``` - -If you wish to migrate your data, a database update is then necessary: - -``` -UPDATE datasetfield df -SET datasetfieldtype_id = (SELECT id FROM datasetfieldtype WHERE name = 'keywordTermURI') -FROM datasetfieldvalue dfv -WHERE dfv.datasetfield_id = df.id -AND df.datasetfieldtype_id = (SELECT id FROM datasetfieldtype WHERE name = 'keywordValue') -AND dfv.value ILIKE 'http%'; -``` - -A ['Reindex in Place'](https://guides.dataverse.org/en/latest/admin/solr-search-index.html#reindex-in-place) will be required and ReExportAll will need to be run to update the metadata exports of the dataset. Follow the directions in the [Admin Guide](http://guides.dataverse.org/en/latest/admin/metadataexport.html#batch-exports-through-the-api). \ No newline at end of file diff --git a/doc/release-notes/10316_cvoc_http_headers.md b/doc/release-notes/10316_cvoc_http_headers.md deleted file mode 100644 index 4b557383a2e..00000000000 --- a/doc/release-notes/10316_cvoc_http_headers.md +++ /dev/null @@ -1,5 +0,0 @@ -You are now able to add HTTP request headers required by the External Vocabulary Services you are implementing. - -A combined documentation can be found on pull request [#10404](https://github.com/IQSS/dataverse/pull/10404). - -For more information, see issue [#10316](https://github.com/IQSS/dataverse/issues/10316) and pull request [gddc/dataverse-external-vocab-support#19](https://github.com/gdcc/dataverse-external-vocab-support/pull/19). diff --git a/doc/release-notes/10330-api-change-latest-version-status.md b/doc/release-notes/10330-api-change-latest-version-status.md deleted file mode 100644 index 6e6a018fe12..00000000000 --- a/doc/release-notes/10330-api-change-latest-version-status.md +++ /dev/null @@ -1 +0,0 @@ -The API endpoint for getting the Dataset version has been extended to include latestVersionPublishingStatus. \ No newline at end of file diff --git a/doc/release-notes/10339-workflow.md b/doc/release-notes/10339-workflow.md deleted file mode 100644 index 90d08dabb1f..00000000000 --- a/doc/release-notes/10339-workflow.md +++ /dev/null @@ -1,3 +0,0 @@ -The computational workflow metadata block has been updated to present a clickable link for the External Code Repository URL field. - -Release notes should include the usual instructions, for those who have installed this optional block, to update the computational_workflow block. (PR#10441) \ No newline at end of file diff --git a/doc/release-notes/10389-metadatablocks-api-extension.md b/doc/release-notes/10389-metadatablocks-api-extension.md deleted file mode 100644 index 9b14100d33c..00000000000 --- a/doc/release-notes/10389-metadatablocks-api-extension.md +++ /dev/null @@ -1,6 +0,0 @@ -New optional query parameters added to ``api/metadatablocks`` and ``api/dataverses/{id}/metadatablocks`` endpoints: - -- ``returnDatasetFieldTypes``: Whether or not to return the dataset field types present in each metadata block. If not set, the default value is false. -- ``onlyDisplayedOnCreate``: Whether or not to return only the metadata blocks that are displayed on dataset creation. If ``returnDatasetFieldTypes`` is true, only the dataset field types shown on dataset creation will be returned within each metadata block. If not set, the default value is false. - -Added new ``displayOnCreate`` field to the MetadataBlock and DatasetFieldType payloads. diff --git a/doc/release-notes/10415-fix-api-performance-issues-on-large-datasets.md b/doc/release-notes/10415-fix-api-performance-issues-on-large-datasets.md deleted file mode 100644 index e8840e9d4f7..00000000000 --- a/doc/release-notes/10415-fix-api-performance-issues-on-large-datasets.md +++ /dev/null @@ -1,4 +0,0 @@ -For scenarios involving API calls related to large datasets (Numerous files, for example: ~10k) it has been optimized: - -- The search API endpoint. -- The permission checking logic present in PermissionServiceBean. diff --git a/doc/release-notes/10425-add-MIT-License.md b/doc/release-notes/10425-add-MIT-License.md deleted file mode 100644 index 95d6fb38ded..00000000000 --- a/doc/release-notes/10425-add-MIT-License.md +++ /dev/null @@ -1,3 +0,0 @@ -A new file has been added to import the MIT License to Dataverse: licenseMIT.json. - -Documentation has been added to explain the procedure for adding new licenses to the guides. diff --git a/doc/release-notes/10464-add-name-harvesting-client-facet.md b/doc/release-notes/10464-add-name-harvesting-client-facet.md deleted file mode 100644 index 1fc0bb47caf..00000000000 --- a/doc/release-notes/10464-add-name-harvesting-client-facet.md +++ /dev/null @@ -1,3 +0,0 @@ -The Metadata Source facet has been updated to show the name of the harvesting client rather than grouping all such datasets under 'harvested' - -TODO: for the v6.13 release note: Please add a full re-index using http://localhost:8080/api/admin/index to the upgrade instructions. diff --git a/doc/release-notes/10466-math-challenge-403-error-page.md b/doc/release-notes/10466-math-challenge-403-error-page.md deleted file mode 100644 index 160c760dc9d..00000000000 --- a/doc/release-notes/10466-math-challenge-403-error-page.md +++ /dev/null @@ -1 +0,0 @@ -On forbidden access error page, also know as 403 error page, the math challenge is now correctly display to submit the contact form. diff --git a/doc/release-notes/10468-doc-datalad-integration.md b/doc/release-notes/10468-doc-datalad-integration.md deleted file mode 100644 index cd4d2d53a5f..00000000000 --- a/doc/release-notes/10468-doc-datalad-integration.md +++ /dev/null @@ -1 +0,0 @@ -DataLad has been integrated with Dataverse. For more information, see https://dataverse-guide--10470.org.readthedocs.build/en/10470/admin/integrations.html#datalad diff --git a/doc/release-notes/10477-metadatablocks-api-extension-input-levels.md b/doc/release-notes/10477-metadatablocks-api-extension-input-levels.md deleted file mode 100644 index 77cc7f59773..00000000000 --- a/doc/release-notes/10477-metadatablocks-api-extension-input-levels.md +++ /dev/null @@ -1,3 +0,0 @@ -Changed ``api/dataverses/{id}/metadatablocks`` so that setting the query parameter ``onlyDisplayedOnCreate=true`` also returns metadata blocks with dataset field type input levels configured as required on the General Information page of the collection, in addition to the metadata blocks and their fields with the property ``displayOnCreate=true`` (which was the original behavior). - -A new endpoint ``api/dataverses/{id}/inputLevels`` has been created for updating the dataset field type input levels of a collection via API. diff --git a/doc/release-notes/10491-add-isreleased-to-get-dataverse-response.md b/doc/release-notes/10491-add-isreleased-to-get-dataverse-response.md deleted file mode 100644 index 5293c7267d0..00000000000 --- a/doc/release-notes/10491-add-isreleased-to-get-dataverse-response.md +++ /dev/null @@ -1,22 +0,0 @@ -The Dataverse object returned by /api/dataverses has been extended to include "isReleased": {boolean}. -```javascript -{ - "status": "OK", - "data": { - "id": 32, - "alias": "dv6f645bb5", - "name": "dv6f645bb5", - "dataverseContacts": [ - { - "displayOrder": 0, - "contactEmail": "54180268@mailinator.com" - } - ], - "permissionRoot": true, - "dataverseType": "UNCATEGORIZED", - "ownerId": 1, - "creationDate": "2024-04-12T18:05:59Z", - "isReleased": true - } -} -``` \ No newline at end of file diff --git a/doc/release-notes/10494-payara-upgrade.md b/doc/release-notes/10494-payara-upgrade.md deleted file mode 100644 index 23ee0e698f7..00000000000 --- a/doc/release-notes/10494-payara-upgrade.md +++ /dev/null @@ -1,119 +0,0 @@ -# Upgrade Payara to v6.2024.6 - -With this version of Dataverse, we encourage you to upgrade to version 6.2024.6. -This will address security issues accumulated since the release of 6.2023.8, which was required since Dataverse release 6.0. - -## Instructions for Upgrading - -If you are using GDCC containers, this upgrade is included when pulling new release images. -No manual intervention is necessary. - -We recommend you ensure you followed all update instructions from the past releases regarding Payara. -(Latest Payara update was for [v6.0](https://github.com/IQSS/dataverse/releases/tag/v6.0)) - -Upgrading requires a maintenance window and downtime. Please plan ahead, create backups of your database, etc. - -The steps below are a simple matter of reusing your existing domain directory with the new distribution. -But we also recommend that you review the Payara upgrade instructions as it could be helpful during any troubleshooting: -[Payara Release Notes](https://docs.payara.fish/community/docs/Release%20Notes/Release%20Notes%206.2024.6.html) -We assume you are already on a Dataverse 6.x installation, using a Payara 6.x release. - -```shell -export PAYARA=/usr/local/payara6 -``` - -(or `setenv PAYARA /usr/local/payara6` if you are using a `csh`-like shell) - -1\. Undeploy the previous version - -```shell - $PAYARA/bin/asadmin list-applications - $PAYARA/bin/asadmin undeploy dataverse<-version> -``` - -2\. Stop Payara - -```shell - service payara stop - rm -rf $PAYARA/glassfish/domains/domain1/generated - rm -rf $PAYARA/glassfish/domains/domain1/osgi-cache - rm -rf $PAYARA/glassfish/domains/domain1/lib/databases -``` - -3\. Move the current Payara directory out of the way - -```shell - mv $PAYARA $PAYARA.MOVED -``` - -4\. Download the new Payara version (6.2024.6), and unzip it in its place - -5\. Replace the brand new payara/glassfish/domains/domain1 with your old, preserved domain1 - -6\. Make sure that you have the following `--add-opens` options in your domain.xml. If not present, add them: - -```diff ---- payara-6.2023.8/glassfish/domains/domain1/config/domain.xml -+++ payara-6.2024.6/glassfish/domains/domain1/config/domain.xml -@@ -212,12 +212,16 @@ - --add-opens=java.naming/javax.naming.spi=ALL-UNNAMED - --add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED - --add-opens=java.logging/java.util.logging=ALL-UNNAMED -+ --add-opens=java.management/javax.management=ALL-UNNAMED -+ --add-opens=java.management/javax.management.openmbean=ALL-UNNAMED - [17|]--add-exports=java.base/sun.net.www=ALL-UNNAMED - [17|]--add-exports=java.base/sun.security.util=ALL-UNNAMED - [17|]--add-opens=java.base/java.lang.invoke=ALL-UNNAMED - [17|]--add-opens=java.desktop/java.beans=ALL-UNNAMED - [17|]--add-exports=jdk.naming.dns/com.sun.jndi.dns=ALL-UNNAMED - [17|]--add-exports=java.naming/com.sun.jndi.ldap=ALL-UNNAMED -+ [17|]--add-opens=java.base/java.io=ALL-UNNAMED -+ [21|]--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED - -Xmx512m - -XX:NewRatio=2 - -XX:+UnlockDiagnosticVMOptions -@@ -447,12 +451,16 @@ - --add-opens=java.naming/javax.naming.spi=ALL-UNNAMED - --add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED - --add-opens=java.logging/java.util.logging=ALL-UNNAMED -+ --add-opens=java.management/javax.management=ALL-UNNAMED -+ --add-opens=java.management/javax.management.openmbean=ALL-UNNAMED - [17|]--add-exports=java.base/sun.net.www=ALL-UNNAMED - [17|]--add-exports=java.base/sun.security.util=ALL-UNNAMED - [17|]--add-opens=java.base/java.lang.invoke=ALL-UNNAMED - [17|]--add-opens=java.desktop/java.beans=ALL-UNNAMED - [17|]--add-exports=jdk.naming.dns/com.sun.jndi.dns=ALL-UNNAMED - [17|]--add-exports=java.naming/com.sun.jndi.ldap=ALL-UNNAMED -+ [17|]--add-opens=java.base/java.io=ALL-UNNAMED -+ [21|]--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED - -Xmx512m - -XX:NewRatio=2 - -XX:+UnlockDiagnosticVMOptions -``` -(You can also save this as a patch file and try to apply it.) - -TODO: For the combined 6.3 release note, I would consider replacing the patch format above with just the 4 specific options, for clarity etc. (L.A.) As in: -``` - --add-opens=java.management/javax.management=ALL-UNNAMED - --add-opens=java.management/javax.management.openmbean=ALL-UNNAMED - [17|]--add-opens=java.base/java.io=ALL-UNNAMED - [21|]--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED -``` - -7\. Start Payara - -```shell - service payara start -``` - -8\. Deploy this version. - -```shell - $PAYARA/bin/asadmin deploy dataverse-6.3.war -``` - -9\. Restart payara - -```shell - service payara stop - service payara start diff --git a/doc/release-notes/10503-cvoc-hidden-html-fields.md b/doc/release-notes/10503-cvoc-hidden-html-fields.md deleted file mode 100644 index e3ea0463fb8..00000000000 --- a/doc/release-notes/10503-cvoc-hidden-html-fields.md +++ /dev/null @@ -1,11 +0,0 @@ -## Release Highlights - -### Updates on Support for External Vocabulary Services - -#### Hidden HTML Fields - -External Controlled Vocabulary scripts, configured via [:CVocConf](https://guides.dataverse.org/en/6.3/installation/config.html#cvocconf), can now access the values of managed fields as well as the term-uri-field for use in constructing the metadata view for a dataset. - -Those values are hidden and can be found with the html attribute `data-cvoc-metadata-name`. - -For more information, see [#10503](https://github.com/IQSS/dataverse/pull/10503). diff --git a/doc/release-notes/10531-contrib.md b/doc/release-notes/10531-contrib.md deleted file mode 100644 index 6cfbe988992..00000000000 --- a/doc/release-notes/10531-contrib.md +++ /dev/null @@ -1 +0,0 @@ -A new [Contributor Guide](https://dataverse-guide--10532.org.readthedocs.build/en/10532/contributor/index.html) has been added by the UX Working Group (#10531 and #10532). diff --git a/doc/release-notes/10547-solr-updates.md b/doc/release-notes/10547-solr-updates.md deleted file mode 100644 index a21809c6369..00000000000 --- a/doc/release-notes/10547-solr-updates.md +++ /dev/null @@ -1 +0,0 @@ -Multiple improvements have ben made to they way Solr indexing and searching is done. Response times should be significantly improved. See the individual PRs in this release for details. \ No newline at end of file diff --git a/doc/release-notes/10554-avoid-solr-join-guest.md b/doc/release-notes/10554-avoid-solr-join-guest.md deleted file mode 100644 index 956c658dbed..00000000000 --- a/doc/release-notes/10554-avoid-solr-join-guest.md +++ /dev/null @@ -1,5 +0,0 @@ -Two experimental features flag called "add-publicobject-solr-field" and "avoid-expensive-solr-join" have been added to change how Solr documents are indexed for public objects and how Solr queries are constructed to accommodate access to restricted content (drafts, etc.). It is hoped that it will help with performance, especially on large instances and under load. - -Before the search feature flag ("avoid-expensive...") can be turned on, the indexing flag must be enabled, and a full reindex performed. Otherwise publicly available objects are NOT going to be shown in search results. - -For details see https://dataverse-guide--10555.org.readthedocs.build/en/10555/installation/config.html#feature-flags and #10555. diff --git a/doc/release-notes/10561-3dviewer.md b/doc/release-notes/10561-3dviewer.md deleted file mode 100644 index 47da10f8837..00000000000 --- a/doc/release-notes/10561-3dviewer.md +++ /dev/null @@ -1 +0,0 @@ -3DViewer by openforestdata.pl has been added to the list of external tools: https://preview.guides.gdcc.io/en/develop/admin/external-tools.html#inventory-of-external-tools diff --git a/doc/release-notes/10565-banner-test-improvements.md b/doc/release-notes/10565-banner-test-improvements.md deleted file mode 100644 index d9030f2a0c3..00000000000 --- a/doc/release-notes/10565-banner-test-improvements.md +++ /dev/null @@ -1 +0,0 @@ -The endpoint `api/admin/bannerMessage` has been extended so the ID is returned when created \ No newline at end of file diff --git a/doc/release-notes/10568-Fix File Reingest.md b/doc/release-notes/10568-Fix File Reingest.md deleted file mode 100644 index 354aa847f01..00000000000 --- a/doc/release-notes/10568-Fix File Reingest.md +++ /dev/null @@ -1 +0,0 @@ -A bug that prevented the Ingest option in the File page Edit File menu from working has been fixed \ No newline at end of file diff --git a/doc/release-notes/10570-extra-facet-settings.md b/doc/release-notes/10570-extra-facet-settings.md deleted file mode 100644 index 9d68defc9a3..00000000000 --- a/doc/release-notes/10570-extra-facet-settings.md +++ /dev/null @@ -1,4 +0,0 @@ -Extra settings have been added giving an instance admin more choices in -selectively limiting the availability of search facets on the Collection and Dataset pages. -See the [Disable Solr Facets](https://guides.dataverse.org/en/6.3/installation/config.html#DisableSolrFacets) sections of the Config Guide for more info. - diff --git a/doc/release-notes/10579-avoid-solr-deletes.md b/doc/release-notes/10579-avoid-solr-deletes.md deleted file mode 100644 index 1062a2fb78f..00000000000 --- a/doc/release-notes/10579-avoid-solr-deletes.md +++ /dev/null @@ -1,9 +0,0 @@ -A features flag called "reduce-solr-deletes" has been added to improve how datafiles are indexed. When the flag is enabled, -Dataverse wil avoid pre-emptively deleting existing solr documents for the files prior to sending updated information. This -should improve performance and will allow additional optimizations going forward. - -The /api/admin/index/status and /api/admin/index/clear-orphans calls -(see https://guides.dataverse.org/en/latest/admin/solr-search-index.html#index-and-database-consistency) -will now find and remove (respectively) additional permissions related solr documents that were not being detected before. -Reducing the overall number of documents will improve solr performance and large sites may wish to periodically call the -clear-orphans API. \ No newline at end of file diff --git a/doc/release-notes/10611-harvested-origin-facet.md b/doc/release-notes/10611-harvested-origin-facet.md deleted file mode 100644 index 89ab6eb7639..00000000000 --- a/doc/release-notes/10611-harvested-origin-facet.md +++ /dev/null @@ -1,10 +0,0 @@ -NOTE that this release note supercedes the 10464-add-name-harvesting-client-facet.md note from the PR 10464. - -An option has been added to index the name of the Harvesting Client as the "Metadata Source" of harvested datasets and files; if enabled, the Metadata Source facet will be showing separate entries for the content harvested from different sources, instead of the current, default behavior where there is one "Harvested" facet for all such content. - - -TODO: for the v6.3 release note: -If you choose to enable the extended "Metadata Souce" facet for harvested content, set the optional feature flage (jvm option) `dataverse.feature.index-harvested-metadata-source=true` before reindexing. - -[Please note that the upgrade instruction in 6.3 will contain a suggestion to run full reindex, as part of the Solr upgrade, so the sentence above will need to be added to that section] - diff --git a/doc/release-notes/10637-fix-dataverse-metadatablocks-api.md b/doc/release-notes/10637-fix-dataverse-metadatablocks-api.md deleted file mode 100644 index c8c9c4fa66f..00000000000 --- a/doc/release-notes/10637-fix-dataverse-metadatablocks-api.md +++ /dev/null @@ -1,2 +0,0 @@ -dataverses/{id}/metadatablocks API endpoint has been fixed, since the fields returned for each metadata block when returnDatasetTypes query parameter is set to true was not correct. - diff --git a/doc/release-notes/5621_dataset image in header.md b/doc/release-notes/5621_dataset image in header.md deleted file mode 100644 index 34b445fd9e1..00000000000 --- a/doc/release-notes/5621_dataset image in header.md +++ /dev/null @@ -1 +0,0 @@ -Dataverse will use the Dataset thumbnail, if one is defined, rather than the generic Dataverse logo in the Open Graph metadata header. This means the image will be seen when, for example, the dataset is referenced in Facebook. diff --git a/doc/release-notes/6.3-release-notes.md b/doc/release-notes/6.3-release-notes.md new file mode 100644 index 00000000000..6cad513690d --- /dev/null +++ b/doc/release-notes/6.3-release-notes.md @@ -0,0 +1,470 @@ +# Dataverse 6.3 + +Please note: To read these instructions in full, please go to https://github.com/IQSS/dataverse/releases/tag/v6.3 rather than the list of releases, which will cut them off. + +This release brings new features, enhancements, and bug fixes to Dataverse. Thank you to all of the community members who contributed code, suggestions, bug reports, and other assistance across the project. + +# Table of Contents +- [Release Highlights](#release-highlights) +- [Features](#features) +- [Bug Fixes](#bug-fixes) +- [API](#api) +- [Settings](#settings) +- [Complete List of Changes](#complete-list-of-changes) +- [Getting Help](#getting-help) +- [Upgrade instructions](#upgrade-instructions) + +## Release Highlights + +### Solr Search and Indexing Improvements + +Multiple improvements have been made to the way Solr indexing and searching is done. Response times should be significantly improved. + +- Two experimental features flag called "add-publicobject-solr-field" and "avoid-expensive-solr-join" have been added to change how Solr documents are indexed for public objects and how Solr queries are constructed to accommodate access to restricted content (drafts, etc.). It is hoped that it will help with performance, especially on large instances and under load. + +- Before the search feature flag ("avoid-expensive...") can be turned on, the indexing flag must be enabled, and a full reindex performed. Otherwise publicly available objects are NOT going to be shown in search results. + +- A feature flag called "reduce-solr-deletes" has been added to improve how datafiles are indexed. When the flag is enabled, Dataverse will avoid pre-emptively deleting existing Solr documents for the files prior to sending updated information. This +should improve performance and will allow additional optimizations going forward. + +- The /api/admin/index/status and /api/admin/index/clear-orphans calls +(see https://guides.dataverse.org/en/latest/admin/solr-search-index.html#index-and-database-consistency) +will now find and remove (respectively) additional permissions related Solr documents that were not being detected before. +Reducing the overall number of documents will improve Solr performance and large sites may wish to periodically call the "clear-orphans" API. + +- Dataverse now relies on the autoCommit and autoSoftCommit settings in the Solr configuration instead of explicitly committing documents to the Solr index. This improves indexing speed. + +See also #10554, #10654, and #10579. + +### File Retention Period + +Dataverse now supports file-level retention periods. The ability to set retention periods, with a minimum duration (in months), can be configured by a Dataverse installation administrator. For more information, see the [Retention Periods section](https://guides.dataverse.org/en/6.3/user/dataset-management.html#retention-periods) of the User Guide. + +- Users can configure a specific retention period, defined by an end date and a short reason, on a set of selected files or an individual file, by selecting the "Retention Period" menu item and entering information in a popup dialog. Retention periods can only be set, changed, or removed before a file has been published. After publication, only Dataverse installation administrators can make changes, using an API. + +- After the retention period expires, files can not be previewed or downloaded (as if restricted, with no option to allow access requests). The file (landing) page and all the metadata remains available. + +## Features + +### Large Datasets Improvements + +For scenarios involving API calls related to large datasets (numerous files, for example: ~10k) the following have been been optimized: + +- The Search API endpoint. +- The permission checking logic present in PermissionServiceBean. + +See also [#10415](https://github.com/IQSS/dataverse/pull/10415). + +### Improved Controlled Vocabulary for Citation Block + +The Controlled Vocabuary Values list for the "Language" metadata field in the citation block has been improved, with some missing two- and three-letter ISO 639 codes added, as well as more alternative names for some of the languages, making all these extra language identifiers importable. See also [#8243](https://github.com/IQSS/dataverse/pull/8243). + +### Updates on Support for External Vocabulary Services + +Multiple extensions of the external vocabulary mechanism have been added. These extensions allow interaction with services based on the Ontoportal software and are expected to be generally useful for other service types. + +These changes include: + +- *Improved Indexing with Compound Fields:* When using an external vocabulary service with compound fields, you can now specify which field(s) will include additional indexed information, such as translations of an entry into other languages. This is done by adding the `indexIn` in `retrieval-filtering`. See also [#10505](https://github.com/IQSS/dataverse/pull/10505) and [GDCC/dataverse-external-vocab-support documentation](https://github.com/gdcc/dataverse-external-vocab-support/tree/main/docs). + +- *Broader Support for Indexing Service Responses:* Indexing of the results from `retrieval-filtering` responses can now handle additional formats including JSON arrays of strings and values from arbitrary keys within a JSON Object. See [#10505](https://github.com/IQSS/dataverse/pull/10505). + +- *HTTP Headers:* You are now able to add HTTP request headers required by the service you are implementing. See [#10331](https://github.com/IQSS/dataverse/pull/10331). + +- *Flexible params in retrievalUri:* You can now use `managed-fields` field names as well as the `term-uri-field` field name as parameters in the `retrieval-uri` when configuring an external vocabulary service. `{0}` as an alternative to using the `term-uri-field` name is still supported for backward compatibility. Also you can specify if the value must be url encoded with `encodeUrl:`. See [#10404](https://github.com/IQSS/dataverse/pull/10404). + + For example : `"retrieval-uri": "https://data.agroportal.lirmm.fr/ontologies/{keywordVocabulary}/classes/{encodeUrl:keywordermURL}"` + +- *Hidden HTML Fields* External controlled vocabulary scripts, configured via [:CVocConf](https://guides.dataverse.org/en/6.3/installation/config.html#cvocconf), can now access the values of managed fields as well as the term-uri-field for use in constructing the metadata view for a dataset. These values are now added as hidden elements in the HTML and can be found with the HTML attribute `data-cvoc-metadata-name`. See also [#10503](https://github.com/IQSS/dataverse/pull/10503). + +### A Contributor Guide is now available + +A new [Contributor Guide](https://guides.dataverse.org/en/6.3/contributor/index.html) has been added by the UX Working Group (#10531 and #10532). + +### URL Validation Is More Permissive + +URL validation now allows two slashes in the path component of the URL. +Among other things, this allows metadata fields of `url` type to be filled with more complex url such as https://archive.softwareheritage.org/browse/directory/561bfe6698ca9e58b552b4eb4e56132cac41c6f9/?origin_url=https://github.com/gem-pasteur/macsyfinder&revision=868637fce184865d8e0436338af66a2648e8f6e1&snapshot=1bde3cb370766b10132c4e004c7cb377979928d1 + +See also #9750 and [#9739](https://github.com/IQSS/dataverse/pull/9739) + +### Improved Detection of RO-Crate Files + +Detection of mime-types based on a filename with extension and detection of the RO-Crate metadata files. + +From now on, filenames with extensions can be added into `MimeTypeDetectionByFileName.properties` file. Filenames added there will take precedence over simply recognizing files by extensions. For example, two new filenames are added into that file: +``` +ro-crate-metadata.json=application/ld+json; profile="http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate" +ro-crate-metadata.jsonld=application/ld+json; profile="http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate" +``` + +Therefore, files named `ro-crate-metadata.json` will be then detected as RO-Crated metadata files from now on, instead as generic `JSON` files. +For more information on the RO-Crate specifications, see https://www.researchobject.org/ro-crate + +See also [#10015](https://github.com/IQSS/dataverse/pull/10015). + +### New S3 Tagging Configuration Option + +If your S3 store does not support tagging and gives an error if you configure direct upload, you can disable the tagging by using the `dataverse.files..disable-tagging` JVM option. For more details, see the section on [S3 tags](https://guides.dataverse.org/en/6.3/developers/big-data-support.html#s3-tags) in the guides, #10022 and #10029. + +### Feature Flag To Remove the Required "Reason" Field in the "Return to Author" Dialog + +A reason field, that is required to not be empty, was added to the "Return to Author" dialog in v6.2. Installations that handle author communications through email or another system may prefer to not be required to use this new field. v6.3 includes a new +disable-return-to-author-reason feature flag that can be enabled to drop the reason field from the dialog and make sending a reason optional in the api/datasets/{id}/returnToAuthor call. See also #10655. + +### Improved Use of Dataverse Thumbnail + +Dataverse will use the dataset thumbnail, if one is defined, rather than the generic Dataverse logo in the Open Graph metadata header. This means the image will be seen when, for example, the dataset is referenced on Facebook. See also [#5621](https://github.com/IQSS/dataverse/pull/5621). + +### Improved Email Notifications When Guestbook is Used for File Access Requests + +Multiple improvements to guestbook response emails making it easier to organize and process them. The subject line of the notification email now includes the name and user identifier of the requestor. Additionally, the body of the email now includes the user id of the requestor. Finally the guestbook responses have been sorted and spaced to improve readability. See also [#10581](https://github.com/IQSS/dataverse/issues/10581). + +### New keywordTermURI Metadata Field in the Citation Metadata Block + +A new metadata field - `keywordTermURI`, has been added in the citation metadata block (as a fourth child field under the `keyword` parent field). This has been done to improve usability and to facilitate the integration of controlled vocabulary services, adding the possibility of saving the "term" and/or its associated URI. For more information, see #10288 and PR #10371. + +### Updated Computational Workflow Metadata Block + +The computational workflow metadata block has been updated to present a clickable link for the External Code Repository URL field. See also [#10339](https://github.com/IQSS/dataverse/pull/10339). + +### Metadata Source Facet Added + +An option has been added to index the name of the harvesting client as the "Metadata Source" of harvested datasets and files; if enabled, the Metadata Source facet will show separate entries for the content harvested from different sources, instead of the current, default behavior where there is one "Harvested" facet for all such content. + +Tho enable this feature, set the optional feature flage (jvm option) `dataverse.feature.index-harvested-metadata-source=true` before reindexing. + +See also [#10611](https://github.com/IQSS/dataverse/pull/10611) and #10651. + +### Additional Facet Settings + +Extra settings have been added giving an instance admin more choices in selectively limiting the availability of search facets on the collection and dataset pages. + +See [Disable Solr Facets](https://guides.dataverse.org/en/6.3/installation/config.html#DisableSolrFacets) under the configuration section of the Installation Guide for more info as well as [#10570](https://github.com/IQSS/dataverse/pull/10570). + +### Sitemap Now Supports More Than 50k Items + +Dataverse can now handle more than 50,000 items when generating sitemap files, splitting the content across multiple files to comply with the Sitemap protocol. For details, see the [sitemap section](https://guides.dataverse.org/en/6.3/installation/config.html#creating-a-sitemap-and-submitting-it-to-search-engines) of the Installation Guide. See also [#8936](https://github.com/IQSS/dataverse/pull/8936) and [#10321](https://github.com/IQSS/dataverse/pull/10321). + +### MIT and Apache 2.0 Licenses Added + +New files have been added to import the MIT and Apache 2.0 Licenses to Dataverse: + +- licenseMIT.json +- licenseApache-2.0.json + +Guidance has been added to the [guides](https://guides.dataverse.org/en/6.2/installation/config.html#adding-custom-licenses) to explain the procedure for adding new licenses to Dataverse. + +See also [#10425](https://github.com/IQSS/dataverse/pull/10425). + +### 3D Viewer by Open Forest Data + +3DViewer by openforestdata.pl has been added to the [list of external tools](https://guides.dataverse.org/en/6.3/admin/external-tools.html#inventory-of-external-tools). See also [#10561](https://github.com/IQSS/dataverse/pull/10561). + +### Datalad Integration With Dataverse + +DataLad has been integrated with Dataverse. For more information, see the [integrations](https://guides.dataverse.org/en/6.3/admin/integrations.html#datalad) section of the guides. See also [#10468](https://github.com/IQSS/dataverse/pull/10468). + +### Rsync Support Has Been Deprecated + +Support for rsync has been deprecated. Information has been removed from the guides for rsync and related software such as Data Capture Module (DCM) and Repository Storage Abstraction Layer (RSAL). You can still find this information in [older versions](https://guides.dataverse.org/en/6.2/developers/big-data-support.html#data-capture-module-dcm) of the guides. See [Settings](#database-settings), below, for deprecated settings. See also [#8985](https://github.com/IQSS/dataverse/pull/8985). + +[↑ Table of Contents](#table-of-contents) + +## Bug Fixes + +### OpenAPI Re-Enabled + +In Dataverse 6.0 when Payara was updated it caused the url `/openapi` to stop working: + +- https://github.com/IQSS/dataverse/issues/9981 +- https://github.com/payara/Payara/issues/6369 + +In addition to fixing the `/openapi` URL, we are also making some changes on how we provide the OpenAPI document: + +When it worked in Dataverse 5.x, the `/openapi` output was generated automatically by Payara, but in this release we have switched to OpenAPI output produced by the [SmallRye OpenAPI plugin](https://github.com/smallrye/smallrye-open-api/tree/main/tools/maven-plugin). This gives us finer control over the output. + +For more information, see the section on [OpenAPI](https://guides.dataverse.org/en/6.3/getting-started.html#openapi) in the API Guide and #10328. + +### Re-Addition of "Cell Counting" to Life Sciences Block + +In the Life Sciences metadata block under the "Measurement Type" field the value `cell counting` was accidentally removed in v5.1. It has been restored. See also #8655 and #9735. + +### Math Challenge Fixed on 403 Error Page + +On the "forbidden" (403) error page, the math challenge now correctly displays so that the contact form can be submitted. See also [#10466](https://github.com/IQSS/dataverse/pull/10466). + +### Ingest Option Bug Fixed + +A bug that prevented the "Ingest" option in the file page "Edit File" menu from working has been fixed. See also [#10568](https://github.com/IQSS/dataverse/pull/10568). + +### Incomplete Metadata Bug Fix + +A bug was fixed where the `incomplete metadata` label was being shown for published dataset with incomplete metadata in certain scenarios. This label will now be shown for draft versions of such datasets and published datasets that the user can edit. This label can also be made invisible for published datasets (regardless of edit rights) with the new option ``dataverse.ui.show-validity-label-when-published`` set to `false`. See also [#10116](https://github.com/IQSS/dataverse/pull/10116). + +### Identical Role Error Message + +An error is now correctly reported when an attempt is made to assign an identical role to the same collection, dataset, or file. See also [#9729](https://github.com/IQSS/dataverse/pull/9729) and #10465. + +[↑ Table of Contents](#table-of-contents) + +## API + +### Superuser Endpoint + +The existing API endpoint for toggling the superuser status of a user has been deprecated in favor of a new API endpoint that allows you to explicitly and idempotently set the status as true or false. For details, see the [API Guide](https://guides.dataverse.org/en/6.3/api/native-api.html#set-superuser-status), [#9887](https://github.com/IQSS/dataverse/pull/9887) and [#10440](https://github.com/IQSS/dataverse/pull/10440). + +### New Featured Collections Endpoints + +New API endpoints have been added to allow you to add or remove featured collections from a collection. + +See also the sections on [listing, setting, and removing](https://guides.dataverse.org/en/6.3/api/native-api.html#list-featured-collections-for-a-dataverse-collection) featured collections in the API Guide, [#10242](https://github.com/IQSS/dataverse/pull/10242) and #10459. + +### Dataset Version Endpoint Extended + +The API endpoint for getting the Dataset version has been extended to include latestVersionPublishingStatus. See also [#10330](https://github.com/IQSS/dataverse/pull/10330). + +### New Optional Query Parameters for Metadatablocks Endpoints + +New optional query parameters have been added to `api/metadatablocks` and `api/dataverses/{id}/metadatablocks` endpoints: + +- `returnDatasetFieldTypes`: Whether or not to return the dataset field types present in each metadata block. If not set, the default value is false. +- Setting the query parameter `onlyDisplayedOnCreate=true` also returns metadata blocks with dataset field type input levels configured as required on the General Information page of the collection, in addition to the metadata blocks and their fields with the property ``displayOnCreate=true``. + +See also [#10389](https://github.com/IQSS/dataverse/pull/10389) + +### Dataverse Payload Includes Release Status + +The Dataverse object returned by /api/dataverses has been extended to include "isReleased": {boolean}. See also [#10491](https://github.com/IQSS/dataverse/pull/10491). + +### New Field Type Input Level Endpoint + +A new endpoint ``api/dataverses/{id}/inputLevels`` has been created for updating the dataset field type input levels of a collection via API. See also [#10477](https://github.com/IQSS/dataverse/pull/10477). + +### Banner Message Endpoint Extended + +The endpoint `api/admin/bannerMessage` has been extended so the ID is returned when created. See also [#10565](https://github.com/IQSS/dataverse/pull/10565). + +[↑ Table of Contents](#table-of-contents) + +## Settings + +### Database Settings: + +***New:*** + +- :DisableSolrFacets + +***Deprecated (used with rsync):*** + +- :DataCaptureModuleUrl +- :DownloadMethods +- :LocalDataAccessPath +- :RepositoryStorageAbstractionLayerUrl + +### New Configuration Options + +- `dataverse.files..disable-tagging` +- `dataverse.feature.add-publicobject-solr-field` +- `dataverse.feature.avoid-expensive-solr-join` +- `dataverse.feature.reduce-solr-deletes` +- `dataverse.feature.disable-return-to-author-reason` +- `dataverse.feature.index-harvested-metadata-source` +- `dataverse.ui.show-validity-label-when-published` + +[↑ Table of Contents](#table-of-contents) + +## Complete List of Changes + +For the complete list of code changes in this release, see the [6.3 Milestone](https://github.com/IQSS/dataverse/issues?q=milestone%3A6.3+is%3Aclosed) in GitHub. + +[↑ Table of Contents](#table-of-contents) + +## Getting Help + +For help with upgrading, installing, or general questions please post to the [Dataverse Community Google Group](https://groups.google.com/g/dataverse-community) or email support@dataverse.org. + +[↑ Table of Contents](#table-of-contents) + +## Upgrade Instructions + +Upgrading requires a maintenance window and downtime. Please plan accordingly, create backups of your database, etc. + +These instructions assume that you've already upgraded through all the 5.x releases and are now running Dataverse 6.2. + +0\. These instructions assume that you are upgrading from the immediate previous version. If you are running an earlier version, the only supported way to upgrade is to progress through the upgrades to all the releases in between before attempting the upgrade to this version. + +If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. Use `sudo` to change to that user first. For example, `sudo -i -u dataverse` if `dataverse` is your dedicated application user. + +In the following commands, we assume that Payara 6 is installed in `/usr/local/payara6`. If not, adjust as needed. + +`export PAYARA=/usr/local/payara6` + +(or `setenv PAYARA /usr/local/payara6` if you are using a `csh`-like shell) + +1\. Undeploy the previous version. + +- `$PAYARA/bin/asadmin undeploy dataverse-6.2` + +2\. Stop Payara and remove the following directories: + +```shell +service payara stop +rm -rf $PAYARA/glassfish/domains/domain1/generated +rm -rf $PAYARA/glassfish/domains/domain1/osgi-cache +rm -rf $PAYARA/glassfish/domains/domain1/lib/databases +``` + +3\. Upgrade Payara to v6.2024.6 + +With this version of Dataverse, we encourage you to upgrade to version 6.2024.6. +This will address security issues accumulated since the release of 6.2023.8. + +Note that if you are using GDCC containers, this upgrade is included when pulling new release images. +No manual intervention is necessary. + +The steps below are a simple matter of reusing your existing domain directory with the new distribution. +But we recommend that you review the Payara upgrade instructions as it could be helpful during any troubleshooting: +[Payara Release Notes](https://docs.payara.fish/community/docs/Release%20Notes/Release%20Notes%206.2024.6.html). +We also recommend you ensure you followed all update instructions from the past releases regarding Payara. +(The latest Payara update was for [v6.0](https://github.com/IQSS/dataverse/releases/tag/v6.0).) + +Move the current Payara directory out of the way: + +```shell +mv $PAYARA $PAYARA.6.2023.8 +``` + +Download the new Payara version 6.2024.6 (from https://www.payara.fish/downloads/payara-platform-community-edition/), and unzip it in its place: + +```shell +cd /usr/local +unzip payara-6.2024.6.zip +``` + +Replace the brand new `payara/glassfish/domains/domain1` with your old, preserved domain1: + +```shell +mv payara6/glassfish/domains/domain1 payara6/glassfish/domains/domain1_DIST +mv payara6-2023.8/glassfish/domains/domain1 payara6/glassfish/domains/ +``` + +Make sure that you have the following `--add-opens` options in your `payara6/glassfish/domains/domain1/config/domain.xml`. If not present, add them: + +``` +--add-opens=java.management/javax.management=ALL-UNNAMED +--add-opens=java.management/javax.management.openmbean=ALL-UNNAMED +[17|]--add-opens=java.base/java.io=ALL-UNNAMED +[21|]--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED +``` + +(Note that you likely already have the `java.base/java.io` option there, but without the `[17|]` prefix. Make sure to replace it with the version above) + +Start Payara: + +```shell +sudo service payara start +``` + +4\. Deploy this version. + +```shell +$PAYARA/bin/asadmin deploy dataverse-6.3.war +``` + +5\. For installations with internationalization: + +- Please remember to update translations via [Dataverse language packs](https://github.com/GlobalDataverseCommunityConsortium/dataverse-language-packs). + +6\. Restart Payara + +```shell +service payara stop +service payara start +``` + +7\. Update the following metadata blocks to reflect the incremental improvements made to the handling of core metadata fields: + +```shell +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.3/scripts/api/data/metadatablocks/citation.tsv + +curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file citation.tsv + +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.3/scripts/api/data/metadatablocks/biomedical.tsv + +curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file biomedical.tsv + +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.3/scripts/api/data/metadatablocks/computational_workflow.tsv + +curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file computational_workflow.tsv + +``` + +8\. Upgrade Solr + +Solr 9.4.1 is now the version recommended in our Installation Guide and used with automated testing. There is a known security issue in the previously recommended version 9.3.0: https://nvd.nist.gov/vuln/detail/CVE-2023-36478. While the risk of an exploit should not be significant unless the Solr instance is accessible from outside networks (which we have always recommended against), we recommend to upgrade. + +Install Solr 9.4.1 following the [instructions](https://guides.dataverse.org/en/6.3/installation/prerequisites.html#solr) from the Installation Guide. + +The instructions in the guide suggest to use the config files from the installer zip bundle. Upgrading an existing instance, it may be easier to download them from the source tree: + +```shell +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.3/conf/solr/solrconfig.xml +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.3/conf/solr/schema.xml +cp solrconfig.xml schema.xml /usr/local/solr/solr-9.4.1/server/solr/collection1/conf +``` + +8a\. For installations with custom or experimental metadata blocks: + +- Stop Solr instance (usually `service solr stop`, depending on Solr installation/OS, see the [Installation Guide](https://guides.dataverse.org/en/6.3/installation/prerequisites.html#solr-init-script)). + +- Run the `update-fields.sh` script that we supply, as in the example below (modify the command lines as needed to reflect the correct path of your Solr installation): + +```shell +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.3/conf/solr/update-fields.sh +chmod +x update-fields.sh +curl "http://localhost:8080/api/admin/index/solr/schema" | ./update-fields.sh /usr/local/solr/solr-9.4.1/server/solr/collection1/conf/schema.xml +``` + +- Start Solr instance (usually `service solr start` depending on Solr/OS). + +9\. Enable the Metadata Source facet for harvested content (Optional): + +If you choose to enable this new feature, set the optional feature flag (jvm option) `dataverse.feature.index-harvested-metadata-source=true` before reindexing. + +10\. Reindex Solr, if you upgraded Solr (recommended), or chose to enable any options that require a reindex: + +```shell +curl http://localhost:8080/api/admin/index +``` + +Note: if you choose to perform a migration of your `keywordValue` metadata fields (section below), that will require a reindex as well, so do that first. + +## Notes for Dataverse Installation Administrators + +### Data migration to the new `keywordTermURI` field + +You can migrate your `keywordValue` data containing URIs to the new `keywordTermURI` field. +In case of data migration, view the affected data with the following database query: + +```sql +SELECT value FROM datasetfieldvalue dfv +INNER JOIN datasetfield df ON df.id = dfv.datasetfield_id +WHERE df.datasetfieldtype_id = (SELECT id FROM datasetfieldtype WHERE name = 'keywordValue') +AND value ILIKE 'http%'; +``` + +If you wish to migrate your data, a database update is then necessary: + +```sql +UPDATE datasetfield df +SET datasetfieldtype_id = (SELECT id FROM datasetfieldtype WHERE name = 'keywordTermURI') +FROM datasetfieldvalue dfv +WHERE dfv.datasetfield_id = df.id +AND df.datasetfieldtype_id = (SELECT id FROM datasetfieldtype WHERE name = 'keywordValue') +AND dfv.value ILIKE 'http%'; +``` + +A [reindex in place](https://guides.dataverse.org/en/latest/admin/solr-search-index.html#reindex-in-place) will be required. ReExportAll will need to be run to update the metadata exports of the dataset. Follow the directions in the [Admin Guide](http://guides.dataverse.org/en/latest/admin/metadataexport.html#batch-exports-through-the-api). + +[↑ Table of Contents](#table-of-contents) diff --git a/doc/release-notes/8243-improve-language-controlled-vocab.md b/doc/release-notes/8243-improve-language-controlled-vocab.md deleted file mode 100644 index 15b2b46c02d..00000000000 --- a/doc/release-notes/8243-improve-language-controlled-vocab.md +++ /dev/null @@ -1,11 +0,0 @@ -The Controlled Vocabuary Values list for the metadata field Language in the Citation block has been improved, with some missing two- and three-letter ISO 639 codes added, as well as more alternative names for some of the languages, making all these extra language identifiers importable. - -To be added to the 6.3 release instructions: - -Update the Citation block, to incorporate the improved controlled vocabulary for language [plus whatever other improvements may be made to the block in other PRs]: - -``` -wget https://raw.githubusercontent.com/IQSS/dataverse/v6.3/scripts/api/data/metadatablocks/citation.tsv -curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file citation.tsv -``` - diff --git a/doc/release-notes/8655-re-add-cell-counting-biomedical-tsv.md b/doc/release-notes/8655-re-add-cell-counting-biomedical-tsv.md deleted file mode 100644 index 295f206871f..00000000000 --- a/doc/release-notes/8655-re-add-cell-counting-biomedical-tsv.md +++ /dev/null @@ -1,12 +0,0 @@ -## Release Highlights - -### Life Science Metadata - -Re-adding value `cell counting` to Life Science metadatablock's Measurement Type vocabularies accidentally removed in `v5.1`. - -## Upgrade Instructions - -### Update the Life Science metadata block - -- `wget https://github.com/IQSS/dataverse/releases/download/v6.3/biomedical.tsv` -- `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @biomedical.tsv -H "Content-type: text/tab-separated-values"` \ No newline at end of file diff --git a/doc/release-notes/8936-more-than-50000-entries-in-sitemap.md b/doc/release-notes/8936-more-than-50000-entries-in-sitemap.md deleted file mode 100644 index 7b367e328c1..00000000000 --- a/doc/release-notes/8936-more-than-50000-entries-in-sitemap.md +++ /dev/null @@ -1,11 +0,0 @@ -Dataverse can now handle more than 50,000 items when generating sitemap files, splitting the content across multiple files to comply with the Sitemap protocol. - -For details see https://dataverse-guide--10321.org.readthedocs.build/en/10321/installation/config.html#creating-a-sitemap-and-submitting-it-to-search-engines #8936 and #10321. - -## Upgrade instructions - -If your installation has more than 50,000 entries, you should re-submit your sitemap URL to Google or other search engines. The file in the URL will change from ``sitemap.xml`` to ``sitemap_index.xml``. - -As explained at https://dataverse-guide--10321.org.readthedocs.build/en/10321/installation/config.html#creating-a-sitemap-and-submitting-it-to-search-engines this is the command for regenerating your sitemap: - -`curl -X POST http://localhost:8080/api/admin/sitemap` diff --git a/doc/release-notes/8985-deprecate-rsync.md b/doc/release-notes/8985-deprecate-rsync.md deleted file mode 100644 index 44563f292fd..00000000000 --- a/doc/release-notes/8985-deprecate-rsync.md +++ /dev/null @@ -1,8 +0,0 @@ -Support for rsync has been deprecated. Information has been removed from the guides for rsync and related software such as Data Capture Module (DCM) and Repository Storage Abstraction Layer (RSAL). You can still find this information in [older versions](https://guides.dataverse.org/en/6.2/developers/big-data-support.html#data-capture-module-dcm) of the guides. - -The following related database settings have been deprecated as well: - -- :DataCaptureModuleUrl -- :DownloadMethods -- :LocalDataAccessPath -- :RepositoryStorageAbstractionLayerUrl diff --git a/doc/release-notes/9276-allow-flexible-params-in-retrievaluri-cvoc.md b/doc/release-notes/9276-allow-flexible-params-in-retrievaluri-cvoc.md deleted file mode 100644 index 5e18007e8ae..00000000000 --- a/doc/release-notes/9276-allow-flexible-params-in-retrievaluri-cvoc.md +++ /dev/null @@ -1,14 +0,0 @@ -## Release Highlights - -### Updates on Support for External Vocabulary Services - -#### HTTP Headers - -You are now able to add HTTP request headers required by the service you are implementing (#10331) - -#### Flexible params in retrievalUri - -You can now use `managed-fields` field names as well as the `term-uri-field` field name as parameters in the `retrieval-uri` when configuring an external vocabulary service. `{0}` as an alternative to using the `term-uri-field` name is still supported for backward compatibility. -Also you can specify if the value must be url encoded with `encodeUrl:`. (#10404) - -For example : `"retrieval-uri": "https://data.agroportal.lirmm.fr/ontologies/{keywordVocabulary}/classes/{encodeUrl:keywordTermURL}"` \ No newline at end of file diff --git a/doc/release-notes/9276-doc-cvoc-index-in.md b/doc/release-notes/9276-doc-cvoc-index-in.md deleted file mode 100644 index 78289201511..00000000000 --- a/doc/release-notes/9276-doc-cvoc-index-in.md +++ /dev/null @@ -1,18 +0,0 @@ -## Release Highlights - -### Updates on Support for External Vocabulary Services - -Multiple extensions of the External Vocabulary mechanism have been added. These extensions allow interaction with services based on the Ontoportal software and are expected to be generally useful for other service types. - -These changes include: - -#### Improved Indexing with Compound Fields - -When using an external vocabulary service with compound fields, you can now specify which field(s) will include additional indexed information, such as translations of an entry into other languages. This is done by adding the `indexIn` in `retrieval-filtering`. (#10505) -For more information, please check [GDCC/dataverse-external-vocab-support documentation](https://github.com/gdcc/dataverse-external-vocab-support/tree/main/docs). - -#### Broader Support for Indexing Service Responses - -Indexing of the results from `retrieval-filtering` responses can now handle additional formats including Json Arrays of Strings and values from arbitrary keys within a JSON Object. (#10505) - -**** This documentation must be merged with 9276-allow-flexible-params-in-retrievaluri-cvoc.md (#10404) \ No newline at end of file diff --git a/doc/release-notes/9375-retention-period.md b/doc/release-notes/9375-retention-period.md deleted file mode 100644 index a088cabf138..00000000000 --- a/doc/release-notes/9375-retention-period.md +++ /dev/null @@ -1,8 +0,0 @@ -The Dataverse Software now supports file-level retention periods. The ability to set retention periods, with a minimum duration (in months), can be configured by a Dataverse installation administrator. For more information, see the [Retention Periods section](https://guides.dataverse.org/en/6.3/user/dataset-management.html#retention-periods) of the Dataverse Software Guides. - -- Users can configure a specific retention period, defined by an end date and a short reason, on a set of selected files or an individual file, by selecting the 'Retention Period' menu item and entering information in a popup dialog. Retention Periods can only be set, changed, or removed before a file has been published. After publication, only Dataverse installation administrators can make changes, using an API. - -- After the retention period expires, files can not be previewed or downloaded (as if restricted, with no option to allow access requests). The file (landing) page and all the metadata remains available. - - -Release notes should mention that a Solr schema update is needed. diff --git a/doc/release-notes/9729-release-notes.md b/doc/release-notes/9729-release-notes.md deleted file mode 100644 index 9dc27995405..00000000000 --- a/doc/release-notes/9729-release-notes.md +++ /dev/null @@ -1 +0,0 @@ -An error is now correctly reported when an attempt is made to assign an identical role to the same collection, dataset, or file. #9729 #10465 \ No newline at end of file diff --git a/doc/release-notes/9739-url-validator.md b/doc/release-notes/9739-url-validator.md deleted file mode 100644 index ad149c54459..00000000000 --- a/doc/release-notes/9739-url-validator.md +++ /dev/null @@ -1,7 +0,0 @@ -## Release Highlights - -### URL validation is more permissive - -Url validation now allows two slashes in the path component of the URL. (#9750) -Among other things, this allows metadata fields of `url` type to be filled with more complex url such as https://archive.softwareheritage.org/browse/directory/561bfe6698ca9e58b552b4eb4e56132cac41c6f9/?origin_url=https://github.com/gem-pasteur/macsyfinder&revision=868637fce184865d8e0436338af66a2648e8f6e1&snapshot=1bde3cb370766b10132c4e004c7cb377979928d1 - diff --git a/doc/release-notes/9887-new-superuser-status-endpoint.md b/doc/release-notes/9887-new-superuser-status-endpoint.md deleted file mode 100644 index 01b1f539f7a..00000000000 --- a/doc/release-notes/9887-new-superuser-status-endpoint.md +++ /dev/null @@ -1 +0,0 @@ -The existing API endpoint for toggling the superuser status of a user has been deprecated in favor of a new API endpoint that allows you to explicitly and idempotently set the status as true or false. For details, see [the guides](https://dataverse-guide--10440.org.readthedocs.build/en/10440/api/native-api.html), #9887 and #10440. \ No newline at end of file diff --git a/doc/release-notes/solr-9.4.1.md b/doc/release-notes/solr-9.4.1.md deleted file mode 100644 index 13624a272ab..00000000000 --- a/doc/release-notes/solr-9.4.1.md +++ /dev/null @@ -1,14 +0,0 @@ -Solr 9.4.1 is now the version recommended in our installation guides and used with automated testing. There is a known security issue in the previously recommended version 9.3.0: https://nvd.nist.gov/vuln/detail/CVE-2023-36478. While the risk of an exploit should not be significant unless the Solr instance is accessible from the outside networks (which we have always recommended against), existing Dataverse installations should consider upgrading. - -For the upgrade instructions section: - -[note that 6.3 will contain other solr-related changes, so the instructions may need to contain information merged from multiple release notes!] - -If you are upgrading Solr: - - Install solr-9.4.1 following the instructions from the Installation guide. - - Run a full reindex to populate the search catalog. - - Note that it may be possible to skip the reindexing step by simply moving the existing `.../server/solr/collection1/` under the new `solr-9.4.1` installation directory. This however has not been thoroughly tested and is not officially supported. - - - - diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 2d0dc714132..75ee5a51f90 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -585,6 +585,8 @@ The fully expanded example above (without environment variables) looks like this Note: you must have "Add Dataset" permission in the given collection to invoke this endpoint. +.. _featured-collections: + List Featured Collections for a Dataverse Collection ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From c9538ab20fd745d5ffce65300a60f7c74d36cc31 Mon Sep 17 00:00:00 2001 From: diptechexpo <78333467+diptechexpo@users.noreply.github.com> Date: Sat, 6 Jul 2024 10:17:32 -0500 Subject: [PATCH 141/145] Added dataverse.db.parameters from long list of JVM options Addressing documentation issue #10669. Updated the config.rst file under doc/sphinx-guides/source/installation to add missing documentation about dataverse.db.parameters under JVM Options section --- doc/sphinx-guides/source/installation/config.rst | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 9e4a5e0ee7b..57bf2d72c81 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -781,7 +781,7 @@ Basic Database Settings - | ``dataverse`` | (installer sets to ``dvndb``) * - dataverse.db.parameters - - Connection parameters, such as ``sslmode=require``. See `Postgres JDBC docs `_ + - Connection parameters, such as ``sslmode=require``. See `Postgres JDBC docs ` Note: you don't need to provide the initial "?". - *Empty string* @@ -2442,6 +2442,15 @@ Defaults to ``5432``, the default PostgreSQL port. Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_DB_PORT``. +dataverse.db.parameters ++++++++++++++++++++++++ + +The PostgreSQL server connection parameters. + +Defaults to *Empty string* + +Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_DB_PARAMETERS``. + .. _dataverse.solr.host: dataverse.solr.host From 5fc0df576344e3d6707a5b95a297a3ff39f2a615 Mon Sep 17 00:00:00 2001 From: diptechexpo <78333467+diptechexpo@users.noreply.github.com> Date: Sat, 6 Jul 2024 14:48:30 -0500 Subject: [PATCH 142/145] Update doc/sphinx-guides/source/installation/config.rst Co-authored-by: Oliver Bertuch --- doc/sphinx-guides/source/installation/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 57bf2d72c81..4b1ac1f1521 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2447,7 +2447,7 @@ dataverse.db.parameters The PostgreSQL server connection parameters. -Defaults to *Empty string* +Defaults to *empty string* Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_DB_PARAMETERS``. From 86815ecd3d0c1aa62286d9112eeedbc31dc22709 Mon Sep 17 00:00:00 2001 From: diptechexpo <78333467+diptechexpo@users.noreply.github.com> Date: Sat, 6 Jul 2024 14:56:05 -0500 Subject: [PATCH 143/145] Update config.rst Added back underscore(_) character to maintain the link. --- doc/sphinx-guides/source/installation/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 4b1ac1f1521..ba92a4180ae 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -781,7 +781,7 @@ Basic Database Settings - | ``dataverse`` | (installer sets to ``dvndb``) * - dataverse.db.parameters - - Connection parameters, such as ``sslmode=require``. See `Postgres JDBC docs ` + - Connection parameters, such as ``sslmode=require``. See `Postgres JDBC docs `_ Note: you don't need to provide the initial "?". - *Empty string* From bfcebb879e8e7d343464e73447084f22662591a4 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 10 Jul 2024 12:05:17 -0400 Subject: [PATCH 144/145] tweak docs and add release note #10606 --- .../10606-dataverse-in-windows-wsl.md | 1 + .../source/developers/windows.rst | 45 +++++++++++-------- 2 files changed, 27 insertions(+), 19 deletions(-) create mode 100644 doc/release-notes/10606-dataverse-in-windows-wsl.md diff --git a/doc/release-notes/10606-dataverse-in-windows-wsl.md b/doc/release-notes/10606-dataverse-in-windows-wsl.md new file mode 100644 index 00000000000..9501d6e3090 --- /dev/null +++ b/doc/release-notes/10606-dataverse-in-windows-wsl.md @@ -0,0 +1 @@ +New instructions have been added for developers on Windows trying to run a Dataverse development environment using Windows Subsystem for Linux (WSL). See https://dataverse-guide--10608.org.readthedocs.build/en/10608/developers/windows.html #10606 and #10608. diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst index 54a30e95aef..699b64c1e1f 100755 --- a/doc/sphinx-guides/source/developers/windows.rst +++ b/doc/sphinx-guides/source/developers/windows.rst @@ -12,19 +12,23 @@ Running Dataverse in Docker on Windows See the `post `_ by Akio Sone for additional details, but please observe the following: -- In git, the line-ending setting should be set to always LF (line feed, ``core.autocrlf=input``) - You must have jq installed: https://jqlang.github.io/jq/download/ +- In git, the line-ending setting should be set to always LF (line feed, ``core.autocrlf=input``). Update: This should have been fixed by https://github.com/IQSS/dataverse/pull/10092. -One the above is all set you can move on to :doc:`/container/dev-usage` in the Container Guide. +Once the above is all set you can move on to :doc:`/container/dev-usage` in the Container Guide. + +Generally speaking, if you're having trouble running a Dataverse dev environment in Docker on Windows, you are highly encouraged to post about it in the #containers channel on Zulip (https://chat.dataverse.org) and join a Containerization Working Group meeting (https://ct.gdcc.io). See also :doc:`/container/intro` in the Container Guide. Running Dataverse in Windows WSL -------------------------------- -It is possible to run Dataverse in Windows 10 and 11 through WSL (Windows subsystem for Linux) +It is possible to run Dataverse in Windows 10 and 11 through WSL (Windows Subsystem for Linux). + +Please note: these instructions have not been extensively tested. If you find any problems, please open an issue at https://github.com/IQSS/dataverse/issues. Install WSL ~~~~~~~~~~~ -If you have Docker already installed, you should already have WSL installed, otherwise open PowerShell and run: +If you have Docker already installed, you should already have WSL installed. Otherwise open PowerShell and run: .. code-block:: powershell @@ -45,13 +49,13 @@ Choose the distribution you would like. Then run the following command. These in wsl --install -d You will be asked to create a Linux user. -After the installation of Linux is complete, check that you have an internet connection: +After the installation of Linux is complete, check that you have an Internet connection: .. code-block:: bash ping www.google.com -If you do not have an internet connection try adding it in ``/etc/wsl.conf`` +If you do not have an Internet connection, try adding it in ``/etc/wsl.conf`` .. code-block:: bash @@ -75,32 +79,35 @@ Install Dataverse ~~~~~~~~~~~~~~~~~ Now you can install Dataverse in WSL following the instructions for :doc:`classic-dev-env` -At the end check that you have ``-Ddataverse.pid.default-provider=fake`` in jvm-options. +At the end, check that you have ``-Ddataverse.pid.default-provider=fake`` in jvm-options. Now you can access Dataverse in your Windows browser (Edge, Chrome, etc.): - - http://localhost:8080 - - username: dataverseAdmin - - password: admin +- http://localhost:8080 +- username: dataverseAdmin +- password: admin IDE for Dataverse in Windows ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Files in WSL are accessible from Windows for editing using ``\\wsl.localhost`` or ``\\wsl$`` path. Windows files are accessible under Linux in the ``/mnt/c/`` directory. Therefore one can use one's favorite editor or IDE to edit Dataverse project files. Then one can build using ``mvn`` in WSL and deploy manually in WSL using ``asadmin``. -It is still though possible to use a full-strength IDE. The following instructions are for Intellij users. +It is still though possible to use a full-strength IDE. The following instructions are for IntelliJ users. - Install Intelij in Windows. -You can open the project through ``\\wsl.localhost`` and navigate to Dataverse project. -You can try to build the project in Intellij. You may get a message ``Cannot establish network connection from WSL to Windows host (could be blocked by firewall).`` In that case you can try -to disable WSL Hyperviser from firewall. -After that you should be able to build the project in Intellij. -It seems that at present it is impossible to deploy the Glassfish application in Intellij. You can try to add Glassfish plugin through Settings->Plugins and in Run->Edit Configurations configure Application Server from WSL ``/usr/localhost/payara6`` with URL http://localhost:8080 and Server Domain as domain1, but it may fail since Intellij confuses the Windows and Linux paths. +You can open the project through ``\\wsl.localhost`` and navigate to the Dataverse project. +You can try to build the project in IntelliJ. You may get a message ``Cannot establish network connection from WSL to Windows host (could be blocked by the firewall).`` In that case you can try +to disable WSL Hyperviser from the firewall. +After that you should be able to build the project in IntelliJ. +It seems that at present it is impossible to deploy the Glassfish application in IntelliJ. You can try to add a Glassfish plugin through Settings->Plugins and in Run->Edit Configurations configure Application Server from WSL ``/usr/localhost/payara6`` with URL http://localhost:8080 and Server Domain as domain1, but it may fail since IntelliJ confuses the Windows and Linux paths. -To use the full strength of Intelij with build, deployment and debugging, one will need to use Intelij ``Remote development``. Close all the projects in Intellij and go to ``Remote development->WSL`` and press ``New Project``. In WSL instance choose your Linux distribution and press ``Next``. In ``Prpject Directory`` navigate to WSL Dataverse project. Then press ``Download IDE and Connect``. This will install Intellij in WSL in ``~/.cache/JetBrains/``. Now in Intellij you should see your project opened in a new Intellij window. After adding Glassfish plugin and editing configuration you should be able to build the project and run the project. +To use the full strength of Intelij with build, deployment and debugging, one will need to use Intelij ``Remote development``. Close all the projects in IntelliJ and go to ``Remote development->WSL`` and press ``New Project``. In WSL instance choose your Linux distribution and press ``Next``. In ``Project Directory`` navigate to WSL Dataverse project. Then press ``Download IDE and Connect``. This will install IntelliJ in WSL in ``~/.cache/JetBrains/``. Now in IntelliJ you should see your project opened in a new IntelliJ window. After adding the Glassfish plugin and editing your configuration you should be able to build the project and run the project. -PgAdmin in Windows for Dataverse +pgAdmin in Windows for Dataverse ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -You can access The Dataverse database from Windows. Install pgAdmin https://www.pgadmin.org/download/pgadmin-4-windows/ In pgAdmin register a server using 127.0.0.1 with port 5432, database dvndb and dvnapp as username with secret password. Now you will be able to access and update Dataverse database. +You can access the Dataverse database from Windows. + +Install pgAdmin from https://www.pgadmin.org/download/pgadmin-4-windows/ +In pgAdmin, register a server using 127.0.0.1 with port 5432, database dvndb and dvnapp as username with secret password. Now you will be able to access and update the Dataverse database. From 5ba74e8a8e75fecec7f4b477abb920453212c53e Mon Sep 17 00:00:00 2001 From: Ben Companjen Date: Fri, 12 Jul 2024 16:28:28 +0200 Subject: [PATCH 145/145] Compare classes, not classnames in tests (#9014) * Use instanceof to compare types, not class names * Use instanceof in FileDataProviderFactoryTest Instead of comparing class names, the tests assert that `result instanceof Class` (for each appropriate class). --- src/test/java/edu/harvard/iq/dataverse/api/SwordIT.java | 2 +- .../util/bagit/data/FileDataProviderFactoryTest.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SwordIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SwordIT.java index 4df6c89411d..518431bfa2d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SwordIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SwordIT.java @@ -462,7 +462,7 @@ public void testCreateAndDeleteDatasetInRoot() { assertNull(attemptToGetFileId); } catch (Exception ex) { System.out.println("We expect an exception here because we can no longer find the file because deleted it: " + ex); - assertTrue(ex.getClass().getName().equals(ArrayIndexOutOfBoundsException.class.getName())); + assertTrue(ex instanceof ArrayIndexOutOfBoundsException); } String newTitle = "A New Hope"; diff --git a/src/test/java/edu/harvard/iq/dataverse/util/bagit/data/FileDataProviderFactoryTest.java b/src/test/java/edu/harvard/iq/dataverse/util/bagit/data/FileDataProviderFactoryTest.java index f43a0c78284..9fac4d42bcd 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/bagit/data/FileDataProviderFactoryTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/bagit/data/FileDataProviderFactoryTest.java @@ -23,21 +23,21 @@ public class FileDataProviderFactoryTest { public void should_return_FolderDataProvider_when_parameter_is_path() { FileDataProvider result = target.getFileDataProvider(Path.of(UUID.randomUUID().toString())); - MatcherAssert.assertThat(result.getClass().getName(), Matchers.is(FolderDataProvider.class.getName())); + MatcherAssert.assertThat("should return FolderDataProvider when parameter is path", result instanceof FolderDataProvider); } @Test public void should_return_ZipFileDataProvider_when_parameter_is_file() throws IOException { FileDataProvider result = target.getFileDataProvider(Path.of(FIXTURE_DIRECTORY, "FileDataProviderFactoryTest.zip").toFile()); - MatcherAssert.assertThat(result.getClass().getName(), Matchers.is(ZipFileDataProvider.class.getName())); + MatcherAssert.assertThat("should return ZipFileDataProvider when parameter is file", result instanceof ZipFileDataProvider); } @Test public void should_return_DataFileDataProvider_when_parameter_is_datafiles() { FileDataProvider result = target.getFileDataProvider("test-name", Collections.emptyList()); - MatcherAssert.assertThat(result.getClass().getName(), Matchers.is(DataFileDataProvider.class.getName())); + MatcherAssert.assertThat("should return DataFileDataProvider when parameter is datafiles", result instanceof DataFileDataProvider); } } \ No newline at end of file