6.8.x release

README.md

@@ -1,7 +1,6 @@
 # DROID (Digital Record Object Identification) 
 
 [![CI](https://github.com/digital-preservation/droid/workflows/CI/badge.svg)](https://github.com/digital-preservation/droid/actions?query=workflow%3ACI)
-[![Build status](https://ci.appveyor.com/api/projects/status/hrr6c3ckbghjvd7h/branch/master?svg=true)](https://ci.appveyor.com/project/dpreservation/droid/branch/master)
 [![Coverage Status](https://coveralls.io/repos/github/digital-preservation/droid/badge.svg?branch=master)](https://coveralls.io/github/digital-preservation/droid?branch=master)
 [![Maven Central](https://maven-badges.herokuapp.com/maven-central/uk.gov.nationalarchives/droid/badge.svg)](https://search.maven.org/search?q=g:uk.gov.nationalarchives)
 
@@ -111,17 +110,17 @@ Once the code is cloned into a folder (e.g. `droid`), executing `mvn clean insta
 
 ### Linux / OSX users
 
-You will need JAVA 8 to 11 installed to run DROID.
+You will need JAVA 8 to 17 installed to run DROID.
 
-Unpack the archive `droid-binary-${VERSION}-bin-unix.zip`, then use the `droid.sh` script to run the application.
+Unpack the archive `droid-binary-${VERSION}-bin.zip`, then use the `droid.sh` script to run the application.
 
 ### Windows users
 Archive  `droid-binary-${VERSION}-bin-win64-with-jre.zip`
 
 
-You will need JAVA 8 to 11 installed to run DROID. For Windows users who might not be able to install JAVA, the provided bundle includes JAVA 11.
+You will need JAVA 8 to 17 installed to run DROID. For Windows users who might not be able to install JAVA, the provided bundle includes JAVA 17.
 
-Unpack the archive `droid-binary-${VERSION}-bin-win32-with-jre.zip`, then use the `droid.bat` script to run the application.
+Unpack the archive `droid-binary-${VERSION}-bin-win64-with-jre.zip`, then use the `droid.bat` script to run the application.
 
 ## Signatures
 

droid-api/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<artifactId>droid-parent</artifactId>
 		<groupId>uk.gov.nationalarchives</groupId>
-		<version>6.8.0-SNAPSHOT</version>
+		<version>6.8.1-SNAPSHOT</version>
 		<relativePath>../droid-parent</relativePath>
 	</parent>
 

droid-binary/bin/ChangeLog

@@ -1,3 +1,14 @@
+Version 6.8.0
+ * Feature: Export Templates - Ability to customise the Droid CSV export by defining templates
+ * #932 Improvement: Prompt for a new profile creation on closing the preferences dialog
+ * #938 Improvement: The Windows distributable now includes a 64 bit Java 17 JRE
+ * #1063 Improvement: Updated commons-httpclient dependecy used for signature download
+ * #561 Bugfix: On Windows, when a profile is closed, the associated profile folder is not deleted
+ * #1072 Bugfix: Empty columns are not populated with a blank when a file has more than one format identification
+ * #1011 Bugfix: Mismatch in the format name between GUI and API when identification method is 'Container'
+ * #1100 Bugfix: Cancelling the preferences dialog still makes changes to the preferences
+ * #1072 Bugfix: Fewer elements in data row when exporting profiles per file when some of them have more than one identification
+
 Version 6.7.0
 * #898 Improvement: Sigtool uses the version of java bundled with Droid on the Windows OS
 * #938 Improvement: Internal API result includes whether there is an extension mismatch in identification

droid-binary/bin/droid.bat

@@ -94,14 +94,14 @@ REM ---------------
 REM This is the maximum memory DROID can use in megabytes.
 REM Remove the "REM " from the line below and set the maximum memory after the "=".
 REM Also configure this property using the environment variable: droidMemory.
-REM SET droidMemory=512
+REM SET droidMemory=1024
 
 
 
 REM Assemble options
 REM ================
-REM Default to using 512 megabytes of memory if no other settings provided:
-SET DROID_OPTIONS="-Xmx512m"
+REM Default to using 1024 megabytes of memory if no other settings provided:
+SET DROID_OPTIONS="-Xmx1024m"
 
 IF "%droidMemory%"=="" GOTO UserDir
 SET DROID_OPTIONS="-Xmx%droidMemory%m"

droid-binary/bin/droid.sh

@@ -86,7 +86,7 @@ logLevel=""
 # Max memory: droidMemory
 # -----------------------
 # The maximum memory for DROID to use in megabytes.
-droidMemory="512m"
+droidMemory="1024m"
 
 
 # Run DROID:

droid-binary/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <artifactId>droid-parent</artifactId>
         <groupId>uk.gov.nationalarchives</groupId>
-        <version>6.8.0-SNAPSHOT</version>
+        <version>6.8.1-SNAPSHOT</version>
   	    <relativePath>../droid-parent</relativePath>
     </parent>
 
@@ -34,7 +34,7 @@
                         </goals>
                         <configuration>
                             <!-- To update, retrieve link for 64 bit Windows jre from https://adoptium.net -->
-                            <url>https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.9%2B9.1/OpenJDK17U-jre_x64_windows_hotspot_17.0.9_9.zip</url>
+                            <url>https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.11%2B9/OpenJDK17U-jre_x64_windows_hotspot_17.0.11_9.zip</url>
                             <unpack>false</unpack>
                             <outputDirectory>${project.build.directory}</outputDirectory>
                         </configuration>
@@ -55,9 +55,9 @@
                         <configuration>
                             <target>
                                 <!-- Update binary from https://adoptopenjdk.net/releases.html -->
-                                <unzip dest="${project.build.directory}/jre_tmp/" src="${project.build.directory}/OpenJDK17U-jre_x64_windows_hotspot_17.0.9_9.zip" />
+                                <unzip dest="${project.build.directory}/jre_tmp/" src="${project.build.directory}/OpenJDK17U-jre_x64_windows_hotspot_17.0.11_9.zip" />
                                 <move todir="${project.build.directory}/jre-windows/">
-                                    <fileset dir="${project.build.directory}/jre_tmp/jdk-17.0.9+9-jre/">
+                                    <fileset dir="${project.build.directory}/jre_tmp/jdk-17.0.11+9-jre/">
                                         <include name="**/*" />
                                     </fileset>
                                 </move>

droid-build-tools/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <artifactId>droid-parent</artifactId>
         <groupId>uk.gov.nationalarchives</groupId>
-        <version>6.8.0-SNAPSHOT</version>
+        <version>6.8.1-SNAPSHOT</version>
         <relativePath>../droid-parent</relativePath>
     </parent>
 

droid-command-line/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <artifactId>droid-parent</artifactId>
         <groupId>uk.gov.nationalarchives</groupId>
-        <version>6.8.0-SNAPSHOT</version>
+        <version>6.8.1-SNAPSHOT</version>
         <relativePath>../droid-parent</relativePath>
     </parent>
 

droid-container/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <artifactId>droid-parent</artifactId>
         <groupId>uk.gov.nationalarchives</groupId>
-        <version>6.8.0-SNAPSHOT</version>
+        <version>6.8.1-SNAPSHOT</version>
   	    <relativePath>../droid-parent</relativePath>
     </parent>
 

droid-core-interfaces/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <artifactId>droid-parent</artifactId>
         <groupId>uk.gov.nationalarchives</groupId>
-        <version>6.8.0-SNAPSHOT</version>
+        <version>6.8.1-SNAPSHOT</version>
         <relativePath>../droid-parent</relativePath>
     </parent>
 

droid-core/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <artifactId>droid-parent</artifactId>
         <groupId>uk.gov.nationalarchives</groupId>
-        <version>6.8.0-SNAPSHOT</version>
+        <version>6.8.1-SNAPSHOT</version>
         <relativePath>../droid-parent</relativePath>
     </parent>
 

droid-export-interfaces/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <artifactId>droid-parent</artifactId>
         <groupId>uk.gov.nationalarchives</groupId>
-        <version>6.8.0-SNAPSHOT</version>
+        <version>6.8.1-SNAPSHOT</version>
    <relativePath>../droid-parent</relativePath>
     </parent>
 

droid-export-interfaces/src/main/java/uk/gov/nationalarchives/droid/export/interfaces/ExportTemplate.java

@@ -35,11 +35,11 @@
 
 /**
  * ExportTemplate interface.
- * There is only one method which returns a map of integer -> ExportTemplateColumnDef
+ * There is only one method which returns a map of integer - ExportTemplateColumnDef
  */
 public interface ExportTemplate {
     /**
-     * Get the map of Integer -> ExportTemplateColumnDef representing column order.
+     * Get the map of Integer - ExportTemplateColumnDef representing column order.
      * @return Map with keys as column order and values as column definition
      */
     Map<Integer, ExportTemplateColumnDef> getColumnOrderMap() ;

droid-export/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <artifactId>droid-parent</artifactId>
         <groupId>uk.gov.nationalarchives</groupId>
-        <version>6.8.0-SNAPSHOT</version>
+        <version>6.8.1-SNAPSHOT</version>
         <relativePath>../droid-parent</relativePath>
     </parent>
 

droid-export/src/main/java/uk/gov/nationalarchives/droid/export/template/ExportTemplateBuilder.java

@@ -111,15 +111,25 @@ private Map<Integer, ExportTemplateColumnDef> parseExportTemplateV1(List<String>
 
             if (token2.startsWith(DATA_COLUMN_PREFIX)) {
                 columnMap.put(i, createProfileNodeDef(header, token2));
-            } else if ((token2.isEmpty()) || (token2.startsWith(DOUBLE_QUOTES))) {
-                columnMap.put(i, createConstantStringDef(header, token2));
-            } else {
+            } else if (isExpressionForDataModification(token2)) {
                 columnMap.put(i, createDataModifierDef(header, token2));
+            } else {
+                columnMap.put(i, createConstantStringDef(header, token2));
             }
         }
         return columnMap;
     }
 
+    private boolean isExpressionForDataModification(String expressionParam) {
+        List<String> operations = Arrays.stream(
+                        ExportTemplateColumnDef.DataModification.values()).map(v -> v.toString() + OPENING_BRACKET).
+                collect(Collectors.toList());
+        String expression = expressionParam.trim();
+
+        List<String> possibleOperations = operations.stream().filter(op -> expression.startsWith(op)).collect(Collectors.toList());
+        return possibleOperations.size() > 0;
+    }
+
     private ExportTemplateColumnDef createDataModifierDef(String header, String param2) {
 
         assertDataModifierSyntaxValid(param2);
@@ -168,10 +178,11 @@ private ExportTemplateColumnDef createConstantStringDef(String header, String pa
         if (param2.isEmpty()) {
             return new ConstantStringColumnDef("", header);
         } else {
-            if (!param2.endsWith(DOUBLE_QUOTES)) {
-                throw new ExportTemplateParseException("The line with a constant value ('" + param2 + "') in template definition does not have closing quotes");
+            if (param2.startsWith(DOUBLE_QUOTES) && param2.endsWith(DOUBLE_QUOTES)) {
+                return new ConstantStringColumnDef(param2.substring(1, param2.length() - 1), header);
+            } else {
+                throw new ExportTemplateParseException("The line with a constant value ('" + param2 + "') is not fully enclosed in quotes");
             }
-            return new ConstantStringColumnDef(param2.substring(1, param2.length() - 1), header);
         }
     }
 

droid-export/src/test/java/uk/gov/nationalarchives/droid/export/template/ExportTemplateBuilderTest.java

@@ -109,6 +109,7 @@ public void should_treat_missing_value_as_empty_string_constant_in_export_templa
         ExportTemplate template = builder.buildExportTemplate(tempFile.getAbsolutePath());
         assertNotNull(template);
         assertTrue(template.getColumnOrderMap().get(1) instanceof ConstantStringColumnDef);
+        assertEquals("", template.getColumnOrderMap().get(1).getDataValue());
         assertTrue(template.getColumnOrderMap().get(2) instanceof DataModifierColumnDef);
         assertEquals("FILE_PATH", template.getColumnOrderMap().get(2).getOriginalColumnName());
     }
@@ -166,7 +167,37 @@ public void should_throw_an_exception_if_the_constant_string_value_does_not_have
         Files.write(tempFile.toPath(), data, StandardOpenOption.WRITE);
 
         ExportTemplateParseException ex = assertThrows(ExportTemplateParseException.class, () -> builder.buildExportTemplate(tempFile.getAbsolutePath()));
-        assertEquals("The line with a constant value ('\"English') in template definition does not have closing quotes", ex.getMessage());
+        assertEquals("The line with a constant value ('\"English') is not fully enclosed in quotes", ex.getMessage());
+    }
+
+    @Test
+    public void should_throw_an_exception_if_the_constant_string_value_does_not_have_opening_double_quotes() throws IOException {
+        ExportTemplateBuilder builder = new ExportTemplateBuilder();
+        File tempFile = temporaryFolder.newFile("export-task-test-default-encoding");
+        List<String> data = Arrays.asList(
+                "version 1.0",
+                "",
+                "Language: English\"",
+                "");
+        Files.write(tempFile.toPath(), data, StandardOpenOption.WRITE);
+
+        ExportTemplateParseException ex = assertThrows(ExportTemplateParseException.class, () -> builder.buildExportTemplate(tempFile.getAbsolutePath()));
+        assertEquals("The line with a constant value ('English\"') is not fully enclosed in quotes", ex.getMessage());
+    }
+
+    @Test
+    public void should_throw_an_exception_if_the_constant_string_value_does_not_have_fully_enclosing_quotes() throws IOException {
+        ExportTemplateBuilder builder = new ExportTemplateBuilder();
+        File tempFile = temporaryFolder.newFile("export-task-test-default-encoding");
+        List<String> data = Arrays.asList(
+                "version 1.0",
+                "",
+                "Language: En\"gli\"sh",
+                "");
+        Files.write(tempFile.toPath(), data, StandardOpenOption.WRITE);
+
+        ExportTemplateParseException ex = assertThrows(ExportTemplateParseException.class, () -> builder.buildExportTemplate(tempFile.getAbsolutePath()));
+        assertEquals("The line with a constant value ('En\"gli\"sh') is not fully enclosed in quotes", ex.getMessage());
     }
 
     @Test
@@ -196,7 +227,7 @@ public void should_throw_an_exception_when_the_operation_is_unknown() throws IOE
         Files.write(tempFile.toPath(), data, StandardOpenOption.WRITE);
 
         ExportTemplateParseException ex = assertThrows(ExportTemplateParseException.class, () -> builder.buildExportTemplate(tempFile.getAbsolutePath()));
-        assertEquals("Undefined operation 'Lower' encountered in export template", ex.getMessage());
+        assertEquals("The line with a constant value ('Lower($ID)') is not fully enclosed in quotes", ex.getMessage());
     }
 
     @Test
@@ -228,6 +259,21 @@ public void should_support_colon_in_the_constant_string_value() throws IOExcepti
         assertEquals("http://www.knowingwhere.com", template.getColumnOrderMap().get(0).getDataValue());
     }
 
+    @Test
+    public void should_support_constant_value_which_includes_operation_name() throws IOException {
+        ExportTemplateBuilder builder = new ExportTemplateBuilder();
+        File tempFile = temporaryFolder.newFile("export-task-test-default-encoding");
+        List<String> data = Arrays.asList(
+                "version 1.0",
+                "     MyWebsite     : \"LCASE($ID)\"",
+                "");
+        Files.write(tempFile.toPath(), data, StandardOpenOption.WRITE);
+
+        ExportTemplate template =  builder.buildExportTemplate(tempFile.getAbsolutePath());
+        assertEquals("MyWebsite", template.getColumnOrderMap().get(0).getHeaderLabel());
+        assertEquals("LCASE($ID)", template.getColumnOrderMap().get(0).getDataValue());
+    }
+
     @Test
     public void should_throw_an_exception_when_an_operation_cannot_be_located() throws IOException {
         ExportTemplateBuilder builder = new ExportTemplateBuilder();
@@ -239,7 +285,7 @@ public void should_throw_an_exception_when_an_operation_cannot_be_located() thro
         Files.write(tempFile.toPath(), data, StandardOpenOption.WRITE);
 
         ExportTemplateParseException ex = assertThrows(ExportTemplateParseException.class, () -> builder.buildExportTemplate(tempFile.getAbsolutePath()));
-        assertEquals("Invalid syntax in data modifier expression 'Crown Copyright (C)', expecting '$' after '('", ex.getMessage());
+        assertEquals("The line with a constant value ('Crown Copyright (C)') is not fully enclosed in quotes", ex.getMessage());
     }
 
     @Test
@@ -253,7 +299,7 @@ public void should_throw_an_exception_when_a_constant_is_not_enclosed_in_double_
         Files.write(tempFile.toPath(), data, StandardOpenOption.WRITE);
 
         ExportTemplateParseException ex = assertThrows(ExportTemplateParseException.class, () -> builder.buildExportTemplate(tempFile.getAbsolutePath()));
-        assertEquals("Invalid syntax in data modifier expression 'Crown Copyright', expecting exactly one occurrence of '('", ex.getMessage());
+        assertEquals("The line with a constant value ('Crown Copyright') is not fully enclosed in quotes", ex.getMessage());
     }
 
     @Test

droid-help/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <artifactId>droid-parent</artifactId>
         <groupId>uk.gov.nationalarchives</groupId>
-        <version>6.8.0-SNAPSHOT</version>
+        <version>6.8.1-SNAPSHOT</version>
   	    <relativePath>../droid-parent</relativePath>
     </parent>