Merge branch 'master' into dependabot/maven/joda-time-joda-time-2.12.7

digital-preservation · Apr 17, 2024 · 5b79165 · 5b79165
2 parents 03e1b36 + 8a4318c
commit 5b79165
Show file tree

Hide file tree

Showing 62 changed files with 3,840 additions and 761 deletions.
diff --git a/droid-api/pom.xml b/droid-api/pom.xml
@@ -4,7 +4,7 @@
 	<parent>
 		<artifactId>droid-parent</artifactId>
 		<groupId>uk.gov.nationalarchives</groupId>
-		<version>6.7.1-SNAPSHOT</version>
+		<version>6.8.0-SNAPSHOT</version>
 		<relativePath>../droid-parent</relativePath>
 	</parent>
 

diff --git a/droid-binary/pom.xml b/droid-binary/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <artifactId>droid-parent</artifactId>
         <groupId>uk.gov.nationalarchives</groupId>
-        <version>6.7.1-SNAPSHOT</version>
+        <version>6.8.0-SNAPSHOT</version>
   	    <relativePath>../droid-parent</relativePath>
     </parent>
 
@@ -24,7 +24,7 @@
             <plugin>
                 <groupId>com.googlecode.maven-download-plugin</groupId>
                 <artifactId>download-maven-plugin</artifactId>
-                <version>1.6.8</version>
+                <version>1.9.0</version>
                 <executions>
                     <execution>
                         <id>download-files-windows</id>

diff --git a/droid-build-tools/pom.xml b/droid-build-tools/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <artifactId>droid-parent</artifactId>
         <groupId>uk.gov.nationalarchives</groupId>
-        <version>6.7.1-SNAPSHOT</version>
+        <version>6.8.0-SNAPSHOT</version>
         <relativePath>../droid-parent</relativePath>
     </parent>
 

diff --git a/droid-command-line/pom.xml b/droid-command-line/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <artifactId>droid-parent</artifactId>
         <groupId>uk.gov.nationalarchives</groupId>
-        <version>6.7.1-SNAPSHOT</version>
+        <version>6.8.0-SNAPSHOT</version>
         <relativePath>../droid-parent</relativePath>
     </parent>
 

diff --git a/...d-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandFactoryImpl.java b/...d-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandFactoryImpl.java
@@ -115,6 +115,10 @@ public DroidCommand getExportFileCommand(final CommandLine cli) throws CommandLi
             cmd.setColumnsToWrite(columns);
         }
 
+        if (cli.hasOption(CommandLineParam.EXPORT_TEMPLATE.getLongName())) {
+            cmd.setExportTemplate(cli.getOptionValue(CommandLineParam.EXPORT_TEMPLATE.getLongName()));
+        }
+
         if (cli.hasOption(CommandLineParam.ALL_FILTER.toString())) {
             cmd.setFilter(createFilter(cli.getOptionValues(CommandLineParam.ALL_FILTER.toString()), true));
         }
@@ -153,6 +157,10 @@ public DroidCommand getExportFormatCommand(final CommandLine cli) throws Command
             cmd.setColumnsToWrite(columns);
         }
 
+        if (cli.hasOption(CommandLineParam.EXPORT_TEMPLATE.getLongName())) {
+            cmd.setExportTemplate(cli.getOptionValue(CommandLineParam.EXPORT_TEMPLATE.getLongName()));
+        }
+
         if (cli.hasOption(CommandLineParam.ALL_FILTER.toString())) {
             cmd.setFilter(createFilter(cli.getOptionValues(CommandLineParam.ALL_FILTER.toString()), true));
         }

diff --git a/...and-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandLineParam.java b/...and-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandLineParam.java
@@ -237,6 +237,16 @@ public DroidCommand getCommand(CommandFactory commandFactory, CommandLine cli) {
         }
     },
 
+    /**
+     * Specifies the absolute path for the export template to be used.
+     */
+    EXPORT_TEMPLATE("et", "export-template", true, -1, I18N.EXPORT_TEMPLATE_HELP, "template-file") {
+        @Override
+        public DroidCommand getCommand(CommandFactory commandFactory, CommandLine cli) {
+            return null;
+        }
+    },
+
     /**
      * Specifies that a row per identification should be written out in a CSV file or console output.
      */
@@ -503,6 +513,7 @@ public static Options options() {
 
         options.addOptionGroup(getFilterOptionGroup());
         options.addOptionGroup(getFileFilterOptionGroup());
+        options.addOptionGroup(getExportOptionGroup());
         options.addOptionGroup(topGroup);
 
         return options;
@@ -523,6 +534,13 @@ private static OptionGroup getFilterOptionGroup() {
         return filterOptions;
     }
 
+    private static OptionGroup getExportOptionGroup() {
+        OptionGroup exportOptions = new OptionGroup();
+        exportOptions.addOption(COLUMNS_TO_WRITE.newOption());
+        exportOptions.addOption(EXPORT_TEMPLATE.newOption());
+        return exportOptions;
+    }
+
     /**
      * Single Options.
      *
@@ -603,6 +621,7 @@ public static Options exportSubOptions() {
         options.addOption(BOM.newOption());
         options.addOption(QUOTE_COMMAS.newOption());
         options.addOption(COLUMNS_TO_WRITE.newOption());
+        options.addOption(EXPORT_TEMPLATE.newOption());
         return options;
     }
 

diff --git a/...ommand-line/src/main/java/uk/gov/nationalarchives/droid/command/action/ExportCommand.java b/...ommand-line/src/main/java/uk/gov/nationalarchives/droid/command/action/ExportCommand.java
@@ -39,6 +39,7 @@
 import java.util.concurrent.Future;
 
 import uk.gov.nationalarchives.droid.core.interfaces.filter.Filter;
+import uk.gov.nationalarchives.droid.export.interfaces.ExportDetails;
 import uk.gov.nationalarchives.droid.export.interfaces.ExportManager;
 import uk.gov.nationalarchives.droid.export.interfaces.ExportOptions;
 import uk.gov.nationalarchives.droid.profile.ProfileInstance;
@@ -60,7 +61,9 @@ public class ExportCommand implements DroidCommand {
     private boolean bom;
     private boolean quoteAllFields = true;
     private String columnsToWrite;
-
+
+    private String exportTemplate;
+
     /**
      * {@inheritDoc}
      */
@@ -86,9 +89,7 @@ public void onProgress(Integer progress) {
         // Run the export
         try {
             //default to UTF-8
-            final String outputEncoding = "UTF-8"; //TODO set encoding from command line option
-            final Future<?> fProfiles = exportManager.exportProfiles(profileIds, destination, filter,
-                    options, outputEncoding, bom, quoteAllFields, columnsToWrite);
+            final Future<?> fProfiles = exportManager.exportProfiles(profileIds, destination, filter, getExportDetails());
             fProfiles.get();
         } catch (InterruptedException e) {
             throw new CommandExecutionException(e);
@@ -218,4 +219,37 @@ public void setColumnsToWrite(String columnNames) {
     public String getColumnsToWrite() {
         return columnsToWrite;
     }
+
+    /**
+     * @return Absolute path of export template.
+     */
+    public String getExportTemplate() {
+        return exportTemplate;
+    }
+
+    /**
+     * @param exportTemplate Absolute path of export template.
+     */
+    public void setExportTemplate(String exportTemplate) {
+        this.exportTemplate = exportTemplate;
+    }
+
+
+    /**
+     *
+     * @return the export details for this export command.
+     * For an export from CLI,
+     * OutputEncoding is always defaulted to UTF-8
+     */
+    private ExportDetails getExportDetails() {
+        ExportDetails.ExportDetailsBuilder builder = new ExportDetails.ExportDetailsBuilder();
+
+        return builder.withExportOptions(getExportOptions())
+                .withOutputEncoding("UTF-8") //default
+                .withBomFlag(isBom())
+                .withQuotingAllFields(getQuoteAllFields())
+                .withColumnsToWrite(getColumnsToWrite())
+                .withExportTemplatePath(getExportTemplate())
+                .build();
+    }
 }
diff --git a/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/i18n/I18N.java b/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/i18n/I18N.java
@@ -71,6 +71,11 @@ public final class I18N {
      */
     public static final String COLUMNS_TO_WRITE_HELP = "profile.columnsToWrite.help";
 
+    /**
+     * Sets the absolute path of an export template.
+     */
+    public static final String EXPORT_TEMPLATE_HELP = "profile.exportTemplate.help";
+
     /**
      * Sets CSV output to write a row per format (rather than a row per file which is the default).
      */
@@ -221,5 +226,4 @@ public static String getResource(final String key, Object... params) {
         String pattern = getResource(key);
         return MessageFormat.format(pattern, params);
     }
-
 }
diff --git a/droid-command-line/src/main/resources/options.properties b/droid-command-line/src/main/resources/options.properties
@@ -92,7 +92,8 @@ filter.field.help=List the available fields to use in filters and the operators
 profile.run.help=Add resources to a new profile and run it.  Resources are the file path of any file or folder you want to profile.  The file paths should be given surrounded in double quotes, and separated by spaces from each other.  The profile results will be saved to a single file specified using the -p option. \n For example: droid -a "C:\\Files\\A Folder" "C:\\Files\\file.xxx" -p "C:\\Results\\result1.droid" \n Note: You cannot use reporting, filtering and exporting when using the -a option.
 profile.outputfile.help=Outputs a profile as a CSV file to the path supplied.  If "stdout" is specified, then output goes to the console.  If no profile or output file is specified, then output defaults to the console.
 profile.quoteCommasOnly.help=Sets CSV output to only quote fields that have a comma in them.
-profile.columnsToWrite.help=A space separated list of columns to write out in CSV output. Valid columns are:\nID PARENT_ID URI FILE_PATH NAME METHOD STATUS SIZE TYPE EXT LAST_MODIFIED EXTENSION_MISMATCH HASH FORMAT_COUNT PUID MIME_TYPE FORMAT_NAME FORMAT_VERSION
+profile.columnsToWrite.help=[Optional] A space separated list of columns to write out in CSV output. Valid columns are:\nID PARENT_ID URI FILE_PATH NAME METHOD STATUS SIZE TYPE EXT LAST_MODIFIED EXTENSION_MISMATCH HASH FORMAT_COUNT PUID MIME_TYPE FORMAT_NAME FORMAT_VERSION. If omitted, all columns are exported. 
+profile.exportTemplate.help=[Optional] Absolute path to the export template file to be used for this export. If omitted, export falls back to -co option.
 profile.rowsPerFormat.help=Outputs a row per format for CSV, rather than a row per file which is the default.
 profile.run.file.help=Adds resources to a new profile which is outputted to a CSV file (or console).  Resources are the file path of any file or folder you want to profile.  The file paths should be given surrounded in double quotes, and separated by spaces from each other.  The profile results will be saved to a single file specified using the -p option. \n For example: droid -Na "C:\\Files\\A Folder" "C:\\Files\\file.xxx" \n Note: You cannot use reporting, filtering and exporting when using the -Na option.
 no_profile.run.help=Identify either a specific file, or all files in a folder, without the use of a profile.  The file or folder path should be bounded by double quotes.  The scan results will be sent to standard output. \n For example: droid -Nr "C:\\Files\\A Folder" \n Note: You cannot use reporting, filtering and exporting when using the -Nr option.

diff --git a/...nd-line/src/test/java/uk/gov/nationalarchives/droid/command/action/ExportCommandTest.java b/...nd-line/src/test/java/uk/gov/nationalarchives/droid/command/action/ExportCommandTest.java
@@ -41,6 +41,7 @@
 import uk.gov.nationalarchives.droid.core.interfaces.filter.CriterionOperator;
 import uk.gov.nationalarchives.droid.core.interfaces.filter.Filter;
 import uk.gov.nationalarchives.droid.core.interfaces.filter.FilterCriterion;
+import uk.gov.nationalarchives.droid.export.interfaces.ExportDetails;
 import uk.gov.nationalarchives.droid.export.interfaces.ExportManager;
 import uk.gov.nationalarchives.droid.export.interfaces.ExportOptions;
 import uk.gov.nationalarchives.droid.profile.ProfileInstance;
@@ -55,6 +56,7 @@
 
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.ArgumentMatchers.isNull;
@@ -92,7 +94,7 @@ public void testExportThreeProfiles() throws Exception {
         when(profileManager.open(eq(Paths.get("foo3")), any(ProgressObserver.class))).thenReturn(profile3);
 
         Future future = mock(Future.class);
-        when(exportManager.exportProfiles(any(List.class), eq(destination), (Filter) isNull(), eq(ExportOptions.ONE_ROW_PER_FORMAT), eq("UTF-8"), eq(false), eq(true), eq(null))).thenReturn(future);
+        when(exportManager.exportProfiles(any(List.class), eq(destination), (Filter) isNull(), any(ExportDetails.class))).thenReturn(future);
 
         ExportCommand command = new ExportCommand();
 
@@ -108,8 +110,18 @@ public void testExportThreeProfiles() throws Exception {
         String[] expectedExportedProfiles = new String[] {
             "profile1", "profile2", "profile3",
         };
-
-        verify(exportManager).exportProfiles(Arrays.asList(expectedExportedProfiles), destination, null, ExportOptions.ONE_ROW_PER_FORMAT, "UTF-8", false, true, null);
+
+        ArgumentCaptor<ExportDetails> exportDetailsCaptor = ArgumentCaptor.forClass(ExportDetails.class);
+        ArgumentCaptor<Filter> filterCaptor = ArgumentCaptor.forClass(Filter.class);
+
+        verify(exportManager).exportProfiles(eq(Arrays.asList(expectedExportedProfiles)), eq(destination), filterCaptor.capture(), exportDetailsCaptor.capture());
+
+        ExportDetails details = exportDetailsCaptor.getValue();
+        assertEquals(ExportOptions.ONE_ROW_PER_FORMAT, details.getExportOptions());
+        assertEquals("UTF-8", details.getOutputEncoding());
+        assertEquals(false, details.bomFlag());
+        assertEquals(true, details.quoteAllFields());
+        assertNull(details.getColumnsToWrite());
     }
 
     @Test
@@ -124,7 +136,7 @@ public void testExportProfileWithNarrowFilter() throws Exception {
         when(profileManager.open(eq(Paths.get("foo1")), any(ProgressObserver.class))).thenReturn(profile1);
 
         Future future = mock(Future.class);
-        when(exportManager.exportProfiles(any(List.class), eq("destination"), any(Filter.class), eq(ExportOptions.ONE_ROW_PER_FORMAT), any(String.class), eq(false), eq(true), eq(null))).thenReturn(future);
+        when(exportManager.exportProfiles(any(List.class), eq("destination"), any(Filter.class), any(ExportDetails.class))).thenReturn(future);
 
         ExportCommand command = new ExportCommand();
 
@@ -146,8 +158,15 @@ public void testExportProfileWithNarrowFilter() throws Exception {
         };
 
         ArgumentCaptor<Filter> filterCaptor = ArgumentCaptor.forClass(Filter.class);
+        ArgumentCaptor<ExportDetails> exportDetailsCaptor = ArgumentCaptor.forClass(ExportDetails.class);
         verify(exportManager).exportProfiles(eq(Arrays.asList(expectedExportedProfiles)), 
-                eq("destination"), filterCaptor.capture(), eq(ExportOptions.ONE_ROW_PER_FORMAT), any(String.class), eq(false), eq(true), eq(null));
+                eq("destination"), filterCaptor.capture(), exportDetailsCaptor.capture());
+
+        ExportDetails details = exportDetailsCaptor.getValue();
+        assertEquals(ExportOptions.ONE_ROW_PER_FORMAT, details.getExportOptions());
+        assertEquals(false, details.bomFlag());
+        assertEquals(true, details.quoteAllFields());
+        assertNull(details.getColumnsToWrite());
 
         Filter filter = filterCaptor.getValue();
         final List<FilterCriterion> criteria = filter.getCriteria();

diff --git a/droid-container/pom.xml b/droid-container/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <artifactId>droid-parent</artifactId>
         <groupId>uk.gov.nationalarchives</groupId>
-        <version>6.7.1-SNAPSHOT</version>
+        <version>6.8.0-SNAPSHOT</version>
   	    <relativePath>../droid-parent</relativePath>
     </parent>
 
@@ -79,9 +79,14 @@
             <artifactId>commons-configuration</artifactId>
         </dependency>
         <dependency>
-            <groupId>commons-httpclient</groupId>
-            <artifactId>commons-httpclient</artifactId>
-            <version>3.1</version>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <version>4.5.14</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpcore</artifactId>
+            <version>4.4.16</version>
         </dependency>
         <dependency>
             <groupId>net.java.truevfs</groupId>