Implementation for the export template work. (#1081)

* Initial commit for export templates. Added template classes for basic wiring and changed writer to use maps instead of lists to use column names as we will be unable to rely cannot rely on column order

* Column order through template implmented for per file as well as per format export

* Created column def classes and used them through into csv writer

* Multiple column defs and fixed a bug in existing code where blank values were not written for additional headers with suffix

* Header labels used from the template

* Constant string data column writing functionality added to template writing

* DataModifier column added and wired up in the backend

* UI and CLI wired up to convert individual params into wrapper object to redue number of parameters on ExportProfiles method

* Added panels to UI and made the UI selectable between columns and templates. Wired up template selection to template use for export

* Added panels to UI and made the UI selectable between columns and templates. Wired up template selection to template use for export

* Constant string data added to the per format export

* UI updated to adjust right hand panel border

* Few corrections and added a new class for combobox sorting

* default selection code for combo box moved to the consumer of ComboBox

* Refactored the logic for data writing into separate classes to get cleaner separation

* Minor name changes

* Added javadoc and removed some hardcoded, unused lines.

* Added unit test for combo box and some ui changes.

* Updated a few javadoc comments

* Updated a few javadoc comments

* missing algorithm added

* New screenshot and explanation in the help pages

* removed the commented leftover code

* Added CLI support

* More changes CLI for export template

* Documentation update

* Fixed NPE when underlying data was null, added few tests

* documentation link mismatch fixed for CLI

* blank space header was not producing error.

* Few unit tests related to colon in the values.

* Modified to throw clearer error messages

* Modified to throw clearer error messages

* clarified help for export template

* Typo corrected

* Changes following code review

---------

Co-authored-by: Sam Palmer <[email protected]>

droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandFactoryImpl.java

@@ -115,6 +115,10 @@ public DroidCommand getExportFileCommand(final CommandLine cli) throws CommandLi
             cmd.setColumnsToWrite(columns);
         }
 
+        if (cli.hasOption(CommandLineParam.EXPORT_TEMPLATE.getLongName())) {
+            cmd.setExportTemplate(cli.getOptionValue(CommandLineParam.EXPORT_TEMPLATE.getLongName()));
+        }
+
         if (cli.hasOption(CommandLineParam.ALL_FILTER.toString())) {
             cmd.setFilter(createFilter(cli.getOptionValues(CommandLineParam.ALL_FILTER.toString()), true));
         }
@@ -153,6 +157,10 @@ public DroidCommand getExportFormatCommand(final CommandLine cli) throws Command
             cmd.setColumnsToWrite(columns);
         }
 
+        if (cli.hasOption(CommandLineParam.EXPORT_TEMPLATE.getLongName())) {
+            cmd.setExportTemplate(cli.getOptionValue(CommandLineParam.EXPORT_TEMPLATE.getLongName()));
+        }
+
         if (cli.hasOption(CommandLineParam.ALL_FILTER.toString())) {
             cmd.setFilter(createFilter(cli.getOptionValues(CommandLineParam.ALL_FILTER.toString()), true));
         }

droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandLineParam.java

@@ -237,6 +237,16 @@ public DroidCommand getCommand(CommandFactory commandFactory, CommandLine cli) {
         }
     },
 
+    /**
+     * Specifies the absolute path for the export template to be used.
+     */
+    EXPORT_TEMPLATE("et", "export-template", true, -1, I18N.EXPORT_TEMPLATE_HELP, "template-file") {
+        @Override
+        public DroidCommand getCommand(CommandFactory commandFactory, CommandLine cli) {
+            return null;
+        }
+    },
+
     /**
      * Specifies that a row per identification should be written out in a CSV file or console output.
      */
@@ -503,6 +513,7 @@ public static Options options() {
 
         options.addOptionGroup(getFilterOptionGroup());
         options.addOptionGroup(getFileFilterOptionGroup());
+        options.addOptionGroup(getExportOptionGroup());
         options.addOptionGroup(topGroup);
 
         return options;
@@ -523,6 +534,13 @@ private static OptionGroup getFilterOptionGroup() {
         return filterOptions;
     }
 
+    private static OptionGroup getExportOptionGroup() {
+        OptionGroup exportOptions = new OptionGroup();
+        exportOptions.addOption(COLUMNS_TO_WRITE.newOption());
+        exportOptions.addOption(EXPORT_TEMPLATE.newOption());
+        return exportOptions;
+    }
+
     /**
      * Single Options.
      *
@@ -603,6 +621,7 @@ public static Options exportSubOptions() {
         options.addOption(BOM.newOption());
         options.addOption(QUOTE_COMMAS.newOption());
         options.addOption(COLUMNS_TO_WRITE.newOption());
+        options.addOption(EXPORT_TEMPLATE.newOption());
         return options;
     }
 

droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/ExportCommand.java

@@ -39,6 +39,7 @@
 import java.util.concurrent.Future;
 
 import uk.gov.nationalarchives.droid.core.interfaces.filter.Filter;
+import uk.gov.nationalarchives.droid.export.interfaces.ExportDetails;
 import uk.gov.nationalarchives.droid.export.interfaces.ExportManager;
 import uk.gov.nationalarchives.droid.export.interfaces.ExportOptions;
 import uk.gov.nationalarchives.droid.profile.ProfileInstance;
@@ -60,7 +61,9 @@ public class ExportCommand implements DroidCommand {
     private boolean bom;
     private boolean quoteAllFields = true;
     private String columnsToWrite;
-
+
+    private String exportTemplate;
+
     /**
      * {@inheritDoc}
      */
@@ -86,9 +89,7 @@ public void onProgress(Integer progress) {
         // Run the export
         try {
             //default to UTF-8
-            final String outputEncoding = "UTF-8"; //TODO set encoding from command line option
-            final Future<?> fProfiles = exportManager.exportProfiles(profileIds, destination, filter,
-                    options, outputEncoding, bom, quoteAllFields, columnsToWrite);
+            final Future<?> fProfiles = exportManager.exportProfiles(profileIds, destination, filter, getExportDetails());
             fProfiles.get();
         } catch (InterruptedException e) {
             throw new CommandExecutionException(e);
@@ -218,4 +219,37 @@ public void setColumnsToWrite(String columnNames) {
     public String getColumnsToWrite() {
         return columnsToWrite;
     }
+
+    /**
+     * @return Absolute path of export template.
+     */
+    public String getExportTemplate() {
+        return exportTemplate;
+    }
+
+    /**
+     * @param exportTemplate Absolute path of export template.
+     */
+    public void setExportTemplate(String exportTemplate) {
+        this.exportTemplate = exportTemplate;
+    }
+
+
+    /**
+     *
+     * @return the export details for this export command.
+     * For an export from CLI,
+     * OutputEncoding is always defaulted to UTF-8
+     */
+    private ExportDetails getExportDetails() {
+        ExportDetails.ExportDetailsBuilder builder = new ExportDetails.ExportDetailsBuilder();
+
+        return builder.withExportOptions(getExportOptions())
+                .withOutputEncoding("UTF-8") //default
+                .withBomFlag(isBom())
+                .withQuotingAllFields(getQuoteAllFields())
+                .withColumnsToWrite(getColumnsToWrite())
+                .withExportTemplatePath(getExportTemplate())
+                .build();
+    }
 }

droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/i18n/I18N.java

@@ -71,6 +71,11 @@ public final class I18N {
      */
     public static final String COLUMNS_TO_WRITE_HELP = "profile.columnsToWrite.help";
 
+    /**
+     * Sets the absolute path of an export template.
+     */
+    public static final String EXPORT_TEMPLATE_HELP = "profile.exportTemplate.help";
+
     /**
      * Sets CSV output to write a row per format (rather than a row per file which is the default).
      */
@@ -221,5 +226,4 @@ public static String getResource(final String key, Object... params) {
         String pattern = getResource(key);
         return MessageFormat.format(pattern, params);
     }
-
 }

droid-command-line/src/main/resources/options.properties

@@ -92,7 +92,8 @@ filter.field.help=List the available fields to use in filters and the operators
 profile.run.help=Add resources to a new profile and run it.  Resources are the file path of any file or folder you want to profile.  The file paths should be given surrounded in double quotes, and separated by spaces from each other.  The profile results will be saved to a single file specified using the -p option. \n For example: droid -a "C:\\Files\\A Folder" "C:\\Files\\file.xxx" -p "C:\\Results\\result1.droid" \n Note: You cannot use reporting, filtering and exporting when using the -a option.
 profile.outputfile.help=Outputs a profile as a CSV file to the path supplied.  If "stdout" is specified, then output goes to the console.  If no profile or output file is specified, then output defaults to the console.
 profile.quoteCommasOnly.help=Sets CSV output to only quote fields that have a comma in them.
-profile.columnsToWrite.help=A space separated list of columns to write out in CSV output. Valid columns are:\nID PARENT_ID URI FILE_PATH NAME METHOD STATUS SIZE TYPE EXT LAST_MODIFIED EXTENSION_MISMATCH HASH FORMAT_COUNT PUID MIME_TYPE FORMAT_NAME FORMAT_VERSION
+profile.columnsToWrite.help=[Optional] A space separated list of columns to write out in CSV output. Valid columns are:\nID PARENT_ID URI FILE_PATH NAME METHOD STATUS SIZE TYPE EXT LAST_MODIFIED EXTENSION_MISMATCH HASH FORMAT_COUNT PUID MIME_TYPE FORMAT_NAME FORMAT_VERSION. If omitted, all columns are exported. 
+profile.exportTemplate.help=[Optional] Absolute path to the export template file to be used for this export. If omitted, export falls back to -co option.
 profile.rowsPerFormat.help=Outputs a row per format for CSV, rather than a row per file which is the default.
 profile.run.file.help=Adds resources to a new profile which is outputted to a CSV file (or console).  Resources are the file path of any file or folder you want to profile.  The file paths should be given surrounded in double quotes, and separated by spaces from each other.  The profile results will be saved to a single file specified using the -p option. \n For example: droid -Na "C:\\Files\\A Folder" "C:\\Files\\file.xxx" \n Note: You cannot use reporting, filtering and exporting when using the -Na option.
 no_profile.run.help=Identify either a specific file, or all files in a folder, without the use of a profile.  The file or folder path should be bounded by double quotes.  The scan results will be sent to standard output. \n For example: droid -Nr "C:\\Files\\A Folder" \n Note: You cannot use reporting, filtering and exporting when using the -Nr option.

droid-command-line/src/test/java/uk/gov/nationalarchives/droid/command/action/ExportCommandTest.java

@@ -41,6 +41,7 @@
 import uk.gov.nationalarchives.droid.core.interfaces.filter.CriterionOperator;
 import uk.gov.nationalarchives.droid.core.interfaces.filter.Filter;
 import uk.gov.nationalarchives.droid.core.interfaces.filter.FilterCriterion;
+import uk.gov.nationalarchives.droid.export.interfaces.ExportDetails;
 import uk.gov.nationalarchives.droid.export.interfaces.ExportManager;
 import uk.gov.nationalarchives.droid.export.interfaces.ExportOptions;
 import uk.gov.nationalarchives.droid.profile.ProfileInstance;
@@ -55,6 +56,7 @@
 
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.ArgumentMatchers.isNull;
@@ -92,7 +94,7 @@ public void testExportThreeProfiles() throws Exception {
         when(profileManager.open(eq(Paths.get("foo3")), any(ProgressObserver.class))).thenReturn(profile3);
 
         Future future = mock(Future.class);
-        when(exportManager.exportProfiles(any(List.class), eq(destination), (Filter) isNull(), eq(ExportOptions.ONE_ROW_PER_FORMAT), eq("UTF-8"), eq(false), eq(true), eq(null))).thenReturn(future);
+        when(exportManager.exportProfiles(any(List.class), eq(destination), (Filter) isNull(), any(ExportDetails.class))).thenReturn(future);
 
         ExportCommand command = new ExportCommand();
 
@@ -108,8 +110,18 @@ public void testExportThreeProfiles() throws Exception {
         String[] expectedExportedProfiles = new String[] {
             "profile1", "profile2", "profile3",
         };
-
-        verify(exportManager).exportProfiles(Arrays.asList(expectedExportedProfiles), destination, null, ExportOptions.ONE_ROW_PER_FORMAT, "UTF-8", false, true, null);
+
+        ArgumentCaptor<ExportDetails> exportDetailsCaptor = ArgumentCaptor.forClass(ExportDetails.class);
+        ArgumentCaptor<Filter> filterCaptor = ArgumentCaptor.forClass(Filter.class);
+
+        verify(exportManager).exportProfiles(eq(Arrays.asList(expectedExportedProfiles)), eq(destination), filterCaptor.capture(), exportDetailsCaptor.capture());
+
+        ExportDetails details = exportDetailsCaptor.getValue();
+        assertEquals(ExportOptions.ONE_ROW_PER_FORMAT, details.getExportOptions());
+        assertEquals("UTF-8", details.getOutputEncoding());
+        assertEquals(false, details.bomFlag());
+        assertEquals(true, details.quoteAllFields());
+        assertNull(details.getColumnsToWrite());
     }
 
     @Test
@@ -124,7 +136,7 @@ public void testExportProfileWithNarrowFilter() throws Exception {
         when(profileManager.open(eq(Paths.get("foo1")), any(ProgressObserver.class))).thenReturn(profile1);
 
         Future future = mock(Future.class);
-        when(exportManager.exportProfiles(any(List.class), eq("destination"), any(Filter.class), eq(ExportOptions.ONE_ROW_PER_FORMAT), any(String.class), eq(false), eq(true), eq(null))).thenReturn(future);
+        when(exportManager.exportProfiles(any(List.class), eq("destination"), any(Filter.class), any(ExportDetails.class))).thenReturn(future);
 
         ExportCommand command = new ExportCommand();
 
@@ -146,8 +158,15 @@ public void testExportProfileWithNarrowFilter() throws Exception {
         };
 
         ArgumentCaptor<Filter> filterCaptor = ArgumentCaptor.forClass(Filter.class);
+        ArgumentCaptor<ExportDetails> exportDetailsCaptor = ArgumentCaptor.forClass(ExportDetails.class);
         verify(exportManager).exportProfiles(eq(Arrays.asList(expectedExportedProfiles)), 
-                eq("destination"), filterCaptor.capture(), eq(ExportOptions.ONE_ROW_PER_FORMAT), any(String.class), eq(false), eq(true), eq(null));
+                eq("destination"), filterCaptor.capture(), exportDetailsCaptor.capture());
+
+        ExportDetails details = exportDetailsCaptor.getValue();
+        assertEquals(ExportOptions.ONE_ROW_PER_FORMAT, details.getExportOptions());
+        assertEquals(false, details.bomFlag());
+        assertEquals(true, details.quoteAllFields());
+        assertNull(details.getColumnsToWrite());
 
         Filter filter = filterCaptor.getValue();
         final List<FilterCriterion> criteria = filter.getCriteria();

droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/config/DroidGlobalConfig.java

@@ -95,7 +95,9 @@ public class DroidGlobalConfig {
     private Path textSignatureFileDir;
     private Path reportDefinitionDir;
     private Path filterDir;
-
+
+    private Path exportTemplatesDir;
+
     private PropertiesConfiguration props;
     private PropertiesConfiguration defaultProps;
 
@@ -160,6 +162,9 @@ public DroidGlobalConfig() throws IOException {
 
         tempDir = Paths.get(droidTempPath, "tmp");
         Files.createDirectories(tempDir);
+
+        exportTemplatesDir = droidWorkDir.resolve("export_templates");
+        Files.createDirectories(exportTemplatesDir);
     }
 
     /**
@@ -357,7 +362,12 @@ public Path getFilterDir() {
     public Path getTempDir() {
         return tempDir;
     }
-
+
+    /**
+     * @return directory for the export templates.
+     */
+    public Path getExportTemplatesDir() { return exportTemplatesDir; }
+
     private void createResourceFile(final Path resourceDir, final String fileName, final String resourceName) throws IOException {
         final Path resourcefile = resourceDir.resolve(fileName);
         if (!Files.exists(resourcefile)) {