diff --git a/droid-api/pom.xml b/droid-api/pom.xml index 4176f046f..70098f8be 100644 --- a/droid-api/pom.xml +++ b/droid-api/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.7.1-SNAPSHOT + 6.8.0-SNAPSHOT ../droid-parent diff --git a/droid-binary/pom.xml b/droid-binary/pom.xml index 736138e1a..26cebe13b 100644 --- a/droid-binary/pom.xml +++ b/droid-binary/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.7.1-SNAPSHOT + 6.8.0-SNAPSHOT ../droid-parent @@ -24,7 +24,7 @@ com.googlecode.maven-download-plugin download-maven-plugin - 1.6.8 + 1.9.0 download-files-windows diff --git a/droid-build-tools/pom.xml b/droid-build-tools/pom.xml index 8dc61eefe..ab6a4c2e6 100644 --- a/droid-build-tools/pom.xml +++ b/droid-build-tools/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.7.1-SNAPSHOT + 6.8.0-SNAPSHOT ../droid-parent diff --git a/droid-command-line/pom.xml b/droid-command-line/pom.xml index 946d9d538..b287f52fc 100644 --- a/droid-command-line/pom.xml +++ b/droid-command-line/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.7.1-SNAPSHOT + 6.8.0-SNAPSHOT ../droid-parent diff --git a/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandFactoryImpl.java b/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandFactoryImpl.java index def0293b0..af5c9cd10 100644 --- a/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandFactoryImpl.java +++ b/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandFactoryImpl.java @@ -115,6 +115,10 @@ public DroidCommand getExportFileCommand(final CommandLine cli) throws CommandLi cmd.setColumnsToWrite(columns); } + if (cli.hasOption(CommandLineParam.EXPORT_TEMPLATE.getLongName())) { + cmd.setExportTemplate(cli.getOptionValue(CommandLineParam.EXPORT_TEMPLATE.getLongName())); + } + if (cli.hasOption(CommandLineParam.ALL_FILTER.toString())) { cmd.setFilter(createFilter(cli.getOptionValues(CommandLineParam.ALL_FILTER.toString()), true)); } @@ -153,6 +157,10 @@ public DroidCommand getExportFormatCommand(final CommandLine cli) throws Command cmd.setColumnsToWrite(columns); } + if (cli.hasOption(CommandLineParam.EXPORT_TEMPLATE.getLongName())) { + cmd.setExportTemplate(cli.getOptionValue(CommandLineParam.EXPORT_TEMPLATE.getLongName())); + } + if (cli.hasOption(CommandLineParam.ALL_FILTER.toString())) { cmd.setFilter(createFilter(cli.getOptionValues(CommandLineParam.ALL_FILTER.toString()), true)); } diff --git a/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandLineParam.java b/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandLineParam.java index 6794ea2fd..89c51349e 100644 --- a/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandLineParam.java +++ b/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandLineParam.java @@ -237,6 +237,16 @@ public DroidCommand getCommand(CommandFactory commandFactory, CommandLine cli) { } }, + /** + * Specifies the absolute path for the export template to be used. + */ + EXPORT_TEMPLATE("et", "export-template", true, -1, I18N.EXPORT_TEMPLATE_HELP, "template-file") { + @Override + public DroidCommand getCommand(CommandFactory commandFactory, CommandLine cli) { + return null; + } + }, + /** * Specifies that a row per identification should be written out in a CSV file or console output. */ @@ -503,6 +513,7 @@ public static Options options() { options.addOptionGroup(getFilterOptionGroup()); options.addOptionGroup(getFileFilterOptionGroup()); + options.addOptionGroup(getExportOptionGroup()); options.addOptionGroup(topGroup); return options; @@ -523,6 +534,13 @@ private static OptionGroup getFilterOptionGroup() { return filterOptions; } + private static OptionGroup getExportOptionGroup() { + OptionGroup exportOptions = new OptionGroup(); + exportOptions.addOption(COLUMNS_TO_WRITE.newOption()); + exportOptions.addOption(EXPORT_TEMPLATE.newOption()); + return exportOptions; + } + /** * Single Options. * @@ -603,6 +621,7 @@ public static Options exportSubOptions() { options.addOption(BOM.newOption()); options.addOption(QUOTE_COMMAS.newOption()); options.addOption(COLUMNS_TO_WRITE.newOption()); + options.addOption(EXPORT_TEMPLATE.newOption()); return options; } diff --git a/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/ExportCommand.java b/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/ExportCommand.java index ac200e9a8..7ab872022 100644 --- a/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/ExportCommand.java +++ b/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/ExportCommand.java @@ -39,6 +39,7 @@ import java.util.concurrent.Future; import uk.gov.nationalarchives.droid.core.interfaces.filter.Filter; +import uk.gov.nationalarchives.droid.export.interfaces.ExportDetails; import uk.gov.nationalarchives.droid.export.interfaces.ExportManager; import uk.gov.nationalarchives.droid.export.interfaces.ExportOptions; import uk.gov.nationalarchives.droid.profile.ProfileInstance; @@ -60,7 +61,9 @@ public class ExportCommand implements DroidCommand { private boolean bom; private boolean quoteAllFields = true; private String columnsToWrite; - + + private String exportTemplate; + /** * {@inheritDoc} */ @@ -86,9 +89,7 @@ public void onProgress(Integer progress) { // Run the export try { //default to UTF-8 - final String outputEncoding = "UTF-8"; //TODO set encoding from command line option - final Future fProfiles = exportManager.exportProfiles(profileIds, destination, filter, - options, outputEncoding, bom, quoteAllFields, columnsToWrite); + final Future fProfiles = exportManager.exportProfiles(profileIds, destination, filter, getExportDetails()); fProfiles.get(); } catch (InterruptedException e) { throw new CommandExecutionException(e); @@ -218,4 +219,37 @@ public void setColumnsToWrite(String columnNames) { public String getColumnsToWrite() { return columnsToWrite; } + + /** + * @return Absolute path of export template. + */ + public String getExportTemplate() { + return exportTemplate; + } + + /** + * @param exportTemplate Absolute path of export template. + */ + public void setExportTemplate(String exportTemplate) { + this.exportTemplate = exportTemplate; + } + + + /** + * + * @return the export details for this export command. + * For an export from CLI, + * OutputEncoding is always defaulted to UTF-8 + */ + private ExportDetails getExportDetails() { + ExportDetails.ExportDetailsBuilder builder = new ExportDetails.ExportDetailsBuilder(); + + return builder.withExportOptions(getExportOptions()) + .withOutputEncoding("UTF-8") //default + .withBomFlag(isBom()) + .withQuotingAllFields(getQuoteAllFields()) + .withColumnsToWrite(getColumnsToWrite()) + .withExportTemplatePath(getExportTemplate()) + .build(); + } } diff --git a/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/i18n/I18N.java b/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/i18n/I18N.java index 61cfce787..456c0f50f 100644 --- a/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/i18n/I18N.java +++ b/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/i18n/I18N.java @@ -71,6 +71,11 @@ public final class I18N { */ public static final String COLUMNS_TO_WRITE_HELP = "profile.columnsToWrite.help"; + /** + * Sets the absolute path of an export template. + */ + public static final String EXPORT_TEMPLATE_HELP = "profile.exportTemplate.help"; + /** * Sets CSV output to write a row per format (rather than a row per file which is the default). */ @@ -221,5 +226,4 @@ public static String getResource(final String key, Object... params) { String pattern = getResource(key); return MessageFormat.format(pattern, params); } - } diff --git a/droid-command-line/src/main/resources/options.properties b/droid-command-line/src/main/resources/options.properties index 8fc34fcc9..5f0765329 100644 --- a/droid-command-line/src/main/resources/options.properties +++ b/droid-command-line/src/main/resources/options.properties @@ -92,7 +92,8 @@ filter.field.help=List the available fields to use in filters and the operators profile.run.help=Add resources to a new profile and run it. Resources are the file path of any file or folder you want to profile. The file paths should be given surrounded in double quotes, and separated by spaces from each other. The profile results will be saved to a single file specified using the -p option. \n For example: droid -a "C:\\Files\\A Folder" "C:\\Files\\file.xxx" -p "C:\\Results\\result1.droid" \n Note: You cannot use reporting, filtering and exporting when using the -a option. profile.outputfile.help=Outputs a profile as a CSV file to the path supplied. If "stdout" is specified, then output goes to the console. If no profile or output file is specified, then output defaults to the console. profile.quoteCommasOnly.help=Sets CSV output to only quote fields that have a comma in them. -profile.columnsToWrite.help=A space separated list of columns to write out in CSV output. Valid columns are:\nID PARENT_ID URI FILE_PATH NAME METHOD STATUS SIZE TYPE EXT LAST_MODIFIED EXTENSION_MISMATCH HASH FORMAT_COUNT PUID MIME_TYPE FORMAT_NAME FORMAT_VERSION +profile.columnsToWrite.help=[Optional] A space separated list of columns to write out in CSV output. Valid columns are:\nID PARENT_ID URI FILE_PATH NAME METHOD STATUS SIZE TYPE EXT LAST_MODIFIED EXTENSION_MISMATCH HASH FORMAT_COUNT PUID MIME_TYPE FORMAT_NAME FORMAT_VERSION. If omitted, all columns are exported. +profile.exportTemplate.help=[Optional] Absolute path to the export template file to be used for this export. If omitted, export falls back to -co option. profile.rowsPerFormat.help=Outputs a row per format for CSV, rather than a row per file which is the default. profile.run.file.help=Adds resources to a new profile which is outputted to a CSV file (or console). Resources are the file path of any file or folder you want to profile. The file paths should be given surrounded in double quotes, and separated by spaces from each other. The profile results will be saved to a single file specified using the -p option. \n For example: droid -Na "C:\\Files\\A Folder" "C:\\Files\\file.xxx" \n Note: You cannot use reporting, filtering and exporting when using the -Na option. no_profile.run.help=Identify either a specific file, or all files in a folder, without the use of a profile. The file or folder path should be bounded by double quotes. The scan results will be sent to standard output. \n For example: droid -Nr "C:\\Files\\A Folder" \n Note: You cannot use reporting, filtering and exporting when using the -Nr option. diff --git a/droid-command-line/src/test/java/uk/gov/nationalarchives/droid/command/action/ExportCommandTest.java b/droid-command-line/src/test/java/uk/gov/nationalarchives/droid/command/action/ExportCommandTest.java index 11f97c536..ef2241c7a 100644 --- a/droid-command-line/src/test/java/uk/gov/nationalarchives/droid/command/action/ExportCommandTest.java +++ b/droid-command-line/src/test/java/uk/gov/nationalarchives/droid/command/action/ExportCommandTest.java @@ -41,6 +41,7 @@ import uk.gov.nationalarchives.droid.core.interfaces.filter.CriterionOperator; import uk.gov.nationalarchives.droid.core.interfaces.filter.Filter; import uk.gov.nationalarchives.droid.core.interfaces.filter.FilterCriterion; +import uk.gov.nationalarchives.droid.export.interfaces.ExportDetails; import uk.gov.nationalarchives.droid.export.interfaces.ExportManager; import uk.gov.nationalarchives.droid.export.interfaces.ExportOptions; import uk.gov.nationalarchives.droid.profile.ProfileInstance; @@ -55,6 +56,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNull; @@ -92,7 +94,7 @@ public void testExportThreeProfiles() throws Exception { when(profileManager.open(eq(Paths.get("foo3")), any(ProgressObserver.class))).thenReturn(profile3); Future future = mock(Future.class); - when(exportManager.exportProfiles(any(List.class), eq(destination), (Filter) isNull(), eq(ExportOptions.ONE_ROW_PER_FORMAT), eq("UTF-8"), eq(false), eq(true), eq(null))).thenReturn(future); + when(exportManager.exportProfiles(any(List.class), eq(destination), (Filter) isNull(), any(ExportDetails.class))).thenReturn(future); ExportCommand command = new ExportCommand(); @@ -108,8 +110,18 @@ public void testExportThreeProfiles() throws Exception { String[] expectedExportedProfiles = new String[] { "profile1", "profile2", "profile3", }; - - verify(exportManager).exportProfiles(Arrays.asList(expectedExportedProfiles), destination, null, ExportOptions.ONE_ROW_PER_FORMAT, "UTF-8", false, true, null); + + ArgumentCaptor exportDetailsCaptor = ArgumentCaptor.forClass(ExportDetails.class); + ArgumentCaptor filterCaptor = ArgumentCaptor.forClass(Filter.class); + + verify(exportManager).exportProfiles(eq(Arrays.asList(expectedExportedProfiles)), eq(destination), filterCaptor.capture(), exportDetailsCaptor.capture()); + + ExportDetails details = exportDetailsCaptor.getValue(); + assertEquals(ExportOptions.ONE_ROW_PER_FORMAT, details.getExportOptions()); + assertEquals("UTF-8", details.getOutputEncoding()); + assertEquals(false, details.bomFlag()); + assertEquals(true, details.quoteAllFields()); + assertNull(details.getColumnsToWrite()); } @Test @@ -124,7 +136,7 @@ public void testExportProfileWithNarrowFilter() throws Exception { when(profileManager.open(eq(Paths.get("foo1")), any(ProgressObserver.class))).thenReturn(profile1); Future future = mock(Future.class); - when(exportManager.exportProfiles(any(List.class), eq("destination"), any(Filter.class), eq(ExportOptions.ONE_ROW_PER_FORMAT), any(String.class), eq(false), eq(true), eq(null))).thenReturn(future); + when(exportManager.exportProfiles(any(List.class), eq("destination"), any(Filter.class), any(ExportDetails.class))).thenReturn(future); ExportCommand command = new ExportCommand(); @@ -146,8 +158,15 @@ public void testExportProfileWithNarrowFilter() throws Exception { }; ArgumentCaptor filterCaptor = ArgumentCaptor.forClass(Filter.class); + ArgumentCaptor exportDetailsCaptor = ArgumentCaptor.forClass(ExportDetails.class); verify(exportManager).exportProfiles(eq(Arrays.asList(expectedExportedProfiles)), - eq("destination"), filterCaptor.capture(), eq(ExportOptions.ONE_ROW_PER_FORMAT), any(String.class), eq(false), eq(true), eq(null)); + eq("destination"), filterCaptor.capture(), exportDetailsCaptor.capture()); + + ExportDetails details = exportDetailsCaptor.getValue(); + assertEquals(ExportOptions.ONE_ROW_PER_FORMAT, details.getExportOptions()); + assertEquals(false, details.bomFlag()); + assertEquals(true, details.quoteAllFields()); + assertNull(details.getColumnsToWrite()); Filter filter = filterCaptor.getValue(); final List criteria = filter.getCriteria(); diff --git a/droid-container/pom.xml b/droid-container/pom.xml index 14b94add7..47f8ac863 100644 --- a/droid-container/pom.xml +++ b/droid-container/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.7.1-SNAPSHOT + 6.8.0-SNAPSHOT ../droid-parent @@ -79,9 +79,14 @@ commons-configuration - commons-httpclient - commons-httpclient - 3.1 + org.apache.httpcomponents + httpclient + 4.5.14 + + + org.apache.httpcomponents + httpcore + 4.4.16 net.java.truevfs diff --git a/droid-container/src/main/java/uk/gov/nationalarchives/droid/container/httpservice/ContainerSignatureHttpService.java b/droid-container/src/main/java/uk/gov/nationalarchives/droid/container/httpservice/ContainerSignatureHttpService.java index 1a132b5fa..40aab21c7 100644 --- a/droid-container/src/main/java/uk/gov/nationalarchives/droid/container/httpservice/ContainerSignatureHttpService.java +++ b/droid-container/src/main/java/uk/gov/nationalarchives/droid/container/httpservice/ContainerSignatureHttpService.java @@ -31,23 +31,18 @@ */ package uk.gov.nationalarchives.droid.container.httpservice; -import java.io.IOException; -import java.net.UnknownHostException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; - import org.apache.commons.configuration.event.ConfigurationEvent; -import org.apache.commons.httpclient.Header; -import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.HttpMethod; -import org.apache.commons.httpclient.HttpStatus; -import org.apache.commons.httpclient.methods.GetMethod; -import org.apache.commons.httpclient.util.DateParseException; -import org.apache.commons.httpclient.util.DateUtil; - +import org.apache.http.Header; +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.utils.DateUtils; +import org.apache.http.conn.routing.HttpRoutePlanner; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.DefaultProxyRoutePlanner; import uk.gov.nationalarchives.droid.core.interfaces.config.DroidGlobalConfig; import uk.gov.nationalarchives.droid.core.interfaces.config.DroidGlobalProperty; import uk.gov.nationalarchives.droid.core.interfaces.signature.ProxySettings; @@ -56,6 +51,14 @@ import uk.gov.nationalarchives.droid.core.interfaces.signature.SignatureType; import uk.gov.nationalarchives.droid.core.interfaces.signature.SignatureUpdateService; +import java.io.IOException; +import java.net.UnknownHostException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + /** * @author rflitcroft * @@ -72,7 +75,7 @@ public class ContainerSignatureHttpService implements SignatureUpdateService { + "server at\n[%s]"; private String endpointUrl; - private HttpClient client = new HttpClient(); + private CloseableHttpClient client = HttpClientBuilder.create().build(); /** * Empty bean constructor. @@ -90,13 +93,14 @@ public ContainerSignatureHttpService(String endpointUrl) { @Override public SignatureFileInfo getLatestVersion(int currentVersion) throws SignatureServiceException { - GetMethod get = new GetMethod(endpointUrl); + HttpGet get = new HttpGet(endpointUrl); try { Date versionDate = getDateFromVersion(currentVersion); - String dateString = DateUtil.formatDate(versionDate); - get.addRequestHeader("If-Modified-Since", dateString); - - int statusCode = client.executeMethod(get); + String dateString = DateUtils.formatDate(versionDate); + get.addHeader("If-Modified-Since", dateString); + + HttpResponse response = client.execute(get); + int statusCode = response.getStatusLine().getStatusCode(); if (statusCode == HttpStatus.SC_NOT_FOUND) { throw new SignatureServiceException( String.format(FILE_NOT_FOUND_404, endpointUrl)); @@ -104,7 +108,7 @@ public SignatureFileInfo getLatestVersion(int currentVersion) throws SignatureSe throw new SignatureServiceException( String.format(ERROR_MESSAGE_PATTERN, endpointUrl, statusCode)); } - int version = getVersion(get); + int version = getVersion(response); return new SignatureFileInfo(version, false, SignatureType.CONTAINER); } catch (UnknownHostException e) { throw new SignatureServiceException( @@ -116,12 +120,15 @@ public SignatureFileInfo getLatestVersion(int currentVersion) throws SignatureSe } } - private static int getVersion(HttpMethod httpMethod) throws DateParseException { + private static int getVersion(HttpResponse httpResponse) throws DateParseException { int version = 0; - Header header = httpMethod.getResponseHeader(LAST_MODIFIED_HEADER); + Header header = httpResponse.getFirstHeader(LAST_MODIFIED_HEADER); if (header != null) { String lastModified = header.getValue(); - Date lastModifiedDate = DateUtil.parseDate(lastModified); + Date lastModifiedDate = DateUtils.parseDate(lastModified); + if (lastModifiedDate == null) { + throw new DateParseException(lastModified); + } SimpleDateFormat sdf = new SimpleDateFormat(DATE_PATTERN); version = Integer.parseInt(sdf.format(lastModifiedDate)); } @@ -136,10 +143,11 @@ private static Date getDateFromVersion(int versionNumber) throws ParseException @Override public SignatureFileInfo importSignatureFile(final Path targetDir) throws SignatureServiceException { - final GetMethod get = new GetMethod(endpointUrl); + final HttpGet get = new HttpGet(endpointUrl); try { - final int statusCode = client.executeMethod(get); + final HttpResponse response = client.execute(get); + final int statusCode = response.getStatusLine().getStatusCode(); if (statusCode == HttpStatus.SC_NOT_FOUND) { throw new SignatureServiceException( String.format(FILE_NOT_FOUND_404, endpointUrl)); @@ -148,13 +156,13 @@ public SignatureFileInfo importSignatureFile(final Path targetDir) throws Signat String.format(ERROR_MESSAGE_PATTERN, endpointUrl, statusCode)); } - final int version = getVersion(get); + final int version = getVersion(response); final SignatureFileInfo signatureFileInfo = new SignatureFileInfo(version, false, SignatureType.CONTAINER); final String fileName = String.format(FILENAME_PATTERN, version); final Path targetFile = targetDir.resolve(fileName); - Files.copy(get.getResponseBodyAsStream(), targetFile); + Files.copy(response.getEntity().getContent(), targetFile); signatureFileInfo.setFile(targetFile); return signatureFileInfo; @@ -189,10 +197,11 @@ public void configurationChanged(ConfigurationEvent evt) { public void onProxyChange(ProxySettings proxySettings) { if (proxySettings.isEnabled()) { - client = new HttpClient(); - client.getHostConfiguration().setProxy(proxySettings.getProxyHost(), proxySettings.getProxyPort()); + HttpRoutePlanner proxyRoutePlanner = new DefaultProxyRoutePlanner( + new HttpHost(proxySettings.getProxyHost(), proxySettings.getProxyPort())); + client = HttpClients.custom().setRoutePlanner(proxyRoutePlanner).build(); } else { - client = new HttpClient(); + client = HttpClientBuilder.create().build(); } } diff --git a/droid-container/src/main/java/uk/gov/nationalarchives/droid/container/httpservice/DateParseException.java b/droid-container/src/main/java/uk/gov/nationalarchives/droid/container/httpservice/DateParseException.java new file mode 100644 index 000000000..9dce54a50 --- /dev/null +++ b/droid-container/src/main/java/uk/gov/nationalarchives/droid/container/httpservice/DateParseException.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.container.httpservice; + +public class DateParseException extends RuntimeException { + private static final String COULD_NOT_PARSE_DATE = "Could not parse date string %s"; + + public DateParseException() { + this(""); + } + + public DateParseException(String unparseableString) { + super(String.format(COULD_NOT_PARSE_DATE, unparseableString)); + } + } diff --git a/droid-core-interfaces/pom.xml b/droid-core-interfaces/pom.xml index 4d5ab3c19..20734fad5 100644 --- a/droid-core-interfaces/pom.xml +++ b/droid-core-interfaces/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.7.1-SNAPSHOT + 6.8.0-SNAPSHOT ../droid-parent diff --git a/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/config/DroidGlobalConfig.java b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/config/DroidGlobalConfig.java index f56e56ad0..19bce37f2 100644 --- a/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/config/DroidGlobalConfig.java +++ b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/config/DroidGlobalConfig.java @@ -95,7 +95,9 @@ public class DroidGlobalConfig { private Path textSignatureFileDir; private Path reportDefinitionDir; private Path filterDir; - + + private Path exportTemplatesDir; + private PropertiesConfiguration props; private PropertiesConfiguration defaultProps; @@ -160,6 +162,9 @@ public DroidGlobalConfig() throws IOException { tempDir = Paths.get(droidTempPath, "tmp"); Files.createDirectories(tempDir); + + exportTemplatesDir = droidWorkDir.resolve("export_templates"); + Files.createDirectories(exportTemplatesDir); } /** @@ -357,7 +362,12 @@ public Path getFilterDir() { public Path getTempDir() { return tempDir; } - + + /** + * @return directory for the export templates. + */ + public Path getExportTemplatesDir() { return exportTemplatesDir; } + private void createResourceFile(final Path resourceDir, final String fileName, final String resourceName) throws IOException { final Path resourcefile = resourceDir.resolve(fileName); if (!Files.exists(resourcefile)) { diff --git a/droid-core/pom.xml b/droid-core/pom.xml index 35f6fba83..6dedd9de4 100644 --- a/droid-core/pom.xml +++ b/droid-core/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.7.1-SNAPSHOT + 6.8.0-SNAPSHOT ../droid-parent diff --git a/droid-export-interfaces/pom.xml b/droid-export-interfaces/pom.xml index ca9ee2d3f..ad22bbe82 100644 --- a/droid-export-interfaces/pom.xml +++ b/droid-export-interfaces/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.7.1-SNAPSHOT + 6.8.0-SNAPSHOT ../droid-parent diff --git a/droid-export-interfaces/src/main/java/uk/gov/nationalarchives/droid/export/interfaces/ExportDetails.java b/droid-export-interfaces/src/main/java/uk/gov/nationalarchives/droid/export/interfaces/ExportDetails.java new file mode 100644 index 000000000..f0cc9e6b7 --- /dev/null +++ b/droid-export-interfaces/src/main/java/uk/gov/nationalarchives/droid/export/interfaces/ExportDetails.java @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.export.interfaces; + +/** + * Wrapper class to wrap various options used for export. + * e.g. + * - whether the export is per file or per format + * - whether the data should be enclosed in quotes + * - whether BOM is to be used + * - Which columns to export + * - Is there an export template to be used + * + */ +public final class ExportDetails { + + private final ExportOptions exportOptions; + private final String outputEncoding; + private final boolean bomFlag; + private final boolean quoteAllFields; + private final String columnsToWrite; + private final String exportTemplatePath; + + /** + * Private constructor. The consumer can get the ExportDetails instance using ExportDetailsBuilder. + * @param exportOptions whether it is a "per row" export or "per format" export + * @param outputEncoding encoding to be used + * @param bomFlag whether to use BOM + * @param quoteAllFields whether the export fields should be enclosed in double quotes + * @param columnsToWrite List of columns to write + * @param exportTemplatePath absolute path to an export template, if one is being used. + */ + private ExportDetails(ExportOptions exportOptions, String outputEncoding, boolean bomFlag, boolean quoteAllFields, String columnsToWrite, String exportTemplatePath) { + this.exportOptions = exportOptions; + this.outputEncoding = outputEncoding; + this.bomFlag = bomFlag; + this.quoteAllFields = quoteAllFields; + this.columnsToWrite = columnsToWrite; + this.exportTemplatePath = exportTemplatePath; + } + + /** + * + * @return The export options. + */ + public ExportOptions getExportOptions() { + return exportOptions; + } + + /** + * @return The encoding for output. + */ + public String getOutputEncoding() { + return outputEncoding; + } + + /** + * + * @return status of bom. + */ + public boolean bomFlag() { + return bomFlag; + } + + /** + * @return whether all fields are quoted, or just those that contain field separators (commas). + */ + public boolean quoteAllFields() { + return quoteAllFields; + } + + /** + * @return A list of the columns to write, or null if all columns. + */ + public String getColumnsToWrite() { + return columnsToWrite; + } + + /** + * @return A path for export template, null if no template in use. + */ + public String getExportTemplatePath() { + return exportTemplatePath; + } + + /** + * Builder class to build the ExportDetails as a fluent API. + */ + public static class ExportDetailsBuilder { + private ExportOptions exportOptions = ExportOptions.ONE_ROW_PER_FILE; + private String outputEncoding = "UTF-8"; + private boolean bomFlag; + private boolean quoteAllFields = true; + private String columnsToWrite; + private String exportTemplatePath; + + public ExportDetailsBuilder withExportOptions(ExportOptions options) { + this.exportOptions = options; + return this; + } + + public ExportDetailsBuilder withOutputEncoding(String encoding) { + this.outputEncoding = encoding; + return this; + } + + public ExportDetailsBuilder withBomFlag(boolean bom) { + this.bomFlag = bom; + return this; + } + public ExportDetailsBuilder withQuotingAllFields(boolean quoteFields) { + this.quoteAllFields = quoteFields; + return this; + } + + public ExportDetailsBuilder withColumnsToWrite(String columns) { + this.columnsToWrite = columns; + return this; + } + + public ExportDetailsBuilder withExportTemplatePath(String templatePath) { + this.exportTemplatePath = templatePath; + return this; + } + + public ExportDetails build() { + return new ExportDetails(exportOptions, outputEncoding, bomFlag, quoteAllFields, columnsToWrite, exportTemplatePath); + } + } +} diff --git a/droid-export-interfaces/src/main/java/uk/gov/nationalarchives/droid/export/interfaces/ExportManager.java b/droid-export-interfaces/src/main/java/uk/gov/nationalarchives/droid/export/interfaces/ExportManager.java index e5cb6d582..f18032537 100644 --- a/droid-export-interfaces/src/main/java/uk/gov/nationalarchives/droid/export/interfaces/ExportManager.java +++ b/droid-export-interfaces/src/main/java/uk/gov/nationalarchives/droid/export/interfaces/ExportManager.java @@ -45,7 +45,6 @@ public interface ExportManager { /** * Exports one or more profiles to a CSV file. - * * FIXME: * The only reason this interface takes an optional filter * is so that the command line can pass in one of its own @@ -62,15 +61,10 @@ public interface ExportManager { * @param profileIds the list of profiles to export. * @param destination the destination filename * @param filter optional filter - * @param options the options for export. - * @param outputEncoding The character encoding to use in the output, null to use default encoding - * @param bom BOM flag. - * @param quoteAllFields - whether to quote all fields, or just those that contain commas. - * @param columnsToWrite a space separated list of column names to write. If null or empty, all columns are written. + * @param details details about various export preferences * @return future for cancelling the task. */ Future exportProfiles(List profileIds, String destination, - Filter filter, ExportOptions options, String outputEncoding, - boolean bom, boolean quoteAllFields, String columnsToWrite); + Filter filter, ExportDetails details); } diff --git a/droid-export-interfaces/src/main/java/uk/gov/nationalarchives/droid/export/interfaces/ExportTemplate.java b/droid-export-interfaces/src/main/java/uk/gov/nationalarchives/droid/export/interfaces/ExportTemplate.java new file mode 100644 index 000000000..e8a052d92 --- /dev/null +++ b/droid-export-interfaces/src/main/java/uk/gov/nationalarchives/droid/export/interfaces/ExportTemplate.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.export.interfaces; + +import java.util.Map; + +/** + * ExportTemplate interface. + * There is only one method which returns a map of integer -> ExportTemplateColumnDef + */ +public interface ExportTemplate { + /** + * Get the map of Integer -> ExportTemplateColumnDef representing column order. + * @return Map with keys as column order and values as column definition + */ + Map getColumnOrderMap() ; + +} diff --git a/droid-export-interfaces/src/main/java/uk/gov/nationalarchives/droid/export/interfaces/ExportTemplateColumnDef.java b/droid-export-interfaces/src/main/java/uk/gov/nationalarchives/droid/export/interfaces/ExportTemplateColumnDef.java new file mode 100644 index 000000000..a4a66da53 --- /dev/null +++ b/droid-export-interfaces/src/main/java/uk/gov/nationalarchives/droid/export/interfaces/ExportTemplateColumnDef.java @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.export.interfaces; + +/** + * Export template column definition. + */ +public interface ExportTemplateColumnDef { + /** + * Returns the header label to be used in the output for this column. + * @return header + */ + String getHeaderLabel(); + + /** + * Returns the well-known column names from one of the default headers. + * Throws an exception if the definition represents a non-profile column + * @return origianl column name + * @throws RuntimeException Implementing methods may throw a RuntimeException if the method is not relevant + */ + String getOriginalColumnName(); + + /** + * Returns the data value, if any associated with this column definition. + * Throws an exception if the data is coming from profile results + * @return data value + * @throws RuntimeException Implementing methods may throw a RuntimeException if the method is not relevant + */ + String getDataValue(); + + /** + * Returns the column type for this column definition. + * @return column type + */ + ColumnType getColumnType(); + + /** + * Returns the result after performing the specific operation on the input. + * @param input String representing input data + * @return String after performing operation associated with this column definition. + */ + String getOperatedValue(String input); + + /** + * Column type as defined in the ExportTemplate. There are 3 types of columns. + */ + enum ColumnType { + /** + * ProfileResourceNode - The data comes directly from the profile result. + */ + ProfileResourceNode, + /** + * ConstantString - The data comes directly from the constant value in the template. + */ + ConstantString, + /** + * DataModifier - The data is modified as per the operation associated with column. + */ + DataModifier + } + + /** + * Data modification operations for the DataModifier columns type. + * At this time, only case change is supported + */ + enum DataModification { + /** + * Represents a data modification operation to convert given string value to lowercase. + */ + LCASE, + /** + * Represents a data modification operation to convert given string value to uppercase. + */ + UCASE + } +} diff --git a/droid-export-interfaces/src/main/java/uk/gov/nationalarchives/droid/export/interfaces/ItemWriter.java b/droid-export-interfaces/src/main/java/uk/gov/nationalarchives/droid/export/interfaces/ItemWriter.java index 5f8904107..1a06d2b35 100644 --- a/droid-export-interfaces/src/main/java/uk/gov/nationalarchives/droid/export/interfaces/ItemWriter.java +++ b/droid-export-interfaces/src/main/java/uk/gov/nationalarchives/droid/export/interfaces/ItemWriter.java @@ -88,4 +88,11 @@ public interface ItemWriter { */ void setColumnsToWrite(String columnNames); + /** + * Sets the ExportTemplate which can override the column names, column ordering and contents of the columns. + * + * @param template An instance of ExportTemplate. + */ + void setExportTemplate(ExportTemplate template); + } diff --git a/droid-export/pom.xml b/droid-export/pom.xml index 9b751640d..a8293ad60 100644 --- a/droid-export/pom.xml +++ b/droid-export/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.7.1-SNAPSHOT + 6.8.0-SNAPSHOT ../droid-parent diff --git a/droid-export/src/main/java/uk/gov/nationalarchives/droid/export/ExportManagerImpl.java b/droid-export/src/main/java/uk/gov/nationalarchives/droid/export/ExportManagerImpl.java index 92352eff5..057c733ea 100644 --- a/droid-export/src/main/java/uk/gov/nationalarchives/droid/export/ExportManagerImpl.java +++ b/droid-export/src/main/java/uk/gov/nationalarchives/droid/export/ExportManagerImpl.java @@ -31,19 +31,20 @@ */ package uk.gov.nationalarchives.droid.export; -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.FutureTask; - import uk.gov.nationalarchives.droid.core.interfaces.filter.Filter; +import uk.gov.nationalarchives.droid.export.interfaces.ExportDetails; import uk.gov.nationalarchives.droid.export.interfaces.ExportManager; -import uk.gov.nationalarchives.droid.export.interfaces.ExportOptions; import uk.gov.nationalarchives.droid.export.interfaces.ItemWriter; +import uk.gov.nationalarchives.droid.export.template.ExportTemplateBuilder; import uk.gov.nationalarchives.droid.profile.ProfileContextLocator; import uk.gov.nationalarchives.droid.profile.ProfileResourceNode; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.FutureTask; + /** * @author rflitcroft * @@ -74,13 +75,14 @@ public ExportManagerImpl(ProfileContextLocator profileContextLocator, } @Override - public Future exportProfiles(final List profileIds, final String destination, - final Filter filter, final ExportOptions options, final String outputEncoding, final boolean bom, - final boolean quoteAllFields, String columnsToWrite) { - itemWriter.setQuoteAllFields(quoteAllFields); - itemWriter.setColumnsToWrite(columnsToWrite); + public Future exportProfiles(final List profileIds, final String destination, + final Filter filter, final ExportDetails details + ) { + itemWriter.setQuoteAllFields(details.quoteAllFields()); + itemWriter.setColumnsToWrite(details.getColumnsToWrite()); + itemWriter.setExportTemplate(new ExportTemplateBuilder().buildExportTemplate(details.getExportTemplatePath())); final ExportTask exportTask = new ExportTask(destination, - profileIds, filter, options, outputEncoding, bom, itemWriter, profileContextLocator); + profileIds, filter, details.getExportOptions(), details.getOutputEncoding(), details.bomFlag(), itemWriter, profileContextLocator); final FutureTask task = new FutureTask(exportTask, null) { @Override public boolean cancel(final boolean mayInterruptIfRunning) { diff --git a/droid-export/src/main/java/uk/gov/nationalarchives/droid/export/template/ConstantStringColumnDef.java b/droid-export/src/main/java/uk/gov/nationalarchives/droid/export/template/ConstantStringColumnDef.java new file mode 100644 index 000000000..d39b2cad6 --- /dev/null +++ b/droid-export/src/main/java/uk/gov/nationalarchives/droid/export/template/ConstantStringColumnDef.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.export.template; + +import uk.gov.nationalarchives.droid.export.interfaces.ExportTemplateColumnDef; + +/** + * Class for a column definition representing constant string in the export template. + * e.g. foo: "bar" + * In such case, "foo" is the header for this column and "bar" is the value for it. + */ +public class ConstantStringColumnDef implements ExportTemplateColumnDef { + + private final String headerLabel; + private final String dataValue; + + public ConstantStringColumnDef(String dataValue, String headerLabel) { + this.dataValue = dataValue; + this.headerLabel = headerLabel; + } + + /** + * Returns the header label associated with this column definition. + * @return header label + */ + @Override + public String getHeaderLabel() { + return headerLabel; + } + + /** + * This type of column does not have a profile column associated with it. + * As a result, this method simply throws an exception if a consumer tries to get original column name + * @return nothing + */ + @Override + public String getOriginalColumnName() { + throw new RuntimeException("Constant String Columns do not have an associated original column name"); + } + + /** + * Returns the data value associated with this column as defined in the export template. + * @return data value + */ + @Override + public String getDataValue() { + return dataValue; + } + + @Override + public ColumnType getColumnType() { + return ColumnType.ConstantString; + } + + /** + * This type of column does not have any associated operation, hence returns the input value as it is. + * @param input String representing input data + * @return the input value as it is + */ + @Override + public String getOperatedValue(String input) { + return input == null ? "" : input; + } +} diff --git a/droid-export/src/main/java/uk/gov/nationalarchives/droid/export/template/DataModifierColumnDef.java b/droid-export/src/main/java/uk/gov/nationalarchives/droid/export/template/DataModifierColumnDef.java new file mode 100644 index 000000000..5c91da163 --- /dev/null +++ b/droid-export/src/main/java/uk/gov/nationalarchives/droid/export/template/DataModifierColumnDef.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.export.template; + +import uk.gov.nationalarchives.droid.export.interfaces.ExportTemplateColumnDef; + +/** + * Class for a column definition representing data modifier column in the export template. + * e.g. name: LCASE($FILE_PATH) + * In such case, "name" is the header for this column and the value is returned by performing + * an associated operation on the value retrieved from the underlying profile data. + * This type of column definition always has an underlying ProfileResourceNodeColumnDef + */ +public class DataModifierColumnDef implements ExportTemplateColumnDef { + + private final DataModification operation; + private final ProfileResourceNodeColumnDef innerDef; + + public DataModifierColumnDef(ProfileResourceNodeColumnDef innerDef, DataModification operation) { + this.innerDef = innerDef; + this.operation = operation; + } + + /** + * Returns the header label associated with this column definition. + * @return header label + */ + @Override + public String getHeaderLabel() { + return innerDef.getHeaderLabel(); + } + + /** + * Returns the original column name from an underlying definition. + * @return original name of underlying column. + */ + @Override + public String getOriginalColumnName() { + return innerDef.getOriginalColumnName(); + } + + /** + * Return data value from the underlying definition. + * @return data value from underlying definition + */ + @Override + public String getDataValue() { + return innerDef.getDataValue(); + } + + @Override + public ColumnType getColumnType() { + return ColumnType.DataModifier; + } + + /** + * Returns a value after performing an associated operation on the input value. + * e.g. If the operation is LCASE, it will convert the supplied value to lowercase + * @param input String representing input data + * @return String value after performing associated operation. + * Note: + * We only support LCASE and UCASE at the moment, if more operations need to be supported. + * they can be defined in the DataModification enum and appropriate conversion can be + * implemented here. We only support an operation returning String by taking in a single + * String parameter. + * + */ + @Override + public String getOperatedValue(String input) { + if ((input == null) || (input.isEmpty())) { + return ""; + } + switch(operation) { + case LCASE: + return input.toLowerCase(); + case UCASE: + return input.toUpperCase(); + default: + throw new RuntimeException("Value conversion for operation " + operation + " is not implemented"); + } + } +} diff --git a/droid-export/src/main/java/uk/gov/nationalarchives/droid/export/template/ExportTemplateBuilder.java b/droid-export/src/main/java/uk/gov/nationalarchives/droid/export/template/ExportTemplateBuilder.java new file mode 100644 index 000000000..88f0c58e6 --- /dev/null +++ b/droid-export/src/main/java/uk/gov/nationalarchives/droid/export/template/ExportTemplateBuilder.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.export.template; + +import uk.gov.nationalarchives.droid.export.interfaces.ExportTemplate; +import uk.gov.nationalarchives.droid.export.interfaces.ExportTemplateColumnDef; +import uk.gov.nationalarchives.droid.profile.CsvWriterConstants; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Class to build export template from a file. + * The class parses a file in a simple way using delimiters and string manipulations. + */ +public class ExportTemplateBuilder { + + private static final String COLON = ":"; + private static final String UNABLE_TO_PARSE_LINE_MSG = "Unable to parse line: '%s', %s"; + private static final String INVALID_DATA_MODIFIER_SYNTAX_MESSAGE_FORMAT = "Invalid syntax in data modifier expression '%s', %s"; + private static final String DOUBLE_QUOTES = "\""; + private static final String OPENING_BRACKET = "("; + private static final String CLOSING_BRACKET = ")"; + private static final String DATA_COLUMN_PREFIX = "$"; + + /** + * The entry point into building an ExportTemplate object by reading a template file. + * @param pathToTemplate absolute path to the template file + * @return ExportTemplate object + */ + public ExportTemplate buildExportTemplate(String pathToTemplate) { + if (pathToTemplate == null) { + return null; + } + + List templateLines; + try { + templateLines = Files.readAllLines(Paths.get(pathToTemplate)); + Map columnMap = buildColumnMap(templateLines); + return new ExportTemplateImpl(columnMap); + } catch (IOException e) { + throw new RuntimeException("Unable to read export template file at path: " + pathToTemplate); + } + } + + private Map buildColumnMap(List templateLines) { + if ((templateLines == null) || (templateLines.size() == 0)) { + throw new ExportTemplateParseException("Export template is empty"); + } + String versionLine = templateLines.get(0); + String version = parseVersionLine(versionLine); + + //we have only one version at the moment, but future provision for versioning + switch (version) { + case "1.0": + return parseExportTemplateV1(templateLines.subList(1, templateLines.size())); + default: + throw new ExportTemplateParseException("Unsupported version for the export template"); + } + } + + private Map parseExportTemplateV1(List templateLines) { + List columnLines = templateLines.stream().filter(line -> line.trim().length() > 0).collect(Collectors.toList()); + Map columnMap = new HashMap<>(); + for (int i = 0; i < columnLines.size(); i++) { + String line = columnLines.get(i); + if (!line.contains(COLON)) { + throw new ExportTemplateParseException(String.format(UNABLE_TO_PARSE_LINE_MSG, line, "line does not contain ':'")); + } + String header = line.substring(0, line.indexOf(COLON)).trim(); + if (header.isEmpty()) { + throw new ExportTemplateParseException(String.format(UNABLE_TO_PARSE_LINE_MSG, line, "column header is empty")); + } + + String token2 = line.substring(line.indexOf(COLON) + 1).trim(); + + if (token2.startsWith(DATA_COLUMN_PREFIX)) { + columnMap.put(i, createProfileNodeDef(header, token2)); + } else if ((token2.isEmpty()) || (token2.startsWith(DOUBLE_QUOTES))) { + columnMap.put(i, createConstantStringDef(header, token2)); + } else { + columnMap.put(i, createDataModifierDef(header, token2)); + } + } + return columnMap; + } + + private ExportTemplateColumnDef createDataModifierDef(String header, String param2) { + + assertDataModifierSyntaxValid(param2); + + String[] tokens = param2.split("\\("); + String operationName = tokens[0].trim(); + + List operations = Arrays.stream( + ExportTemplateColumnDef.DataModification.values()).map(v -> v.toString()). + collect(Collectors.toList()); + + if (!operations.contains(operationName)) { + throw new ExportTemplateParseException("Undefined operation '" + operationName + "' encountered in export template"); + } + + String column = tokens[1].trim().substring(0, tokens[1].trim().length() - 1); + ProfileResourceNodeColumnDef inner = createProfileNodeDef(header, column); + + ExportTemplateColumnDef.DataModification operation = ExportTemplateColumnDef.DataModification.valueOf(operationName); + return new DataModifierColumnDef(inner, operation); + } + + private void assertDataModifierSyntaxValid(String expression) { + String expressionToTest = expression.trim(); + // valid statement like LCASE($URI) + if (expressionToTest.chars().filter(ch -> ch == '(').count() != 1) { + throw new ExportTemplateParseException(String.format(INVALID_DATA_MODIFIER_SYNTAX_MESSAGE_FORMAT, expression, "expecting exactly one occurrence of '('")); + } + if (expressionToTest.chars().filter(ch -> ch == ')').count() != 1) { + throw new ExportTemplateParseException(String.format(INVALID_DATA_MODIFIER_SYNTAX_MESSAGE_FORMAT, expression, "expecting exactly one occurrence of ')'")); + } + if (expressionToTest.indexOf(OPENING_BRACKET) > expressionToTest.indexOf(CLOSING_BRACKET)) { + throw new ExportTemplateParseException(String.format(INVALID_DATA_MODIFIER_SYNTAX_MESSAGE_FORMAT, expression, "expecting '(' before ')'")); + } + if (expressionToTest.indexOf(OPENING_BRACKET) == 0) { + throw new ExportTemplateParseException(String.format(INVALID_DATA_MODIFIER_SYNTAX_MESSAGE_FORMAT, expression, "expecting an operation definition before '('")); + } + String dataColumnToken = expressionToTest.substring(expressionToTest.indexOf(OPENING_BRACKET) + 1, + expressionToTest.indexOf(CLOSING_BRACKET)).trim(); + if (!dataColumnToken.startsWith(DATA_COLUMN_PREFIX)) { + throw new ExportTemplateParseException(String.format(INVALID_DATA_MODIFIER_SYNTAX_MESSAGE_FORMAT, expression, "expecting '$' after '('")); + } + } + + private ExportTemplateColumnDef createConstantStringDef(String header, String param2) { + if (param2.isEmpty()) { + return new ConstantStringColumnDef("", header); + } else { + if (!param2.endsWith(DOUBLE_QUOTES)) { + throw new ExportTemplateParseException("The line with a constant value ('" + param2 + "') in template definition does not have closing quotes"); + } + return new ConstantStringColumnDef(param2.substring(1, param2.length() - 1), header); + } + } + + private ProfileResourceNodeColumnDef createProfileNodeDef(String header, String param2) { + String messageFormat = "Invalid column name. '%s' does not exist in profile results"; + if (!param2.startsWith(DATA_COLUMN_PREFIX)) { + throw new ExportTemplateParseException(String.format(messageFormat, param2)); + } + String originalColumnName = param2.substring(1); + if (!(Arrays.stream(CsvWriterConstants.HEADERS).collect(Collectors.toList()).contains(originalColumnName))) { + throw new ExportTemplateParseException(String.format(messageFormat, originalColumnName)); + } + return new ProfileResourceNodeColumnDef(originalColumnName, header); + } + + private String parseVersionLine(String versionLine) { + String versionPrefix = "version"; + String versionString = versionLine.trim(); + if (!versionString.startsWith(versionPrefix)) { + throw new ExportTemplateParseException("First line in the template needs to specify version in the form \"version \""); + } + + return versionString.substring(versionPrefix.length()).trim(); + } +} + + diff --git a/droid-export/src/main/java/uk/gov/nationalarchives/droid/export/template/ExportTemplateImpl.java b/droid-export/src/main/java/uk/gov/nationalarchives/droid/export/template/ExportTemplateImpl.java new file mode 100644 index 000000000..ff9cac07e --- /dev/null +++ b/droid-export/src/main/java/uk/gov/nationalarchives/droid/export/template/ExportTemplateImpl.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.export.template; + +import uk.gov.nationalarchives.droid.export.interfaces.ExportTemplate; +import uk.gov.nationalarchives.droid.export.interfaces.ExportTemplateColumnDef; + +import java.util.HashMap; +import java.util.Map; + +/** + * Implementation of ExportTemplate. + */ +public class ExportTemplateImpl implements ExportTemplate { + private final Map columnOrderMap = new HashMap<>(); + + public ExportTemplateImpl(final Map columnOrder) { + this.columnOrderMap.putAll(columnOrder); + } + public Map getColumnOrderMap() { + return columnOrderMap; + } +} diff --git a/droid-export/src/main/java/uk/gov/nationalarchives/droid/export/template/ExportTemplateParseException.java b/droid-export/src/main/java/uk/gov/nationalarchives/droid/export/template/ExportTemplateParseException.java new file mode 100644 index 000000000..ada883f55 --- /dev/null +++ b/droid-export/src/main/java/uk/gov/nationalarchives/droid/export/template/ExportTemplateParseException.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.export.template; + + +public class ExportTemplateParseException extends RuntimeException { + public ExportTemplateParseException(String message) { + super(message); + } + + public ExportTemplateParseException() { + this("Unable to parse export template file"); + } +} diff --git a/droid-export/src/main/java/uk/gov/nationalarchives/droid/export/template/ProfileResourceNodeColumnDef.java b/droid-export/src/main/java/uk/gov/nationalarchives/droid/export/template/ProfileResourceNodeColumnDef.java new file mode 100644 index 000000000..2de4a7986 --- /dev/null +++ b/droid-export/src/main/java/uk/gov/nationalarchives/droid/export/template/ProfileResourceNodeColumnDef.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.export.template; + +import uk.gov.nationalarchives.droid.export.interfaces.ExportTemplateColumnDef; + +/** + * Class for a column definition representing an underlying profile resource node column. + * e.g. identifier: $ID + * In such case, "identifier" is the header for this column and the data is retrieved from the + * ID column in the results . + */ +public class ProfileResourceNodeColumnDef implements ExportTemplateColumnDef { + + private final String originalHeaderLabel; + private final String headerLabel; + + public ProfileResourceNodeColumnDef(String originalHeaderLabel, String headerLabel) { + this.originalHeaderLabel = originalHeaderLabel; + this.headerLabel = headerLabel; + } + + /** + * Returns the header label associated with this column definition. + * @return header label + */ + @Override + public String getHeaderLabel() { + return headerLabel; + } + + /** + * Returns the original column name. + * @return The well-known name of column as it appears in the profile results + */ + @Override + public String getOriginalColumnName() { + return originalHeaderLabel; + } + + /** + * This type of column does not have data associated with it. + * As a result, this method simply throws an exception if a consumer tries to get data from it + * @return nothing + */ + @Override + public String getDataValue() { + throw new RuntimeException("Profile resource node column uses data from the profile results"); + } + + @Override + public ColumnType getColumnType() { + return ColumnType.ProfileResourceNode; + } + + /** + * This type of column does not have any associated operation, hence returns the input value as it is. + * @param input String representing input data + * @return the input value as it is + */ + @Override + public String getOperatedValue(String input) { + return input == null ? "" : input; + } +} diff --git a/droid-export/src/test/java/uk/gov/nationalarchives/droid/export/ExportJobIntegrationTest.java b/droid-export/src/test/java/uk/gov/nationalarchives/droid/export/ExportJobIntegrationTest.java index 14e363d16..fdb5b757d 100644 --- a/droid-export/src/test/java/uk/gov/nationalarchives/droid/export/ExportJobIntegrationTest.java +++ b/droid-export/src/test/java/uk/gov/nationalarchives/droid/export/ExportJobIntegrationTest.java @@ -45,6 +45,7 @@ import org.springframework.test.context.ContextConfiguration; import uk.gov.nationalarchives.droid.core.interfaces.config.DroidGlobalConfig; +import uk.gov.nationalarchives.droid.export.interfaces.ExportDetails; import uk.gov.nationalarchives.droid.export.interfaces.ExportOptions; import uk.gov.nationalarchives.droid.profile.DirectoryProfileResource; import uk.gov.nationalarchives.droid.profile.ProfileContextLocator; @@ -98,10 +99,14 @@ public void testEndToEndExportOfOneProfile() throws Exception { String[] profileIds = new String[] { "test", }; - - exportManager.exportProfiles(Arrays.asList(profileIds), "exports/export.csv", null, - ExportOptions.ONE_ROW_PER_FILE, null, false, true, null); - + + ExportDetails details = new ExportDetails.ExportDetailsBuilder() + .withExportOptions(ExportOptions.ONE_ROW_PER_FILE) + .withQuotingAllFields(true) + .build(); + + exportManager.exportProfiles(Arrays.asList(profileIds), "exports/export.csv", null, details); + profileContextLocator.removeProfileContext("test"); } diff --git a/droid-export/src/test/java/uk/gov/nationalarchives/droid/export/ExportManagerImplTest.java b/droid-export/src/test/java/uk/gov/nationalarchives/droid/export/ExportManagerImplTest.java index 70b08b250..fd5f10cf5 100644 --- a/droid-export/src/test/java/uk/gov/nationalarchives/droid/export/ExportManagerImplTest.java +++ b/droid-export/src/test/java/uk/gov/nationalarchives/droid/export/ExportManagerImplTest.java @@ -31,34 +31,36 @@ */ package uk.gov.nationalarchives.droid.export; +import org.junit.Before; +import org.junit.Test; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import uk.gov.nationalarchives.droid.core.interfaces.filter.Filter; +import uk.gov.nationalarchives.droid.export.interfaces.ExportDetails; +import uk.gov.nationalarchives.droid.export.interfaces.ExportOptions; +import uk.gov.nationalarchives.droid.export.interfaces.ItemReader; +import uk.gov.nationalarchives.droid.export.interfaces.ItemReaderCallback; +import uk.gov.nationalarchives.droid.export.interfaces.ItemWriter; +import uk.gov.nationalarchives.droid.export.interfaces.JobCancellationException; +import uk.gov.nationalarchives.droid.profile.ProfileContextLocator; +import uk.gov.nationalarchives.droid.profile.ProfileInstance; +import uk.gov.nationalarchives.droid.profile.ProfileInstanceManager; +import uk.gov.nationalarchives.droid.profile.ProfileResourceNode; + import java.io.Writer; import java.net.URI; import java.util.ArrayList; import java.util.List; import static org.junit.Assert.fail; -import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.junit.MockitoJUnitRunner; -import org.mockito.stubbing.Answer; - -import uk.gov.nationalarchives.droid.core.interfaces.filter.Filter; -import uk.gov.nationalarchives.droid.export.interfaces.*; -import uk.gov.nationalarchives.droid.profile.ProfileContextLocator; -import uk.gov.nationalarchives.droid.profile.ProfileInstance; -import uk.gov.nationalarchives.droid.profile.ProfileInstanceManager; -import uk.gov.nationalarchives.droid.profile.ProfileResourceNode; - /** * @author rflitcroft * @@ -125,8 +127,14 @@ public Object answer(InvocationOnMock invocation) { List profileIdList = new ArrayList(); profileIdList.add("profile1"); - - exportManager.exportProfiles(profileIdList, "destination", null, ExportOptions.ONE_ROW_PER_FILE, null, false, true, "").get(); + + ExportDetails details = new ExportDetails.ExportDetailsBuilder() + .withExportOptions(ExportOptions.ONE_ROW_PER_FILE) + .withQuotingAllFields(true) + .withColumnsToWrite("") + .build(); + + exportManager.exportProfiles(profileIdList, "destination", null, details).get(); verify(writer).open(any(Writer.class)); verify(writer).write(fis); diff --git a/droid-export/src/test/java/uk/gov/nationalarchives/droid/export/template/ConstantStringColumnDefTest.java b/droid-export/src/test/java/uk/gov/nationalarchives/droid/export/template/ConstantStringColumnDefTest.java new file mode 100644 index 000000000..555c456c0 --- /dev/null +++ b/droid-export/src/test/java/uk/gov/nationalarchives/droid/export/template/ConstantStringColumnDefTest.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.export.template; + +import org.junit.Test; +import uk.gov.nationalarchives.droid.export.interfaces.ExportTemplateColumnDef; + +import static org.junit.Assert.*; + +public class ConstantStringColumnDefTest { + @Test + public void should_operate_on_null_value_and_return_empty_string() { + ConstantStringColumnDef def = new ConstantStringColumnDef("English (UK)", "Language"); + assertEquals("", def.getOperatedValue(null)); + } + + @Test + public void should_return_input_as_it_is_when_asked_for_operated_value() { + ConstantStringColumnDef def = new ConstantStringColumnDef("English (UK)", "Language"); + assertEquals("FoRMat", def.getOperatedValue("FoRMat")); + } + + @Test + public void should_return_header_label_as_set_when_defining_the_column_definition() { + ConstantStringColumnDef def = new ConstantStringColumnDef("English (UK)", "Language"); + assertEquals("Language", def.getHeaderLabel()); + } + @Test + public void should_throw_exception_when_asking_for_original_column_name() { + ConstantStringColumnDef def = new ConstantStringColumnDef("English (UK)", "Language"); + RuntimeException ex = assertThrows(RuntimeException.class, () -> def.getOriginalColumnName()); + assertEquals("Constant String Columns do not have an associated original column name", ex.getMessage()); + } + + @Test + public void should_return_data_as_set_in_the_definition() { + ConstantStringColumnDef def = new ConstantStringColumnDef("English (UK)", "Language"); + assertEquals("English (UK)", def.getDataValue()); + } + + @Test + public void should_return_column_type_as_data_modifier_column_def() { + ConstantStringColumnDef def = new ConstantStringColumnDef("English (UK)", "Language"); + assertEquals(ExportTemplateColumnDef.ColumnType.ConstantString, def.getColumnType()); + } +} \ No newline at end of file diff --git a/droid-export/src/test/java/uk/gov/nationalarchives/droid/export/template/DataModifierColumnDefTest.java b/droid-export/src/test/java/uk/gov/nationalarchives/droid/export/template/DataModifierColumnDefTest.java new file mode 100644 index 000000000..028642d1d --- /dev/null +++ b/droid-export/src/test/java/uk/gov/nationalarchives/droid/export/template/DataModifierColumnDefTest.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.export.template; + +import org.junit.Test; +import uk.gov.nationalarchives.droid.export.interfaces.ExportTemplateColumnDef; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class DataModifierColumnDefTest { + @Test + public void should_operate_on_null_value_and_return_empty_string() { + ProfileResourceNodeColumnDef innerDef = mock(ProfileResourceNodeColumnDef.class); + DataModifierColumnDef def = new DataModifierColumnDef(innerDef, ExportTemplateColumnDef.DataModification.LCASE); + assertEquals("", def.getOperatedValue(null)); + } + + @Test + public void should_return_header_label_as_set_on_the_inner_profile_def() { + ProfileResourceNodeColumnDef innerDef = mock(ProfileResourceNodeColumnDef.class); + when(innerDef.getHeaderLabel()).thenReturn("InnerHeader"); + DataModifierColumnDef def = new DataModifierColumnDef(innerDef, ExportTemplateColumnDef.DataModification.LCASE); + assertEquals("InnerHeader", def.getHeaderLabel()); + } + @Test + public void should_return_original_column_name_as_set_on_the_inner_profile_def() { + ProfileResourceNodeColumnDef innerDef = mock(ProfileResourceNodeColumnDef.class); + when(innerDef.getOriginalColumnName()).thenReturn("FORMAT_NAME"); + DataModifierColumnDef def = new DataModifierColumnDef(innerDef, ExportTemplateColumnDef.DataModification.LCASE); + assertEquals("FORMAT_NAME", def.getOriginalColumnName()); + } + + @Test + public void should_return_column_type_as_data_modifier_column_def() { + ProfileResourceNodeColumnDef innerDef = mock(ProfileResourceNodeColumnDef.class); + DataModifierColumnDef def = new DataModifierColumnDef(innerDef, ExportTemplateColumnDef.DataModification.LCASE); + assertEquals(ExportTemplateColumnDef.ColumnType.DataModifier, def.getColumnType()); + } +} \ No newline at end of file diff --git a/droid-export/src/test/java/uk/gov/nationalarchives/droid/export/template/ExportTemplateBuilderTest.java b/droid-export/src/test/java/uk/gov/nationalarchives/droid/export/template/ExportTemplateBuilderTest.java new file mode 100644 index 000000000..4a81d0836 --- /dev/null +++ b/droid-export/src/test/java/uk/gov/nationalarchives/droid/export/template/ExportTemplateBuilderTest.java @@ -0,0 +1,286 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.export.template; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import uk.gov.nationalarchives.droid.export.interfaces.ExportTemplate; +import uk.gov.nationalarchives.droid.export.interfaces.ExportTemplateColumnDef; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.*; + +public class ExportTemplateBuilderTest { + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void should_read_an_export_template_file_and_construct_export_template_object() throws IOException { + ExportTemplateBuilder builder = new ExportTemplateBuilder(); + File tempFile = temporaryFolder.newFile("export-task-test-default-encoding"); + List data = Arrays.asList( + "version 1.0", + "Identifier: $ID", + "Language: \"Gibberish\"", + "Path: UCASE($FILE_PATH)", + "Size: $SIZE", + "HASH: $HASH"); + Files.write(tempFile.toPath(), data, StandardOpenOption.WRITE); + ExportTemplate template = builder.buildExportTemplate(tempFile.getAbsolutePath()); + assertNotNull(template); + ExportTemplateColumnDef cscd = template.getColumnOrderMap().get(1); + assertTrue(cscd instanceof ConstantStringColumnDef); + assertEquals("Gibberish",template.getColumnOrderMap().get(1).getDataValue()); + + ExportTemplateColumnDef dmcd = template.getColumnOrderMap().get(2); + assertTrue(dmcd instanceof DataModifierColumnDef); + assertEquals("FILE_PATH", dmcd.getOriginalColumnName()); + assertEquals("SMALL", dmcd.getOperatedValue("small")); + } + + @Test + public void should_throw_exception_when_the_column_description_lines_do_not_have_a_colon() throws IOException { + File tempFile = temporaryFolder.newFile("export-task-test-default-encoding"); + List data = Arrays.asList("version 1.0", "myCol $My_COL"); + Files.write(tempFile.toPath(), data, StandardOpenOption.WRITE); + ExportTemplateBuilder builder = new ExportTemplateBuilder(); + ExportTemplateParseException ex = assertThrows(ExportTemplateParseException.class, () -> builder.buildExportTemplate(tempFile.getAbsolutePath())); + assertEquals("Unable to parse line: 'myCol $My_COL', line does not contain ':'", ex.getMessage()); + } + + @Test + public void should_throw_exception_when_the_header_information_is_missing() throws IOException { + File tempFile = temporaryFolder.newFile("export-task-test-default-encoding"); + List data = Arrays.asList("version 1.0", ": $My_COL"); + Files.write(tempFile.toPath(), data, StandardOpenOption.WRITE); + ExportTemplateBuilder builder = new ExportTemplateBuilder(); + ExportTemplateParseException ex = assertThrows(ExportTemplateParseException.class, () -> builder.buildExportTemplate(tempFile.getAbsolutePath())); + assertEquals("Unable to parse line: ': $My_COL', column header is empty", ex.getMessage()); + } + + @Test + public void should_treat_missing_value_as_empty_string_constant_in_export_template() throws IOException { + ExportTemplateBuilder builder = new ExportTemplateBuilder(); + File tempFile = temporaryFolder.newFile("export-task-test-default-encoding"); + List data = Arrays.asList( + "version 1.0", + "Identifier: $ID", + "Language: ", + "Path: UCASE($FILE_PATH)"); + Files.write(tempFile.toPath(), data, StandardOpenOption.WRITE); + ExportTemplate template = builder.buildExportTemplate(tempFile.getAbsolutePath()); + assertNotNull(template); + assertTrue(template.getColumnOrderMap().get(1) instanceof ConstantStringColumnDef); + assertTrue(template.getColumnOrderMap().get(2) instanceof DataModifierColumnDef); + assertEquals("FILE_PATH", template.getColumnOrderMap().get(2).getOriginalColumnName()); + } + + @Test + public void should_throw_exception_when_version_string_is_bad() throws IOException { + File tempFile = temporaryFolder.newFile("export-task-test-default-encoding"); + Files.write(tempFile.toPath(), "versio 1.3".getBytes(StandardCharsets.UTF_8)); + List data = Collections.singletonList("myCol:$My_COL"); + Files.write(tempFile.toPath(), data, StandardOpenOption.APPEND); + ExportTemplateBuilder builder = new ExportTemplateBuilder(); + ExportTemplateParseException ex = assertThrows(ExportTemplateParseException.class, () -> builder.buildExportTemplate(tempFile.getAbsolutePath())); + assertEquals("First line in the template needs to specify version in the form \"version \"", ex.getMessage()); + } + + @Test + public void should_trim_blanks_from_lines_and_tokens_to_produce_a_valid_template() throws IOException { + ExportTemplateBuilder builder = new ExportTemplateBuilder(); + File tempFile = temporaryFolder.newFile("export-task-test-default-encoding"); + List data = Arrays.asList( + "version 1.0", + "", + "Language: \"Marathi\" ", + ""); + Files.write(tempFile.toPath(), data, StandardOpenOption.WRITE); + ExportTemplate template = builder.buildExportTemplate(tempFile.getAbsolutePath()); + assertEquals(1, template.getColumnOrderMap().size()); + assertEquals("Marathi", template.getColumnOrderMap().get(0).getDataValue()); + } + + @Test + public void should_allow_constant_strings_with_double_quotes_in_the_data_value() throws IOException { + ExportTemplateBuilder builder = new ExportTemplateBuilder(); + File tempFile = temporaryFolder.newFile("export-task-test-default-encoding"); + List data = Arrays.asList( + "version 1.0", + "", + "Language: \"Star trek: \"Klingon\"\"", + ""); + Files.write(tempFile.toPath(), data, StandardOpenOption.WRITE); + ExportTemplate template = builder.buildExportTemplate(tempFile.getAbsolutePath()); + assertEquals(1, template.getColumnOrderMap().size()); + assertEquals("Star trek: \"Klingon\"", template.getColumnOrderMap().get(0).getDataValue()); + } + + @Test + public void should_throw_an_exception_if_the_constant_string_value_does_not_have_closing_double_quotes() throws IOException { + ExportTemplateBuilder builder = new ExportTemplateBuilder(); + File tempFile = temporaryFolder.newFile("export-task-test-default-encoding"); + List data = Arrays.asList( + "version 1.0", + "", + "Language: \"English", + ""); + Files.write(tempFile.toPath(), data, StandardOpenOption.WRITE); + + ExportTemplateParseException ex = assertThrows(ExportTemplateParseException.class, () -> builder.buildExportTemplate(tempFile.getAbsolutePath())); + assertEquals("The line with a constant value ('\"English') in template definition does not have closing quotes", ex.getMessage()); + } + + @Test + public void should_throw_an_exception_if_the_column_name_given_for_profile_resource_column_is_not_a_well_known_header() throws IOException { + ExportTemplateBuilder builder = new ExportTemplateBuilder(); + File tempFile = temporaryFolder.newFile("export-task-test-default-encoding"); + List data = Arrays.asList( + "version 1.0", + "", + "Identifier: $UUID", + ""); + Files.write(tempFile.toPath(), data, StandardOpenOption.WRITE); + + ExportTemplateParseException ex = assertThrows(ExportTemplateParseException.class, () -> builder.buildExportTemplate(tempFile.getAbsolutePath())); + assertEquals("Invalid column name. 'UUID' does not exist in profile results", ex.getMessage()); + } + + @Test + public void should_throw_an_exception_when_the_operation_is_unknown() throws IOException { + ExportTemplateBuilder builder = new ExportTemplateBuilder(); + File tempFile = temporaryFolder.newFile("export-task-test-default-encoding"); + List data = Arrays.asList( + "version 1.0", + "", + "Identifier: Lower($ID)", + ""); + Files.write(tempFile.toPath(), data, StandardOpenOption.WRITE); + + ExportTemplateParseException ex = assertThrows(ExportTemplateParseException.class, () -> builder.buildExportTemplate(tempFile.getAbsolutePath())); + assertEquals("Undefined operation 'Lower' encountered in export template", ex.getMessage()); + } + + @Test + public void should_trim_leading_and_trailing_spaces_from_a_header_label() throws IOException { + ExportTemplateBuilder builder = new ExportTemplateBuilder(); + File tempFile = temporaryFolder.newFile("export-task-test-default-encoding"); + List data = Arrays.asList( + "version 1.0", + " Identifier : \"Something\"", + ""); + Files.write(tempFile.toPath(), data, StandardOpenOption.WRITE); + + ExportTemplate template = builder.buildExportTemplate(tempFile.getAbsolutePath()); + assertEquals("Identifier", template.getColumnOrderMap().get(0).getHeaderLabel()); + } + + @Test + public void should_support_colon_in_the_constant_string_value() throws IOException { + ExportTemplateBuilder builder = new ExportTemplateBuilder(); + File tempFile = temporaryFolder.newFile("export-task-test-default-encoding"); + List data = Arrays.asList( + "version 1.0", + " MyWebsite : \"http://www.knowingwhere.com\"", + ""); + Files.write(tempFile.toPath(), data, StandardOpenOption.WRITE); + + ExportTemplate template = builder.buildExportTemplate(tempFile.getAbsolutePath()); + assertEquals("MyWebsite", template.getColumnOrderMap().get(0).getHeaderLabel()); + assertEquals("http://www.knowingwhere.com", template.getColumnOrderMap().get(0).getDataValue()); + } + + @Test + public void should_throw_an_exception_when_an_operation_cannot_be_located() throws IOException { + ExportTemplateBuilder builder = new ExportTemplateBuilder(); + File tempFile = temporaryFolder.newFile("export-task-test-default-encoding"); + List data = Arrays.asList( + "version 1.0", + " Copyright : Crown Copyright (C)", + ""); + Files.write(tempFile.toPath(), data, StandardOpenOption.WRITE); + + ExportTemplateParseException ex = assertThrows(ExportTemplateParseException.class, () -> builder.buildExportTemplate(tempFile.getAbsolutePath())); + assertEquals("Invalid syntax in data modifier expression 'Crown Copyright (C)', expecting '$' after '('", ex.getMessage()); + } + + @Test + public void should_throw_an_exception_when_a_constant_is_not_enclosed_in_double_quotes() throws IOException { + ExportTemplateBuilder builder = new ExportTemplateBuilder(); + File tempFile = temporaryFolder.newFile("export-task-test-default-encoding"); + List data = Arrays.asList( + "version 1.0", + " Copyright : Crown Copyright", + ""); + Files.write(tempFile.toPath(), data, StandardOpenOption.WRITE); + + ExportTemplateParseException ex = assertThrows(ExportTemplateParseException.class, () -> builder.buildExportTemplate(tempFile.getAbsolutePath())); + assertEquals("Invalid syntax in data modifier expression 'Crown Copyright', expecting exactly one occurrence of '('", ex.getMessage()); + } + + @Test + public void should_throw_an_exception_when_a_column_name_for_modofication_is_invalid() throws IOException { + ExportTemplateBuilder builder = new ExportTemplateBuilder(); + File tempFile = temporaryFolder.newFile("export-task-test-default-encoding"); + List data = Arrays.asList( + "version 1.0", + " Copyright : LCASE($CROWN)", + ""); + Files.write(tempFile.toPath(), data, StandardOpenOption.WRITE); + + ExportTemplateParseException ex = assertThrows(ExportTemplateParseException.class, () -> builder.buildExportTemplate(tempFile.getAbsolutePath())); + assertEquals("Invalid column name. 'CROWN' does not exist in profile results", ex.getMessage()); + } + + @Test + public void should_throw_an_exception_when_a_column_name_is_not_prefixed_with_dollar_sign() throws IOException { + ExportTemplateBuilder builder = new ExportTemplateBuilder(); + File tempFile = temporaryFolder.newFile("export-task-test-default-encoding"); + List data = Arrays.asList( + "version 1.0", + " Copyright : LCASE(PUID)", + ""); + Files.write(tempFile.toPath(), data, StandardOpenOption.WRITE); + + ExportTemplateParseException ex = assertThrows(ExportTemplateParseException.class, () -> builder.buildExportTemplate(tempFile.getAbsolutePath())); + assertEquals("Invalid syntax in data modifier expression 'LCASE(PUID)', expecting '$' after '('", ex.getMessage()); + } +} \ No newline at end of file diff --git a/droid-export/src/test/java/uk/gov/nationalarchives/droid/export/template/ProfileResourceNodeColumnDefTest.java b/droid-export/src/test/java/uk/gov/nationalarchives/droid/export/template/ProfileResourceNodeColumnDefTest.java new file mode 100644 index 000000000..a8b8a9def --- /dev/null +++ b/droid-export/src/test/java/uk/gov/nationalarchives/droid/export/template/ProfileResourceNodeColumnDefTest.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.export.template; + +import org.junit.Test; +import uk.gov.nationalarchives.droid.export.interfaces.ExportTemplateColumnDef; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +public class ProfileResourceNodeColumnDefTest { + @Test + public void should_operate_on_null_value_and_return_empty_string() { + ProfileResourceNodeColumnDef def = new ProfileResourceNodeColumnDef("FORMAT_NAME", "Format"); + assertEquals("", def.getOperatedValue(null)); + } + + @Test + public void should_return_input_as_it_is_when_asked_for_operated_value() { + ProfileResourceNodeColumnDef def = new ProfileResourceNodeColumnDef("FORMAT_NAME", "Format"); + assertEquals("FoRMat", def.getOperatedValue("FoRMat")); + } + + @Test + public void should_return_header_label_as_set_when_defining_the_column_definition() { + ProfileResourceNodeColumnDef def = new ProfileResourceNodeColumnDef("FORMAT_NAME", "Format"); + assertEquals("Format", def.getHeaderLabel()); + } + @Test + public void should_return_original_column_name_as_set_when_defining_the_column_definition() { + ProfileResourceNodeColumnDef def = new ProfileResourceNodeColumnDef("FORMAT_NAME", "Format"); + assertEquals("FORMAT_NAME", def.getOriginalColumnName()); + } + + @Test + public void should_throw_exception_when_asking_for_data_from_the_definition() { + ProfileResourceNodeColumnDef def = new ProfileResourceNodeColumnDef("FORMAT_NAME", "Format"); + RuntimeException ex = assertThrows(RuntimeException.class, () -> def.getDataValue()); + assertEquals("Profile resource node column uses data from the profile results", ex.getMessage()); + } + + @Test + public void should_return_column_type_as_data_modifier_column_def() { + ProfileResourceNodeColumnDef def = new ProfileResourceNodeColumnDef("FORMAT_NAME", "Format"); + assertEquals(ExportTemplateColumnDef.ColumnType.ProfileResourceNode, def.getColumnType()); + } +} \ No newline at end of file diff --git a/droid-help/pom.xml b/droid-help/pom.xml index 910278ac0..36afff4d1 100644 --- a/droid-help/pom.xml +++ b/droid-help/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.7.1-SNAPSHOT + 6.8.0-SNAPSHOT ../droid-parent diff --git a/droid-help/src/main/resources/Images/Export dialog template.png b/droid-help/src/main/resources/Images/Export dialog template.png new file mode 100644 index 000000000..cd9b243b9 Binary files /dev/null and b/droid-help/src/main/resources/Images/Export dialog template.png differ diff --git a/droid-help/src/main/resources/Images/Export dialog.png b/droid-help/src/main/resources/Images/Export dialog.png index 3e60ee79e..20895ee0b 100644 Binary files a/droid-help/src/main/resources/Images/Export dialog.png and b/droid-help/src/main/resources/Images/Export dialog.png differ diff --git a/droid-help/src/main/resources/Web pages/Command line control.html b/droid-help/src/main/resources/Web pages/Command line control.html index 938290f02..f62f4c571 100644 --- a/droid-help/src/main/resources/Web pages/Command line control.html +++ b/droid-help/src/main/resources/Web pages/Command line control.html @@ -97,7 +97,7 @@
Filtering results once identified
  • - List filtering options + List filtering options
  • Specifying the signature files to use @@ -216,7 +216,7 @@

    Format identification

    - Writing to a file

    + Writing to a file
    • To output the CSV results to a file rather than the console, you can use the -o option: @@ -534,6 +534,28 @@

    Exporting profiles to CSV

    +

    + If you want to export a subset of columns, simply specify a space separated list of those columns after the -co option: + + + + +
    +  droid -p "C:\Results\myprofile1.droid" -e "C:\Exports\myprofiles.csv" -co FILE_PATH STATUS HASH PUID + +     +
    +

    + If you have defined an export template, simply specify the absolute path to the export template after the -et option: + + + + +
    +  droid -p "C:\Results\myprofile1.droid" -e "C:\Exports\myprofiles.csv" -et "C:\Templates\MyExport.template" + +     +

    A final option for export is to control whether a Byte Order Mark (BOM) is written at the beginning of the file. To write a BOM, use the -B option: @@ -1223,6 +1245,15 @@

    Command options

    droid “C:\Files\A Folder” "C:\Files\A File” -co FILE_PATH STATUS HASH PUID + + -et + --export-template + Absolute path to the export template + Specifies that CSV output should be written using the customisations defined in the Export Template. + + droid “C:\Files\A Folder” "C:\Files\A File” -et "C:\Templates\MyExport.template" + + -e --export-file diff --git a/droid-help/src/main/resources/Web pages/Exporting profiles.html b/droid-help/src/main/resources/Web pages/Exporting profiles.html index 504324105..4ab16af2d 100644 --- a/droid-help/src/main/resources/Web pages/Exporting profiles.html +++ b/droid-help/src/main/resources/Web pages/Exporting profiles.html @@ -75,37 +75,34 @@
    export a profile, press the    Export   button, or select the File / Export - all... menu item.  This will bring up the export dialog window: -

    -

    - + all... menu item. This shows the export dialog. The profiles you have open are listed + in the export window.  If a profile is empty, or in the process of running, it is greyed out. +  Select all the profiles you want to export into a single CSV file by checking the boxes next to + them.   If any of your profiles have active filters, then + the results will also be filtered.  Each profile can have different filters defined and enabled.

    - The profiles you have open are listed in the export window.  If a profile is empty, or - in the process of running, it will be greyed out.  Select all the profiles you want to - export into a single CSV file by checking the boxes next to them.   If any of your - profiles have active filters, then the results will - also be filtered.  Each profile can have different filters defined and enabled. + There are two possible ways of exporting the profiles. +

    - You can choose which columns to be included in the export as well as whether the values - should be enclosed in quotes when exporting. You can also select whether the export - should produce one row per file, or one row per format.  When exporting one row per - file, each row in the CSV file will represent a single file, folder or archival file - profiled with DROID.  If exporting one row per format, each row in the CSV file will be - a single format identificaiton made by DROID. Since a file can be identified as being more - than one possible format, this option will produce CSV files with multiple rows for the - same file (but with different identifications for it). +

    + Selecting the columns for export +

    - The characters of the export will be encoded as UTF-8 by default. If you need to set this - to the encoding used on your local machine instead, select 'Platform specific' instead. -

    -

    - When you are happy you want to export your profiles, press the    Export - profiles...   button.  This will bring up a standard file-save - dialog, in which you can specify where you want your CSV file to be saved.   + When the "Use export template" checkbox is unchecked, you are presented with an option to select + one or more columns that you wish to export, as shown below: +

    +

    +

    CSV File Columns @@ -248,11 +245,11 @@

    The date and time on which a resource was last modified.

    - MD5, SHA1, or SHA256 hash + MD5, SHA1, SHA256 or SHA512 hash

    If you have enabled hash generation in - the preferences, then this column will contain the MD5, SHA1, or SHA256 hash for each file and archival file + the preferences, then this column will contain the MD5, SHA1, SHA256 or SHA512 hash for each file and archival file processed.  See "Detecting duplicate files" for more information on hashes.

    @@ -298,7 +295,70 @@

    blank even when a file has a PUID.

    -   +

    + Selecting a predefined Export Template for export +

    +

    +

    + If you have configured any export templates for use with DROID, the "Use export template" checkbox + is enabled. Checking this checkbox shows you a view to select one of the configured templates + to be used for export as shown below: +

    +

    + +

    +

    + An export template is a simple text file, with a .template extension, which defines customisations of + columns to be exported. Using a template, you can customise headers for the data columns, add new columns + to the export, convert the data in a column to be uppercase / lowercase and change the order in which columns + appear in the export. You can make an export template available to Droid by copying it into the + ".droid6\export_templates" folder. An example template is shown below: +

    +

    + 

    +        version 1.0
+        Identifier: $ID
+        LowerName: LCASE($NAME)
+        Language: "Simplified English"
+        Submitter:
+
    +

    +

    + The above template indicates: +

      +
    1. The template is defined in version 1.0 of template syntax
      2. +
    2. There are 4 columns to be exported in the order Identifier, LowerName, Language and Submitter
      3. +
    3. Values in the column "Identifier" are populated using values in the "ID" column
      4. +
    4. Values in the columns "LowerName" are populated by converting the values from the "NAME" column to lowercase
      5. +
    5. There is a non-profile column called "Language" and the values are hardcoded to "Simplified English"
      6. +
    6. There is a non-profile column called "Submitter" and the values are left empty
      7. +
    +

    +

    +

    + Other options +

    +

    +

    + Once you have selected the columns to export or selected a template that you wish to use, + you can also select other options such as whether the values should be enclosed in quotes + when exporting. You can also select whether the export should produce one row per file, + or one row per format.  When exporting one row per file, each row in the CSV file will + represent a single file, folder or archival file profiled with DROID.  If exporting one + row per format, each row in the CSV file will be a single format identificaiton made by + DROID. Since a file can be identified as being more than one possible format, this option + will produce CSV files with multiple rows for the same file (but with different identifications + for it). +

    +

    + The characters of the export will be encoded as UTF-8 by default. If you need to set this + to the encoding used on your local machine instead, select 'Platform specific' instead. +

    +

    + When you are happy you want to export your profiles, press the    Export + profiles...   button.  This will bring up a standard file-save + dialog, in which you can specify where you want your CSV file to be saved.  

    diff --git a/droid-parent/pom.xml b/droid-parent/pom.xml index af8ae9c18..1a0bc3004 100644 --- a/droid-parent/pom.xml +++ b/droid-parent/pom.xml @@ -11,7 +11,7 @@ uk.gov.nationalarchivesdroid-parent - 6.7.1-SNAPSHOT + 6.8.0-SNAPSHOTpomdroid-parent @@ -162,7 +162,7 @@ org.apache.maven.plugins maven-surefire-plugin - 3.0.0-M7 + 3.2.3 org.apache.maven.plugins @@ -177,13 +177,13 @@ org.apache.maven.plugins maven-source-plugin - 3.3.0 + 3.3.1 org.apache.maven.plugins maven-javadoc-plugin - 3.5.0 + 3.6.3 @@ -608,7 +608,7 @@ Copyright © ${project.inceptionYear}-{currentYear} PER_FORMAT_HEADERS = Arrays.asList( - HEADER_NAME_PUID, HEADER_NAME_MIME_TYPE, HEADER_NAME_FORMAT_NAME, HEADER_NAME_FORMAT_VERSION); - - private static final String FILE_URI_SCHEME = "file"; - - /* - * Indexes of the headers used in the CSV output. - */ - private static final int ID_ARRAY_INDEX = 0; - private static final int PARENT_ID_ARRAY_INDEX = 1; - private static final int URI_ARRAY_INDEX = 2; - private static final int FILE_PATH_ARRAY_INDEX = 3; - private static final int FILE_NAME_ARRAY_INDEX = 4; - private static final int ID_METHOD_ARRAY_INDEX = 5; - private static final int STATUS_ARRAY_INDEX = 6; - private static final int SIZE_ARRAY_INDEX = 7; - private static final int RESOURCE_ARRAY_INDEX = 8; - private static final int EXTENSION_ARRAY_INDEX = 9; - private static final int LAST_MODIFIED_ARRAY_INDEX = 10; - private static final int EXTENSION_MISMATCH_ARRAY_INDEX = 11; - private static final int HASH_ARRAY_INDEX = 12; - private static final int ID_COUNT_ARRAY_INDEX = 13; - private static final int PUID_ARRAY_INDEX = 14; - private static final int MIME_TYPE_ARRAY_INDEX = 15; - private static final int FORMAT_NAME_ARRAY_INDEX = 16; - private static final int FORMAT_VERSION_ARRAY_INDEX = 17; private static final String BLANK_SPACE_DELIMITER = " "; + private final Map columnsToWriteMap = new HashMap<>(); private final Logger log = LoggerFactory.getLogger(getClass()); private CsvWriter csvWriter; - private final FastDateFormat dateFormat = DateFormatUtils.ISO_DATETIME_FORMAT; private ExportOptions options = ExportOptions.ONE_ROW_PER_FILE; - private String[] headers; + private String[] allHeaders; private boolean quoteAllFields; - private final boolean[] columnsToWrite; - private int numColumnsToWrite; + private ExportTemplate exportTemplate; /** * Empty bean constructor. @@ -167,73 +86,69 @@ public CsvItemWriter() { public CsvItemWriter(CsvWriter writer) { this.csvWriter = writer; this.quoteAllFields = true; - numColumnsToWrite = HEADERS.length; - columnsToWrite = new boolean[HEADERS.length]; - Arrays.fill(columnsToWrite, true); + populateDefaultColumnsToWrite(); + } + + private void populateDefaultColumnsToWrite() { + columnsToWriteMap.put(CsvWriterConstants.HEADER_NAME_ID, true); + columnsToWriteMap.put(CsvWriterConstants.HEADER_NAME_PARENT_ID, true); + columnsToWriteMap.put(CsvWriterConstants.HEADER_NAME_URI, true); + columnsToWriteMap.put(CsvWriterConstants.HEADER_NAME_FILE_PATH, true); + columnsToWriteMap.put(CsvWriterConstants.HEADER_NAME_NAME, true); + columnsToWriteMap.put(CsvWriterConstants.HEADER_NAME_METHOD, true); + columnsToWriteMap.put(CsvWriterConstants.HEADER_NAME_STATUS, true); + columnsToWriteMap.put(CsvWriterConstants.HEADER_NAME_SIZE, true); + columnsToWriteMap.put(CsvWriterConstants.HEADER_NAME_TYPE, true); + columnsToWriteMap.put(CsvWriterConstants.HEADER_NAME_EXT, true); + columnsToWriteMap.put(CsvWriterConstants.HEADER_NAME_LAST_MODIFIED, true); + columnsToWriteMap.put(CsvWriterConstants.HEADER_NAME_EXTENSION_MISMATCH, true); + columnsToWriteMap.put(CsvWriterConstants.HEADER_NAME_HASH, true); + columnsToWriteMap.put(CsvWriterConstants.HEADER_NAME_FORMAT_COUNT, true); + columnsToWriteMap.put(CsvWriterConstants.HEADER_NAME_PUID, true); + columnsToWriteMap.put(CsvWriterConstants.HEADER_NAME_MIME_TYPE, true); + columnsToWriteMap.put(CsvWriterConstants.HEADER_NAME_FORMAT_NAME, true); + columnsToWriteMap.put(CsvWriterConstants.HEADER_NAME_FORMAT_VERSION, true); } @Override public void write(List nodes) { + FormattedDataWriter dataWriter = DataWriterProvider.getDataWriter(columnsToWriteMap, exportTemplate); switch (options) { case ONE_ROW_PER_FILE: { - writeOneRowPerFile(nodes); + writeOneRowPerFile(nodes, dataWriter); break; } case ONE_ROW_PER_FORMAT: { - writeOneRowPerFormat(nodes); + writeOneRowPerFormat(nodes, dataWriter); break; } default: { //what? unknown option log.warn("Unable to handle ExportOptions = " + options + ", was there a new option created?"); options = ExportOptions.ONE_ROW_PER_FILE; - writeOneRowPerFile(nodes); + writeOneRowPerFile(nodes, dataWriter); } } } - private void writeOneRowPerFile(List nodes) { + private void writeOneRowPerFile(List nodes, FormattedDataWriter dataWriter) { if (csvWriter.getRecordCount() == 0) { - writeHeadersForOneRowPerFileExport(nodes); + dataWriter.writeHeadersForOneRowPerFile(nodes, allHeaders, csvWriter); } try { - for (ProfileResourceNode node : nodes) { - List nodeEntries = new ArrayList<>(); - addNodeColumns(nodeEntries, node); - for (Format format : node.getFormatIdentifications()) { - addColumn(nodeEntries, PUID_ARRAY_INDEX, format.getPuid()); - addColumn(nodeEntries, MIME_TYPE_ARRAY_INDEX, format.getMimeType()); - addColumn(nodeEntries, FORMAT_NAME_ARRAY_INDEX, format.getName()); - addColumn(nodeEntries, FORMAT_VERSION_ARRAY_INDEX, format.getVersion()); - } - csvWriter.writeRow(nodeEntries); - } - csvWriter.flush(); + dataWriter.writeDataRowsForOneRowPerFile(nodes, csvWriter); } catch (final TextWritingException e) { log.error(e.getRecordCharacters(), e); throw new RuntimeException(e.getMessage(), e); } } - - private void writeOneRowPerFormat(List nodes) { - List headersToWrite = Arrays.stream(getHeadersToWrite(headers)).collect(Collectors.toList()) ; + + private void writeOneRowPerFormat(List nodes, FormattedDataWriter dataWriter) { if (csvWriter.getRecordCount() == 0) { - csvWriter.writeHeaders(headersToWrite); + dataWriter.writeHeadersForOneRowPerFormat(nodes, allHeaders, csvWriter); } - try { - for (ProfileResourceNode node : nodes) { - for (Format format : node.getFormatIdentifications()) { - List nodeEntries = new ArrayList<>(); - addNodeColumns(nodeEntries, node); - addColumn(nodeEntries, PUID_ARRAY_INDEX, format.getPuid()); - addColumn(nodeEntries, MIME_TYPE_ARRAY_INDEX, format.getMimeType()); - addColumn(nodeEntries, FORMAT_NAME_ARRAY_INDEX, format.getName()); - addColumn(nodeEntries, FORMAT_VERSION_ARRAY_INDEX, format.getVersion()); - csvWriter.writeRow(nodeEntries); - } - } - csvWriter.flush(); + dataWriter.writeDataRowsForOneRowPerFormat(nodes, csvWriter); } catch (final TextWritingException e) { log.error(e.getRecordCharacters(), e); throw new RuntimeException(e.getMessage(), e); @@ -252,46 +167,15 @@ public void open(final Writer writer) { final CsvWriterSettings csvWriterSettings = new CsvWriterSettings(); csvWriterSettings.setQuoteAllFields(quoteAllFields); CsvFormat format = new CsvFormat(); - // following Unix convention on line separators as previously + // following Unix convention about line separators as previously format.setLineSeparator("\n"); csvWriterSettings.setFormat(format); csvWriter = new CsvWriter(writer, csvWriterSettings); - if (headers == null) { - headers = Arrays.copyOf(HEADERS, HEADERS.length) ; + if (allHeaders == null) { + allHeaders = Arrays.copyOf(CsvWriterConstants.HEADERS, CsvWriterConstants.HEADERS.length) ; } } - private void writeHeadersForOneRowPerFileExport(List nodes) { - - if (options != ExportOptions.ONE_ROW_PER_FILE) { - throw new RuntimeException("Unexpectedly called per file header creation. Unable to proceed"); - } - - List headersToWrite = Arrays.stream(getHeadersToWrite(headers)).collect(Collectors.toList()) ; - - //if we are writing one row per file, then we tag the "per format" fields as additional columns, - //if such columns need to be added, we create appropriate headers with a running suffix and write - //them to the file - if (!Collections.disjoint(headersToWrite, PER_FORMAT_HEADERS)) { - Optional maxIdentificationsOption = nodes.stream().map(ProfileResourceNode::getIdentificationCount).collect(Collectors.toList()).stream().filter(Objects::nonNull).max(Integer::compare); - int maxIdentifications = 0; - if (maxIdentificationsOption.isPresent()) { - maxIdentifications = maxIdentificationsOption.get(); - } - if (maxIdentifications > 1) { //add headers - for (int newColumnSuffix = 1; newColumnSuffix < maxIdentifications; newColumnSuffix++) { - //"PUID","MIME_TYPE","FORMAT_NAME","FORMAT_VERSION" - for (String headerEntry : PER_FORMAT_HEADERS) { - if (headersToWrite.contains(headerEntry)) { - headersToWrite.add(headerEntry + newColumnSuffix); - } - } - } - } - } - csvWriter.writeHeaders(headersToWrite); - } - @Override public void setOptions(ExportOptions options) { this.options = options; @@ -305,44 +189,6 @@ public void close() { csvWriter.close(); } - private static String nullSafeName(Enum value) { - return value == null ? "" : value.toString(); - } - - private static String nullSafeNumber(Number number) { - return number == null ? "" : number.toString(); - } - - private static String nullSafeDate(Date date, FastDateFormat format) { - return date == null ? "" : format.format(date); - } - - private static String toFileName(String name) { - return FilenameUtils.getName(name); - } - - private String toFilePath(URI uri) { - if (uri == null) { - log.warn("[URI not set]"); - return ""; - } - if (FILE_URI_SCHEME.equals(uri.getScheme())) { - return Paths.get(uri).toAbsolutePath().toString(); - } - - // for URIs that have other than "file" scheme - String result = java.net.URLDecoder.decode(uri.toString()).replaceAll("file://", ""); - result = result.replace("/", File.separator); - - // Handle substitution of 7z - final String sevenZedIdentifier = "sevenz:"; - if (result.startsWith(sevenZedIdentifier)) { - result = "7z:" + result.substring(sevenZedIdentifier.length()); - } - - return result; - } - /** * No config is needed by this class, but it's retained temporarily for backwards compatibility purposes. * @param config the config to set @@ -356,14 +202,28 @@ public void setConfig(DroidGlobalConfig config) { @Override public void setHeaders(Map headersToSet) { - if (this.headers == null) { - this.headers = Arrays.copyOf(HEADERS, HEADERS.length); + if (this.allHeaders == null) { + this.allHeaders = Arrays.copyOf(CsvWriterConstants.HEADERS, CsvWriterConstants.HEADERS.length); } // The header for hash is modified based on algorithm used to generate the hash String hashHeader = headersToSet.get("hash"); if (hashHeader != null) { - this.headers[HASH_ARRAY_INDEX] = hashHeader; + if (exportTemplate != null) { + Map columnPositions = exportTemplate.getColumnOrderMap(); + List> profileCols = columnPositions.entrySet() + .stream() + .filter(e -> e.getValue().getColumnType() == ExportTemplateColumnDef.ColumnType.ProfileResourceNode) + .collect(Collectors.toList()); + List> hashEntry = profileCols.stream() + .filter(entry -> entry.getValue().getOriginalColumnName().equals(CsvWriterConstants.HEADER_NAME_HASH)) + .collect(Collectors.toList()); + if (!hashEntry.isEmpty()) { + this.allHeaders[CsvWriterConstants.HASH_ARRAY_INDEX] = hashHeader; + } + } else { + this.allHeaders[CsvWriterConstants.HASH_ARRAY_INDEX] = hashHeader; + } } } @@ -376,26 +236,22 @@ public void setQuoteAllFields(boolean quoteAll) { public void setColumnsToWrite(String columnNames) { Set headersToWrite = getColumnsToWrite(columnNames); if (headersToWrite == null) { - Arrays.fill(columnsToWrite, true); - numColumnsToWrite = columnsToWrite.length; + populateDefaultColumnsToWrite(); } else { - int numberToWrite = 0; - for (int i = 0; i < HEADERS.length; i++) { - if (headersToWrite.contains(HEADERS[i])) { - columnsToWrite[i] = true; - numberToWrite++; - headersToWrite.remove(HEADERS[i]); + for (int i = 0; i < CsvWriterConstants.HEADERS.length; i++) { + String currentHeader = CsvWriterConstants.HEADERS[i]; + if (headersToWrite.contains(currentHeader)) { + columnsToWriteMap.put(currentHeader, true); + headersToWrite.remove(currentHeader); } else { - columnsToWrite[i] = false; + columnsToWriteMap.put(currentHeader, false); } } // Defence: if no valid column names were specified, log a warning then write them all out: - if (numberToWrite == 0) { - numColumnsToWrite = columnsToWrite.length; - Arrays.fill(columnsToWrite, true); + if (!columnsToWriteMap.containsValue(true)) { + populateDefaultColumnsToWrite(); log.warn("-co option: no CSV columns specified are valid, writing all columns: " + columnNames); } else { - numColumnsToWrite = numberToWrite; // If there are some columns specified left over, they aren't valid columns - log a warning: if (headersToWrite.size() > 0) { String invalidHeaders = String.join(BLANK_SPACE_DELIMITER, headersToWrite); @@ -405,6 +261,11 @@ public void setColumnsToWrite(String columnNames) { } } + @Override + public void setExportTemplate(ExportTemplate template) { + this.exportTemplate = template; + } + private Set getColumnsToWrite(String columnNames) { if (columnNames != null && !columnNames.isEmpty()) { String[] columns = columnNames.split(BLANK_SPACE_DELIMITER); @@ -414,43 +275,5 @@ private Set getColumnsToWrite(String columnNames) { } return null; } - - private String[] getHeadersToWrite(String[] headersToWrite) { - if (numColumnsToWrite < columnsToWrite.length) { - String[] newHeaders = new String[numColumnsToWrite]; - int newHeaderIndex = 0; - for (int i = 0; i < columnsToWrite.length; i++) { - if (columnsToWrite[i]) { - newHeaders[newHeaderIndex++] = headers[i]; - } - } - return newHeaders; - } - return headersToWrite; - } - - private void addNodeColumns(List row, ProfileResourceNode node) { - NodeMetaData metaData = node.getMetaData(); - addColumn(row, ID_ARRAY_INDEX, nullSafeNumber(node.getId())); - addColumn(row, PARENT_ID_ARRAY_INDEX, nullSafeNumber(node.getParentId())); - addColumn(row, URI_ARRAY_INDEX, DroidUrlFormat.format(node.getUri())); - addColumn(row, FILE_PATH_ARRAY_INDEX, toFilePath(node.getUri())); - addColumn(row, FILE_NAME_ARRAY_INDEX, toFileName(metaData.getName())); - addColumn(row, ID_METHOD_ARRAY_INDEX, nullSafeName(metaData.getIdentificationMethod())); - addColumn(row, STATUS_ARRAY_INDEX, metaData.getNodeStatus().getStatus()); - addColumn(row, SIZE_ARRAY_INDEX, nullSafeNumber(metaData.getSize())); - addColumn(row, RESOURCE_ARRAY_INDEX, metaData.getResourceType().getResourceType()); - addColumn(row, EXTENSION_ARRAY_INDEX, metaData.getExtension()); - addColumn(row, LAST_MODIFIED_ARRAY_INDEX, nullSafeDate(metaData.getLastModifiedDate(), dateFormat)); - addColumn(row, EXTENSION_MISMATCH_ARRAY_INDEX, node.getExtensionMismatch().toString()); - addColumn(row, HASH_ARRAY_INDEX, metaData.getHash()); - addColumn(row, ID_COUNT_ARRAY_INDEX, nullSafeNumber(node.getIdentificationCount())); - } - - private void addColumn(List row, int columnIndex, String value) { - if (columnsToWrite[columnIndex]) { - row.add(value); - } - } } diff --git a/droid-results/src/main/java/uk/gov/nationalarchives/droid/profile/CsvWriterConstants.java b/droid-results/src/main/java/uk/gov/nationalarchives/droid/profile/CsvWriterConstants.java new file mode 100644 index 000000000..3aa8ea43c --- /dev/null +++ b/droid-results/src/main/java/uk/gov/nationalarchives/droid/profile/CsvWriterConstants.java @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.profile; + +import org.apache.commons.lang.time.DateFormatUtils; +import org.apache.commons.lang.time.FastDateFormat; + +import java.util.Arrays; +import java.util.List; + +public final class CsvWriterConstants { + /** + * Default header for the corresponding column. + */ + public static final String HEADER_NAME_ID = "ID"; + /** + * Default header for the corresponding column. + */ + public static final String HEADER_NAME_PARENT_ID = "PARENT_ID"; + /** + * Default header for the corresponding column. + */ + public static final String HEADER_NAME_URI = "URI"; + /** + * Default header for the corresponding column. + */ + public static final String HEADER_NAME_FILE_PATH = "FILE_PATH"; + /** + * Default header for the corresponding column. + */ + public static final String HEADER_NAME_NAME = "NAME"; + /** + * Default header for the corresponding column. + */ + public static final String HEADER_NAME_METHOD = "METHOD"; + /** + * Default header for the corresponding column. + */ + public static final String HEADER_NAME_STATUS = "STATUS"; + /** + * Default header for the corresponding column. + */ + public static final String HEADER_NAME_SIZE = "SIZE"; + /** + * Default header for the corresponding column. + */ + public static final String HEADER_NAME_TYPE = "TYPE"; + /** + * Default header for the corresponding column. + */ + public static final String HEADER_NAME_EXT = "EXT"; + /** + * Default header for the corresponding column. + */ + public static final String HEADER_NAME_LAST_MODIFIED = "LAST_MODIFIED"; + /** + * Default header for the corresponding column. + */ + public static final String HEADER_NAME_EXTENSION_MISMATCH = "EXTENSION_MISMATCH"; + /** + * Default header for the corresponding column. + */ + public static final String HEADER_NAME_HASH = "HASH"; + /** + * Default header for the corresponding column. + */ + public static final String HEADER_NAME_FORMAT_COUNT = "FORMAT_COUNT"; + /** + * Default header for the corresponding column. + */ + public static final String HEADER_NAME_PUID = "PUID"; + /** + * Default header for the corresponding column. + */ + public static final String HEADER_NAME_MIME_TYPE = "MIME_TYPE"; + /** + * Default header for the corresponding column. + */ + public static final String HEADER_NAME_FORMAT_NAME = "FORMAT_NAME"; + /** + * Default header for the corresponding column. + */ + public static final String HEADER_NAME_FORMAT_VERSION = "FORMAT_VERSION"; + + /** + * Default dateTime format to be used for writing CSVs. + */ + public static final FastDateFormat DATE_FORMAT = DateFormatUtils.ISO_DATETIME_FORMAT; + + + /** + * Default index of the hash column, since this column header may be modified. + * This is the only column that we may need to access using index. + */ + public static final int HASH_ARRAY_INDEX = 12; + + /** + * String array for all the default headers. + */ + public static final String[] HEADERS = { + HEADER_NAME_ID, + HEADER_NAME_PARENT_ID, + HEADER_NAME_URI, + HEADER_NAME_FILE_PATH, + HEADER_NAME_NAME, + HEADER_NAME_METHOD, + HEADER_NAME_STATUS, + HEADER_NAME_SIZE, + HEADER_NAME_TYPE, + HEADER_NAME_EXT, + HEADER_NAME_LAST_MODIFIED, + HEADER_NAME_EXTENSION_MISMATCH, + HEADER_NAME_HASH, + HEADER_NAME_FORMAT_COUNT, + HEADER_NAME_PUID, + HEADER_NAME_MIME_TYPE, + HEADER_NAME_FORMAT_NAME, + HEADER_NAME_FORMAT_VERSION, + }; + + /** + * List of headers that appear more than once if a file matches more than one format. + */ + public static final List PER_FORMAT_HEADERS = Arrays.asList( + HEADER_NAME_PUID, HEADER_NAME_MIME_TYPE, HEADER_NAME_FORMAT_NAME, HEADER_NAME_FORMAT_VERSION); + + /** + * Empty string constant. + */ + public static final String EMPTY_STRING = ""; + + /** + * File uri scheme, used as prefix. + */ + public static final String FILE_URI_SCHEME = "file"; + + private CsvWriterConstants() { + //hidden constructor + } + +} diff --git a/droid-results/src/main/java/uk/gov/nationalarchives/droid/profile/datawriter/ColumnBasedDataWriter.java b/droid-results/src/main/java/uk/gov/nationalarchives/droid/profile/datawriter/ColumnBasedDataWriter.java new file mode 100644 index 000000000..6d7b38425 --- /dev/null +++ b/droid-results/src/main/java/uk/gov/nationalarchives/droid/profile/datawriter/ColumnBasedDataWriter.java @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.profile.datawriter; + +import com.univocity.parsers.csv.CsvWriter; +import uk.gov.nationalarchives.droid.core.interfaces.util.DroidUrlFormat; +import uk.gov.nationalarchives.droid.profile.CsvWriterConstants; +import uk.gov.nationalarchives.droid.profile.NodeMetaData; +import uk.gov.nationalarchives.droid.profile.ProfileResourceNode; +import uk.gov.nationalarchives.droid.profile.referencedata.Format; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * This class implements the methods for writing headers and data when the export is done using selection of various columns. + */ +public class ColumnBasedDataWriter extends FormattedDataWriter { + private final Map columnsToWriteMap; + + public ColumnBasedDataWriter(Map columnsToWriteMap) { + this.columnsToWriteMap = columnsToWriteMap; + } + + @Override + public void writeDataRowsForOneRowPerFile(List nodes, CsvWriter csvWriter) { + int maxIdCount = getMaxIdentificationCount(nodes); + for (ProfileResourceNode node : nodes) { + List nodeEntries = new ArrayList<>(); + addNodeColumnsInDefaultOrder(nodeEntries, node); + List formatIdentifications = node.getFormatIdentifications(); + for (int i = 0; i < maxIdCount; i++) { + addColumn(nodeEntries, CsvWriterConstants.HEADER_NAME_PUID, (i < formatIdentifications.size()) ? formatIdentifications.get(i).getPuid() : CsvWriterConstants.EMPTY_STRING); + addColumn(nodeEntries, CsvWriterConstants.HEADER_NAME_MIME_TYPE, (i < formatIdentifications.size()) ? formatIdentifications.get(i).getMimeType() : CsvWriterConstants.EMPTY_STRING); + addColumn(nodeEntries, CsvWriterConstants.HEADER_NAME_FORMAT_NAME, (i < formatIdentifications.size()) ? formatIdentifications.get(i).getName() : CsvWriterConstants.EMPTY_STRING); + addColumn(nodeEntries, CsvWriterConstants.HEADER_NAME_FORMAT_VERSION, (i < formatIdentifications.size()) ? formatIdentifications.get(i).getVersion(): CsvWriterConstants.EMPTY_STRING); + } + csvWriter.writeRow(nodeEntries); + } + csvWriter.flush(); + } + + @Override + public void writeDataRowsForOneRowPerFormat(List nodes, CsvWriter csvWriter) { + for (ProfileResourceNode node : nodes) { + for (Format format : node.getFormatIdentifications()) { + List nodeEntries = new ArrayList<>(); + addNodeColumnsInDefaultOrder(nodeEntries, node); + addColumn(nodeEntries, CsvWriterConstants.HEADER_NAME_PUID, format.getPuid()); + addColumn(nodeEntries, CsvWriterConstants.HEADER_NAME_MIME_TYPE, format.getMimeType()); + addColumn(nodeEntries, CsvWriterConstants.HEADER_NAME_FORMAT_NAME, format.getName()); + addColumn(nodeEntries, CsvWriterConstants.HEADER_NAME_FORMAT_VERSION, format.getVersion()); + csvWriter.writeRow(nodeEntries); + } + } + csvWriter.flush(); + } + + @Override + public void writeHeadersForOneRowPerFile(List nodes, String[] headers, CsvWriter csvWriter) { + super.setCustomisedHeaders(headers); + List headersToWrite = new ArrayList<>(getHeadersToWrite(getCustomisedHeaders())); + int maxIdCount = getMaxIdentificationCount(nodes); + + //if we are writing one row per file, then we tag the "per format" fields as additional columns, + //if such columns need to be added, we create appropriate headers with a running suffix and write + //them to the file + if (!Collections.disjoint(headersToWrite, CsvWriterConstants.PER_FORMAT_HEADERS) && (maxIdCount > 1)) { //add headers + for (int newColumnSuffix = 1; newColumnSuffix < maxIdCount; newColumnSuffix++) { + //"PUID","MIME_TYPE","FORMAT_NAME","FORMAT_VERSION" + for (String headerEntry : CsvWriterConstants.PER_FORMAT_HEADERS) { + if (headersToWrite.contains(headerEntry)) { + headersToWrite.add(headerEntry + newColumnSuffix); + } + } + } + } + + csvWriter.writeHeaders(headersToWrite); + csvWriter.flush(); + } + + @Override + public void writeHeadersForOneRowPerFormat(List nodes, String[] headers, CsvWriter csvWriter) { + super.setCustomisedHeaders(headers); + List headersToWrite = new ArrayList<>(getHeadersToWrite(getCustomisedHeaders())); + csvWriter.writeHeaders(headersToWrite); + csvWriter.flush(); + } + + private void addNodeColumnsInDefaultOrder(List row, ProfileResourceNode node) { + NodeMetaData metaData = node.getMetaData(); + addColumn(row, CsvWriterConstants.HEADER_NAME_ID, nullSafeNumber(node.getId())); + addColumn(row, CsvWriterConstants.HEADER_NAME_PARENT_ID, nullSafeNumber(node.getParentId())); + addColumn(row, CsvWriterConstants.HEADER_NAME_URI, DroidUrlFormat.format(node.getUri())); + addColumn(row, CsvWriterConstants.HEADER_NAME_FILE_PATH, toFilePath(node.getUri())); + addColumn(row, CsvWriterConstants.HEADER_NAME_NAME, toFileName(metaData.getName())); + addColumn(row, CsvWriterConstants.HEADER_NAME_METHOD, nullSafeName(metaData.getIdentificationMethod())); + addColumn(row, CsvWriterConstants.HEADER_NAME_STATUS, metaData.getNodeStatus().getStatus()); + addColumn(row, CsvWriterConstants.HEADER_NAME_SIZE, nullSafeNumber(metaData.getSize())); + addColumn(row, CsvWriterConstants.HEADER_NAME_TYPE, metaData.getResourceType().getResourceType()); + addColumn(row, CsvWriterConstants.HEADER_NAME_EXT, metaData.getExtension()); + addColumn(row, CsvWriterConstants.HEADER_NAME_LAST_MODIFIED, nullSafeDate(metaData.getLastModifiedDate(), CsvWriterConstants.DATE_FORMAT)); + addColumn(row, CsvWriterConstants.HEADER_NAME_EXTENSION_MISMATCH, node.getExtensionMismatch().toString()); + addColumn(row, CsvWriterConstants.HEADER_NAME_HASH, metaData.getHash()); + addColumn(row, CsvWriterConstants.HEADER_NAME_FORMAT_COUNT, nullSafeNumber(node.getIdentificationCount())); + } + + private void addColumn(List row, String columnName, String value) { + if (columnsToWriteMap.get(columnName)) { + row.add(value); + } + } + + private List getHeadersToWrite(String[] headersToWrite) { + if (columnsToWriteMap.containsValue(false)) { + int numColumnsToWrite = (int) columnsToWriteMap.values().stream().filter(eachVal -> eachVal).count(); + String[] newHeaders = new String[numColumnsToWrite]; + int newHeaderIndex = 0; + for (int i = 0; i < CsvWriterConstants.HEADERS.length; i++) { + if (columnsToWriteMap.get(CsvWriterConstants.HEADERS[i])) { + newHeaders[newHeaderIndex++] = getCustomisedHeaders()[i]; + } + } + return Arrays.stream(newHeaders).collect(Collectors.toList()); + } + return Arrays.stream(headersToWrite).collect(Collectors.toList()); + } + +} diff --git a/droid-results/src/main/java/uk/gov/nationalarchives/droid/profile/datawriter/DataWriterProvider.java b/droid-results/src/main/java/uk/gov/nationalarchives/droid/profile/datawriter/DataWriterProvider.java new file mode 100644 index 000000000..e760c588b --- /dev/null +++ b/droid-results/src/main/java/uk/gov/nationalarchives/droid/profile/datawriter/DataWriterProvider.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.profile.datawriter; + +import uk.gov.nationalarchives.droid.export.interfaces.ExportTemplate; + +import java.util.Map; + +/** + * Abstracts the creation of specific type of data writer away from the user. + */ +public final class DataWriterProvider { + + private DataWriterProvider() { + //hidden constructor + } + + /** + * Instantiae and return specific data writer (either column based, or template based). + * @param columnsToWriteMap Map with a list of columns to write + * @param exportTemplate exportTemplate + * @return Specific data writer. + */ + public static FormattedDataWriter getDataWriter(Map columnsToWriteMap, ExportTemplate exportTemplate) { + if (exportTemplate != null) { + return new TemplateBasedDataWriter(exportTemplate); + } else { + return new ColumnBasedDataWriter(columnsToWriteMap); + } + } +} diff --git a/droid-results/src/main/java/uk/gov/nationalarchives/droid/profile/datawriter/FormattedDataWriter.java b/droid-results/src/main/java/uk/gov/nationalarchives/droid/profile/datawriter/FormattedDataWriter.java new file mode 100644 index 000000000..ddb33c4fd --- /dev/null +++ b/droid-results/src/main/java/uk/gov/nationalarchives/droid/profile/datawriter/FormattedDataWriter.java @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.profile.datawriter; + +import com.univocity.parsers.csv.CsvWriter; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang.time.FastDateFormat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import uk.gov.nationalarchives.droid.profile.CsvWriterConstants; +import uk.gov.nationalarchives.droid.profile.ProfileResourceNode; + +import java.io.File; +import java.net.URI; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Abstract class used fow writing headers and lines to the CSV writer. + * This class implements some common methods and other specific data writers are responsible for overriding + * the methods for writing headers and data + */ +public abstract class FormattedDataWriter { + private final Logger log = LoggerFactory.getLogger(getClass()); + + private String[] customisedHeaders; + + public abstract void writeHeadersForOneRowPerFile(List nodes, String[] headers, CsvWriter csvWriter); + public abstract void writeDataRowsForOneRowPerFile(List nodes, CsvWriter csvWriter); + public abstract void writeHeadersForOneRowPerFormat(List nodes, String[] headers, CsvWriter csvWriter); + public abstract void writeDataRowsForOneRowPerFormat(List nodes, CsvWriter csvWriter); + + protected static String nullSafeName(Enum value) { + return value == null ? CsvWriterConstants.EMPTY_STRING : value.toString(); + } + + protected static String nullSafeNumber(Number number) { + return number == null ? CsvWriterConstants.EMPTY_STRING : number.toString(); + } + + protected static String nullSafeDate(Date date, FastDateFormat format) { + return date == null ? CsvWriterConstants.EMPTY_STRING : format.format(date); + } + + protected static String toFileName(String name) { + return FilenameUtils.getName(name); + } + + protected String toFilePath(URI uri) { + if (uri == null) { + log.warn("[URI not set]"); + return CsvWriterConstants.EMPTY_STRING; + } + if (CsvWriterConstants.FILE_URI_SCHEME.equals(uri.getScheme())) { + return Paths.get(uri).toAbsolutePath().toString(); + } + + // for URIs that have other than "file" scheme + String result = java.net.URLDecoder.decode(uri.toString()).replaceAll("file://", CsvWriterConstants.EMPTY_STRING); + result = result.replace("/", File.separator); + + // Handle substitution of 7z + final String sevenZedIdentifier = "sevenz:"; + if (result.startsWith(sevenZedIdentifier)) { + result = "7z:" + result.substring(sevenZedIdentifier.length()); + } + + return result; + } + + protected int getMaxIdentificationCount(List nodes) { + Optional maxIdentificationsOption = nodes.stream().map(ProfileResourceNode::getIdentificationCount).collect(Collectors.toList()).stream().filter(Objects::nonNull).max(Integer::compare); + int maxIdentifications = 0; + if (maxIdentificationsOption.isPresent()) { + maxIdentifications = maxIdentificationsOption.get(); + } + return maxIdentifications; + } + + + protected void setCustomisedHeaders(final String[] customisedHeaders) { + this.customisedHeaders = Arrays.copyOf(customisedHeaders, customisedHeaders.length); + } + + protected String[] getCustomisedHeaders() { + return this.customisedHeaders; + } +} diff --git a/droid-results/src/main/java/uk/gov/nationalarchives/droid/profile/datawriter/TemplateBasedDataWriter.java b/droid-results/src/main/java/uk/gov/nationalarchives/droid/profile/datawriter/TemplateBasedDataWriter.java new file mode 100644 index 000000000..c3c9f3132 --- /dev/null +++ b/droid-results/src/main/java/uk/gov/nationalarchives/droid/profile/datawriter/TemplateBasedDataWriter.java @@ -0,0 +1,228 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.profile.datawriter; + +import com.univocity.parsers.csv.CsvWriter; +import uk.gov.nationalarchives.droid.core.interfaces.util.DroidUrlFormat; +import uk.gov.nationalarchives.droid.export.interfaces.ExportTemplate; +import uk.gov.nationalarchives.droid.export.interfaces.ExportTemplateColumnDef; +import uk.gov.nationalarchives.droid.profile.CsvWriterConstants; +import uk.gov.nationalarchives.droid.profile.NodeMetaData; +import uk.gov.nationalarchives.droid.profile.ProfileResourceNode; +import uk.gov.nationalarchives.droid.profile.referencedata.Format; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * This class implements the methods for writing headers and data when the export is done using an export template. + */ +public class TemplateBasedDataWriter extends FormattedDataWriter{ + private final ExportTemplate template; + public TemplateBasedDataWriter(ExportTemplate exportTemplate) { + this.template = exportTemplate; + } + + @Override + public void writeDataRowsForOneRowPerFile(List nodes, CsvWriter csvWriter) { + int maxIdCount = getMaxIdentificationCount(nodes); + Map columnPositions = template.getColumnOrderMap(); + int maxCols = columnPositions.keySet().stream().max(Integer::compare).get(); + for (ProfileResourceNode node : nodes) { + List nodeEntries = new ArrayList<>(); + for (int i = 0; i <= maxCols; i++) { + ExportTemplateColumnDef def = columnPositions.get(i); + if (def.getColumnType() == ExportTemplateColumnDef.ColumnType.ConstantString) { + nodeEntries.add(def.getDataValue()); + } else { + String columnName = def.getOriginalColumnName(); + if (CsvWriterConstants.PER_FORMAT_HEADERS.contains(columnName)) { + addFormatColumnTemplate(nodeEntries, node, def, maxIdCount); + } else { + addNodeColumn(nodeEntries, node, def); + } + } + } + csvWriter.writeRow(nodeEntries); + } + csvWriter.flush(); + } + + @Override + public void writeDataRowsForOneRowPerFormat(List nodes, CsvWriter csvWriter) { + Map columnPositions = template.getColumnOrderMap(); + int maxCols = columnPositions.keySet().stream().max(Integer::compare).get(); + for (ProfileResourceNode node : nodes) { + for (Format format : node.getFormatIdentifications()) { + List nodeEntries = new ArrayList<>(); + for (int i = 0; i <= maxCols; i++) { + ExportTemplateColumnDef def = columnPositions.get(i); + if (def.getColumnType() == ExportTemplateColumnDef.ColumnType.ConstantString) { + nodeEntries.add(def.getDataValue()); + } else { + String columnName = def.getOriginalColumnName(); + if (CsvWriterConstants.PER_FORMAT_HEADERS.contains(columnName)) { + String columnValue = getFormatValue(columnName, format); + nodeEntries.add(columnValue); + } else { + addNodeColumn(nodeEntries, node, def); + } + } + } + csvWriter.writeRow(nodeEntries); + } + } + csvWriter.flush(); + } + + @Override + public void writeHeadersForOneRowPerFile(List nodes, String[] headers, CsvWriter csvWriter) { + super.setCustomisedHeaders(headers); + int maxIdCount = getMaxIdentificationCount(nodes); + List headersToWrite = getHeadersToWrite(maxIdCount); + csvWriter.writeHeaders(headersToWrite); + csvWriter.flush(); + } + + @Override + public void writeHeadersForOneRowPerFormat(List nodes, String[] headers, CsvWriter csvWriter) { + super.setCustomisedHeaders(headers); + int maxIdColumns = 1; //for "per format" export, the additional identified formats are written as new row hence no additional id headers + List headersToWrite = getHeadersToWrite(maxIdColumns); + csvWriter.writeHeaders(headersToWrite); + csvWriter.flush(); + } + + private void addFormatColumnTemplate(List nodeEntries, ProfileResourceNode node, ExportTemplateColumnDef def, int maxIdCount) { + List formats = node.getFormatIdentifications(); + for (int i = 0; i < maxIdCount; i++) { + Format format = i < formats.size() ? formats.get(i) : null; + nodeEntries.add(def.getOperatedValue(getFormatValue(def.getOriginalColumnName(), format))); + } + } + + //CHECKSTYLE:OFF Swith without default + private String getFormatValue(String columnName, Format format) { + String retVal = CsvWriterConstants.EMPTY_STRING; + switch (columnName) { + case CsvWriterConstants.HEADER_NAME_PUID: + retVal = format == null ? CsvWriterConstants.EMPTY_STRING : format.getPuid(); + break; + case CsvWriterConstants.HEADER_NAME_MIME_TYPE: + retVal = format == null ? CsvWriterConstants.EMPTY_STRING : format.getMimeType(); + break; + case CsvWriterConstants.HEADER_NAME_FORMAT_NAME: + retVal = format == null ? CsvWriterConstants.EMPTY_STRING : format.getName(); + break; + case CsvWriterConstants.HEADER_NAME_FORMAT_VERSION: + retVal = format == null ? CsvWriterConstants.EMPTY_STRING : format.getVersion(); + } + return retVal; + } + //CHECKSTYLE:ON + + //CHECKSTYLE:OFF - cyclomatic complexity is too high but we have to go through so many column names!! + private void addNodeColumn(List nodeEntries, ProfileResourceNode node, ExportTemplateColumnDef def) { + NodeMetaData metaData = node.getMetaData(); + + String columnValue = ""; + switch (def.getOriginalColumnName()) { + case CsvWriterConstants.HEADER_NAME_ID: + columnValue = nullSafeNumber(node.getId()); + break; + case CsvWriterConstants.HEADER_NAME_PARENT_ID: + columnValue = nullSafeNumber(node.getParentId()); + break; + case CsvWriterConstants.HEADER_NAME_URI: + columnValue = DroidUrlFormat.format(node.getUri()); + break; + case CsvWriterConstants.HEADER_NAME_FILE_PATH: + columnValue = toFilePath(node.getUri()); + break; + case CsvWriterConstants.HEADER_NAME_NAME: + columnValue = toFileName(metaData.getName()); + break; + case CsvWriterConstants.HEADER_NAME_METHOD: + columnValue = nullSafeName(metaData.getIdentificationMethod()); + break; + case CsvWriterConstants.HEADER_NAME_STATUS: + columnValue = metaData.getNodeStatus().getStatus(); + break; + case CsvWriterConstants.HEADER_NAME_SIZE: + columnValue = nullSafeNumber(metaData.getSize()); + break; + case CsvWriterConstants.HEADER_NAME_TYPE: + columnValue = metaData.getResourceType().getResourceType(); + break; + case CsvWriterConstants.HEADER_NAME_EXT: + columnValue = metaData.getExtension(); + break; + case CsvWriterConstants.HEADER_NAME_LAST_MODIFIED: + columnValue = nullSafeDate(metaData.getLastModifiedDate(), CsvWriterConstants.DATE_FORMAT); + break; + case CsvWriterConstants.HEADER_NAME_EXTENSION_MISMATCH: + columnValue = node.getExtensionMismatch().toString(); + break; + case CsvWriterConstants.HEADER_NAME_HASH: + columnValue = metaData.getHash(); + break; + case CsvWriterConstants.HEADER_NAME_FORMAT_COUNT: + columnValue = nullSafeNumber(node.getIdentificationCount()); + break; + } + nodeEntries.add(def.getOperatedValue(columnValue)); + } + //CHECKSTYLE:ON + + private List getHeadersToWrite(int maxIdCount) { + if (template == null) { + throw new IllegalArgumentException("Export template does not exist, unable to get headers from template"); + } + List retVal = new LinkedList<>(); + + Map columnOrderMap = template.getColumnOrderMap(); + for (int i = 0; i < columnOrderMap.size(); i++) { + ExportTemplateColumnDef def = columnOrderMap.get(i); + retVal.add(def.getHeaderLabel()); + if (def.getColumnType() != ExportTemplateColumnDef.ColumnType.ConstantString) { + if (CsvWriterConstants.PER_FORMAT_HEADERS.contains(def.getOriginalColumnName())) { + for (int newColumnSuffix = 1; newColumnSuffix < maxIdCount; newColumnSuffix++) { + retVal.add(def.getHeaderLabel() + newColumnSuffix); + } + } + } + } + return retVal; + } +} diff --git a/droid-results/src/test/java/uk/gov/nationalarchives/droid/profile/CsvItemWriterTest.java b/droid-results/src/test/java/uk/gov/nationalarchives/droid/profile/CsvItemWriterTest.java index b855248c1..5eec88c5b 100644 --- a/droid-results/src/test/java/uk/gov/nationalarchives/droid/profile/CsvItemWriterTest.java +++ b/droid-results/src/test/java/uk/gov/nationalarchives/droid/profile/CsvItemWriterTest.java @@ -45,6 +45,8 @@ import uk.gov.nationalarchives.droid.core.interfaces.config.DroidGlobalConfig; import uk.gov.nationalarchives.droid.core.interfaces.config.DroidGlobalProperty; import uk.gov.nationalarchives.droid.export.interfaces.ExportOptions; +import uk.gov.nationalarchives.droid.export.interfaces.ExportTemplate; +import uk.gov.nationalarchives.droid.export.interfaces.ExportTemplateColumnDef; import uk.gov.nationalarchives.droid.profile.referencedata.Format; import java.io.File; @@ -54,9 +56,12 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -80,7 +85,12 @@ public class CsvItemWriterTest { private CsvItemWriter itemWriter; private DroidGlobalConfig config; private String testDateTimeString; - + + private static final String defaultHeaders = toCsvRow(new String[] { + "ID","PARENT_ID","URI","FILE_PATH","NAME","METHOD","STATUS","SIZE","TYPE","EXT","LAST_MODIFIED","EXTENSION_MISMATCH","HASH","FORMAT_COUNT", + "PUID","MIME_TYPE","FORMAT_NAME","FORMAT_VERSION" + }); + @Before public void setup() { File dir = new File("exports"); @@ -105,7 +115,7 @@ private static String toCsvRow(final String[] values) { } @Test - public void testWriteNoNodes() throws IOException { + public void should_write_only_headers_when_there_are_no_nodes_to_be_written() throws IOException { when(config.getBooleanProperty(DroidGlobalProperty.CSV_EXPORT_ROW_PER_FORMAT)).thenReturn(false); try(final Writer writer = new StringWriter()) { @@ -113,12 +123,14 @@ public void testWriteNoNodes() throws IOException { itemWriter.open(writer); itemWriter.write(nodes); - assertEquals(1, writer.toString().split("\n").length); + String[] writtenLines = writer.toString().split(LINE_SEPARATOR); + assertEquals(1, writtenLines.length); + assertEquals(defaultHeaders, writtenLines[0]); } } @Test - public void testWriteOneNode() throws IOException { + public void should_write_one_node_as_one_row_per_format_export() throws IOException { when(config.getBooleanProperty(DroidGlobalProperty.CSV_EXPORT_ROW_PER_FORMAT)).thenReturn(false); try(final Writer writer = new StringWriter()) { @@ -153,6 +165,7 @@ public void testWriteOneNode() throws IOException { final String[] lines = writer.toString().split(LINE_SEPARATOR); assertEquals(2, lines.length); + assertEquals(defaultHeaders, lines[0]); assertEquals(expectedEntry, lines[1]); } } @@ -299,6 +312,274 @@ public void shouldCreateAdditionalHeadersForIdentificationFormatsWhenWritingAnEn } } + @Test + public void should_create_additional_headers_for_identification_format_in_order_of_template_when_writing_entries_per_file() throws IOException { + when(config.getBooleanProperty(DroidGlobalProperty.CSV_EXPORT_ROW_PER_FORMAT)).thenReturn(false); + ExportTemplate template = mock(ExportTemplate.class); + + Map columnPositions = new HashMap<>(); + + ExportTemplateColumnDef def1 = getMockColumnDef("ID", "Identifier", ExportTemplateColumnDef.ColumnType.ProfileResourceNode); + ExportTemplateColumnDef def2 = getMockColumnDef("PUID", "Puid", ExportTemplateColumnDef.ColumnType.ProfileResourceNode); + ExportTemplateColumnDef def3 = getMockColumnDef("HASH", "Hash123", ExportTemplateColumnDef.ColumnType.ProfileResourceNode); + ExportTemplateColumnDef def4 = getMockColumnDef("FORMAT_NAME", "Format_Name", ExportTemplateColumnDef.ColumnType.ProfileResourceNode); + ExportTemplateColumnDef def5 = getMockColumnDef("Simple Column", "Simple_Header", ExportTemplateColumnDef.ColumnType.ConstantString); + + columnPositions.put(0, def1); + columnPositions.put(1, def2); + columnPositions.put(2, def3); + columnPositions.put(3, def4); + columnPositions.put(4, def5); + + when(template.getColumnOrderMap()).thenReturn(columnPositions); + + try(final Writer writer = new StringWriter()) { + List nodes = new ArrayList<>(); + + Format id1 = buildFormat(1); + Format id2 = buildFormat(2); + + ProfileResourceNode node = buildProfileResourceNode(1, 1000L); + node.addFormatIdentification(id1); + node.addFormatIdentification(id2); + nodes.add(node); + + itemWriter.setOptions(ExportOptions.ONE_ROW_PER_FILE); + itemWriter.setExportTemplate(template); + itemWriter.open(writer); + itemWriter.write(nodes); + + final String expectedHeaders = toCsvRow(new String[] { + "Identifier", "Puid", "Puid1", "Hash123", "Format_Name", "Format_Name1", "Simple_Header" + }); + + final String[] lines = writer.toString().split(LINE_SEPARATOR); + + assertEquals(2, lines.length); + assertEquals(expectedHeaders, lines[0]); + assertEquals(7, lines[1].split(",").length); + } + } + + @Test + public void should_write_a_column_with_pre_defined_constant_value() throws IOException { + when(config.getBooleanProperty(DroidGlobalProperty.CSV_EXPORT_ROW_PER_FORMAT)).thenReturn(false); + ExportTemplate template = mock(ExportTemplate.class); + + Map columnPositions = new HashMap<>(); + + ExportTemplateColumnDef def1 = getMockColumnDef("ID", "Identifier", ExportTemplateColumnDef.ColumnType.ProfileResourceNode); + ExportTemplateColumnDef def2 = getMockColumnDef("Simplified English", "Language", ExportTemplateColumnDef.ColumnType.ConstantString); + ExportTemplateColumnDef def3 = getMockColumnDef("FORMAT_NAME", "Format_Name", ExportTemplateColumnDef.ColumnType.ProfileResourceNode); + + columnPositions.put(0, def1); + columnPositions.put(1, def2); + columnPositions.put(2, def3); + + when(template.getColumnOrderMap()).thenReturn(columnPositions); + + try(final Writer writer = new StringWriter()) { + List nodes = new ArrayList<>(); + + Format id1 = buildFormat(1); + + ProfileResourceNode node = buildProfileResourceNode(1, 1000L); + node.addFormatIdentification(id1); + nodes.add(node); + + itemWriter.setOptions(ExportOptions.ONE_ROW_PER_FILE); + itemWriter.setExportTemplate(template); + itemWriter.open(writer); + itemWriter.write(nodes); + + final String expectedHeaders = toCsvRow(new String[] { + "Identifier", "Language", "Format_Name" + }); + + final String expectedEntry = toCsvRow(new String[] { + "", + "Simplified English", + "Plain Text" + }); + + + final String[] lines = writer.toString().split(LINE_SEPARATOR); + + assertEquals(2, lines.length); + assertEquals(expectedHeaders, lines[0]); + assertEquals(expectedEntry, lines[1]); + } + + } + + @Test + public void should_write_a_column_with_modified_value_and_a_pre_defined_constant_value() throws IOException { + when(config.getBooleanProperty(DroidGlobalProperty.CSV_EXPORT_ROW_PER_FORMAT)).thenReturn(false); + ExportTemplate template = mock(ExportTemplate.class); + + Map columnPositions = new HashMap<>(); + + ExportTemplateColumnDef def1 = getMockColumnDef("ID", "Identifier", ExportTemplateColumnDef.ColumnType.ProfileResourceNode); + ExportTemplateColumnDef def2 = getMockColumnDef("Simplified English", "Language", ExportTemplateColumnDef.ColumnType.ConstantString); + ExportTemplateColumnDef def3 = getMockColumnDef("FORMAT_NAME", "Format_Name", ExportTemplateColumnDef.ColumnType.DataModifier); + + + columnPositions.put(0, def1); + columnPositions.put(1, def2); + columnPositions.put(2, def3); + + when(template.getColumnOrderMap()).thenReturn(columnPositions); + + try(final Writer writer = new StringWriter()) { + List nodes = new ArrayList<>(); + + Format id1 = buildFormat(1); + + ProfileResourceNode node = buildProfileResourceNode(1, 1000L); + node.addFormatIdentification(id1); + nodes.add(node); + + itemWriter.setOptions(ExportOptions.ONE_ROW_PER_FILE); + itemWriter.setExportTemplate(template); + itemWriter.open(writer); + itemWriter.write(nodes); + + final String expectedHeaders = toCsvRow(new String[] { + "Identifier", "Language", "Format_Name" + }); + + final String expectedEntry = toCsvRow(new String[] { + "", + "Simplified English", + "plain text" + }); + + + final String[] lines = writer.toString().split(LINE_SEPARATOR); + + assertEquals(2, lines.length); + assertEquals(expectedHeaders, lines[0]); + assertEquals(expectedEntry, lines[1]); + } + + } + + @Test + public void should_restrict_to_limited_number_of_columns_when_the_column_names_are_set_on_the_writer_writing_per_file() throws IOException { + + try(final Writer writer = new StringWriter()) { + List nodes = new ArrayList<>(); + + Format id1 = buildFormat(1); + Format id2 = buildFormat(2); + + ProfileResourceNode node = buildProfileResourceNode(1, 1000L); + node.addFormatIdentification(id1); + node.addFormatIdentification(id2); + + nodes.add(node); + itemWriter.setOptions(ExportOptions.ONE_ROW_PER_FILE); + itemWriter.setColumnsToWrite("ID FORMAT_NAME"); + itemWriter.open(writer); + itemWriter.write(nodes); + + final String expectedHeaders = toCsvRow(new String[] { + "ID", "FORMAT_NAME", "FORMAT_NAME1" //per format columns + }); + + final String expectedEntry1 = toCsvRow(new String[] { + "", + "Plain Text", + "Plain Text" + }); + + final String[] lines = writer.toString().split(LINE_SEPARATOR); + + assertEquals(2, lines.length); + assertEquals(expectedHeaders, lines[0]); + assertEquals(expectedEntry1, lines[1]); + } + } + + @Test + public void should_restrict_to_limited_number_of_columns_when_the_column_names_are_set_on_the_writer_writing_per_format() throws IOException { + + try(final Writer writer = new StringWriter()) { + List nodes = new ArrayList<>(); + + Format id1 = buildFormat(1); + Format id2 = buildFormat(2); + + ProfileResourceNode node = buildProfileResourceNode(1, 1000L); + node.addFormatIdentification(id1); + node.addFormatIdentification(id2); + + nodes.add(node); + itemWriter.setOptions(ExportOptions.ONE_ROW_PER_FORMAT); + itemWriter.setColumnsToWrite("ID NAME FORMAT_NAME FORMAT_VERSION"); + itemWriter.open(writer); + itemWriter.write(nodes); + + final String expectedHeaders = toCsvRow(new String[] { + "ID", "NAME", "FORMAT_NAME", "FORMAT_VERSION" //per format columns + }); + + final String expectedEntry1 = toCsvRow(new String[] { + "", + "file1.txt", + "Plain Text", + "1.0" + }); + + final String[] lines = writer.toString().split(LINE_SEPARATOR); + + assertEquals(3, lines.length); + assertEquals(expectedHeaders, lines[0]); + assertEquals(expectedEntry1, lines[1]); + assertEquals(expectedEntry1, lines[2]); + } + } + + @Test + public void should_do_header_customisations_for_hash_column_when_an_algorithm_is_present() throws IOException { + try(final Writer writer = new StringWriter()) { + List nodes = new ArrayList<>(); + + Format id1 = buildFormat(1); + + ProfileResourceNode node = buildProfileResourceNode(1, 1000L); + node.addFormatIdentification(id1); + + nodes.add(node); + itemWriter.setOptions(ExportOptions.ONE_ROW_PER_FORMAT); + + Map headerCustomisation = new HashMap<>(); + headerCustomisation.put("hash", "XTRA_STRONG_HASH"); + itemWriter.setHeaders(headerCustomisation); + itemWriter.setColumnsToWrite("ID NAME FORMAT_NAME HASH"); + itemWriter.open(writer); + itemWriter.write(nodes); + + final String expectedHeaders = toCsvRow(new String[] { + "ID", "NAME", "XTRA_STRONG_HASH", "FORMAT_NAME" + }); + + final String expectedEntry1 = toCsvRow(new String[] { + "", + "file1.txt", + "11111111111111111111111111111111", + "Plain Text" + }); + + final String[] lines = writer.toString().split(LINE_SEPARATOR); + + assertEquals(2, lines.length); + assertEquals(expectedHeaders, lines[0]); + assertEquals(expectedEntry1, lines[1]); + } + + } + @Test public void testWriteOneNodeWithTwoFormatsWithOneRowPerFormat() throws IOException { when(config.getBooleanProperty(DroidGlobalProperty.CSV_EXPORT_ROW_PER_FORMAT)).thenReturn(true); @@ -307,7 +588,7 @@ public void testWriteOneNodeWithTwoFormatsWithOneRowPerFormat() throws IOExcepti List nodes = new ArrayList<>(); Format id1 = buildFormat(1); - Format id2 = buildFormat(2); + Format id2 = buildFormat(2, "2.0"); ProfileResourceNode node = buildProfileResourceNode(1, 1000L); node.addFormatIdentification(id1); @@ -355,7 +636,7 @@ public void testWriteOneNodeWithTwoFormatsWithOneRowPerFormat() throws IOExcepti "fmt/2", "text/plain", "Plain Text", - "1.0", + "2.0", }); final String[] lines = writer.toString().split(LINE_SEPARATOR); @@ -422,6 +703,88 @@ public void should_write_nodes_inside_archives_using_seaparators_depending_on_OS } } + @Test + public void should_write_data_row_with_additional_blank_elements_when_other_rows_have_more_identified_formats() throws IOException { + when(config.getBooleanProperty(DroidGlobalProperty.CSV_EXPORT_ROW_PER_FORMAT)).thenReturn(false); + + try(final Writer writer = new StringWriter()) { + List nodes = new ArrayList<>(); + + Format id1 = buildFormat(1); + Format id2 = buildFormat(2); + + ProfileResourceNode nodeWithTwoIdentifications = buildProfileResourceNode(1, 1000L); + nodeWithTwoIdentifications.addFormatIdentification(id1); + nodeWithTwoIdentifications.addFormatIdentification(id2); + + ProfileResourceNode nodeWithOneIentifications = buildProfileResourceNode(2, 500L); + Format id3 = buildFormat(3); + nodeWithOneIentifications.addFormatIdentification(id3); + + nodes.add(nodeWithTwoIdentifications); + nodes.add(nodeWithOneIentifications); + + itemWriter.setOptions(ExportOptions.ONE_ROW_PER_FILE); + itemWriter.setColumnsToWrite("ID PUID FORMAT_NAME METHOD"); + itemWriter.open(writer); + itemWriter.write(nodes); + + final String expectedHeaders = toCsvRow(new String[] { + "ID","METHOD","PUID","FORMAT_NAME","PUID1","FORMAT_NAME1" + }); + + final String expectedEntry1 = toCsvRow(new String[] { + "", + "Signature", + "fmt/1", + "Plain Text", + "fmt/2", + "Plain Text" + }); + + final String expectedEntry2 = toCsvRow(new String[] { + "", + "Signature", + "fmt/3", + "Plain Text", + "", + "" + }); + + final String[] lines = writer.toString().split(LINE_SEPARATOR); + + assertEquals(3, lines.length); + assertEquals(expectedHeaders, lines[0]); + assertEquals(expectedEntry1, lines[1]); + assertEquals(expectedEntry2, lines[2]); + } + } + + private static ExportTemplateColumnDef getMockColumnDef(String param1, String header, ExportTemplateColumnDef.ColumnType columnType) { + ExportTemplateColumnDef def = mock(ExportTemplateColumnDef.class); + when(def.getColumnType()).thenReturn(columnType); + switch (columnType) { + case ProfileResourceNode: + when(def.getOriginalColumnName()).thenReturn(param1); + when(def.getHeaderLabel()).thenReturn(header); + when(def.getDataValue()).thenThrow(new RuntimeException("Profile resource node column uses data from the profile results")); + when(def.getOperatedValue(anyString())).thenAnswer(i -> i.getArguments()[0]); + break; + case ConstantString: + when(def.getOriginalColumnName()).thenThrow(new RuntimeException("Constant String Columns do not have an associated original column name")); + when(def.getHeaderLabel()).thenReturn(header); + when(def.getDataValue()).thenReturn(param1); + break; + case DataModifier: + when(def.getOriginalColumnName()).thenReturn(param1); + when(def.getHeaderLabel()).thenReturn(header); + when(def.getDataValue()).thenThrow(new RuntimeException("Profile resource node column uses data from the profile results")); + when(def.getOperatedValue(anyString())).thenAnswer(i -> i.getArguments()[0].toString().toLowerCase()); + break; + } + return def; + } + private static boolean isNotWindows() { return !SystemUtils.IS_OS_WINDOWS; } @@ -451,12 +814,17 @@ private static ProfileResourceNode buildProfileResourceNode(int i, Long size, UR private static Format buildFormat(int i) { + return buildFormat(i, "1.0"); + } + + private static Format buildFormat(int i, String version) { Format format = new Format(); format.setPuid("fmt/" + i); format.setMimeType("text/plain"); format.setName("Plain Text"); - format.setVersion("1.0"); - + format.setVersion(version); + return format; } + } diff --git a/droid-swing-ui/pom.xml b/droid-swing-ui/pom.xml index 39d1fd8c6..4e816ec5c 100644 --- a/droid-swing-ui/pom.xml +++ b/droid-swing-ui/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.7.1-SNAPSHOT + 6.8.0-SNAPSHOT ../droid-parent diff --git a/droid-swing-ui/src/main/java/uk/gov/nationalarchives/droid/gui/DroidMainFrame.java b/droid-swing-ui/src/main/java/uk/gov/nationalarchives/droid/gui/DroidMainFrame.java index 15ef8e7c0..cd8e15b9f 100644 --- a/droid-swing-ui/src/main/java/uk/gov/nationalarchives/droid/gui/DroidMainFrame.java +++ b/droid-swing-ui/src/main/java/uk/gov/nationalarchives/droid/gui/DroidMainFrame.java @@ -1460,6 +1460,9 @@ private void export() { } else { exportOptions.setExportOptions(ExportOptions.ONE_ROW_PER_FILE); } + + exportOptions.setDefaultTemplatesFolder(globalContext.getGlobalConfig().getExportTemplatesDir()); + exportOptions.showDialog(); if (exportOptions.isApproved()) { String columnNames = exportOptions.getColumnsToExport(); @@ -1486,6 +1489,7 @@ private void export() { exportAction.setBom(exportOptions.isBom()); exportAction.setQuoteAllFields(exportOptions.getQuoteAllColumns()); exportAction.setColumnsToWrite(columnNames); + exportAction.setExportTemplatePath(exportOptions.getTemplatePath()); exportAction.setCallback(new ActionDoneCallback() { @Override diff --git a/droid-swing-ui/src/main/java/uk/gov/nationalarchives/droid/gui/export/ExportAction.java b/droid-swing-ui/src/main/java/uk/gov/nationalarchives/droid/gui/export/ExportAction.java index f8684471e..5387362bf 100644 --- a/droid-swing-ui/src/main/java/uk/gov/nationalarchives/droid/gui/export/ExportAction.java +++ b/droid-swing-ui/src/main/java/uk/gov/nationalarchives/droid/gui/export/ExportAction.java @@ -42,6 +42,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import uk.gov.nationalarchives.droid.export.interfaces.ExportDetails; import uk.gov.nationalarchives.droid.export.interfaces.ExportManager; import uk.gov.nationalarchives.droid.export.interfaces.ExportOptions; import uk.gov.nationalarchives.droid.gui.action.ActionDoneCallback; @@ -61,6 +62,7 @@ public class ExportAction extends SwingWorker { private boolean bom; private boolean quoteAllFields; private String columnsToWrite; + private String exportTemplatePath; private List profileIds; private ActionDoneCallback callback; @@ -122,8 +124,7 @@ protected void process(List chunks) { @Override protected Void doInBackground() { //TODO: configure columns to write in UI. - exportTask = exportManager.exportProfiles(profileIds, destination.getPath(), null, options, - outputEncoding, bom, quoteAllFields, columnsToWrite); + exportTask = exportManager.exportProfiles(profileIds, destination.getPath(), null, getExportDetails()); try { exportTask.get(); } catch (InterruptedException e) { @@ -213,4 +214,28 @@ public void setQuoteAllFields(boolean quoteAllFields) { public void setColumnsToWrite(String columnsToWrite) { this.columnsToWrite = columnsToWrite; } + + /** + * Set the absolute path of the export template to the ExportAction. + * @param exportTemplatePath absolute path of the export template + */ + public void setExportTemplatePath(String exportTemplatePath) { + this.exportTemplatePath = exportTemplatePath; + } + + /** + * + * @return the export details for this export command. + */ + private ExportDetails getExportDetails() { + ExportDetails.ExportDetailsBuilder builder = new ExportDetails.ExportDetailsBuilder(); + return builder.withExportOptions(options) + .withOutputEncoding(outputEncoding) + .withBomFlag(bom) + .withQuotingAllFields(quoteAllFields) + .withColumnsToWrite(columnsToWrite) + .withExportTemplatePath(exportTemplatePath) + .build(); + } + } diff --git a/droid-swing-ui/src/main/java/uk/gov/nationalarchives/droid/gui/export/ExportDialog.form b/droid-swing-ui/src/main/java/uk/gov/nationalarchives/droid/gui/export/ExportDialog.form index 5c377af0c..4c8fd6d28 100644 --- a/droid-swing-ui/src/main/java/uk/gov/nationalarchives/droid/gui/export/ExportDialog.form +++ b/droid-swing-ui/src/main/java/uk/gov/nationalarchives/droid/gui/export/ExportDialog.form @@ -10,7 +10,13 @@ + + + + + + @@ -30,33 +36,33 @@ - + - + - + - - - + + + - - + + - - + + - + - - + + @@ -69,7 +75,7 @@ - + @@ -96,7 +102,14 @@ - + + + + + + + + @@ -104,30 +117,34 @@ - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + - @@ -267,269 +284,396 @@ - + + + + + + + + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + - - - - - - - - - - - + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + + + + + - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - - - - + + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + diff --git a/droid-swing-ui/src/main/java/uk/gov/nationalarchives/droid/gui/export/ExportDialog.java b/droid-swing-ui/src/main/java/uk/gov/nationalarchives/droid/gui/export/ExportDialog.java index e2677afca..139773350 100644 --- a/droid-swing-ui/src/main/java/uk/gov/nationalarchives/droid/gui/export/ExportDialog.java +++ b/droid-swing-ui/src/main/java/uk/gov/nationalarchives/droid/gui/export/ExportDialog.java @@ -31,14 +31,23 @@ */ package uk.gov.nationalarchives.droid.gui.export; +import java.awt.BorderLayout; +import java.awt.CardLayout; import java.awt.Component; import java.awt.Dimension; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; +import java.io.IOException; import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; +import java.util.LinkedList; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.swing.BorderFactory; import javax.swing.ButtonGroup; import javax.swing.ComboBoxModel; @@ -57,6 +66,8 @@ import javax.swing.GroupLayout.Alignment; import javax.swing.JComboBox; import javax.swing.LayoutStyle.ComponentPlacement; +import javax.swing.event.ChangeEvent; +import javax.swing.event.ChangeListener; import javax.swing.table.DefaultTableModel; import javax.swing.table.TableCellRenderer; @@ -65,6 +76,7 @@ import uk.gov.nationalarchives.droid.export.interfaces.ExportOptions; import uk.gov.nationalarchives.droid.gui.DroidMainFrame; import uk.gov.nationalarchives.droid.gui.ProfileForm; +import uk.gov.nationalarchives.droid.gui.util.SortedComboBoxModel; /** * @@ -74,12 +86,14 @@ public class ExportDialog extends JDialog { private static final long serialVersionUID = -4598078880004073202L; private static final int CAPACITY = 128; + private static final String EXPORT_TEMPLATE_FILE_EXTENSION = ".template"; - private DroidMainFrame droidMain; + private final DroidMainFrame droidMain; private DefaultTableModel tableModel; private List profilesRowData; private boolean approved; - + private Path exportTemplatesFolder; + /** * Creates new form ReportDialog. * @param parent the dialog's parent @@ -90,7 +104,7 @@ public ExportDialog(final DroidMainFrame parent) { setModal(true); initComponents(); - jScrollPane1.getViewport().setBackground(profileSelectTable.getBackground()); + jScrollPaneProfileSelection.getViewport().setBackground(profileSelectTable.getBackground()); pack(); setLocationRelativeTo(getParent()); @@ -129,9 +143,17 @@ public boolean isCellEditable(int row, int column) { profileSelectTable.setDefaultEditor(ProfileForm.class, new CheckBoxEditor()); profileSelectTable.setDefaultRenderer(ProfileForm.class, new CheckBoxRenderer()); - jScrollPane1.setColumnHeaderView(null); + jScrollPaneProfileSelection.setColumnHeaderView(null); profileSelectTable.setCellSelectionEnabled(false); - + + ComboBoxModel templatesModel = getExportTemplatesModel(); + if (templatesModel.getSize() == 0) { + jCheckBoxUseTemplate.setEnabled(false); + } else { + jComboBox1.setModel(templatesModel); + jComboBox1.setSelectedItem(templatesModel.getElementAt(0)); + } + enableGenerateButton(); approved = false; setVisible(true); @@ -184,7 +206,7 @@ public void setExportOptions(ExportOptions options) { * @return the profilesRowData */ public List getSelectedProfileIds() { - List selectedProfiles = new ArrayList(); + List selectedProfiles = new ArrayList<>(); for (ProfileWrapper profileWrapper : profilesRowData) { if (profileWrapper.isSelected()) { @@ -228,6 +250,19 @@ public String getColumnsToExport() { return builder.toString().trim(); } + public String getTemplatePath() { + if (jCheckBoxUseTemplate.isSelected()) { + ExportTemplateComboBoxItem item = (ExportTemplateComboBoxItem) jComboBox1.getSelectedItem(); + return item.getTemplatePath().toAbsolutePath().toString(); + } else { + return null; + } + } + + public void setDefaultTemplatesFolder(Path templatesFolder) { + this.exportTemplatesFolder = templatesFolder; + } + private void addColumn(String columnName, boolean selected, StringBuilder builder) { if (selected) { builder.append(columnName).append(' '); @@ -251,9 +286,9 @@ private void initComponents() { buttonGroup1 = new ButtonGroup(); profileSelectLabel = new JLabel(); - jScrollPane1 = new JScrollPane(); + jScrollPaneProfileSelection = new JScrollPane(); profileSelectTable = new JTable(); - jPanel1 = new JPanel(); + jPanelBottomControl = new JPanel(); cancelButton = new JButton(); exportButton = new JButton(); RadioOneRowPerFile = new JRadioButton(); @@ -263,7 +298,9 @@ private void initComponents() { jCheckBoxQuoteAll = new JCheckBox(); toggleColumnButton = new JButton(); jButtonSetAllColumns = new JButton(); - jPanel2 = new JPanel(); + jPanelRight = new JPanel(); + jPanelCards = new JPanel(); + jPanelColumnChoices = new JPanel(); profileSelectLabel1 = new JLabel(); jCheckBoxId = new JCheckBox(); jCheckBoxParentId = new JCheckBox(); @@ -283,14 +320,20 @@ private void initComponents() { jCheckBoxFormatVersion = new JCheckBox(); jCheckBoxExtMismatch = new JCheckBox(); jCheckBoxFileHash = new JCheckBox(); + jPanelTemplateChoices = new JPanel(); + jLabel2 = new JLabel(); + jComboBox1 = new JComboBox<>(); + jCheckBoxUseTemplate = new JCheckBox(); setTitle(NbBundle.getMessage(ExportDialog.class, "ExportDialog.title_1")); // NOI18N setAlwaysOnTop(true); + setMinimumSize(new Dimension(1155, 835)); setName("exportDialog"); // NOI18N + setPreferredSize(new Dimension(1055, 760)); profileSelectLabel.setText(NbBundle.getMessage(ExportDialog.class, "ExportDialog.profileSelectLabel.text_1")); // NOI18N - jScrollPane1.setPreferredSize(new Dimension(300, 402)); + jScrollPaneProfileSelection.setPreferredSize(new Dimension(300, 402)); profileSelectTable.setModel(new DefaultTableModel( new Object [][] { @@ -302,7 +345,9 @@ private void initComponents() { )); profileSelectTable.setRowSelectionAllowed(false); profileSelectTable.setTableHeader(null); - jScrollPane1.setViewportView(profileSelectTable); + jScrollPaneProfileSelection.setViewportView(profileSelectTable); + + jPanelBottomControl.setBorder(BorderFactory.createEmptyBorder(1, 1, 10, 1)); cancelButton.setText(NbBundle.getMessage(ExportDialog.class, "ExportDialog.cancelButton.text")); // NOI18N cancelButton.setVerticalAlignment(SwingConstants.BOTTOM); @@ -357,45 +402,47 @@ public void actionPerformed(ActionEvent evt) { } }); - GroupLayout jPanel1Layout = new GroupLayout(jPanel1); - jPanel1.setLayout(jPanel1Layout); - jPanel1Layout.setHorizontalGroup(jPanel1Layout.createParallelGroup(Alignment.LEADING) - .addGroup(jPanel1Layout.createSequentialGroup() + GroupLayout jPanelBottomControlLayout = new GroupLayout(jPanelBottomControl); + jPanelBottomControl.setLayout(jPanelBottomControlLayout); + jPanelBottomControlLayout.setHorizontalGroup(jPanelBottomControlLayout.createParallelGroup(Alignment.LEADING) + .addGroup(jPanelBottomControlLayout.createSequentialGroup() .addContainerGap() - .addGroup(jPanel1Layout.createParallelGroup(Alignment.LEADING) - .addGroup(Alignment.TRAILING, jPanel1Layout.createSequentialGroup() - .addComponent(jLabel1) - .addGap(18, 18, 18) - .addComponent(cmdEncoding, GroupLayout.PREFERRED_SIZE, GroupLayout.DEFAULT_SIZE, GroupLayout.PREFERRED_SIZE) - .addPreferredGap(ComponentPlacement.RELATED, GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .addComponent(exportButton) - .addPreferredGap(ComponentPlacement.UNRELATED) - .addComponent(cancelButton, GroupLayout.PREFERRED_SIZE, 125, GroupLayout.PREFERRED_SIZE)) - .addGroup(jPanel1Layout.createSequentialGroup() + .addGroup(jPanelBottomControlLayout.createParallelGroup(Alignment.LEADING) + .addGroup(jPanelBottomControlLayout.createSequentialGroup() .addComponent(RadioOneRowPerFile) .addGap(18, 18, 18) .addComponent(RadioOneRowPerIdentification) - .addGap(0, 0, Short.MAX_VALUE)) - .addGroup(jPanel1Layout.createSequentialGroup() - .addComponent(jCheckBoxQuoteAll) - .addPreferredGap(ComponentPlacement.RELATED, GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .addComponent(jButtonSetAllColumns) - .addPreferredGap(ComponentPlacement.UNRELATED) - .addComponent(toggleColumnButton))) - .addContainerGap()) + .addContainerGap(GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + .addGroup(Alignment.TRAILING, jPanelBottomControlLayout.createSequentialGroup() + .addGroup(jPanelBottomControlLayout.createParallelGroup(Alignment.TRAILING) + .addGroup(jPanelBottomControlLayout.createSequentialGroup() + .addComponent(jLabel1) + .addGap(18, 18, 18) + .addComponent(cmdEncoding, GroupLayout.PREFERRED_SIZE, GroupLayout.DEFAULT_SIZE, GroupLayout.PREFERRED_SIZE) + .addPreferredGap(ComponentPlacement.RELATED, GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addComponent(exportButton) + .addGap(18, 18, 18) + .addComponent(cancelButton, GroupLayout.PREFERRED_SIZE, 180, GroupLayout.PREFERRED_SIZE)) + .addGroup(jPanelBottomControlLayout.createSequentialGroup() + .addComponent(jCheckBoxQuoteAll) + .addPreferredGap(ComponentPlacement.RELATED, GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addComponent(jButtonSetAllColumns) + .addGap(18, 18, 18) + .addComponent(toggleColumnButton))) + .addGap(27, 27, 27)))) ); - jPanel1Layout.setVerticalGroup(jPanel1Layout.createParallelGroup(Alignment.LEADING) - .addGroup(Alignment.TRAILING, jPanel1Layout.createSequentialGroup() - .addGroup(jPanel1Layout.createParallelGroup(Alignment.BASELINE) + jPanelBottomControlLayout.setVerticalGroup(jPanelBottomControlLayout.createParallelGroup(Alignment.LEADING) + .addGroup(Alignment.TRAILING, jPanelBottomControlLayout.createSequentialGroup() + .addGroup(jPanelBottomControlLayout.createParallelGroup(Alignment.BASELINE) .addComponent(toggleColumnButton) .addComponent(jCheckBoxQuoteAll) .addComponent(jButtonSetAllColumns)) .addPreferredGap(ComponentPlacement.RELATED) - .addGroup(jPanel1Layout.createParallelGroup(Alignment.BASELINE) + .addGroup(jPanelBottomControlLayout.createParallelGroup(Alignment.BASELINE) .addComponent(RadioOneRowPerFile) .addComponent(RadioOneRowPerIdentification)) .addPreferredGap(ComponentPlacement.RELATED, GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .addGroup(jPanel1Layout.createParallelGroup(Alignment.BASELINE) + .addGroup(jPanelBottomControlLayout.createParallelGroup(Alignment.BASELINE) .addComponent(cancelButton) .addComponent(exportButton) .addComponent(cmdEncoding, GroupLayout.PREFERRED_SIZE, GroupLayout.DEFAULT_SIZE, GroupLayout.PREFERRED_SIZE) @@ -403,6 +450,14 @@ public void actionPerformed(ActionEvent evt) { .addContainerGap()) ); + jPanelRight.setBorder(BorderFactory.createEmptyBorder(1, 1, 1, 10)); + jPanelRight.setLayout(new BorderLayout()); + + jPanelCards.setBorder(BorderFactory.createEtchedBorder()); + jPanelCards.setLayout(new CardLayout()); + + jPanelColumnChoices.setBorder(BorderFactory.createEmptyBorder(10, 10, 10, 10)); + profileSelectLabel1.setText(NbBundle.getMessage(ExportDialog.class, "ExportDialog.profileSelectLabel1.text")); // NOI18N jCheckBoxId.setSelected(true); @@ -477,11 +532,11 @@ public void actionPerformed(ActionEvent evt) { jCheckBoxFileHash.setSelected(true); jCheckBoxFileHash.setText(NbBundle.getMessage(ExportDialog.class, "ExportDialog.jCheckBoxFileHash.text")); // NOI18N - GroupLayout jPanel2Layout = new GroupLayout(jPanel2); - jPanel2.setLayout(jPanel2Layout); - jPanel2Layout.setHorizontalGroup(jPanel2Layout.createParallelGroup(Alignment.LEADING) - .addGroup(jPanel2Layout.createSequentialGroup() - .addGroup(jPanel2Layout.createParallelGroup(Alignment.LEADING) + GroupLayout jPanelColumnChoicesLayout = new GroupLayout(jPanelColumnChoices); + jPanelColumnChoices.setLayout(jPanelColumnChoicesLayout); + jPanelColumnChoicesLayout.setHorizontalGroup(jPanelColumnChoicesLayout.createParallelGroup(Alignment.LEADING) + .addGroup(jPanelColumnChoicesLayout.createSequentialGroup() + .addGroup(jPanelColumnChoicesLayout.createParallelGroup(Alignment.LEADING) .addComponent(jCheckBoxId) .addComponent(profileSelectLabel1) .addComponent(jCheckBoxParentId) @@ -493,87 +548,129 @@ public void actionPerformed(ActionEvent evt) { .addComponent(jCheckBoxExtension) .addComponent(jCheckBoxExtMismatch)) .addPreferredGap(ComponentPlacement.RELATED, GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .addGroup(jPanel2Layout.createParallelGroup(Alignment.LEADING) - .addComponent(jCheckBoxFileHash) - .addGroup(jPanel2Layout.createParallelGroup(Alignment.LEADING) - .addComponent(jCheckBoxIdCount, Alignment.TRAILING) - .addComponent(jCheckBoxMIMEtype)) - .addComponent(jCheckBoxFormatName) - .addGroup(jPanel2Layout.createParallelGroup(Alignment.LEADING) - .addGroup(Alignment.TRAILING, jPanel2Layout.createParallelGroup(Alignment.LEADING) + .addGroup(jPanelColumnChoicesLayout.createParallelGroup(Alignment.LEADING) + .addGroup(jPanelColumnChoicesLayout.createParallelGroup(Alignment.LEADING) + .addComponent(jCheckBoxFileHash) + .addGroup(jPanelColumnChoicesLayout.createParallelGroup(Alignment.LEADING) + .addComponent(jCheckBoxIdCount, Alignment.TRAILING) + .addComponent(jCheckBoxMIMEtype)) + .addGroup(Alignment.TRAILING, jPanelColumnChoicesLayout.createParallelGroup(Alignment.LEADING) .addComponent(jCheckBoxIdMethod) .addComponent(jCheckBoxStatus) .addComponent(jCheckBoxFormatVersion)) - .addComponent(jCheckBoxPUID)) - .addComponent(jCheckBoxResourceType)) + .addComponent(jCheckBoxResourceType)) + .addComponent(jCheckBoxFormatName) + .addComponent(jCheckBoxPUID)) .addGap(0, 0, Short.MAX_VALUE)) ); - jPanel2Layout.setVerticalGroup(jPanel2Layout.createParallelGroup(Alignment.LEADING) - .addGroup(jPanel2Layout.createSequentialGroup() + jPanelColumnChoicesLayout.setVerticalGroup(jPanelColumnChoicesLayout.createParallelGroup(Alignment.LEADING) + .addGroup(jPanelColumnChoicesLayout.createSequentialGroup() .addComponent(profileSelectLabel1) .addPreferredGap(ComponentPlacement.RELATED) - .addGroup(jPanel2Layout.createParallelGroup(Alignment.BASELINE) + .addGroup(jPanelColumnChoicesLayout.createParallelGroup(Alignment.BASELINE) .addComponent(jCheckBoxId) .addComponent(jCheckBoxPUID)) .addPreferredGap(ComponentPlacement.RELATED) - .addGroup(jPanel2Layout.createParallelGroup(Alignment.BASELINE) + .addGroup(jPanelColumnChoicesLayout.createParallelGroup(Alignment.BASELINE) .addComponent(jCheckBoxParentId) .addComponent(jCheckBoxFormatName)) .addPreferredGap(ComponentPlacement.RELATED) - .addGroup(jPanel2Layout.createParallelGroup(Alignment.BASELINE) + .addGroup(jPanelColumnChoicesLayout.createParallelGroup(Alignment.BASELINE) .addComponent(jCheckBoxURI) .addComponent(jCheckBoxFormatVersion)) .addPreferredGap(ComponentPlacement.RELATED) - .addGroup(jPanel2Layout.createParallelGroup(Alignment.BASELINE) + .addGroup(jPanelColumnChoicesLayout.createParallelGroup(Alignment.BASELINE) .addComponent(jCheckBoxFilePath) .addComponent(jCheckBoxMIMEtype)) .addPreferredGap(ComponentPlacement.RELATED) - .addGroup(jPanel2Layout.createParallelGroup(Alignment.BASELINE) + .addGroup(jPanelColumnChoicesLayout.createParallelGroup(Alignment.BASELINE) .addComponent(jCheckBoxFileName) .addComponent(jCheckBoxIdCount)) .addPreferredGap(ComponentPlacement.RELATED) - .addGroup(jPanel2Layout.createParallelGroup(Alignment.BASELINE) + .addGroup(jPanelColumnChoicesLayout.createParallelGroup(Alignment.BASELINE) .addComponent(jCheckBoxFileSize, GroupLayout.PREFERRED_SIZE, 24, GroupLayout.PREFERRED_SIZE) .addComponent(jCheckBoxIdMethod)) .addPreferredGap(ComponentPlacement.RELATED) - .addGroup(jPanel2Layout.createParallelGroup(Alignment.BASELINE) + .addGroup(jPanelColumnChoicesLayout.createParallelGroup(Alignment.BASELINE) .addComponent(jCheckBoxLastModified) .addComponent(jCheckBoxStatus)) .addPreferredGap(ComponentPlacement.RELATED) - .addGroup(jPanel2Layout.createParallelGroup(Alignment.BASELINE) + .addGroup(jPanelColumnChoicesLayout.createParallelGroup(Alignment.BASELINE) .addComponent(jCheckBoxExtension) .addComponent(jCheckBoxResourceType)) .addPreferredGap(ComponentPlacement.RELATED) - .addGroup(jPanel2Layout.createParallelGroup(Alignment.BASELINE) + .addGroup(jPanelColumnChoicesLayout.createParallelGroup(Alignment.BASELINE) .addComponent(jCheckBoxExtMismatch) .addComponent(jCheckBoxFileHash)) - .addGap(0, 9, Short.MAX_VALUE)) + .addGap(0, 52, Short.MAX_VALUE)) ); + jPanelColumnChoicesLayout.linkSize(SwingConstants.VERTICAL, new Component[] {jCheckBoxExtMismatch, jCheckBoxExtension, jCheckBoxFileHash, jCheckBoxFileName, jCheckBoxFilePath, jCheckBoxFileSize, jCheckBoxFormatName, jCheckBoxFormatVersion, jCheckBoxId, jCheckBoxIdCount, jCheckBoxIdMethod, jCheckBoxLastModified, jCheckBoxMIMEtype, jCheckBoxPUID, jCheckBoxParentId, jCheckBoxResourceType, jCheckBoxStatus, jCheckBoxURI}); + + jPanelCards.add(jPanelColumnChoices, "card5"); + + jPanelTemplateChoices.setBorder(BorderFactory.createEmptyBorder(10, 10, 10, 10)); + + jLabel2.setText(NbBundle.getMessage(ExportDialog.class, "ExportDialog.jLabel2.text")); // NOI18N + + jComboBox1.setModel(new DefaultComboBoxModel<>(new String[] { "Item 1", "Item 2", "Item 3", "Item 4" })); + + GroupLayout jPanelTemplateChoicesLayout = new GroupLayout(jPanelTemplateChoices); + jPanelTemplateChoices.setLayout(jPanelTemplateChoicesLayout); + jPanelTemplateChoicesLayout.setHorizontalGroup(jPanelTemplateChoicesLayout.createParallelGroup(Alignment.LEADING) + .addGroup(jPanelTemplateChoicesLayout.createSequentialGroup() + .addContainerGap() + .addGroup(jPanelTemplateChoicesLayout.createParallelGroup(Alignment.LEADING) + .addComponent(jLabel2) + .addComponent(jComboBox1, 0, 638, Short.MAX_VALUE)) + .addGap(25, 25, 25)) + ); + jPanelTemplateChoicesLayout.setVerticalGroup(jPanelTemplateChoicesLayout.createParallelGroup(Alignment.LEADING) + .addGroup(jPanelTemplateChoicesLayout.createSequentialGroup() + .addGap(25, 25, 25) + .addComponent(jLabel2) + .addGap(32, 32, 32) + .addComponent(jComboBox1, GroupLayout.PREFERRED_SIZE, GroupLayout.DEFAULT_SIZE, GroupLayout.PREFERRED_SIZE) + .addContainerGap(389, Short.MAX_VALUE)) + ); + + jPanelCards.add(jPanelTemplateChoices, "card3"); + + jPanelRight.add(jPanelCards, BorderLayout.CENTER); + + jCheckBoxUseTemplate.setText(NbBundle.getMessage(ExportDialog.class, "ExportDialog.jCheckBoxUseTemplate.text")); // NOI18N + jCheckBoxUseTemplate.setBorder(BorderFactory.createEmptyBorder(1, 15, 1, 1)); + jCheckBoxUseTemplate.addChangeListener(new ChangeListener() { + public void stateChanged(ChangeEvent evt) { + jCheckBoxUseTemplateStateChanged(evt); + } + }); + jPanelRight.add(jCheckBoxUseTemplate, BorderLayout.PAGE_START); + GroupLayout layout = new GroupLayout(getContentPane()); getContentPane().setLayout(layout); layout.setHorizontalGroup(layout.createParallelGroup(Alignment.LEADING) - .addComponent(jPanel1, Alignment.TRAILING, GroupLayout.DEFAULT_SIZE, GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addComponent(jPanelBottomControl, Alignment.TRAILING, GroupLayout.DEFAULT_SIZE, GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) .addGroup(layout.createSequentialGroup() .addContainerGap() .addGroup(layout.createParallelGroup(Alignment.LEADING) .addComponent(profileSelectLabel) - .addComponent(jScrollPane1, GroupLayout.PREFERRED_SIZE, 300, GroupLayout.PREFERRED_SIZE)) - .addGap(18, 18, Short.MAX_VALUE) - .addComponent(jPanel2, GroupLayout.PREFERRED_SIZE, GroupLayout.DEFAULT_SIZE, GroupLayout.PREFERRED_SIZE) + .addComponent(jScrollPaneProfileSelection, GroupLayout.DEFAULT_SIZE, 375, Short.MAX_VALUE)) + .addGap(18, 18, 18) + .addComponent(jPanelRight, GroupLayout.DEFAULT_SIZE, GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) .addContainerGap()) ); layout.setVerticalGroup(layout.createParallelGroup(Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addContainerGap() - .addGroup(layout.createParallelGroup(Alignment.LEADING) + .addGroup(layout.createParallelGroup(Alignment.TRAILING) .addGroup(layout.createSequentialGroup() .addComponent(profileSelectLabel) .addPreferredGap(ComponentPlacement.RELATED) - .addComponent(jScrollPane1, GroupLayout.PREFERRED_SIZE, 0, Short.MAX_VALUE)) - .addComponent(jPanel2, GroupLayout.DEFAULT_SIZE, GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + .addComponent(jScrollPaneProfileSelection, GroupLayout.DEFAULT_SIZE, GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + .addComponent(jPanelRight, GroupLayout.DEFAULT_SIZE, GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) .addPreferredGap(ComponentPlacement.RELATED) - .addComponent(jPanel1, GroupLayout.PREFERRED_SIZE, GroupLayout.DEFAULT_SIZE, GroupLayout.PREFERRED_SIZE)) + .addComponent(jPanelBottomControl, GroupLayout.PREFERRED_SIZE, GroupLayout.DEFAULT_SIZE, GroupLayout.PREFERRED_SIZE)) ); }// //GEN-END:initComponents @@ -635,6 +732,13 @@ private void jButtonSetAllColumnsActionPerformed(ActionEvent evt) {//GEN-FIRST:e jCheckBoxFileHash.setSelected(true); }//GEN-LAST:event_jButtonSetAllColumnsActionPerformed + private void jCheckBoxUseTemplateStateChanged(ChangeEvent evt) {//GEN-FIRST:event_jCheckBoxUseTemplateStateChanged + jPanelTemplateChoices.setVisible(jCheckBoxUseTemplate.isSelected()); + jPanelColumnChoices.setVisible(!jCheckBoxUseTemplate.isSelected()); + jButtonSetAllColumns.setEnabled(!jCheckBoxUseTemplate.isSelected()); + toggleColumnButton.setEnabled(!jCheckBoxUseTemplate.isSelected()); + }//GEN-LAST:event_jCheckBoxUseTemplateStateChanged + /** * @param evt The event that triggers the action. */ @@ -673,10 +777,16 @@ protected void exportButtonActionPerformed(ActionEvent evt) { private JCheckBox jCheckBoxResourceType; private JCheckBox jCheckBoxStatus; private JCheckBox jCheckBoxURI; + private JCheckBox jCheckBoxUseTemplate; + private JComboBox jComboBox1; private JLabel jLabel1; - private JPanel jPanel1; - private JPanel jPanel2; - private JScrollPane jScrollPane1; + private JLabel jLabel2; + private JPanel jPanelBottomControl; + private JPanel jPanelCards; + private JPanel jPanelColumnChoices; + private JPanel jPanelRight; + private JPanel jPanelTemplateChoices; + private JScrollPane jScrollPaneProfileSelection; private JLabel profileSelectLabel; private JLabel profileSelectLabel1; private JTable profileSelectTable; @@ -775,7 +885,54 @@ private ComboBoxModel getOutputEncodings() { return model; } - + + class ExportTemplateComboBoxItem implements Comparable{ + private final String label; + private final Path templatePath; + + ExportTemplateComboBoxItem(Path templateFilePath) { + String templateFileName = templateFilePath.getFileName().toString(); + this.label = templateFileName.substring(0, templateFileName.length() - EXPORT_TEMPLATE_FILE_EXTENSION.length()); + this.templatePath = templateFilePath; + } + + public String getLabel() { + return label; + } + + public Path getTemplatePath() { + return templatePath; + } + + @Override + public String toString() { + return this.label; + } + + @Override + public int compareTo(Object o) { + if (!(o instanceof ExportTemplateComboBoxItem)) { + throw new ClassCastException("Invalid item for comparison " + o.getClass().getName()); + } + ExportTemplateComboBoxItem that = (ExportTemplateComboBoxItem) o; + return this.getLabel().toUpperCase().compareTo(that.getLabel().toUpperCase()); + } + } + + private ComboBoxModel getExportTemplatesModel() { + List items = new LinkedList<>(); + try (Stream fileStream = Files.list(exportTemplatesFolder)) { + List templates = fileStream.collect(Collectors.toList()); + for (Path template : templates) { + if (!Files.isDirectory(template) && (template.getFileName().toString().endsWith(EXPORT_TEMPLATE_FILE_EXTENSION))) { + items.add(new ExportTemplateComboBoxItem(template)); + } + } + return new SortedComboBoxModel(items) ; + } catch (IOException e) { + throw new RuntimeException(e); + } + } private final class CheckBoxRenderer extends JCheckBox implements TableCellRenderer { diff --git a/droid-swing-ui/src/main/java/uk/gov/nationalarchives/droid/gui/util/SortedComboBoxModel.java b/droid-swing-ui/src/main/java/uk/gov/nationalarchives/droid/gui/util/SortedComboBoxModel.java new file mode 100644 index 000000000..0041fd3c2 --- /dev/null +++ b/droid-swing-ui/src/main/java/uk/gov/nationalarchives/droid/gui/util/SortedComboBoxModel.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.gui.util; + +import javax.swing.DefaultComboBoxModel; +import java.util.Collections; +import java.util.List; + +/** + * Model to make the items in the combo box appear sorted. + * Items need to implement Comparable + * @param ComboBox Item that implements comparable + */ +public class SortedComboBoxModel> extends DefaultComboBoxModel { + public SortedComboBoxModel() { + super(); + } + + public SortedComboBoxModel(List items) { + Collections.sort(items); + for (int i = 0; i < items.size(); i++) { + super.addElement(items.get(i)); + } + } +} diff --git a/droid-swing-ui/src/main/resources/uk/gov/nationalarchives/droid/gui/export/Bundle.properties b/droid-swing-ui/src/main/resources/uk/gov/nationalarchives/droid/gui/export/Bundle.properties index cfd52f5b7..7b9e569dd 100644 --- a/droid-swing-ui/src/main/resources/uk/gov/nationalarchives/droid/gui/export/Bundle.properties +++ b/droid-swing-ui/src/main/resources/uk/gov/nationalarchives/droid/gui/export/Bundle.properties @@ -71,3 +71,5 @@ ExportDialog.toggleColumnButton.toolTipText=Toggles columns to export between se ExportDialog.toggleColumnButton.text=Toggle columns ExportDialog.jButtonSetAllColumns.text=Set all columns ExportDialog.jButtonSetAllColumns.toolTipText=Sets all columns for export +ExportDialog.jLabel2.text=Select an export template: +ExportDialog.jCheckBoxUseTemplate.text=Use export template diff --git a/droid-swing-ui/src/test/java/uk/gov/nationalarchives/droid/gui/util/SortedComboBoxModelTest.java b/droid-swing-ui/src/test/java/uk/gov/nationalarchives/droid/gui/util/SortedComboBoxModelTest.java new file mode 100644 index 000000000..ab4428d78 --- /dev/null +++ b/droid-swing-ui/src/test/java/uk/gov/nationalarchives/droid/gui/util/SortedComboBoxModelTest.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.gui.util; + +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class SortedComboBoxModelTest { + @Test + public void should_show_the_string_entries_in_sorted_order() { + List items = Arrays.asList("First Item", "0th item", "last item", "First item"); + SortedComboBoxModel model = new SortedComboBoxModel<>(items); + assertEquals("0th item", model.getElementAt(0)); + assertEquals("First Item", model.getElementAt(1)); + assertEquals("First item", model.getElementAt(2)); + assertEquals("last item", model.getElementAt(3)); + } + + @Test + public void should_show_the_numeric_entries_in_sorted_order() { + List items = Arrays.asList(12, 3, 4, 0); + SortedComboBoxModel model = new SortedComboBoxModel<>(items); + assertEquals(0, model.getElementAt(0)); + assertEquals(3, model.getElementAt(1)); + assertEquals(4, model.getElementAt(2)); + assertEquals(12, model.getElementAt(3)); + } + + @Test + public void should_show_the_entries_based_on_custom_sorted_order_based_on_comparable_implemntation() { + List items = Arrays.asList(new IntAsStringSortedItem(12), + new IntAsStringSortedItem(3), + new IntAsStringSortedItem(4), + new IntAsStringSortedItem(0)); + SortedComboBoxModel model = new SortedComboBoxModel<>(items); + assertEquals("0", ((IntAsStringSortedItem)model.getElementAt(0)).valAsString); + assertEquals("12", ((IntAsStringSortedItem)model.getElementAt(1)).valAsString); + assertEquals("3", ((IntAsStringSortedItem)model.getElementAt(2)).valAsString); + assertEquals("4", ((IntAsStringSortedItem)model.getElementAt(3)).valAsString); + } + + static class IntAsStringSortedItem implements Comparable { + private final String valAsString; + + IntAsStringSortedItem(Integer someVal) { + this.valAsString = someVal.toString(); + } + @Override + public int compareTo(Object o) { + IntAsStringSortedItem that = (IntAsStringSortedItem) o; + return this.valAsString.compareTo(that.valAsString); + } + } +} \ No newline at end of file diff --git a/droid-tools/pom.xml b/droid-tools/pom.xml index 7724b271e..833427ba1 100644 --- a/droid-tools/pom.xml +++ b/droid-tools/pom.xml @@ -5,7 +5,7 @@ droid-parent uk.gov.nationalarchives - 6.7.1-SNAPSHOT + 6.8.0-SNAPSHOT ../droid-parent diff --git a/pom.xml b/pom.xml index 66777efd2..876184059 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.7.1-SNAPSHOT + 6.8.0-SNAPSHOT droid-parent