diff --git a/.gitignore b/.gitignore index 356845ca..3d657b30 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ common/ _layouts/ public/ log.txt +/api-key.txt diff --git a/README.md b/README.md index 0f3e4db4..040ca4be 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ This repository is our development basecamp. If you find a bug or have questions ### Apache Maven DataSync uses Maven for building and package management. For more information: [What is Maven?](http://maven.apache.org/what-is-maven.html) -To build the project run: +To build the project, first you'll need to create an application token on your profile page. Put the random string it produces in a file called "api-key.txt" in the root directory of this project, then run ``` mvn clean install ``` @@ -57,5 +57,6 @@ java -jar DataSync-1.8.2-jar-with-dependencies.jar ### Java SDK -DataSync can be used as a Java SDK, for detailed documentation refer to: +DataSync can be used as a Java SDK, for detailed documentation refer +to: [http://socrata.github.io/datasync/guides/datasync-library-sdk.html](http://socrata.github.io/datasync/guides/datasync-library-sdk.html) diff --git a/pom.xml b/pom.xml index 932db3c8..59821784 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ 4.0.0 DataSync DataSync - 1.8.2 + 1.9.0 Ayn Leslie-Cook @@ -60,11 +60,25 @@ images/ + + . + true + + api-key.txt + + src/test/java + + . + true + + api-key.txt + + @@ -97,7 +111,7 @@ com.socrata soda-api-java - 0.9.12 + 0.10.1 com.socrata @@ -110,14 +124,19 @@ 1.4.7 - org.codehaus.jackson - jackson-core-asl - 1.9.13 + com.fasterxml.jackson.core + jackson-core + 2.8.6 + + + com.fasterxml.jackson.core + jackson-databind + 2.8.6 - org.codehaus.jackson - jackson-mapper-asl - 1.9.13 + com.fasterxml.jackson.core + jackson-annotations + 2.8.6 org.tukaani @@ -180,5 +199,10 @@ javac2 7.0.3 + + info.debatty + java-string-similarity + 1.1.0 + diff --git a/src/main/java/com/socrata/datasync/DatasetUtils.java b/src/main/java/com/socrata/datasync/DatasetUtils.java index 4aa48cb8..78ae2a42 100644 --- a/src/main/java/com/socrata/datasync/DatasetUtils.java +++ b/src/main/java/com/socrata/datasync/DatasetUtils.java @@ -1,8 +1,10 @@ package com.socrata.datasync; +import au.com.bytecode.opencsv.CSVReader; import com.socrata.datasync.config.userpreferences.UserPreferences; import com.socrata.model.importer.Column; import com.socrata.model.importer.Dataset; +import com.socrata.model.importer.GeoDataset; import com.socrata.model.importer.DatasetInfo; import org.apache.http.HttpException; import org.apache.http.HttpStatus; @@ -14,42 +16,50 @@ import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.ResponseHandler; import org.apache.http.client.utils.URIBuilder; -import org.codehaus.jackson.map.DeserializationConfig; -import org.codehaus.jackson.map.ObjectMapper; +import com.fasterxml.jackson.databind.DeserializationConfig; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; +import java.io.StringReader; import java.net.URI; import java.net.URISyntaxException; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; public class DatasetUtils { + private static final String LOCATION_DATATYPE_NAME = "location"; - private static class DatasetInfoResponseHandler implements ResponseHandler { - @Override - public DatasetInfo handleResponse(final HttpResponse response) - throws ClientProtocolException, IOException { - - StatusLine statusLine = response.getStatusLine(); - int status = statusLine.getStatusCode(); - if (status >= 200 && status < 300) { - HttpEntity entity = response.getEntity(); - return entity != null ? mapper.readValue(entity.getContent(), DatasetInfo.class) : null; - } else { - throw new ClientProtocolException(statusLine.toString()); - } - } - } + private static ObjectMapper mapper = new ObjectMapper().enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY); + public static Dataset getDatasetInfo(UserPreferences userPrefs, String viewId) throws URISyntaxException, IOException, HttpException { + Dataset ds = getDatasetInfoReflective(userPrefs, viewId, Dataset.class); + removeSystemAndComputedColumns(ds); + return ds; + } - private static final String LOCATION_DATATYPE_NAME = "location"; + public static GeoDataset getGeoDatasetInfo(UserPreferences userPrefs, String viewId) throws URISyntaxException, IOException, HttpException { + return getDatasetInfoReflective(userPrefs, viewId, GeoDataset.class); + } - private static ObjectMapper mapper = new ObjectMapper().enable(DeserializationConfig.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY); + private static void removeSystemAndComputedColumns(Dataset ds) { + List columns = ds.getColumns(); + Iterator it = columns.iterator(); + while(it.hasNext()) { + Column c = it.next(); + if(c.getFieldName().startsWith(":") || c.getComputationStrategy() != null) { + it.remove(); + } + } + ds.setColumns(columns); + } - public static T getDatasetInfo(UserPreferences userPrefs, String viewId, final Class typ) throws URISyntaxException, IOException, HttpException { + private static T getDatasetInfoReflective(UserPreferences userPrefs, String viewId, final Class typ) throws URISyntaxException, IOException, HttpException { String justDomain = getDomainWithoutScheme(userPrefs); URI absolutePath = new URIBuilder() .setScheme("https") @@ -78,12 +88,12 @@ public T handleResponse( return datasetInfo; } - public static String getDatasetSample(UserPreferences userPrefs, String viewId, int rowsToSample) throws URISyntaxException, IOException, HttpException { + public static List> getDatasetSample(UserPreferences userPrefs, Dataset dataset, int rowsToSample) throws URISyntaxException, IOException, HttpException { String justDomain = getDomainWithoutScheme(userPrefs); URI absolutePath = new URIBuilder() .setScheme("https") .setHost(justDomain) - .setPath("/resource/" + viewId + ".csv") + .setPath("/resource/" + dataset.getId() + ".csv") .addParameter("$limit",""+rowsToSample) .build(); @@ -105,7 +115,37 @@ public String handleResponse( HttpUtility util = new HttpUtility(userPrefs, true); String sample = util.get(absolutePath, "application/csv", handler); util.close(); - return sample; + + CSVReader reader = new CSVReader(new StringReader(sample)); + + List> results = new ArrayList<>(); + + Set expectedFieldNames = new HashSet(); + for(Column c : dataset.getColumns()) { + expectedFieldNames.add(c.getFieldName()); + } + String[] row = reader.readNext(); + boolean[] keep = new boolean[row.length]; + for(int i = 0; i != row.length; ++i) { + keep[i] = expectedFieldNames.contains(row[i]); + } + results.add(filter(keep, row)); + + while((row = reader.readNext()) != null) { + results.add(filter(keep, row)); + } + + return results; + } + + private static List filter(boolean[] filter, String[] elems) { + List result = new ArrayList<>(); + + for(int i = 0; i != elems.length; ++i) { + if(filter[i]) result.add(elems[i]); + } + + return result; } public static String getDomainWithoutScheme(UserPreferences userPrefs){ @@ -140,7 +180,7 @@ public static String getRowIdentifierName(Dataset schema) { * @return list of field names or null if there */ public static String getFieldNamesString(UserPreferences userPrefs, String datasetId) throws HttpException, IOException, URISyntaxException { - Dataset datasetInfo = getDatasetInfo(userPrefs, datasetId, Dataset.class); + Dataset datasetInfo = getDatasetInfo(userPrefs, datasetId); return getFieldNamesString(datasetInfo); } diff --git a/src/main/java/com/socrata/datasync/DatasyncGithubRelease.java b/src/main/java/com/socrata/datasync/DatasyncGithubRelease.java index a34d1d5e..954bb17a 100644 --- a/src/main/java/com/socrata/datasync/DatasyncGithubRelease.java +++ b/src/main/java/com/socrata/datasync/DatasyncGithubRelease.java @@ -1,8 +1,8 @@ package com.socrata.datasync; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.annotate.JsonSerialize; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) @JsonIgnoreProperties(ignoreUnknown=true) diff --git a/src/main/java/com/socrata/datasync/HttpUtility.java b/src/main/java/com/socrata/datasync/HttpUtility.java index 150b635b..d58e5c4b 100644 --- a/src/main/java/com/socrata/datasync/HttpUtility.java +++ b/src/main/java/com/socrata/datasync/HttpUtility.java @@ -70,7 +70,7 @@ public HttpUtility(UserPreferences userPrefs, boolean useAuth, int maxRetries, d HttpClientBuilder clientBuilder = HttpClients.custom(); if (useAuth) { authHeader = getAuthHeader(userPrefs.getUsername(), userPrefs.getPassword()); - appToken = userPrefs.getAPIKey(); + appToken = userPrefs.getConnectionInfo().getToken(); } authRequired = useAuth; if(userPrefs != null) { diff --git a/src/main/java/com/socrata/datasync/Main.java b/src/main/java/com/socrata/datasync/Main.java index 4f52b3c8..98ff93dd 100644 --- a/src/main/java/com/socrata/datasync/Main.java +++ b/src/main/java/com/socrata/datasync/Main.java @@ -1,129 +1,129 @@ -package com.socrata.datasync; - -import com.socrata.datasync.job.Job; -import com.socrata.datasync.job.Jobs; -import com.socrata.datasync.job.LoadPreferencesJob; -import com.socrata.datasync.job.PortJob; -import com.socrata.datasync.job.GISJob; -import com.socrata.datasync.job.GISJob.ControlDisagreementException; -import com.socrata.datasync.config.CommandLineOptions; -import com.socrata.datasync.config.userpreferences.UserPreferences; -import com.socrata.datasync.config.userpreferences.UserPreferencesFile; -import com.socrata.datasync.config.userpreferences.UserPreferencesJava; -import com.socrata.datasync.ui.SimpleIntegrationWizard; - -import java.io.File; -import java.io.IOException; -import java.util.Arrays; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.ParseException; -import org.codehaus.jackson.map.ObjectMapper; - -public class Main { - /** - * Loads an instance of the SimpleIntegrationWizard in command line - * mode (if arguments are given) or as a GUI (if no arguments are given). - */ - public static void main(String[] args) throws ParseException, ControlDisagreementException { - if(args.length == 0) { - // Open GUI (default) - new SimpleIntegrationWizard(); - } else if(args.length == 1) { - if (args[0].equals("-?") || args[0].equals("--help")) { - printHelp(); - } else if (args[0].equals("-v") || args[0].equals("--version")) { - System.out.println("DataSync version " + VersionProvider.getThisVersion()); - } else { - // Run a job file (.sij) in command-line mode - String jobFileToRun = args[0]; - - new SimpleIntegrationRunner(jobFileToRun); - } - } else { - // generate & run job from command line args - checkVersion(); - - CommandLineOptions options = new CommandLineOptions(); - CommandLine cmd = options.getCommandLine(args); - UserPreferences userPrefs = null; - try { - userPrefs = loadUserPreferences(options, cmd); - } catch (IOException e) { - System.err.println("Failed to load configuration: " + e.toString()); - System.exit(1); - } - - String jobTypeFlag = options.JOB_TYPE_FLAG; - String jobType = cmd.getOptionValue(jobTypeFlag, options.DEFAULT_JOBTYPE); - - Job jobToRun = new com.socrata.datasync.job.IntegrationJob(userPrefs); - if(jobType.equals(Jobs.PORT_JOB.toString())) { - jobToRun = new PortJob(userPrefs); - } else if(jobType.equals(Jobs.GIS_JOB.toString())){ - jobToRun = new GISJob(userPrefs); - } else if(jobType.equals(Jobs.LOAD_PREFERENCES_JOB.toString())) { - jobToRun = new LoadPreferencesJob(userPrefs); - } else if (!jobType.equals(Jobs.INTEGRATION_JOB.toString())){ - System.err.println("Invalid " + jobTypeFlag + ": " + cmd.getOptionValue(jobTypeFlag) + - " (must be " + Arrays.toString(Jobs.values()) + ")"); - System.exit(1); - } - - if (jobToRun.validateArgs(cmd)) { - jobToRun.configure(cmd); - new SimpleIntegrationRunner(jobToRun); - } else { - printHelp(); - System.exit(1); - } - } - } - - private static void printHelp() { - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("DataSync", CommandLineOptions.options); - } - - private static void checkVersion() { - if(VersionProvider.isLatestMajorVersion() == VersionProvider.VersionStatus.NOT_LATEST) { - String newDownloadLink = VersionProvider.getDownloadUrlForLatestVersion(); - String newVersionDownloadMessage = newDownloadLink == null ? "\n" : - "Download the new version (" + VersionProvider.getThisVersion() + ") here:\n" + - newDownloadLink + "\n"; - System.err.println("\nWARNING: DataSync is out-of-date. " + newVersionDownloadMessage); - } - } - - // TODO: move the method below to UserPreferences when I get those set of interfaces/classes consolidated. - - /** - * Returns a UserPreferences object which either loads User Prefs from a JSON file - * or the Java Preferences class (previously saved from GUI mode input) - * - * @param cmd - * @return UserPreferences object containing global preferences - * @throws IOException - */ - private static UserPreferences loadUserPreferences(CommandLineOptions options, CommandLine cmd) throws IOException { - UserPreferences userPrefs; - if (cmd.getOptionValue(options.CONFIG_FLAG) != null) { - // load user preferences from given JSON config file - File configFile = new File(cmd.getOptionValue("config")); - ObjectMapper mapper = new ObjectMapper(); - userPrefs = mapper.readValue(configFile, UserPreferencesFile.class); - String proxyUsername = cmd.getOptionValue(options.PROXY_USERNAME_FLAG); - String proxyPassword = cmd.getOptionValue(options.PROXY_PASSWORD_FLAG); - String jobType = cmd.getOptionValue(options.JOB_TYPE_FLAG, options.DEFAULT_JOBTYPE); - if (proxyUsername != null && proxyPassword != null && !jobType.equals(Jobs.LOAD_PREFERENCES_JOB.toString())) { - userPrefs.setProxyUsername(proxyUsername); - userPrefs.setProxyPassword(proxyPassword); - } - } else { - // load user preferences from Java preferences class - userPrefs = new UserPreferencesJava(); - } - return userPrefs; - } -} +package com.socrata.datasync; + +import com.socrata.datasync.job.Job; +import com.socrata.datasync.job.Jobs; +import com.socrata.datasync.job.LoadPreferencesJob; +import com.socrata.datasync.job.PortJob; +import com.socrata.datasync.job.GISJob; +import com.socrata.datasync.job.GISJob.ControlDisagreementException; +import com.socrata.datasync.config.CommandLineOptions; +import com.socrata.datasync.config.userpreferences.UserPreferences; +import com.socrata.datasync.config.userpreferences.UserPreferencesFile; +import com.socrata.datasync.config.userpreferences.UserPreferencesJava; +import com.socrata.datasync.ui.SimpleIntegrationWizard; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.ParseException; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class Main { + /** + * Loads an instance of the SimpleIntegrationWizard in command line + * mode (if arguments are given) or as a GUI (if no arguments are given). + */ + public static void main(String[] args) throws ParseException, ControlDisagreementException { + if(args.length == 0) { + // Open GUI (default) + SimpleIntegrationWizard.get(); + } else if(args.length == 1) { + if (args[0].equals("-?") || args[0].equals("--help")) { + printHelp(); + } else if (args[0].equals("-v") || args[0].equals("--version")) { + System.out.println("DataSync version " + VersionProvider.getThisVersion()); + } else { + // Run a job file (.sij) in command-line mode + String jobFileToRun = args[0]; + + new SimpleIntegrationRunner(jobFileToRun); + } + } else { + // generate & run job from command line args + checkVersion(); + + CommandLineOptions options = new CommandLineOptions(); + CommandLine cmd = options.getCommandLine(args); + UserPreferences userPrefs = null; + try { + userPrefs = loadUserPreferences(options, cmd); + } catch (IOException e) { + System.err.println("Failed to load configuration: " + e.toString()); + System.exit(1); + } + + String jobTypeFlag = options.JOB_TYPE_FLAG; + String jobType = cmd.getOptionValue(jobTypeFlag, options.DEFAULT_JOBTYPE); + + Job jobToRun = new com.socrata.datasync.job.IntegrationJob(userPrefs); + if(jobType.equals(Jobs.PORT_JOB.toString())) { + jobToRun = new PortJob(userPrefs); + } else if(jobType.equals(Jobs.GIS_JOB.toString())){ + jobToRun = new GISJob(userPrefs); + } else if(jobType.equals(Jobs.LOAD_PREFERENCES_JOB.toString())) { + jobToRun = new LoadPreferencesJob(userPrefs); + } else if (!jobType.equals(Jobs.INTEGRATION_JOB.toString())){ + System.err.println("Invalid " + jobTypeFlag + ": " + cmd.getOptionValue(jobTypeFlag) + + " (must be " + Arrays.toString(Jobs.values()) + ")"); + System.exit(1); + } + + if (jobToRun.validateArgs(cmd)) { + jobToRun.configure(cmd); + new SimpleIntegrationRunner(jobToRun); + } else { + printHelp(); + System.exit(1); + } + } + } + + private static void printHelp() { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("DataSync", CommandLineOptions.options); + } + + private static void checkVersion() { + if(VersionProvider.isLatestMajorVersion() == VersionProvider.VersionStatus.NOT_LATEST) { + String newDownloadLink = VersionProvider.getDownloadUrlForLatestVersion(); + String newVersionDownloadMessage = newDownloadLink == null ? "\n" : + "Download the new version (" + VersionProvider.getThisVersion() + ") here:\n" + + newDownloadLink + "\n"; + System.err.println("\nWARNING: DataSync is out-of-date. " + newVersionDownloadMessage); + } + } + + // TODO: move the method below to UserPreferences when I get those set of interfaces/classes consolidated. + + /** + * Returns a UserPreferences object which either loads User Prefs from a JSON file + * or the Java Preferences class (previously saved from GUI mode input) + * + * @param cmd + * @return UserPreferences object containing global preferences + * @throws IOException + */ + private static UserPreferences loadUserPreferences(CommandLineOptions options, CommandLine cmd) throws IOException { + UserPreferences userPrefs; + if (cmd.getOptionValue(options.CONFIG_FLAG) != null) { + // load user preferences from given JSON config file + File configFile = new File(cmd.getOptionValue("config")); + ObjectMapper mapper = new ObjectMapper(); + userPrefs = mapper.readValue(configFile, UserPreferencesFile.class); + String proxyUsername = cmd.getOptionValue(options.PROXY_USERNAME_FLAG); + String proxyPassword = cmd.getOptionValue(options.PROXY_PASSWORD_FLAG); + String jobType = cmd.getOptionValue(options.JOB_TYPE_FLAG, options.DEFAULT_JOBTYPE); + if (proxyUsername != null && proxyPassword != null && !jobType.equals(Jobs.LOAD_PREFERENCES_JOB.toString())) { + userPrefs.setProxyUsername(proxyUsername); + userPrefs.setProxyPassword(proxyPassword); + } + } else { + // load user preferences from Java preferences class + userPrefs = new UserPreferencesJava(); + } + return userPrefs; + } +} diff --git a/src/main/java/com/socrata/datasync/PortUtility.java b/src/main/java/com/socrata/datasync/PortUtility.java index acfc5bfa..c87121db 100644 --- a/src/main/java/com/socrata/datasync/PortUtility.java +++ b/src/main/java/com/socrata/datasync/PortUtility.java @@ -1,5 +1,6 @@ package com.socrata.datasync; +import com.socrata.api.DatasetDestination; import com.socrata.api.HttpLowLevel; import com.socrata.api.Soda2Consumer; import com.socrata.api.Soda2Producer; @@ -13,15 +14,17 @@ import com.socrata.model.importer.Dataset; import com.socrata.model.importer.DatasetInfo; import com.socrata.model.soql.SoqlQuery; -import com.sun.jersey.api.client.ClientResponse; -import org.codehaus.jackson.JsonGenerator; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.type.TypeReference; +import javax.ws.rs.core.Response; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.core.type.TypeReference; import java.io.File; import java.io.FileInputStream; import java.io.FileWriter; import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -36,17 +39,26 @@ private PortUtility() { public static String portSchema(SodaDdl loader, SodaDdl creator, final String sourceSetID, final String destinationDatasetTitle, - final boolean useNewBackend) + boolean actuallyCopySchema) throws SodaError, InterruptedException { System.out.print("Copying schema from dataset " + sourceSetID); Dataset sourceSet = (Dataset) loader.loadDatasetInfo(sourceSetID); + if(destinationDatasetTitle != null && !destinationDatasetTitle.equals("")) sourceSet.setName(destinationDatasetTitle); - adaptSchemaForAggregates(sourceSet); + DatasetDestination destination = + sourceSet.isNewBackend() ? DatasetDestination.NBE + : DatasetDestination.OBE; + + if(actuallyCopySchema) { + adaptSchemaForAggregates(sourceSet); + } else { + sourceSet.setColumns(Collections.emptyList()); + } - DatasetInfo sinkSet = creator.createDataset(sourceSet, useNewBackend); + DatasetInfo sinkSet = creator.createDataset(sourceSet, destination); String sinkSetID = sinkSet.getId(); System.out.println(" to dataset " + sinkSetID); @@ -86,7 +98,7 @@ private static void upsertContents(Soda2Consumer streamExporter, Soda2Producer s // Limit of 1000 rows per export, so page through dataset using $offset int offset = 0; int rowsUpserted = 0; - ClientResponse response; + Response response; ObjectMapper mapper = new ObjectMapper(); List> rowSet; @@ -95,7 +107,7 @@ private static void upsertContents(Soda2Consumer streamExporter, Soda2Producer s for(int retries = 0;; ++retries) { try { response = streamExporter.query(sourceSetID, HttpLowLevel.JSON_TYPE, myQuery); - rowSet = mapper.readValue(response.getEntityInputStream(), new TypeReference>>() {}); + rowSet = mapper.readValue(response.readEntity(InputStream.class), new TypeReference>>() {}); break; } catch(SodaError|IOException e) { if(retries == retryLimit) throw e; @@ -131,7 +143,7 @@ private static void replaceContents(Soda2Consumer streamExporter, Soda2Producer int offset = 0; int batchesRead = 0; SoqlQuery myQuery; - ClientResponse response; + Response response; ObjectMapper mapper = new ObjectMapper().configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false); List> rowSet; final File tempFile = File.createTempFile("replacement_dataset", ".json"); @@ -143,7 +155,7 @@ private static void replaceContents(Soda2Consumer streamExporter, Soda2Producer do { myQuery = new SoqlQueryBuilder().setOffset(offset).build(); response = streamExporter.query(sourceSetID, HttpLowLevel.JSON_TYPE, myQuery); - rowSet = mapper.readValue(response.getEntityInputStream(), + rowSet = mapper.readValue(response.readEntity(InputStream.class), new TypeReference>>() { } ); diff --git a/src/main/java/com/socrata/datasync/PublishMethod.java b/src/main/java/com/socrata/datasync/PublishMethod.java index 07df1fad..e09cc081 100644 --- a/src/main/java/com/socrata/datasync/PublishMethod.java +++ b/src/main/java/com/socrata/datasync/PublishMethod.java @@ -1,11 +1,11 @@ -package com.socrata.datasync; - -/** - * @author Adrian Laurenzi - * - * Holds the various methods that can be used to - * publish data using the Socrata publisher API - */ -public enum PublishMethod { - replace, upsert, append, delete -} +package com.socrata.datasync; + +/** + * @author Adrian Laurenzi + * + * Holds the various methods that can be used to + * publish data using the Socrata publisher API + */ +public enum PublishMethod { + replace, upsert, append, delete +} diff --git a/src/main/java/com/socrata/datasync/SimpleIntegrationRunner.java b/src/main/java/com/socrata/datasync/SimpleIntegrationRunner.java index b3315f4c..4d0520ba 100644 --- a/src/main/java/com/socrata/datasync/SimpleIntegrationRunner.java +++ b/src/main/java/com/socrata/datasync/SimpleIntegrationRunner.java @@ -1,89 +1,89 @@ -package com.socrata.datasync; - -import com.socrata.datasync.job.IntegrationJob; -import com.socrata.datasync.job.Job; -import com.socrata.datasync.job.JobStatus; -import com.socrata.datasync.job.PortJob; -import com.socrata.datasync.job.GISJob; -import com.socrata.datasync.job.GISJob.ControlDisagreementException; -import com.socrata.datasync.ui.GISJobTab; -import com.socrata.datasync.job.MetadataJob; -import com.socrata.datasync.ui.MetadataJobTab; -import com.socrata.datasync.ui.PortJobTab; - -import java.io.File; -import java.io.IOException; - -public class SimpleIntegrationRunner { - /** - * @author Adrian Laurenzi - * - * A command-line interface to DataSync - * @throws ControlDisagreementException - */ - - public SimpleIntegrationRunner(String jobFileToRun) throws ControlDisagreementException { - File jobFile = new File(jobFileToRun); - if(jobFile.exists()) { - try { - Job job; - //TODO BW: Follow how port jobs are run from command line? - if (jobFileToRun.endsWith(MetadataJobTab.JOB_FILE_EXTENSION)) { - job = new MetadataJob(jobFileToRun); - } else if(jobFileToRun.endsWith(GISJobTab.JOB_FILE_EXTENSION)) { - job = new GISJob(jobFileToRun); - } else if(jobFileToRun.endsWith(PortJobTab.JOB_FILE_EXTENSION)) { - job = new PortJob(jobFileToRun); - } else { - job = new IntegrationJob(jobFileToRun); - } - JobStatus status = job.run(); - if(status.isError()) { - System.err.print("Job completed with errors: "); - System.err.println(status.getMessage()); - System.exit(1); - } else { - // job ran successfully! - System.out.println("Job completed successfully"); - if(job.getClass() == PortJob.class) { - if(status == JobStatus.SUCCESS) System.out.print("Success. "); - System.out.println("Your newly created dataset is at:\n" + - ((PortJob)job).getSinkSiteDomain() + "/d/" + ((PortJob)job).getSinkSetID()); - } - System.out.println(status.getMessage()); - } - } catch (IOException | IntegrationJob.ControlDisagreementException e) { - System.err.println("Error running " + jobFileToRun + ":\n " + e.toString()); - System.exit(1); - } - } else { - // TODO record error in DataSync log? - System.err.println("Error running " + jobFileToRun + ": job file does not exist."); - System.exit(1); - } - } - - public SimpleIntegrationRunner(Job job) { - JobStatus status; - try { - status = job.run(); - if(status.isError()) { - System.err.print("Job completed with errors: "); - System.err.println(status.getMessage()); - System.exit(1); - } else { - System.out.println("Job completed successfully"); - if(job.getClass() == PortJob.class) { - if(status == JobStatus.SUCCESS) System.out.print("Success. "); - System.out.println("Your newly created dataset is at:\n" + - ((PortJob)job).getSinkSiteDomain() + "/d/" + ((PortJob)job).getSinkSetID()); - } - System.out.println(status.getMessage()); - } - } catch (IOException e) { - System.err.println(e.getMessage()); - System.exit(1); - } - } -} - +package com.socrata.datasync; + +import com.socrata.datasync.job.IntegrationJob; +import com.socrata.datasync.job.Job; +import com.socrata.datasync.job.JobStatus; +import com.socrata.datasync.job.PortJob; +import com.socrata.datasync.job.GISJob; +import com.socrata.datasync.job.GISJob.ControlDisagreementException; +import com.socrata.datasync.ui.GISJobTab; +import com.socrata.datasync.job.MetadataJob; +import com.socrata.datasync.ui.MetadataJobTab; +import com.socrata.datasync.ui.PortJobTab; + +import java.io.File; +import java.io.IOException; + +public class SimpleIntegrationRunner { + /** + * @author Adrian Laurenzi + * + * A command-line interface to DataSync + * @throws ControlDisagreementException + */ + + public SimpleIntegrationRunner(String jobFileToRun) throws ControlDisagreementException { + File jobFile = new File(jobFileToRun); + if(jobFile.exists()) { + try { + Job job; + //TODO BW: Follow how port jobs are run from command line? + if (jobFileToRun.endsWith(MetadataJobTab.JOB_FILE_EXTENSION)) { + job = new MetadataJob(jobFileToRun); + } else if(jobFileToRun.endsWith(GISJobTab.JOB_FILE_EXTENSION)) { + job = new GISJob(jobFileToRun); + } else if(jobFileToRun.endsWith(PortJobTab.JOB_FILE_EXTENSION)) { + job = new PortJob(jobFileToRun); + } else { + job = new IntegrationJob(jobFileToRun); + } + JobStatus status = job.run(); + if(status.isError()) { + System.err.print("Job completed with errors: "); + System.err.println(status.getMessage()); + System.exit(1); + } else { + // job ran successfully! + System.out.println("Job completed successfully"); + if(job.getClass() == PortJob.class) { + if(status == JobStatus.SUCCESS) System.out.print("Success. "); + System.out.println("Your newly created dataset is at:\n" + + ((PortJob)job).getSinkSiteDomain() + "/d/" + ((PortJob)job).getSinkSetID()); + } + System.out.println(status.getMessage()); + } + } catch (IOException | IntegrationJob.ControlDisagreementException e) { + System.err.println("Error running " + jobFileToRun + ":\n " + e.toString()); + System.exit(1); + } + } else { + // TODO record error in DataSync log? + System.err.println("Error running " + jobFileToRun + ": job file does not exist."); + System.exit(1); + } + } + + public SimpleIntegrationRunner(Job job) { + JobStatus status; + try { + status = job.run(); + if(status.isError()) { + System.err.print("Job completed with errors: "); + System.err.println(status.getMessage()); + System.exit(1); + } else { + System.out.println("Job completed successfully"); + if(job.getClass() == PortJob.class) { + if(status == JobStatus.SUCCESS) System.out.print("Success. "); + System.out.println("Your newly created dataset is at:\n" + + ((PortJob)job).getSinkSiteDomain() + "/d/" + ((PortJob)job).getSinkSetID()); + } + System.out.println(status.getMessage()); + } + } catch (IOException e) { + System.err.println(e.getMessage()); + System.exit(1); + } + } +} + diff --git a/src/main/java/com/socrata/datasync/SocrataConnectionInfo.java b/src/main/java/com/socrata/datasync/SocrataConnectionInfo.java index 456a4b67..80afb2ca 100644 --- a/src/main/java/com/socrata/datasync/SocrataConnectionInfo.java +++ b/src/main/java/com/socrata/datasync/SocrataConnectionInfo.java @@ -1,5 +1,8 @@ package com.socrata.datasync; +import java.io.*; +import java.nio.charset.StandardCharsets; + /** * This class contains all the information needed to connect to a * Socrata site. @@ -9,14 +12,13 @@ public class SocrataConnectionInfo public String url; public String user; public String password; - public String token; + private static String token; - public SocrataConnectionInfo(String url, String user, String password, String token) + public SocrataConnectionInfo(String url, String user, String password) { this.url = url; this.user = user; this.password = password; - this.token = token; } public String getUrl() @@ -34,8 +36,14 @@ public String getPassword() return password; } - public String getToken() - { + public String getToken() { + if(token == null) { + try(InputStream stream = SocrataConnectionInfo.class.getResourceAsStream("/api-key.txt")) { + token = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)).readLine().trim(); + } catch (Exception e) { + throw new RuntimeException("Unable to load API key! If this isn't an official build, you must create\na file called \"api-key.txt\" containing your app token before building."); + } + } return token; } } diff --git a/src/main/java/com/socrata/datasync/Utils.java b/src/main/java/com/socrata/datasync/Utils.java index a675e755..256eb889 100644 --- a/src/main/java/com/socrata/datasync/Utils.java +++ b/src/main/java/com/socrata/datasync/Utils.java @@ -18,7 +18,10 @@ import java.nio.charset.Charset; import java.nio.charset.UnsupportedCharsetException; import java.nio.file.Paths; +import java.util.Arrays; +import java.util.ArrayList; import java.util.UUID; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -246,4 +249,52 @@ public static String regionOfDomain(UserPreferences userPrefs, String domain) th } } } + + public static String[] commaSplit(String s) { + List result = new ArrayList(); + s = s.trim(); + + StringBuilder sb = new StringBuilder(); + for(int i = 0; i < s.length(); ++i) { + char c = s.charAt(i); + if(c == '\\' && i != s.length() - 1) { // if the backslash is the last character, we'll just treat it as a literal + sb.append(s.charAt(++i)); + } else if(c == ',') { + result.add(sb.toString().trim()); + sb = new StringBuilder(); + } else { + sb.append(c); + } + } + String finalString = sb.toString().trim(); + if(!result.isEmpty() || finalString.length() != 0) { + result.add(finalString.trim()); + } + return result.toArray(new String[result.size()]); + } + + public static String commaJoin(String[] ss) { + return commaJoin(Arrays.asList(ss)); + } + + public static String commaJoin(List ss) { + StringBuilder sb = new StringBuilder(); + boolean didOne = false; + for(String s : ss) { + if(didOne) sb.append(", "); + else didOne = true; + + s = s.trim(); + if(s.contains(",") || s.contains("\\")) { + for(int i = 0; i < s.length(); ++i) { + char c = s.charAt(i); + if(c == ',' || c == '\\') sb.append('\\'); + sb.append(c); + } + } else { + sb.append(s); + } + } + return sb.toString(); + } } diff --git a/src/main/java/com/socrata/datasync/VersionProvider.java b/src/main/java/com/socrata/datasync/VersionProvider.java index fb5182b8..a75e19de 100644 --- a/src/main/java/com/socrata/datasync/VersionProvider.java +++ b/src/main/java/com/socrata/datasync/VersionProvider.java @@ -2,7 +2,7 @@ import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.entity.ContentType; -import org.codehaus.jackson.map.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import java.net.URI; import java.util.ResourceBundle; diff --git a/src/main/java/com/socrata/datasync/config/controlfile/ColumnOverride.java b/src/main/java/com/socrata/datasync/config/controlfile/ColumnOverride.java index e2d35db2..655fe383 100644 --- a/src/main/java/com/socrata/datasync/config/controlfile/ColumnOverride.java +++ b/src/main/java/com/socrata/datasync/config/controlfile/ColumnOverride.java @@ -1,7 +1,7 @@ package com.socrata.datasync.config.controlfile; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; @JsonSerialize(include= JsonSerialize.Inclusion.NON_NULL) @JsonIgnoreProperties(ignoreUnknown=false) diff --git a/src/main/java/com/socrata/datasync/config/controlfile/ControlFile.java b/src/main/java/com/socrata/datasync/config/controlfile/ControlFile.java index 9f52aa26..48453466 100644 --- a/src/main/java/com/socrata/datasync/config/controlfile/ControlFile.java +++ b/src/main/java/com/socrata/datasync/config/controlfile/ControlFile.java @@ -3,12 +3,13 @@ import com.socrata.datasync.PublishMethod; import com.socrata.datasync.Utils; import com.socrata.datasync.job.IntegrationJob; -import org.codehaus.jackson.annotate.JsonIgnore; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonPropertyOrder; -import org.codehaus.jackson.map.annotate.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import java.util.HashMap; +import java.util.TreeMap; +import java.util.List; @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) @JsonIgnoreProperties(ignoreUnknown=true) @@ -72,7 +73,8 @@ public static ControlFile generateControlFile(final String fileToPublish, final PublishMethod publishMethod, final String[] columns, final boolean useSocrataGeocoding, - final boolean hasHeaderRow) { + final boolean hasHeaderRow, + final List timeFormats) { String fileToPublishExtension = Utils.getFileExtension(fileToPublish); @@ -89,20 +91,18 @@ public static ControlFile generateControlFile(final String fileToPublish, int skip = 0; String separator = isCsv ? "," : "\t"; - //Adding our standard export formats so that a customer can easily round-trip data into the system. - String[] timeFormats = new String[]{"ISO8601", "MM/dd/yy", "MM/dd/yyyy", "dd-MMM-yyyy","MM/dd/yyyy hh:mm:ss a Z","MM/dd/yyyy hh:mm:ss a"}; ftc.emptyTextIsNull(true) .filePath(fileToPublish) .ignoreColumns(new String[]{}) - .fixedTimestampFormat(timeFormats) - .floatingTimestampFormat(timeFormats) + .fixedTimestampFormat(timeFormats.toArray(new String[0])) + .floatingTimestampFormat(timeFormats.toArray(new String[0])) .separator(separator) .skip(skip) .timezone("UTC") .useSocrataGeocoding(useSocrataGeocoding) .trimWhitespace(true) .trimServerWhitespace(false) - .overrides(new HashMap()) + .overrides(new TreeMap()) .setAsideErrors(false); diff --git a/src/main/java/com/socrata/datasync/config/controlfile/FileTypeControl.java b/src/main/java/com/socrata/datasync/config/controlfile/FileTypeControl.java index 2c7cf531..a6be9af6 100644 --- a/src/main/java/com/socrata/datasync/config/controlfile/FileTypeControl.java +++ b/src/main/java/com/socrata/datasync/config/controlfile/FileTypeControl.java @@ -1,12 +1,12 @@ package com.socrata.datasync.config.controlfile; -import org.codehaus.jackson.annotate.JsonIgnore; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonPropertyOrder; -import org.codehaus.jackson.map.annotate.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import javax.xml.stream.Location; -import java.util.HashMap; +import java.util.TreeMap; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -34,10 +34,10 @@ public class FileTypeControl { public String[] fixedTimestampFormat; public String timezone; public String dropUninterpretableRows; - public Map overrides; - public Map analyticLocations; - public Map syntheticLocations; - public Map syntheticPoints; + public TreeMap overrides; + public TreeMap analyticLocations; + public TreeMap syntheticLocations; + public TreeMap syntheticPoints; public Boolean useSocrataGeocoding; public String action; public Boolean columnStatistics; @@ -131,13 +131,13 @@ public Set lookupTimestampFormatting() { public FileTypeControl dropUninterpretableRows(String d) { dropUninterpretableRows = d; return this; } - public FileTypeControl overrides(Map o) { overrides = o; return this; } + public FileTypeControl overrides(TreeMap o) { overrides = o; return this; } - public FileTypeControl syntheticLocations(Map s) { syntheticLocations = s; return this; } + public FileTypeControl syntheticLocations(TreeMap s) { syntheticLocations = s; return this; } - public FileTypeControl analyticLocations(Map s) { analyticLocations = s; return this; } + public FileTypeControl analyticLocations(TreeMap s) { analyticLocations = s; return this; } - public FileTypeControl syntheticPoints(Map s) { syntheticPoints = s; return this; } + public FileTypeControl syntheticPoints(TreeMap s) { syntheticPoints = s; return this; } public FileTypeControl useSocrataGeocoding(boolean u) { useSocrataGeocoding = u; return this; } diff --git a/src/main/java/com/socrata/datasync/config/controlfile/GeocodedPointColumn.java b/src/main/java/com/socrata/datasync/config/controlfile/GeocodedPointColumn.java new file mode 100644 index 00000000..c6232403 --- /dev/null +++ b/src/main/java/com/socrata/datasync/config/controlfile/GeocodedPointColumn.java @@ -0,0 +1,57 @@ +package com.socrata.datasync.config.controlfile; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import java.util.HashMap; +import java.util.Map; + +@JsonSerialize(include= JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown=false) +public class GeocodedPointColumn extends SyntheticPointColumn implements Cloneable { + // all of these fields should be set to the column id of the relevant field + public String address; + public String city; + public String state; + public String zip; + public String country; + public String type = "geocoded"; + + @JsonIgnore + public static final String[] locationFieldNames = new String[] {"address", "city", "state", "zip", "country"}; + + public Map findComponentColumns() { + Map components = new HashMap<>(); + for (String s : locationFieldNames) { + String fieldname = null; + switch (s) { + case "address": fieldname = address; break; + case "city": fieldname = city; break; + case "state": fieldname = state; break; + case "zip": fieldname = zip; break; + case "country": fieldname = country; break; + } + if (fieldname != null && !fieldname.isEmpty()) + components.put(s, fieldname); + } + return components; + } + + + // Builder methods: + + public GeocodedPointColumn address(String a) { address = a; return this; } + + public GeocodedPointColumn city(String c) { city = c; return this; } + + public GeocodedPointColumn state(String s) { state = s; return this; } + + public GeocodedPointColumn zip(String z) { zip = z; return this; } + + public GeocodedPointColumn country(String c) { country = c; return this; } + + public GeocodedPointColumn clone() { + return (GeocodedPointColumn) super.clone(); + } +} diff --git a/src/main/java/com/socrata/datasync/config/controlfile/LocationColumn.java b/src/main/java/com/socrata/datasync/config/controlfile/LocationColumn.java index 9c2f325a..1357ce0b 100644 --- a/src/main/java/com/socrata/datasync/config/controlfile/LocationColumn.java +++ b/src/main/java/com/socrata/datasync/config/controlfile/LocationColumn.java @@ -1,15 +1,15 @@ package com.socrata.datasync.config.controlfile; -import org.codehaus.jackson.annotate.JsonIgnore; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.util.HashMap; import java.util.Map; @JsonSerialize(include= JsonSerialize.Inclusion.NON_NULL) @JsonIgnoreProperties(ignoreUnknown=false) -public class LocationColumn { +public class LocationColumn extends SyntheticColumn implements Cloneable { // all of these fields should be set to the column id of the relevant field public String address; public String city; @@ -53,4 +53,13 @@ public Map findComponentColumns() { public LocationColumn state(String s) { state = s; return this; } public LocationColumn zip(String z) { zip = z; return this; } + + public LocationColumn clone() { + try { + return (LocationColumn) super.clone(); + } catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + } + } diff --git a/src/main/java/com/socrata/datasync/config/controlfile/PortControlFile.java b/src/main/java/com/socrata/datasync/config/controlfile/PortControlFile.java index d8075cc6..08fb6090 100644 --- a/src/main/java/com/socrata/datasync/config/controlfile/PortControlFile.java +++ b/src/main/java/com/socrata/datasync/config/controlfile/PortControlFile.java @@ -1,10 +1,17 @@ package com.socrata.datasync.config.controlfile; +import java.io.IOException; + import com.socrata.datasync.Utils; -import org.codehaus.jackson.annotate.JsonIgnore; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonPropertyOrder; -import org.codehaus.jackson.map.annotate.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; import com.socrata.datasync.PortMethod; @@ -12,12 +19,31 @@ @JsonIgnoreProperties(ignoreUnknown=true) @JsonPropertyOrder(alphabetic=true) public class PortControlFile { - public String destinationDomain; + public String sourceDomain; + public String sourceDataset; public String destinationName; public CopyType copyType; public Boolean publish; public String opaque; + public static class CopyTypeDeserializer extends StdDeserializer { + public CopyTypeDeserializer() { + super(CopyType.class); + } + + public CopyType deserialize(JsonParser jp, DeserializationContext ctx) throws IOException { + JsonNode node = jp.getCodec().readTree(jp); + String type = node.get("type").asText(); + switch(type) { + case "data": return new CopyData(); + case "schema": return new CopySchema(); + case "schema_and_data": return new CopyAll(); + default: return null; + } + } + } + + @JsonDeserialize(using = CopyTypeDeserializer.class) public static interface CopyType {} @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) @@ -25,10 +51,8 @@ public static interface CopyType {} @JsonPropertyOrder(alphabetic=true) public static class CopyData implements CopyType { public final String type; - public String destinationDataset; - public CopyData(String destinationDataset) { + public CopyData() { this.type = "data"; - this.destinationDataset = destinationDataset; } } @@ -37,10 +61,8 @@ public CopyData(String destinationDataset) { @JsonPropertyOrder(alphabetic=true) public static class CopySchema implements CopyType { public final String type; - public boolean toNbe; - public CopySchema(boolean toNbe) { + public CopySchema() { this.type = "schema"; - this.toNbe = toNbe; } } @@ -49,35 +71,33 @@ public CopySchema(boolean toNbe) { @JsonPropertyOrder(alphabetic=true) public static class CopyAll implements CopyType { public final String type; - public boolean toNbe; - public CopyAll(boolean toNbe) { + public CopyAll() { this.type = "schema_and_data"; - this.toNbe = toNbe; } } public PortControlFile() {} - public PortControlFile(String destinationDomain, + public PortControlFile(String sourceDomain, + String sourceDataset, String destinationName, - String destinationDataset, - boolean toNbe, PortMethod copyType, Boolean publish) { - this.destinationDomain = destinationDomain; + this.sourceDomain = sourceDomain; + this.sourceDataset = sourceDataset; this.destinationName = destinationName; this.publish = publish; switch(copyType) { case copy_data: - this.copyType = new CopyData(destinationDataset); + this.copyType = new CopyData(); break; case copy_schema: - this.copyType = new CopySchema(toNbe); + this.copyType = new CopySchema(); break; case copy_all: - this.copyType = new CopyAll(toNbe); + this.copyType = new CopyAll(); break; } } diff --git a/src/main/java/com/socrata/datasync/config/controlfile/ProvidedPointColumn.java b/src/main/java/com/socrata/datasync/config/controlfile/ProvidedPointColumn.java new file mode 100644 index 00000000..cfcb2fdc --- /dev/null +++ b/src/main/java/com/socrata/datasync/config/controlfile/ProvidedPointColumn.java @@ -0,0 +1,45 @@ +package com.socrata.datasync.config.controlfile; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import java.util.HashMap; +import java.util.Map; + +@JsonSerialize(include= JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown=false) +public class ProvidedPointColumn extends SyntheticPointColumn implements Cloneable { + // all of these fields should be set to the column id of the relevant field + public String latitude; + public String longitude; + public String type = "point"; + + @JsonIgnore + public static final String[] locationFieldNames = new String[] {"latitude", "longitude"}; + + public Map findComponentColumns() { + Map components = new HashMap<>(); + for (String s : locationFieldNames) { + String fieldname = null; + switch (s) { + case "latitude": fieldname = latitude; break; + case "longitude": fieldname = longitude; break; + } + if (fieldname != null && !fieldname.isEmpty()) + components.put(s, fieldname); + } + return components; + } + + + // Builder methods: + + public ProvidedPointColumn latitude(String l) { latitude = l; return this; } + + public ProvidedPointColumn longitude(String l) { longitude = l; return this; } + + public ProvidedPointColumn clone() { + return (ProvidedPointColumn) super.clone(); + } +} diff --git a/src/main/java/com/socrata/datasync/config/controlfile/SyntheticColumn.java b/src/main/java/com/socrata/datasync/config/controlfile/SyntheticColumn.java new file mode 100644 index 00000000..c20e2416 --- /dev/null +++ b/src/main/java/com/socrata/datasync/config/controlfile/SyntheticColumn.java @@ -0,0 +1,13 @@ +package com.socrata.datasync.config.controlfile; + +import java.util.HashMap; +import java.util.Map; + +public abstract class SyntheticColumn { + public abstract Map findComponentColumns(); + + @Override + public String toString() { + return findComponentColumns().toString(); + } +} diff --git a/src/main/java/com/socrata/datasync/config/controlfile/SyntheticPointColumn.java b/src/main/java/com/socrata/datasync/config/controlfile/SyntheticPointColumn.java new file mode 100644 index 00000000..8bf7b51c --- /dev/null +++ b/src/main/java/com/socrata/datasync/config/controlfile/SyntheticPointColumn.java @@ -0,0 +1,22 @@ +package com.socrata.datasync.config.controlfile; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonSubTypes; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", include = JsonTypeInfo.As.PROPERTY) + @JsonSubTypes(value = { + @JsonSubTypes.Type(value = GeocodedPointColumn.class, name = "geocoded"), + @JsonSubTypes.Type(value = ProvidedPointColumn.class, name = "point") + }) +public abstract class SyntheticPointColumn extends SyntheticColumn { + public SyntheticPointColumn clone() { + try { + return (SyntheticPointColumn) super.clone(); + } catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/main/java/com/socrata/datasync/config/userpreferences/UserPreferences.java b/src/main/java/com/socrata/datasync/config/userpreferences/UserPreferences.java index e74e8dfe..db85a4f9 100644 --- a/src/main/java/com/socrata/datasync/config/userpreferences/UserPreferences.java +++ b/src/main/java/com/socrata/datasync/config/userpreferences/UserPreferences.java @@ -1,12 +1,19 @@ package com.socrata.datasync.config.userpreferences; import com.socrata.datasync.SocrataConnectionInfo; +import java.util.List; +import java.util.Arrays; +import java.util.Collections; /** * Author: Adrian Laurenzi * Date: 12/2/13 */ public interface UserPreferences { + public static final String[] DEFAULT_TIME_FORMATS = new String[] { + "ISO8601", "MM/dd/yy", "MM/dd/yyyy", "dd-MMM-yyyy","MM/dd/yyyy hh:mm:ss a Z","MM/dd/yyyy hh:mm:ss a" + }; + public String getDomain(); public String getHost(); @@ -15,9 +22,6 @@ public interface UserPreferences { public String getPassword(); - // API key a.k.a. App token - public String getAPIKey(); - public String getProxyHost(); public String getProxyPort(); @@ -46,14 +50,11 @@ public interface UserPreferences { public String getNumRowsPerChunk(); - public String getPortDestinationDomainAppToken(); - - public boolean getUseNewBackend(); - public SocrataConnectionInfo getConnectionInfo(); public void setProxyPassword(String password); public void setProxyUsername(String username); + public List getDefaultTimeFormats(); } diff --git a/src/main/java/com/socrata/datasync/config/userpreferences/UserPreferencesFile.java b/src/main/java/com/socrata/datasync/config/userpreferences/UserPreferencesFile.java index 235911cc..38dc8cc5 100644 --- a/src/main/java/com/socrata/datasync/config/userpreferences/UserPreferencesFile.java +++ b/src/main/java/com/socrata/datasync/config/userpreferences/UserPreferencesFile.java @@ -1,9 +1,14 @@ package com.socrata.datasync.config.userpreferences; import com.socrata.datasync.SocrataConnectionInfo; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.annotate.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import java.util.List; +import java.util.Arrays; +import java.util.ArrayList; +import java.util.Collections; @JsonIgnoreProperties(ignoreUnknown=true) @JsonSerialize(include= JsonSerialize.Inclusion.NON_NULL) @@ -17,7 +22,6 @@ public class UserPreferencesFile implements UserPreferences { private String domain; private String username; private String password; - private String appToken; private String proxyHost; private String proxyPort; private String proxyUsername; @@ -32,8 +36,7 @@ public class UserPreferencesFile implements UserPreferences { private String smtpPassword; private String filesizeChunkingCutoffMB; private String numRowsPerChunk; - private String portDestinationDomainAppToken; - private boolean useNewBackend; + private List timeFormats; // Anytime a @JsonProperty is added/removed/updated in this class add 1 to this value private static final long fileVersionUID = 5L; @@ -54,15 +57,6 @@ public String getPassword() { return password; } - @JsonProperty("appToken") - public String getAppToken() { - return appToken; - } - // Alias for getAppToken - public String getAPIKey() { - return appToken; - } - @JsonProperty("proxyHost") public String getProxyHost() { return proxyHost; @@ -133,20 +127,18 @@ public String getNumRowsPerChunk() { return numRowsPerChunk; } - @JsonProperty("portDestinationDomainAppToken") - public String getPortDestinationDomainAppToken() { - return portDestinationDomainAppToken; - } - - @JsonProperty("useNewBackend") - public boolean getUseNewBackend() { return useNewBackend; } - @JsonProperty("proxyUsername") public void setProxyUsername(String username) { proxyUsername = username; } @JsonProperty("proxyPassword") public void setProxyPassword(String password) { proxyPassword = password; } + @JsonProperty("defaultTimeFormats") + public List getDefaultTimeFormats() { + if(timeFormats == null) return Collections.unmodifiableList(Arrays.asList(DEFAULT_TIME_FORMATS)); + return Collections.unmodifiableList(timeFormats); + } + public String getHost() { if (domain != null) { String[] schemeAndHost = domain.split("//"); @@ -158,7 +150,7 @@ public String getHost() { public SocrataConnectionInfo getConnectionInfo() { return new SocrataConnectionInfo( - this.getDomain(), this.getUsername(), this.getPassword(), this.getAPIKey()); + this.getDomain(), this.getUsername(), this.getPassword()); } } diff --git a/src/main/java/com/socrata/datasync/config/userpreferences/UserPreferencesJava.java b/src/main/java/com/socrata/datasync/config/userpreferences/UserPreferencesJava.java index 5e1066fa..dd6a99b9 100644 --- a/src/main/java/com/socrata/datasync/config/userpreferences/UserPreferencesJava.java +++ b/src/main/java/com/socrata/datasync/config/userpreferences/UserPreferencesJava.java @@ -1,271 +1,266 @@ -package com.socrata.datasync.config.userpreferences; - -import com.socrata.datasync.SocrataConnectionInfo; - -import java.util.prefs.BackingStoreException; -import java.util.prefs.Preferences; - -public class UserPreferencesJava implements UserPreferences { - /** - * @author Adrian Laurenzi - * - * Saves and retrieves (global) user preferences using the - * Java Preferences class (which stores/retrieves data from standard - * locations that vary depending on the platform). - */ - - private static Preferences userPrefs; - // Preference keys for saving user data - private static final String DOMAIN = "domain"; - private static final String USERNAME = "username"; - private static final String PASSWORD = "password"; - private static final String API_KEY = "api_key"; - - private static final String PROXY_USERNAME = "proxy_username"; - private static final String PROXY_PASSWORD = "proxy_password"; - private static final String PROXY_HOST = "proxy_host"; - private static final String PROXY_PORT = "proxy_port"; - - private static final String ADMIN_EMAIL = "admin_email"; - private static final String EMAIL_UPON_ERROR = "email_upon_error"; - private static final String LOG_DATASET_ID = "log_dataset_id"; - - private static final String OUTGOING_MAIL_SERVER = "outgoing_mail_server"; - private static final String SMTP_PORT = "smtp_port"; - // NOTE: if SSL port is set to the empty string then do not use SSL - private static final String SSL_PORT = "ssl_port"; - private static final String SMTP_USERNAME = "smtp_username"; - private static final String SMTP_PASSWORD = "smtp_password"; - - private static final String FILESIZE_CHUNKING_CUTOFF_MB = "filesize_chunking_cutoff_mb"; - private static final String NUM_ROWS_PER_CHUNK = "num_rows_per_chunk"; - - // When a file to be published is larger than this value (in MB), file is chunked - private static final String DEFAULT_FILESIZE_CHUNK_CUTOFF_MB = "10"; - // During chunking files are uploaded NUM_ROWS_PER_CHUNK rows per chunk - private static final String DEFAULT_NUM_ROWS_PER_CHUNK = "10000"; - - private final String DEFAULT_DOMAIN = "https://"; - private final String DEFAULT_SSL_PORT = "465"; - - public UserPreferencesJava() { - userPrefs = Preferences.userRoot().node("SocrataIntegrationPrefs"); - } - - public void clear() throws BackingStoreException { - userPrefs.clear(); - } - - // ------------- Save methods --------------- - public void saveDomain(String domain) { - saveKeyValuePair(DOMAIN, domain); - } - - public void saveUsername(String username) { - saveKeyValuePair(USERNAME, username); - } - - public void savePassword(String password) { - saveKeyValuePair(PASSWORD, password); - } - - // API key a.k.a. App token - public void saveAPIKey(String apiKey) { - saveKeyValuePair(API_KEY, apiKey); - } - - public void saveProxyHost(String host) { saveKeyValuePair(PROXY_HOST, host); } - - public void saveProxyPort(String port) { - saveKeyValuePair(PROXY_PORT, port); - } - - public void saveProxyUsername(String username) { saveKeyValuePair(PROXY_USERNAME, username); } - - public void saveProxyPassword(String password) { - saveKeyValuePair(PROXY_PASSWORD, password); - } - - public void saveAdminEmail(String adminEmail) { saveKeyValuePair(ADMIN_EMAIL, adminEmail); } - - public void saveEmailUponError(boolean emailUponError) { - String prefString = emailUponError ? "YES" : ""; - saveKeyValuePair(EMAIL_UPON_ERROR, prefString); - } - - public void saveLogDatasetID(String datasetID) { - saveKeyValuePair(LOG_DATASET_ID, datasetID); - } - - public void saveOutgoingMailServer(String mailServer) { - saveKeyValuePair(OUTGOING_MAIL_SERVER, mailServer); - } - - public void saveSMTPPort(String port) { - saveKeyValuePair(SMTP_PORT, port); - } - - public void saveSSLPort(String port) { - saveKeyValuePair(SSL_PORT, port); - } - - public void saveFilesizeChunkingCutoffMB(int numMegaBytes) { - saveKeyValuePair(FILESIZE_CHUNKING_CUTOFF_MB, Integer.toString(numMegaBytes)); - } - - public void saveNumRowsPerChunk(int numRows) { - saveKeyValuePair(NUM_ROWS_PER_CHUNK, Integer.toString(numRows)); - } - - public void saveSMTPUsername(String username) { - saveKeyValuePair(SMTP_USERNAME, username); - } - - public void saveSMTPPassword(String password) { - saveKeyValuePair(SMTP_PASSWORD, password); - } - - public void setProxyUsername(String username) {}; // never save proxy credentials - - public void setProxyPassword(String password) {}; // never save proxy credentials - - - // ------------- Get methods --------------- - - public String getDomain() { - String domain = userPrefs.get(DOMAIN, DEFAULT_DOMAIN); - if(domain == null) return null; - return UserPreferencesUtil.prefixDomain(domain.trim()); - } - - public String getUsername() { - return userPrefs.get(USERNAME, ""); - } - - public String getPassword() { - return userPrefs.get(PASSWORD, ""); - } - - // API key a.k.a. App token - public String getAPIKey() { - return userPrefs.get(API_KEY, ""); - } - - public String getProxyHost() { return userPrefs.get(PROXY_HOST, null); } - - public String getProxyPort() { - return userPrefs.get(PROXY_PORT, null); - } - - public String getProxyUsername() { - return userPrefs.get(PROXY_USERNAME, null); - } - - public String getProxyPassword() { - return userPrefs.get(PROXY_PASSWORD, null); - } - - public String getAdminEmail() { - return userPrefs.get(ADMIN_EMAIL, ""); - } - - public boolean emailUponError() { - String emailAdminPref = userPrefs.get(EMAIL_UPON_ERROR, ""); - return (!emailAdminPref.equals("")); - } - - public String getLogDatasetID() { - return userPrefs.get(LOG_DATASET_ID, ""); - } - - public String getOutgoingMailServer() { - return userPrefs.get(OUTGOING_MAIL_SERVER, ""); - } - - public String getSmtpPort() { - return userPrefs.get(SMTP_PORT, ""); - } - - public String getSslPort() { - return userPrefs.get(SSL_PORT, DEFAULT_SSL_PORT); - } - - public String getSmtpUsername() { - return userPrefs.get(SMTP_USERNAME, ""); - } - - public String getSmtpPassword() { - return userPrefs.get(SMTP_PASSWORD, ""); - } - - public String getFilesizeChunkingCutoffMB() { - return userPrefs.get( - FILESIZE_CHUNKING_CUTOFF_MB, DEFAULT_FILESIZE_CHUNK_CUTOFF_MB); - } - - public String getNumRowsPerChunk() { - return userPrefs.get(NUM_ROWS_PER_CHUNK, DEFAULT_NUM_ROWS_PER_CHUNK); - } - - /** - * This preference is for testing usage only (returns empty string because - * portDestinationDomainAppToken should only be set when DataSync is run - * in command-line mode). - */ - public String getPortDestinationDomainAppToken() { - return ""; - } - - /** - * This preference is for internal testing usage only (returns false - * because useNewBackend should only be set when DataSync is run in - * command-line mode). - */ - public boolean getUseNewBackend() { - return false; - } - - public String getHost() { - String domain = getDomain(); - if (domain != null) { - String[] schemeAndHost = domain.split("//"); - return schemeAndHost[schemeAndHost.length - 1]; - } else { - return null; - } - } - - public SocrataConnectionInfo getConnectionInfo() { - return new SocrataConnectionInfo( - this.getDomain(), this.getUsername(), this.getPassword(), this.getAPIKey()); - } - - @Override - public String toString() { - return "domain: " + getDomain() + "\n" + - "username: " + getUsername() + "\n" + - "password: " + getPassword().replaceAll(".", "*") +"\n" + - "appToken: " + getAPIKey() + "\n" + - "proxyHost:" + getProxyHost() + "\n" + - "proxyPort:" + getProxyPort() + "\n" + - "adminEmail: " + getAdminEmail() + "\n" + - "emailUponError: " + emailUponError() + "\n" + - "logDatasetID: " + getLogDatasetID() + "\n" + - "outgoingMailServer: " + getOutgoingMailServer() + "\n" + - "smtpPort: " + getSmtpPort() + "\n" + - "sslPort: " + getSslPort() + "\n" + - "smtpUsername: " + getSmtpUsername() + "\n" + - "smtpPassword: " + getSmtpPassword().replaceAll(".", "*") + "\n" + - "filesizeChunkingCutoffMB: " + getFilesizeChunkingCutoffMB() + "\n" + - "numRowsPerChunk: " + getNumRowsPerChunk() + "\n"; - } - - private void saveKeyValuePair(String key, String value) { - if (value == null) { - userPrefs.remove(key); - } else { - //ONCALL-3204 - Remove leading and trailing whitespace from user entered fields. - userPrefs.put(key, value.trim()); - } - } -} +package com.socrata.datasync.config.userpreferences; + +import com.socrata.datasync.SocrataConnectionInfo; +import com.socrata.datasync.Utils; + +import java.util.prefs.BackingStoreException; +import java.util.prefs.Preferences; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class UserPreferencesJava implements UserPreferences { + /** + * @author Adrian Laurenzi + * + * Saves and retrieves (global) user preferences using the + * Java Preferences class (which stores/retrieves data from standard + * locations that vary depending on the platform). + */ + + private static Preferences userPrefs; + // Preference keys for saving user data + private static final String DOMAIN = "domain"; + private static final String USERNAME = "username"; + private static final String PASSWORD = "password"; + + private static final String PROXY_USERNAME = "proxy_username"; + private static final String PROXY_PASSWORD = "proxy_password"; + private static final String PROXY_HOST = "proxy_host"; + private static final String PROXY_PORT = "proxy_port"; + + private static final String ADMIN_EMAIL = "admin_email"; + private static final String EMAIL_UPON_ERROR = "email_upon_error"; + private static final String LOG_DATASET_ID = "log_dataset_id"; + + private static final String OUTGOING_MAIL_SERVER = "outgoing_mail_server"; + private static final String SMTP_PORT = "smtp_port"; + // NOTE: if SSL port is set to the empty string then do not use SSL + private static final String SSL_PORT = "ssl_port"; + private static final String SMTP_USERNAME = "smtp_username"; + private static final String SMTP_PASSWORD = "smtp_password"; + + private static final String FILESIZE_CHUNKING_CUTOFF_MB = "filesize_chunking_cutoff_mb"; + private static final String NUM_ROWS_PER_CHUNK = "num_rows_per_chunk"; + + // When a file to be published is larger than this value (in MB), file is chunked + private static final String DEFAULT_FILESIZE_CHUNK_CUTOFF_MB = "10"; + // During chunking files are uploaded NUM_ROWS_PER_CHUNK rows per chunk + private static final String DEFAULT_NUM_ROWS_PER_CHUNK = "10000"; + + private static final String TIME_FORMATS = "time_formats"; + + private final String DEFAULT_DOMAIN = "https://"; + private final String DEFAULT_SSL_PORT = "465"; + + public UserPreferencesJava() { + userPrefs = Preferences.userRoot().node("SocrataIntegrationPrefs"); + } + + public void clear() throws BackingStoreException { + userPrefs.clear(); + } + + // ------------- Save methods --------------- + public void saveDomain(String domain) { + saveKeyValuePair(DOMAIN, domain); + } + + public void saveUsername(String username) { + saveKeyValuePair(USERNAME, username); + } + + public void savePassword(String password) { + saveKeyValuePair(PASSWORD, password); + } + + public void saveProxyHost(String host) { saveKeyValuePair(PROXY_HOST, host); } + + public void saveProxyPort(String port) { + saveKeyValuePair(PROXY_PORT, port); + } + + public void saveProxyUsername(String username) { saveKeyValuePair(PROXY_USERNAME, username); } + + public void saveProxyPassword(String password) { + saveKeyValuePair(PROXY_PASSWORD, password); + } + + public void saveAdminEmail(String adminEmail) { saveKeyValuePair(ADMIN_EMAIL, adminEmail); } + + public void saveEmailUponError(boolean emailUponError) { + String prefString = emailUponError ? "YES" : ""; + saveKeyValuePair(EMAIL_UPON_ERROR, prefString); + } + + public void saveLogDatasetID(String datasetID) { + saveKeyValuePair(LOG_DATASET_ID, datasetID); + } + + public void saveOutgoingMailServer(String mailServer) { + saveKeyValuePair(OUTGOING_MAIL_SERVER, mailServer); + } + + public void saveSMTPPort(String port) { + saveKeyValuePair(SMTP_PORT, port); + } + + public void saveSSLPort(String port) { + saveKeyValuePair(SSL_PORT, port); + } + + public void saveFilesizeChunkingCutoffMB(int numMegaBytes) { + saveKeyValuePair(FILESIZE_CHUNKING_CUTOFF_MB, Integer.toString(numMegaBytes)); + } + + public void saveNumRowsPerChunk(int numRows) { + saveKeyValuePair(NUM_ROWS_PER_CHUNK, Integer.toString(numRows)); + } + + public void saveSMTPUsername(String username) { + saveKeyValuePair(SMTP_USERNAME, username); + } + + public void saveSMTPPassword(String password) { + saveKeyValuePair(SMTP_PASSWORD, password); + } + + public void saveDefaultTimeFormats(List defaultTimeFormats) { + saveKeyValuePair(TIME_FORMATS, Utils.commaJoin(defaultTimeFormats)); + } + + public void setProxyUsername(String username) {}; // never save proxy credentials + + public void setProxyPassword(String password) {}; // never save proxy credentials + + + // ------------- Get methods --------------- + + public String getDomain() { + String domain = userPrefs.get(DOMAIN, DEFAULT_DOMAIN); + if(domain == null) return null; + return UserPreferencesUtil.prefixDomain(domain.trim()); + } + + public String getUsername() { + return userPrefs.get(USERNAME, ""); + } + + public String getPassword() { + return userPrefs.get(PASSWORD, ""); + } + + public String getProxyHost() { return userPrefs.get(PROXY_HOST, null); } + + public String getProxyPort() { + return userPrefs.get(PROXY_PORT, null); + } + + public String getProxyUsername() { + return userPrefs.get(PROXY_USERNAME, null); + } + + public String getProxyPassword() { + return userPrefs.get(PROXY_PASSWORD, null); + } + + public String getAdminEmail() { + return userPrefs.get(ADMIN_EMAIL, ""); + } + + public boolean emailUponError() { + String emailAdminPref = userPrefs.get(EMAIL_UPON_ERROR, ""); + return (!emailAdminPref.equals("")); + } + + public String getLogDatasetID() { + return userPrefs.get(LOG_DATASET_ID, ""); + } + + public String getOutgoingMailServer() { + return userPrefs.get(OUTGOING_MAIL_SERVER, ""); + } + + public String getSmtpPort() { + return userPrefs.get(SMTP_PORT, ""); + } + + public String getSslPort() { + return userPrefs.get(SSL_PORT, DEFAULT_SSL_PORT); + } + + public String getSmtpUsername() { + return userPrefs.get(SMTP_USERNAME, ""); + } + + public String getSmtpPassword() { + return userPrefs.get(SMTP_PASSWORD, ""); + } + + public String getFilesizeChunkingCutoffMB() { + return userPrefs.get( + FILESIZE_CHUNKING_CUTOFF_MB, DEFAULT_FILESIZE_CHUNK_CUTOFF_MB); + } + + public String getNumRowsPerChunk() { + return userPrefs.get(NUM_ROWS_PER_CHUNK, DEFAULT_NUM_ROWS_PER_CHUNK); + } + + /** + * This preference is for testing usage only (returns empty string because + * portDestinationDomainAppToken should only be set when DataSync is run + * in command-line mode). + */ + public String getPortDestinationDomainAppToken() { + return ""; + } + + public String getHost() { + String domain = getDomain(); + if (domain != null) { + String[] schemeAndHost = domain.split("//"); + return schemeAndHost[schemeAndHost.length - 1]; + } else { + return null; + } + } + + public SocrataConnectionInfo getConnectionInfo() { + return new SocrataConnectionInfo( + this.getDomain(), this.getUsername(), this.getPassword()); + } + + public List getDefaultTimeFormats() { + return Collections.unmodifiableList(Arrays.asList(Utils.commaSplit(userPrefs.get(TIME_FORMATS, Utils.commaJoin(DEFAULT_TIME_FORMATS))))); + } + + @Override + public String toString() { + return "domain: " + getDomain() + "\n" + + "username: " + getUsername() + "\n" + + "password: " + getPassword().replaceAll(".", "*") +"\n" + + "proxyHost:" + getProxyHost() + "\n" + + "proxyPort:" + getProxyPort() + "\n" + + "adminEmail: " + getAdminEmail() + "\n" + + "emailUponError: " + emailUponError() + "\n" + + "logDatasetID: " + getLogDatasetID() + "\n" + + "outgoingMailServer: " + getOutgoingMailServer() + "\n" + + "smtpPort: " + getSmtpPort() + "\n" + + "sslPort: " + getSslPort() + "\n" + + "smtpUsername: " + getSmtpUsername() + "\n" + + "smtpPassword: " + getSmtpPassword().replaceAll(".", "*") + "\n" + + "filesizeChunkingCutoffMB: " + getFilesizeChunkingCutoffMB() + "\n" + + "numRowsPerChunk: " + getNumRowsPerChunk() + "\n" + + "defaultTimeFormats: " + getDefaultTimeFormats() + "\n"; + } + + private void saveKeyValuePair(String key, String value) { + if (value == null) { + userPrefs.remove(key); + } else { + //ONCALL-3204 - Remove leading and trailing whitespace from user entered fields. + userPrefs.put(key, value.trim()); + } + } +} diff --git a/src/main/java/com/socrata/datasync/config/userpreferences/UserPreferencesLib.java b/src/main/java/com/socrata/datasync/config/userpreferences/UserPreferencesLib.java index fda67fb2..d62542c6 100644 --- a/src/main/java/com/socrata/datasync/config/userpreferences/UserPreferencesLib.java +++ b/src/main/java/com/socrata/datasync/config/userpreferences/UserPreferencesLib.java @@ -3,6 +3,11 @@ import com.socrata.datasync.SocrataConnectionInfo; import com.socrata.datasync.job.LoadPreferencesJob; +import java.util.Arrays; +import java.util.ArrayList; +import java.util.List; +import java.util.Collections; + /** * Author: Adrian Laurenzi * Date: 6/13/14 @@ -14,7 +19,6 @@ public class UserPreferencesLib implements UserPreferences { private String domain; private String username; private String password; - private String appToken; private String adminEmail; private String proxyHost; private String proxyPort; @@ -29,7 +33,8 @@ public class UserPreferencesLib implements UserPreferences { private String smtpPassword; private String filesizeChunkingCutoffMB; private String numRowsPerChunk; - private String portDestinationDomainAppToken; + private boolean useNewBackend; + private List defaultTimeFormats; // When a file to be published is larger than this value (in MB), file is chunked private static final String DEFAULT_FILESIZE_CHUNK_CUTOFF_MB = "10"; @@ -47,7 +52,7 @@ public UserPreferencesLib() { smtpPassword = ""; filesizeChunkingCutoffMB = DEFAULT_FILESIZE_CHUNK_CUTOFF_MB; numRowsPerChunk = DEFAULT_NUM_ROWS_PER_CHUNK; - portDestinationDomainAppToken = ""; + defaultTimeFormats = Arrays.asList(DEFAULT_TIME_FORMATS); } public String getDomain() { @@ -77,16 +82,11 @@ public void setPassword(String password) { public UserPreferencesLib password(String password) { setPassword(password); return this; } - public String getAPIKey() { return appToken; } - - public String getAppToken() { - return appToken; - } - + @Deprecated public void setAppToken(String appToken) { - this.appToken = appToken; } + @Deprecated public UserPreferencesLib appToken(String appToken) { setAppToken(appToken); return this; } public String getProxyHost() { return proxyHost; } @@ -220,20 +220,20 @@ public void setNumRowsPerChunk(String numRowsPerChunk) { public UserPreferencesLib numRowsPerChunk(String numRows) { setNumRowsPerChunk(numRows); return this; } public boolean getUseNewBackend() { - return false; + return useNewBackend; } - public void setPortDestinationDomainAppToken(String portDestinationDomainAppToken) { - this.portDestinationDomainAppToken = portDestinationDomainAppToken; + public void setUseNewBackend(boolean useNewBackend) { + this.useNewBackend = useNewBackend; } - public String getPortDestinationDomainAppToken() { - return portDestinationDomainAppToken; + @Deprecated + public void setPortDestinationDomainAppToken(String portDestinationDomainAppToken) { } public SocrataConnectionInfo getConnectionInfo() { return new SocrataConnectionInfo( - this.getDomain(), this.getUsername(), this.getPassword(), this.getAPIKey()); + this.getDomain(), this.getUsername(), this.getPassword()); } public String getHost() { @@ -245,6 +245,14 @@ public String getHost() { } } + public List getDefaultTimeFormats() { + return Collections.unmodifiableList(defaultTimeFormats); + } + + public void setDefaultTimeForamts(List defaultTimeFormats) { + this.defaultTimeFormats = new ArrayList<>(defaultTimeFormats); + } + public UserPreferencesLib load() { LoadPreferencesJob j = new LoadPreferencesJob(); j.setUserPrefs(this); diff --git a/src/main/java/com/socrata/datasync/deltaimporter2/BlobId.java b/src/main/java/com/socrata/datasync/deltaimporter2/BlobId.java index 0057a8ec..87909ebe 100644 --- a/src/main/java/com/socrata/datasync/deltaimporter2/BlobId.java +++ b/src/main/java/com/socrata/datasync/deltaimporter2/BlobId.java @@ -1,7 +1,7 @@ package com.socrata.datasync.deltaimporter2; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; @JsonSerialize(include= JsonSerialize.Inclusion.NON_NULL) @JsonIgnoreProperties(ignoreUnknown=true) diff --git a/src/main/java/com/socrata/datasync/deltaimporter2/CommitMessage.java b/src/main/java/com/socrata/datasync/deltaimporter2/CommitMessage.java index b118986c..f306fd07 100644 --- a/src/main/java/com/socrata/datasync/deltaimporter2/CommitMessage.java +++ b/src/main/java/com/socrata/datasync/deltaimporter2/CommitMessage.java @@ -1,8 +1,8 @@ package com.socrata.datasync.deltaimporter2; import com.socrata.datasync.config.controlfile.ControlFile; -import org.codehaus.jackson.annotate.JsonPropertyOrder; -import org.codehaus.jackson.map.annotate.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.util.List; diff --git a/src/main/java/com/socrata/datasync/deltaimporter2/DI2Error.java b/src/main/java/com/socrata/datasync/deltaimporter2/DI2Error.java index 52f6f6eb..792b407d 100644 --- a/src/main/java/com/socrata/datasync/deltaimporter2/DI2Error.java +++ b/src/main/java/com/socrata/datasync/deltaimporter2/DI2Error.java @@ -1,12 +1,12 @@ package com.socrata.datasync.deltaimporter2; -import org.codehaus.jackson.JsonParser; -import org.codehaus.jackson.JsonProcessingException; -import org.codehaus.jackson.JsonToken; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.DeserializationContext; -import org.codehaus.jackson.map.JsonDeserializer; -import org.codehaus.jackson.map.annotate.JsonDeserialize; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import java.io.IOException; import java.util.Collections; @@ -52,7 +52,7 @@ public ErrorType deserialize(JsonParser jsonParser, DeserializationContext deser if(jsonParser.getCurrentToken() != JsonToken.VALUE_STRING) throw deserializationContext.wrongTokenException(jsonParser, JsonToken.VALUE_STRING, "Expected string"); String code = jsonParser.getText(); ErrorType result = errorTypes.get(code); - if(result == null) throw deserializationContext.weirdStringException(ErrorType.class, "Unknown error code " + code); + if(result == null) throw deserializationContext.weirdStringException(code, ErrorType.class, "Unknown error code"); return result; } } diff --git a/src/main/java/com/socrata/datasync/deltaimporter2/DatasyncDirectory.java b/src/main/java/com/socrata/datasync/deltaimporter2/DatasyncDirectory.java index e94ae1e4..1182171d 100644 --- a/src/main/java/com/socrata/datasync/deltaimporter2/DatasyncDirectory.java +++ b/src/main/java/com/socrata/datasync/deltaimporter2/DatasyncDirectory.java @@ -5,7 +5,7 @@ import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.utils.URIBuilder; import org.apache.http.entity.ContentType; -import org.codehaus.jackson.map.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.io.UnsupportedEncodingException; diff --git a/src/main/java/com/socrata/datasync/deltaimporter2/JobId.java b/src/main/java/com/socrata/datasync/deltaimporter2/JobId.java index e44e316a..0905cd90 100644 --- a/src/main/java/com/socrata/datasync/deltaimporter2/JobId.java +++ b/src/main/java/com/socrata/datasync/deltaimporter2/JobId.java @@ -1,7 +1,7 @@ package com.socrata.datasync.deltaimporter2; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; @JsonSerialize(include= JsonSerialize.Inclusion.NON_NULL) @JsonIgnoreProperties(ignoreUnknown=true) diff --git a/src/main/java/com/socrata/datasync/deltaimporter2/LogItem.java b/src/main/java/com/socrata/datasync/deltaimporter2/LogItem.java index a4894939..cf9951da 100644 --- a/src/main/java/com/socrata/datasync/deltaimporter2/LogItem.java +++ b/src/main/java/com/socrata/datasync/deltaimporter2/LogItem.java @@ -2,8 +2,8 @@ import com.socrata.datasync.config.controlfile.ControlFile; import com.socrata.datasync.config.controlfile.PortControlFile; -import org.codehaus.jackson.map.annotate.JsonSerialize; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.util.Map; diff --git a/src/main/java/com/socrata/datasync/deltaimporter2/Version.java b/src/main/java/com/socrata/datasync/deltaimporter2/Version.java index e8c20bb9..b3ef2e21 100644 --- a/src/main/java/com/socrata/datasync/deltaimporter2/Version.java +++ b/src/main/java/com/socrata/datasync/deltaimporter2/Version.java @@ -1,7 +1,7 @@ package com.socrata.datasync.deltaimporter2; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; @JsonIgnoreProperties(ignoreUnknown=true) public class Version { diff --git a/src/main/java/com/socrata/datasync/imports2/Blueprint.java b/src/main/java/com/socrata/datasync/imports2/Blueprint.java index 44e9755b..d72cfb14 100644 --- a/src/main/java/com/socrata/datasync/imports2/Blueprint.java +++ b/src/main/java/com/socrata/datasync/imports2/Blueprint.java @@ -1,8 +1,8 @@ package com.socrata.datasync.imports2; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.annotate.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; @JsonIgnoreProperties(ignoreUnknown=true) @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) diff --git a/src/main/java/com/socrata/datasync/imports2/Error.java b/src/main/java/com/socrata/datasync/imports2/Error.java index 3d1c06c6..ade10583 100644 --- a/src/main/java/com/socrata/datasync/imports2/Error.java +++ b/src/main/java/com/socrata/datasync/imports2/Error.java @@ -1,8 +1,8 @@ package com.socrata.datasync.imports2; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.annotate.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; @JsonIgnoreProperties(ignoreUnknown=true) @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) diff --git a/src/main/java/com/socrata/datasync/imports2/Layer.java b/src/main/java/com/socrata/datasync/imports2/Layer.java index d1d58212..b71f604d 100644 --- a/src/main/java/com/socrata/datasync/imports2/Layer.java +++ b/src/main/java/com/socrata/datasync/imports2/Layer.java @@ -1,8 +1,8 @@ package com.socrata.datasync.imports2; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.annotate.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; @JsonIgnoreProperties(ignoreUnknown=true) @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) diff --git a/src/main/java/com/socrata/datasync/imports2/Summary.java b/src/main/java/com/socrata/datasync/imports2/Summary.java index 7b33bee2..b7d15036 100644 --- a/src/main/java/com/socrata/datasync/imports2/Summary.java +++ b/src/main/java/com/socrata/datasync/imports2/Summary.java @@ -1,8 +1,8 @@ package com.socrata.datasync.imports2; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.annotate.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; @JsonIgnoreProperties(ignoreUnknown=true) @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) diff --git a/src/main/java/com/socrata/datasync/job/GISJob.java b/src/main/java/com/socrata/datasync/job/GISJob.java index 0c63457c..0e152664 100644 --- a/src/main/java/com/socrata/datasync/job/GISJob.java +++ b/src/main/java/com/socrata/datasync/job/GISJob.java @@ -15,11 +15,12 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.io.FilenameUtils; import org.apache.http.HttpException; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.DeserializationConfig; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.annotate.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.DeserializationConfig; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.io.*; import java.net.URISyntaxException; @@ -55,7 +56,7 @@ public class GISJob extends Job { private ObjectMapper controlFileMapper = - new ObjectMapper().enable(DeserializationConfig.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY); + new ObjectMapper().enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY); public GISJob() { userPrefs = new UserPreferencesJava(); @@ -240,7 +241,7 @@ public void initializeLayerMapping() throws URISyntaxException, IOException, Htt String fileExtension = FilenameUtils.getExtension(fileToPublish); if (fileExtension.equals(GISJobValidity.ZIP_EXT)) { - GeoDataset dataset = DatasetUtils.getDatasetInfo(userPrefs, getDatasetID(), GeoDataset.class); + GeoDataset dataset = DatasetUtils.getGeoDatasetInfo(userPrefs, getDatasetID()); // Get map of existing layer UIDs/names by looking at child views Map existingLayers = getLayerListFromExistingDataset(userPrefs, dataset); @@ -272,7 +273,7 @@ private static Map getLayerListFromExistingDataset(UserPreferenc for (String uid : existingLayersUids) { try { - Dataset child = DatasetUtils.getDatasetInfo(userPrefs, uid, Dataset.class); + Dataset child = DatasetUtils.getDatasetInfo(userPrefs, uid); existingLayerInfo.put(child.getName(), uid); } catch (Exception e) { // there’s no way for the client to recover, diff --git a/src/main/java/com/socrata/datasync/job/IntegrationJob.java b/src/main/java/com/socrata/datasync/job/IntegrationJob.java index 436ea001..452948bc 100644 --- a/src/main/java/com/socrata/datasync/job/IntegrationJob.java +++ b/src/main/java/com/socrata/datasync/job/IntegrationJob.java @@ -1,639 +1,642 @@ -package com.socrata.datasync.job; - -import com.google.common.collect.ImmutableMap; -import com.socrata.api.Soda2Producer; -import com.socrata.api.SodaImporter; -import com.socrata.datasync.PublishMethod; -import com.socrata.datasync.SMTPMailer; -import com.socrata.datasync.SocrataConnectionInfo; -import com.socrata.datasync.Utils; -import com.socrata.datasync.VersionProvider; -import com.socrata.datasync.config.CommandLineOptions; -import com.socrata.datasync.config.controlfile.ControlFile; -import com.socrata.datasync.config.userpreferences.UserPreferences; -import com.socrata.datasync.config.userpreferences.UserPreferencesJava; -import com.socrata.datasync.publishers.DeltaImporter2Publisher; -import com.socrata.datasync.publishers.FTPDropbox2Publisher; -import com.socrata.datasync.publishers.Soda2Publisher; -import com.socrata.datasync.validation.IntegrationJobValidity; -import com.socrata.exceptions.SodaError; -import com.socrata.model.UpsertError; -import com.socrata.model.UpsertResult; -import org.apache.commons.cli.CommandLine; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.DeserializationConfig; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.annotate.JsonSerialize; -import com.sun.jersey.api.client.ClientHandlerException; - -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.ObjectInput; -import java.io.ObjectInputStream; -import java.net.SocketException; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; - -@JsonIgnoreProperties(ignoreUnknown=true) -@JsonSerialize(include= JsonSerialize.Inclusion.NON_NULL) -public class IntegrationJob extends Job { - - static AtomicInteger jobCounter = new AtomicInteger(0); - int jobNum = jobCounter.getAndIncrement(); - private String defaultJobName = "Unsaved Standard Job" + " (" + jobNum + ")"; - - // to upload entire file as a single chunk (numRowsPerChunk == 0) - private static final int UPLOAD_SINGLE_CHUNK = 0; - public static final int NUM_BYTES_PER_MB = 1048576; - - // Anytime a @JsonProperty is added/removed/updated in this class add 1 to this value - private static final long fileVersionUID = 4L; - - private UserPreferences userPrefs; - private String datasetID = ""; - private String fileToPublish = ""; - private PublishMethod publishMethod = null; - private boolean fileToPublishHasHeaderRow = true; - private String pathToControlFile = null; - private String controlFileContent = null; - private boolean publishViaFTP = false; - private boolean publishViaDi2Http = false; - private ControlFile controlFile = null; - - private String userAgent = "datasync"; - - private String userAgentNameClient = "Client"; - private String userAgentNameCli = "CLI"; - private String userAgentNameSijFile = ".sij File"; - - private ObjectMapper controlFileMapper = - new ObjectMapper().enable(DeserializationConfig.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY); - - public IntegrationJob() { - userPrefs = new UserPreferencesJava(); - } - - /* - * This is a method that enables DataSync preferences to be established - * directly when DataSync is used in "library mode" or "command-line mode" - * (rather than being loaded from Preferences class) - */ - public IntegrationJob(UserPreferences userPrefs) { - this.userPrefs = userPrefs; - } - - /** - * Loads integration job data from a file and - * uses the saved data to populate the fields - * of this object - */ - public IntegrationJob(String pathToFile) throws IOException, ControlDisagreementException { - this(pathToFile, false); - setUserAgentSijFile(); - } - - /** - * Loads integration job data from a file and - * uses the saved data to populate the fields - * of this object - */ - public IntegrationJob(String pathToFile, boolean ignoreControlInconsistencies) throws IOException, ControlDisagreementException { - userPrefs = new UserPreferencesJava(); - - // first try reading the 'current' format - ObjectMapper mapper = new ObjectMapper(); - IntegrationJob loadedJob; - try { - loadedJob = mapper.readValue(new File(pathToFile), IntegrationJob.class); - } catch (IOException e) { - // if reading new format fails...try reading old format into this object - loadOldSijFile(pathToFile); - return; - } - loadedJob.setPathToSavedFile(pathToFile); - String controlPath = loadedJob.getPathToControlFile(); - String controlContent = loadedJob.getControlFileContent(); - try { - setControlFile(controlPath, controlContent); - } catch (ControlDisagreementException e) { - if (!ignoreControlInconsistencies) - throw e; - } - setDatasetID(loadedJob.getDatasetID()); - setFileToPublish(loadedJob.getFileToPublish()); - setPublishMethod(loadedJob.getPublishMethod()); - setFileToPublishHasHeaderRow(loadedJob.getFileToPublishHasHeaderRow()); - setPathToSavedFile(pathToFile); - setPathToControlFile(loadedJob.getPathToControlFile()); - setControlFileContent(loadedJob.getControlFileContent()); - setPublishViaFTP(loadedJob.getPublishViaFTP()); - setPublishViaDi2Http(loadedJob.getPublishViaDi2Http()); - } - - - @JsonProperty("fileVersionUID") - public long getFileVersionUID() { return fileVersionUID; } - - public ControlFile getControlFile() { return controlFile; } - - public void setControlFile(ControlFile cf) { controlFile = cf; } - - @JsonProperty("datasetID") - public void setDatasetID(String newDatasetID) { datasetID = newDatasetID; } - - @JsonProperty("datasetID") - public String getDatasetID() { - return datasetID; - } - - @JsonProperty("fileToPublish") - public void setFileToPublish(String newFileToPublish) { fileToPublish = newFileToPublish; } - - @JsonProperty("fileToPublish") - public String getFileToPublish() { - return fileToPublish; - } - - @JsonProperty("publishMethod") - public void setPublishMethod(PublishMethod newPublishMethod) { - publishMethod = newPublishMethod; - } - - @JsonProperty("publishMethod") - public PublishMethod getPublishMethod() { - return publishMethod; - } - - @JsonProperty("fileToPublishHasHeaderRow") - public boolean getFileToPublishHasHeaderRow() { return fileToPublishHasHeaderRow; } - - @JsonProperty("fileToPublishHasHeaderRow") - public void setFileToPublishHasHeaderRow(boolean has) { fileToPublishHasHeaderRow = has; } - - @JsonProperty("pathToControlFile") - public String getPathToControlFile() { return pathToControlFile; } - - @JsonProperty("pathToControlFile") - public void setPathToControlFile(String path) { pathToControlFile = path; } - - @JsonProperty("pathToFTPControlFile") - public String getFTPPathToControlFile() { return pathToControlFile; } - - @JsonProperty("pathToFTPControlFile") - public void setPathToFTPControlFile(String path) { pathToControlFile = path; } - - @JsonProperty("controlFileContent") - public String getControlFileContent() { return controlFileContent; } - - @JsonProperty("controlFileContent") - public void setControlFileContent(String content) { controlFileContent = content; } - - @JsonProperty("ftpControlFileContent") - public String getFTPControlFileContent() { return controlFileContent; } - - @JsonProperty("ftpControlFileContent") - public void setFTPControlFileContent(String content) { controlFileContent = content; } - - @JsonProperty("publishViaFTP") - public boolean getPublishViaFTP() { return publishViaFTP; } - - @JsonProperty("publishViaFTP") - public void setPublishViaFTP(boolean newPublishViaFTP) { publishViaFTP = newPublishViaFTP; } - - @JsonProperty("publishViaDi2Http") - public boolean getPublishViaDi2Http() { return publishViaDi2Http; } - - @JsonProperty("publishViaDi2Http") - public void setPublishViaDi2Http(boolean newPublishViaDi2Http) { publishViaDi2Http = newPublishViaDi2Http; } - - public String getDefaultJobName() { return defaultJobName; } - - public void setUserAgent(String usrAgentName) { - userAgent = Utils.getUserAgentString(usrAgentName); - } - public void setUserAgentClient() { - userAgent = Utils.getUserAgentString(userAgentNameClient); - } - public void setUserAgentSijFile() { - userAgent = Utils.getUserAgentString(userAgentNameSijFile); - } - - /** - * Checks that the command line arguments are sensible - * NB: it is expected that this is run before 'configure'. - * @param cmd the commandLine object constructed from the user's options - * @return true if the commandLine is approved, false otherwise - */ - public boolean validateArgs(CommandLine cmd) { - return IntegrationJobValidity.validateArgs(cmd); - } - - /** - * Configures an integration job prior to running it; in particular, the fields we need are - * set from the cmd line and the controlFile contents are deserialized - * NB: This should be run after 'validateArgs' and before 'run' - * @param cmd the commandLine object constructed from the user's options - */ - public void configure(CommandLine cmd) { - CommandLineOptions options = new CommandLineOptions(); - String method = cmd.getOptionValue(options.PUBLISH_METHOD_FLAG); - - setDatasetID(cmd.getOptionValue(options.DATASET_ID_FLAG)); - setFileToPublish(cmd.getOptionValue(options.FILE_TO_PUBLISH_FLAG)); - if(method != null) - setPublishMethod(PublishMethod.valueOf(method)); - setFileToPublishHasHeaderRow(Boolean.parseBoolean(cmd.getOptionValue(options.HAS_HEADER_ROW_FLAG, "true"))); - setPublishViaFTP(Boolean.parseBoolean(cmd.getOptionValue(options.PUBLISH_VIA_FTP_FLAG, options.DEFAULT_PUBLISH_VIA_FTP))); - setPublishViaDi2Http(Boolean.parseBoolean(cmd.getOptionValue(options.PUBLISH_VIA_DI2_FLAG, options.DEFAULT_PUBLISH_VIA_DI2))); - String controlFilePath = cmd.getOptionValue(options.PATH_TO_CONTROL_FILE_FLAG); - if (controlFilePath == null) - controlFilePath = cmd.getOptionValue(options.PATH_TO_FTP_CONTROL_FILE_FLAG); - setPathToControlFile(controlFilePath); - - String userAgentName = cmd.getOptionValue(options.USER_AGENT_FLAG); - if(Utils.nullOrEmpty(userAgentName)) { - userAgentName = userAgentNameCli; - } - setUserAgent(userAgentName); - } - - - /** - * Runs an integration job. It is expected that 'configure' was run beforehand. - * @return - * @throws IOException - */ - public JobStatus run() { - SocrataConnectionInfo connectionInfo = userPrefs.getConnectionInfo(); - UpsertResult result = null; - String publishExceptions = ""; - JobStatus runStatus = JobStatus.SUCCESS; - - JobStatus controlDeserialization = deserializeControlFile(); - if (controlDeserialization.isError() && (publishViaDi2Http || publishViaFTP)) { - runStatus = controlDeserialization; - } else { - JobStatus validationStatus = IntegrationJobValidity.validateJobParams(connectionInfo, this); - if (validationStatus.isError()) { - runStatus = validationStatus; - } else { - Soda2Producer producer = null; - try { - File fileToPublishFile = new File(fileToPublish); - if (publishViaDi2Http) { - try (DeltaImporter2Publisher publisher = new DeltaImporter2Publisher(userPrefs, userAgent)) { - String action = controlFile.action == null ? publishMethod.name() : controlFile.action; - // "upsert" == "append" in di2 - if ("upsert".equalsIgnoreCase(action)) - action = "Append"; - controlFile.action = Utils.capitalizeFirstLetter(action); - runStatus = publisher.publishWithDi2OverHttp(datasetID, fileToPublishFile, controlFile); - } - } else if (publishViaFTP) { - runStatus = doPublishViaFTPv2(fileToPublishFile); - } else { - // attach a requestId to all Producer API calls (for error tracking purposes) - String jobRequestId = Utils.generateRequestId(); - producer = Soda2Producer.newProducerWithRequestId( - connectionInfo.getUrl(), connectionInfo.getUser(), connectionInfo.getPassword(), connectionInfo.getToken(), jobRequestId); - final SodaImporter importer = SodaImporter.newImporter(connectionInfo.getUrl(), connectionInfo.getUser(), connectionInfo.getPassword(), connectionInfo.getToken()); - int filesizeChunkingCutoffBytes = userPrefs.getFilesizeChunkingCutoffMB() == null ? 10 * NUM_BYTES_PER_MB : - Integer.parseInt(userPrefs.getFilesizeChunkingCutoffMB()) * NUM_BYTES_PER_MB; - int numRowsPerChunk = userPrefs.getNumRowsPerChunk() == null ? 10000 : - Integer.parseInt(userPrefs.getNumRowsPerChunk()); - switch (publishMethod) { - case upsert: - case append: - result = doAppendOrUpsertViaHTTP( - producer, importer, fileToPublishFile, filesizeChunkingCutoffBytes, numRowsPerChunk); - break; - case replace: - result = Soda2Publisher.replaceNew( - producer, importer, datasetID, fileToPublishFile, fileToPublishHasHeaderRow); - break; - case delete: - result = doDeleteViaHTTP( - producer, importer, fileToPublishFile, filesizeChunkingCutoffBytes, numRowsPerChunk); - break; - default: - runStatus = JobStatus.INVALID_PUBLISH_METHOD; - } - } - - } catch (IOException | SodaError | InterruptedException e) { - publishExceptions = e.getMessage(); - e.printStackTrace(); - } finally { - if (producer != null) producer.close(); - } - } - } - - if (publishExceptions.length() > 0) { - runStatus = JobStatus.PUBLISH_ERROR; - runStatus.setMessage(publishExceptions); - } else if (result != null && result.errorCount() > 0) { // Check for [row-level] SODA 2 errors - runStatus = craftSoda2PublishError(result); - } - - String logPublishingErrorMessage = logRunResults(runStatus, result); - emailAdmin(runStatus, logPublishingErrorMessage); - return runStatus; - } - - - /** - * Adds an entry to specified log dataset with given job run information - * - * @return null if log entry was added successfully, otherwise return an error message as a String - */ - public static String addLogEntry(final String logDatasetID, final SocrataConnectionInfo connectionInfo, - final IntegrationJob job, final JobStatus status, final UpsertResult result) { - final Soda2Producer producer = Soda2Producer.newProducer(connectionInfo.getUrl(), connectionInfo.getUser(), - connectionInfo.getPassword(), connectionInfo.getToken()); - - List> upsertObjects = new ArrayList<>(); - Map newCols = new HashMap<>(); - - // add standard log data - Date currentDateTime = new Date(); - newCols.put("Date", currentDateTime); - newCols.put("DatasetID", job.getDatasetID()); - newCols.put("FileToPublish", job.getFileToPublish()); - if(job.getPublishMethod() != null) - newCols.put("PublishMethod", job.getPublishMethod()); - newCols.put("JobFile", job.getPathToSavedFile()); - if(result != null) { - newCols.put("RowsUpdated", result.rowsUpdated); - newCols.put("RowsCreated", result.rowsCreated); - newCols.put("RowsDeleted", result.rowsDeleted); - } else { - newCols.put("RowsUpdated", (status.rowsUpdated == null ? 0 : status.rowsUpdated)); - newCols.put("RowsCreated", (status.rowsCreated == null ? 0 : status.rowsCreated)); - newCols.put("RowsDeleted", (status.rowsDeleted == null ? 0 : status.rowsDeleted)); - } - if(status.isError()) { - newCols.put("Errors", status.getMessage()); - } else { - newCols.put("Success", true); - } - newCols.put("DataSyncVersion", VersionProvider.getThisVersion()); - upsertObjects.add(ImmutableMap.copyOf(newCols)); - - String logPublishingErrorMessage = null; - - int retryLimit = 10; - boolean retry; - do { - retry = false; - try { - producer.upsert(logDatasetID, upsertObjects); - } - catch (SodaError | InterruptedException e) { - e.printStackTrace(); - logPublishingErrorMessage = e.getMessage(); - } - catch (ClientHandlerException e) { - if(e.getCause() instanceof SocketException) { - System.out.println("Socket exception while updating logging dataset: " + e.getCause().getMessage()); - if(retryLimit-- > 0) { - System.out.println("Retrying"); - retry = true; - } else { - e.printStackTrace(); - logPublishingErrorMessage = e.getMessage(); - } - } else { - throw e; - } - } - } while(retry); - - return logPublishingErrorMessage; - } - - private JobStatus doPublishViaFTPv2(File fileToPublishFile) { - if((pathToControlFile != null && !pathToControlFile.equals(""))) { - return FTPDropbox2Publisher.publishViaFTPDropboxV2( - userPrefs, datasetID, fileToPublishFile, new File(pathToControlFile)); - } else { - return FTPDropbox2Publisher.publishViaFTPDropboxV2( - userPrefs, datasetID, fileToPublishFile, controlFileContent); - } - } - - private void sendErrorNotificationEmail(final String adminEmail, final SocrataConnectionInfo connectionInfo, final JobStatus runStatus, final String runErrorMessage, final String logDatasetID, final String logPublishingErrorMessage) { - String errorEmailMessage = ""; - String urlToLogDataset = connectionInfo.getUrl() + "/d/" + logDatasetID; - if(runStatus.isError()) { - errorEmailMessage += "There was an error updating a dataset.\n" - + "\nDataset: " + connectionInfo.getUrl() + "/d/" + getDatasetID() - + "\nFile to publish: " + fileToPublish - + "\nFile to publish has header row: " + fileToPublishHasHeaderRow - + "\nPublish method: " + publishMethod - + "\nJob File: " + pathToSavedJobFile - + "\nError message: " + runErrorMessage - + "\nLog dataset: " + urlToLogDataset + "\n\n"; - } - if(logPublishingErrorMessage != null) { - errorEmailMessage += "There was an error updating the log dataset: " - + urlToLogDataset + "\n" - + "Error message: " + logPublishingErrorMessage + "\n\n"; - } - if(runStatus.isError() || logPublishingErrorMessage != null) { - try { - SMTPMailer.send(adminEmail, "Socrata DataSync Error", errorEmailMessage); - } catch (Exception e) { - System.out.println("Error sending email to: " + adminEmail + "\n" + e.getMessage()); - } - } - } - - private UpsertResult doAppendOrUpsertViaHTTP(Soda2Producer producer, SodaImporter importer, File fileToPublishFile, int filesizeChunkingCutoffBytes, int numRowsPerChunk) throws SodaError, InterruptedException, IOException { - int numberOfRows = numRowsPerChunk(fileToPublishFile, filesizeChunkingCutoffBytes, numRowsPerChunk); - UpsertResult result = Soda2Publisher.appendUpsert( - producer, importer, datasetID, fileToPublishFile, numberOfRows, fileToPublishHasHeaderRow); - return result; - } - - private UpsertResult doDeleteViaHTTP( - Soda2Producer producer, SodaImporter importer, File fileToPublishFile, int filesizeChunkingCutoffBytes, int numRowsPerChunk) - throws SodaError, InterruptedException, IOException { - int numberOfRows = numRowsPerChunk(fileToPublishFile, filesizeChunkingCutoffBytes, numRowsPerChunk); - UpsertResult result = Soda2Publisher.deleteRows( - producer, importer, datasetID, fileToPublishFile, numberOfRows, fileToPublishHasHeaderRow); - return result; - } - - private int numRowsPerChunk(File fileToPublishFile, int filesizeChunkingCutoffBytes, int numRowsPerChunk) { - int numberOfRows; - if(fileToPublishFile.length() > filesizeChunkingCutoffBytes) { - numberOfRows = numRowsPerChunk; - } else { - numberOfRows = UPLOAD_SINGLE_CHUNK; - } - return numberOfRows; - } - - private JobStatus craftSoda2PublishError(UpsertResult result) { - JobStatus error = JobStatus.PUBLISH_ERROR; - if(result != null && result.errorCount() > 0) { - int lineIndexOffset = (fileToPublishHasHeaderRow) ? 2 : 1; - String errMsg = ""; - for (UpsertError upsertErr : result.getErrors()) { - errMsg += upsertErr.getError() + " (line " + (upsertErr.getIndex() + lineIndexOffset) + " of file) \n"; - } - error.setMessage(errMsg); - } - return error; - } - - private String logRunResults(JobStatus runStatus, UpsertResult result) { - String logDatasetID = userPrefs.getLogDatasetID(); - String logPublishingErrorMessage = null; - SocrataConnectionInfo connectionInfo = userPrefs.getConnectionInfo(); - - if (logDatasetID != null && !logDatasetID.equals("")) { - String logDatasetUrl = userPrefs.getDomain() + "/d/" + userPrefs.getLogDatasetID(); - System.out.println("Publishing results to logging dataset (" + logDatasetUrl + ")..."); - logPublishingErrorMessage = addLogEntry( - logDatasetID, connectionInfo, this, runStatus, result); - if (logPublishingErrorMessage != null) { - System.out.println("Error publishing results to logging dataset (" + logDatasetUrl + "): " + - logPublishingErrorMessage); - } - } - return logPublishingErrorMessage; - } - - private void emailAdmin(JobStatus status, String logPublishingErrorMessage) { - String adminEmail = userPrefs.getAdminEmail(); - String logDatasetID = userPrefs.getLogDatasetID(); - SocrataConnectionInfo connectionInfo = userPrefs.getConnectionInfo(); - - if(userPrefs.emailUponError() && adminEmail != null && !adminEmail.equals("")) { - sendErrorNotificationEmail( - adminEmail, connectionInfo, status, status.getMessage(), logDatasetID, logPublishingErrorMessage); - } - } - - private JobStatus deserializeControlFile() { - if (controlFile != null) - return JobStatus.VALID; - - JobStatus controlDeserialization = null; - if (controlFileContent != null && !controlFileContent.equals("")) - controlDeserialization = deserializeControlFile(controlFileContent); - - if (pathToControlFile != null && !pathToControlFile.equals("")) - controlDeserialization = deserializeControlFile(new File(pathToControlFile)); - - if (controlDeserialization == null) { - JobStatus noControl = JobStatus.PUBLISH_ERROR; - noControl.setMessage("You must generate or select a Control file if publishing via FTP SmartUpdate or delta-importer-2 over HTTP"); - return noControl; - } else if (controlDeserialization.isError()) { - return controlDeserialization; - } - return JobStatus.VALID; - } - - - private JobStatus deserializeControlFile(String contents) { - try { - controlFile = controlFileMapper.readValue(contents, ControlFile.class); - return JobStatus.SUCCESS; - } catch (Exception e) { - JobStatus status = JobStatus.PUBLISH_ERROR; - status.setMessage("Unable to interpret control file contents: " + e); - return status; - } - } - - private JobStatus deserializeControlFile(File controlFilePath) { - try { - controlFile = controlFileMapper.readValue(controlFilePath, ControlFile.class); - return JobStatus.SUCCESS; - } catch (Exception e) { - JobStatus status = JobStatus.PUBLISH_ERROR; - status.setMessage("Unable to read in and interpret control file contents: " + e); - return status; - } - } - - public class ControlDisagreementException extends Exception { - public ControlDisagreementException(String msg) { - super(msg); - } - } - - /** - * Sets up the control file from the two possible sources in an sij file. - * @param controlFilePath the path to the control file - * @param controlFileContent the content of the control file - * @throw ControlDisagreementException the content of the two sources disagrees - */ - private void setControlFile(String controlFilePath, String controlFileContent) throws IOException, ControlDisagreementException { - ControlFile controlFileFromFile = null; - ControlFile controlFileFromContents = null; - - if (!Utils.nullOrEmpty(controlFileContent)) { - controlFileFromContents = controlFileMapper.readValue(controlFileContent, ControlFile.class); - controlFile = controlFileFromContents; - } - if (!Utils.nullOrEmpty(controlFilePath)) { - controlFileFromFile = controlFileMapper.readValue(new File(controlFilePath), ControlFile.class); - controlFile = controlFileFromFile; - } - - if (controlFileFromFile != null && controlFileFromContents != null) { - String controlTextFromFile = controlFileMapper.writeValueAsString(controlFileFromFile); - String controlTextFromContents = controlFileMapper.writeValueAsString(controlFileFromContents); - if(!controlTextFromFile.equals(controlTextFromContents)) { - throw new ControlDisagreementException("The contents of control file \n'" + controlFilePath + - "' differ from the contents in the .sij file"); - } - } - - } - - /** - * This allows backward compatability with DataSync 0.1 .sij file format - * - * @param pathToFile .sij file that uses old serialization format (Java native) - * @throws IOException - */ - private void loadOldSijFile(String pathToFile) throws IOException { - try { - InputStream file = new FileInputStream(pathToFile); - InputStream buffer = new BufferedInputStream(file); - ObjectInput input = new ObjectInputStream (buffer); - try{ - com.socrata.datasync.IntegrationJob loadedJobOld = (com.socrata.datasync.IntegrationJob) input.readObject(); - setDatasetID(loadedJobOld.getDatasetID()); - setFileToPublish(loadedJobOld.getFileToPublish()); - setPublishMethod(loadedJobOld.getPublishMethod()); - setPathToSavedFile(pathToFile); - setFileToPublishHasHeaderRow(true); - setPublishViaFTP(false); - setPathToControlFile(null); - setControlFileContent(null); - } - finally{ - input.close(); - } - } catch(Exception e) { - // TODO add log entry? - throw new IOException(e.toString()); - } - } - -} +package com.socrata.datasync.job; + +import com.google.common.collect.ImmutableMap; +import com.socrata.api.Soda2Producer; +import com.socrata.api.SodaImporter; +import com.socrata.datasync.PublishMethod; +import com.socrata.datasync.SMTPMailer; +import com.socrata.datasync.SocrataConnectionInfo; +import com.socrata.datasync.Utils; +import com.socrata.datasync.VersionProvider; +import com.socrata.datasync.config.CommandLineOptions; +import com.socrata.datasync.config.controlfile.ControlFile; +import com.socrata.datasync.config.userpreferences.UserPreferences; +import com.socrata.datasync.config.userpreferences.UserPreferencesJava; +import com.socrata.datasync.publishers.DeltaImporter2Publisher; +import com.socrata.datasync.publishers.FTPDropbox2Publisher; +import com.socrata.datasync.publishers.Soda2Publisher; +import com.socrata.datasync.validation.IntegrationJobValidity; +import com.socrata.exceptions.SodaError; +import com.socrata.model.UpsertError; +import com.socrata.model.UpsertResult; +import org.apache.commons.cli.CommandLine; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.DeserializationConfig; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.joda.time.LocalDateTime; +import org.joda.time.format.ISODateTimeFormat; +import javax.ws.rs.ProcessingException; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInput; +import java.io.ObjectInputStream; +import java.net.SocketException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +@JsonIgnoreProperties(ignoreUnknown=true) +@JsonSerialize(include= JsonSerialize.Inclusion.NON_NULL) +public class IntegrationJob extends Job { + + static AtomicInteger jobCounter = new AtomicInteger(0); + int jobNum = jobCounter.getAndIncrement(); + private String defaultJobName = "Unsaved Standard Job" + " (" + jobNum + ")"; + + // to upload entire file as a single chunk (numRowsPerChunk == 0) + private static final int UPLOAD_SINGLE_CHUNK = 0; + public static final int NUM_BYTES_PER_MB = 1048576; + + // Anytime a @JsonProperty is added/removed/updated in this class add 1 to this value + private static final long fileVersionUID = 4L; + + private UserPreferences userPrefs; + private String datasetID = ""; + private String fileToPublish = ""; + private PublishMethod publishMethod = null; + private boolean fileToPublishHasHeaderRow = true; + private String pathToControlFile = null; + private String controlFileContent = null; + private boolean publishViaFTP = false; + private boolean publishViaDi2Http = false; + private ControlFile controlFile = null; + + private String userAgent = "datasync"; + + private String userAgentNameClient = "Client"; + private String userAgentNameCli = "CLI"; + private String userAgentNameSijFile = ".sij File"; + + private ObjectMapper controlFileMapper = + new ObjectMapper().enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY); + + public IntegrationJob() { + userPrefs = new UserPreferencesJava(); + } + + /* + * This is a method that enables DataSync preferences to be established + * directly when DataSync is used in "library mode" or "command-line mode" + * (rather than being loaded from Preferences class) + */ + public IntegrationJob(UserPreferences userPrefs) { + this.userPrefs = userPrefs; + } + + /** + * Loads integration job data from a file and + * uses the saved data to populate the fields + * of this object + */ + public IntegrationJob(String pathToFile) throws IOException, ControlDisagreementException { + this(pathToFile, false); + setUserAgentSijFile(); + } + + /** + * Loads integration job data from a file and + * uses the saved data to populate the fields + * of this object + */ + public IntegrationJob(String pathToFile, boolean ignoreControlInconsistencies) throws IOException, ControlDisagreementException { + userPrefs = new UserPreferencesJava(); + + // first try reading the 'current' format + ObjectMapper mapper = new ObjectMapper(); + IntegrationJob loadedJob; + try { + loadedJob = mapper.readValue(new File(pathToFile), IntegrationJob.class); + } catch (IOException e) { + // if reading new format fails...try reading old format into this object + loadOldSijFile(pathToFile); + return; + } + loadedJob.setPathToSavedFile(pathToFile); + String controlPath = loadedJob.getPathToControlFile(); + String controlContent = loadedJob.getControlFileContent(); + try { + setControlFile(controlPath, controlContent); + } catch (ControlDisagreementException e) { + if (!ignoreControlInconsistencies) + throw e; + } + setDatasetID(loadedJob.getDatasetID()); + setFileToPublish(loadedJob.getFileToPublish()); + setPublishMethod(loadedJob.getPublishMethod()); + setFileToPublishHasHeaderRow(loadedJob.getFileToPublishHasHeaderRow()); + setPathToSavedFile(pathToFile); + setPathToControlFile(loadedJob.getPathToControlFile()); + setControlFileContent(loadedJob.getControlFileContent()); + setPublishViaFTP(loadedJob.getPublishViaFTP()); + setPublishViaDi2Http(loadedJob.getPublishViaDi2Http()); + } + + + @JsonProperty("fileVersionUID") + public long getFileVersionUID() { return fileVersionUID; } + + public ControlFile getControlFile() { return controlFile; } + + public void setControlFile(ControlFile cf) { controlFile = cf; } + + @JsonProperty("datasetID") + public void setDatasetID(String newDatasetID) { datasetID = newDatasetID; } + + @JsonProperty("datasetID") + public String getDatasetID() { + return datasetID; + } + + @JsonProperty("fileToPublish") + public void setFileToPublish(String newFileToPublish) { fileToPublish = newFileToPublish; } + + @JsonProperty("fileToPublish") + public String getFileToPublish() { + return fileToPublish; + } + + @JsonProperty("publishMethod") + public void setPublishMethod(PublishMethod newPublishMethod) { + publishMethod = newPublishMethod; + } + + @JsonProperty("publishMethod") + public PublishMethod getPublishMethod() { + return publishMethod; + } + + @JsonProperty("fileToPublishHasHeaderRow") + public boolean getFileToPublishHasHeaderRow() { return fileToPublishHasHeaderRow; } + + @JsonProperty("fileToPublishHasHeaderRow") + public void setFileToPublishHasHeaderRow(boolean has) { fileToPublishHasHeaderRow = has; } + + @JsonProperty("pathToControlFile") + public String getPathToControlFile() { return pathToControlFile; } + + @JsonProperty("pathToControlFile") + public void setPathToControlFile(String path) { pathToControlFile = path; } + + @JsonProperty("pathToFTPControlFile") + public String getFTPPathToControlFile() { return pathToControlFile; } + + @JsonProperty("pathToFTPControlFile") + public void setPathToFTPControlFile(String path) { pathToControlFile = path; } + + @JsonProperty("controlFileContent") + public String getControlFileContent() { return controlFileContent; } + + @JsonProperty("controlFileContent") + public void setControlFileContent(String content) { controlFileContent = content; } + + @JsonProperty("ftpControlFileContent") + public String getFTPControlFileContent() { return controlFileContent; } + + @JsonProperty("ftpControlFileContent") + public void setFTPControlFileContent(String content) { controlFileContent = content; } + + @JsonProperty("publishViaFTP") + public boolean getPublishViaFTP() { return publishViaFTP; } + + @JsonProperty("publishViaFTP") + public void setPublishViaFTP(boolean newPublishViaFTP) { publishViaFTP = newPublishViaFTP; } + + @JsonProperty("publishViaDi2Http") + public boolean getPublishViaDi2Http() { return publishViaDi2Http; } + + @JsonProperty("publishViaDi2Http") + public void setPublishViaDi2Http(boolean newPublishViaDi2Http) { publishViaDi2Http = newPublishViaDi2Http; } + + public String getDefaultJobName() { return defaultJobName; } + + public void setUserAgent(String usrAgentName) { + userAgent = Utils.getUserAgentString(usrAgentName); + } + public void setUserAgentClient() { + userAgent = Utils.getUserAgentString(userAgentNameClient); + } + public void setUserAgentSijFile() { + userAgent = Utils.getUserAgentString(userAgentNameSijFile); + } + + /** + * Checks that the command line arguments are sensible + * NB: it is expected that this is run before 'configure'. + * @param cmd the commandLine object constructed from the user's options + * @return true if the commandLine is approved, false otherwise + */ + public boolean validateArgs(CommandLine cmd) { + return IntegrationJobValidity.validateArgs(cmd); + } + + /** + * Configures an integration job prior to running it; in particular, the fields we need are + * set from the cmd line and the controlFile contents are deserialized + * NB: This should be run after 'validateArgs' and before 'run' + * @param cmd the commandLine object constructed from the user's options + */ + public void configure(CommandLine cmd) { + CommandLineOptions options = new CommandLineOptions(); + String method = cmd.getOptionValue(options.PUBLISH_METHOD_FLAG); + + setDatasetID(cmd.getOptionValue(options.DATASET_ID_FLAG)); + setFileToPublish(cmd.getOptionValue(options.FILE_TO_PUBLISH_FLAG)); + if(method != null) + setPublishMethod(PublishMethod.valueOf(method)); + setFileToPublishHasHeaderRow(Boolean.parseBoolean(cmd.getOptionValue(options.HAS_HEADER_ROW_FLAG, "true"))); + setPublishViaFTP(Boolean.parseBoolean(cmd.getOptionValue(options.PUBLISH_VIA_FTP_FLAG, options.DEFAULT_PUBLISH_VIA_FTP))); + setPublishViaDi2Http(Boolean.parseBoolean(cmd.getOptionValue(options.PUBLISH_VIA_DI2_FLAG, options.DEFAULT_PUBLISH_VIA_DI2))); + String controlFilePath = cmd.getOptionValue(options.PATH_TO_CONTROL_FILE_FLAG); + if (controlFilePath == null) + controlFilePath = cmd.getOptionValue(options.PATH_TO_FTP_CONTROL_FILE_FLAG); + setPathToControlFile(controlFilePath); + + String userAgentName = cmd.getOptionValue(options.USER_AGENT_FLAG); + if(Utils.nullOrEmpty(userAgentName)) { + userAgentName = userAgentNameCli; + } + setUserAgent(userAgentName); + } + + + /** + * Runs an integration job. It is expected that 'configure' was run beforehand. + * @return + * @throws IOException + */ + public JobStatus run() { + SocrataConnectionInfo connectionInfo = userPrefs.getConnectionInfo(); + UpsertResult result = null; + String publishExceptions = ""; + JobStatus runStatus = JobStatus.SUCCESS; + + JobStatus controlDeserialization = deserializeControlFile(); + if (controlDeserialization.isError() && (publishViaDi2Http || publishViaFTP)) { + runStatus = controlDeserialization; + } else { + JobStatus validationStatus = IntegrationJobValidity.validateJobParams(userPrefs, this); + if (validationStatus.isError()) { + runStatus = validationStatus; + } else { + Soda2Producer producer = null; + try { + File fileToPublishFile = new File(fileToPublish); + if (publishViaDi2Http) { + try (DeltaImporter2Publisher publisher = new DeltaImporter2Publisher(userPrefs, userAgent)) { + String action = controlFile.action == null ? publishMethod.name() : controlFile.action; + // "upsert" == "append" in di2 + if ("upsert".equalsIgnoreCase(action)) + action = "Append"; + controlFile.action = Utils.capitalizeFirstLetter(action); + runStatus = publisher.publishWithDi2OverHttp(datasetID, fileToPublishFile, controlFile); + } + } else if (publishViaFTP) { + runStatus = doPublishViaFTPv2(fileToPublishFile); + } else { + // attach a requestId to all Producer API calls (for error tracking purposes) + String jobRequestId = Utils.generateRequestId(); + producer = Soda2Producer.newProducerWithRequestId( + connectionInfo.getUrl(), connectionInfo.getUser(), connectionInfo.getPassword(), connectionInfo.getToken(), jobRequestId); + final SodaImporter importer = SodaImporter.newImporter(connectionInfo.getUrl(), connectionInfo.getUser(), connectionInfo.getPassword(), connectionInfo.getToken()); + int filesizeChunkingCutoffBytes = userPrefs.getFilesizeChunkingCutoffMB() == null ? 10 * NUM_BYTES_PER_MB : + Integer.parseInt(userPrefs.getFilesizeChunkingCutoffMB()) * NUM_BYTES_PER_MB; + int numRowsPerChunk = userPrefs.getNumRowsPerChunk() == null ? 10000 : + Integer.parseInt(userPrefs.getNumRowsPerChunk()); + switch (publishMethod) { + case upsert: + case append: + result = doAppendOrUpsertViaHTTP( + producer, importer, fileToPublishFile, filesizeChunkingCutoffBytes, numRowsPerChunk); + break; + case replace: + result = Soda2Publisher.replaceNew( + producer, importer, datasetID, fileToPublishFile, fileToPublishHasHeaderRow); + break; + case delete: + result = doDeleteViaHTTP( + producer, importer, fileToPublishFile, filesizeChunkingCutoffBytes, numRowsPerChunk); + break; + default: + runStatus = JobStatus.INVALID_PUBLISH_METHOD; + } + } + + } catch (IOException | SodaError | InterruptedException e) { + publishExceptions = e.getMessage(); + e.printStackTrace(); + } finally { + if (producer != null) producer.close(); + } + } + } + + if (publishExceptions.length() > 0) { + runStatus = JobStatus.PUBLISH_ERROR; + runStatus.setMessage(publishExceptions); + } else if (result != null && result.errorCount() > 0) { // Check for [row-level] SODA 2 errors + runStatus = craftSoda2PublishError(result); + } + + String logPublishingErrorMessage = logRunResults(runStatus, result); + emailAdmin(runStatus, logPublishingErrorMessage); + return runStatus; + } + + + /** + * Adds an entry to specified log dataset with given job run information + * + * @return null if log entry was added successfully, otherwise return an error message as a String + */ + public static String addLogEntry(final String logDatasetID, final SocrataConnectionInfo connectionInfo, + final IntegrationJob job, final JobStatus status, final UpsertResult result) { + final Soda2Producer producer = Soda2Producer.newProducer(connectionInfo.getUrl(), connectionInfo.getUser(), + connectionInfo.getPassword(), connectionInfo.getToken()); + + List> upsertObjects = new ArrayList<>(); + Map newCols = new HashMap<>(); + + // add standard log data + LocalDateTime currentDateTime = new LocalDateTime(); + newCols.put("Date", ISODateTimeFormat.dateTime().print(currentDateTime)); + newCols.put("DatasetID", job.getDatasetID()); + newCols.put("FileToPublish", job.getFileToPublish()); + if(job.getPublishMethod() != null) + newCols.put("PublishMethod", job.getPublishMethod()); + newCols.put("JobFile", job.getPathToSavedFile()); + if(result != null) { + newCols.put("RowsUpdated", result.rowsUpdated); + newCols.put("RowsCreated", result.rowsCreated); + newCols.put("RowsDeleted", result.rowsDeleted); + } else { + newCols.put("RowsUpdated", (status.rowsUpdated == null ? 0 : status.rowsUpdated)); + newCols.put("RowsCreated", (status.rowsCreated == null ? 0 : status.rowsCreated)); + newCols.put("RowsDeleted", (status.rowsDeleted == null ? 0 : status.rowsDeleted)); + } + if(status.isError()) { + newCols.put("Errors", status.getMessage()); + } else { + newCols.put("Success", true); + } + newCols.put("DataSyncVersion", VersionProvider.getThisVersion()); + upsertObjects.add(ImmutableMap.copyOf(newCols)); + + String logPublishingErrorMessage = null; + + int retryLimit = 10; + boolean retry; + do { + retry = false; + try { + producer.upsert(logDatasetID, upsertObjects); + } + catch (SodaError | InterruptedException e) { + e.printStackTrace(); + logPublishingErrorMessage = e.getMessage(); + } + catch (ProcessingException e) { + if(e.getCause() instanceof SocketException) { + System.out.println("Socket exception while updating logging dataset: " + e.getCause().getMessage()); + if(retryLimit-- > 0) { + System.out.println("Retrying"); + retry = true; + } else { + e.printStackTrace(); + logPublishingErrorMessage = e.getMessage(); + } + } else { + throw e; + } + } + } while(retry); + + return logPublishingErrorMessage; + } + + private JobStatus doPublishViaFTPv2(File fileToPublishFile) { + if((pathToControlFile != null && !pathToControlFile.equals(""))) { + return FTPDropbox2Publisher.publishViaFTPDropboxV2( + userPrefs, datasetID, fileToPublishFile, new File(pathToControlFile)); + } else { + return FTPDropbox2Publisher.publishViaFTPDropboxV2( + userPrefs, datasetID, fileToPublishFile, controlFileContent); + } + } + + private void sendErrorNotificationEmail(final String adminEmail, final SocrataConnectionInfo connectionInfo, final JobStatus runStatus, final String runErrorMessage, final String logDatasetID, final String logPublishingErrorMessage) { + String errorEmailMessage = ""; + String urlToLogDataset = connectionInfo.getUrl() + "/d/" + logDatasetID; + if(runStatus.isError()) { + errorEmailMessage += "There was an error updating a dataset.\n" + + "\nDataset: " + connectionInfo.getUrl() + "/d/" + getDatasetID() + + "\nFile to publish: " + fileToPublish + + "\nFile to publish has header row: " + fileToPublishHasHeaderRow + + "\nPublish method: " + publishMethod + + "\nJob File: " + pathToSavedJobFile + + "\nError message: " + runErrorMessage + + "\nLog dataset: " + urlToLogDataset + "\n\n"; + } + if(logPublishingErrorMessage != null) { + errorEmailMessage += "There was an error updating the log dataset: " + + urlToLogDataset + "\n" + + "Error message: " + logPublishingErrorMessage + "\n\n"; + } + if(runStatus.isError() || logPublishingErrorMessage != null) { + try { + SMTPMailer.send(adminEmail, "Socrata DataSync Error", errorEmailMessage); + } catch (Exception e) { + System.out.println("Error sending email to: " + adminEmail + "\n" + e.getMessage()); + } + } + } + + private UpsertResult doAppendOrUpsertViaHTTP(Soda2Producer producer, SodaImporter importer, File fileToPublishFile, int filesizeChunkingCutoffBytes, int numRowsPerChunk) throws SodaError, InterruptedException, IOException { + int numberOfRows = numRowsPerChunk(fileToPublishFile, filesizeChunkingCutoffBytes, numRowsPerChunk); + UpsertResult result = Soda2Publisher.appendUpsert( + producer, importer, datasetID, fileToPublishFile, numberOfRows, fileToPublishHasHeaderRow); + return result; + } + + private UpsertResult doDeleteViaHTTP( + Soda2Producer producer, SodaImporter importer, File fileToPublishFile, int filesizeChunkingCutoffBytes, int numRowsPerChunk) + throws SodaError, InterruptedException, IOException { + int numberOfRows = numRowsPerChunk(fileToPublishFile, filesizeChunkingCutoffBytes, numRowsPerChunk); + UpsertResult result = Soda2Publisher.deleteRows( + producer, importer, datasetID, fileToPublishFile, numberOfRows, fileToPublishHasHeaderRow); + return result; + } + + private int numRowsPerChunk(File fileToPublishFile, int filesizeChunkingCutoffBytes, int numRowsPerChunk) { + int numberOfRows; + if(fileToPublishFile.length() > filesizeChunkingCutoffBytes) { + numberOfRows = numRowsPerChunk; + } else { + numberOfRows = UPLOAD_SINGLE_CHUNK; + } + return numberOfRows; + } + + private JobStatus craftSoda2PublishError(UpsertResult result) { + JobStatus error = JobStatus.PUBLISH_ERROR; + if(result != null && result.errorCount() > 0) { + int lineIndexOffset = (fileToPublishHasHeaderRow) ? 2 : 1; + String errMsg = ""; + for (UpsertError upsertErr : result.getErrors()) { + errMsg += upsertErr.getError() + " (line " + (upsertErr.getIndex() + lineIndexOffset) + " of file) \n"; + } + error.setMessage(errMsg); + } + return error; + } + + private String logRunResults(JobStatus runStatus, UpsertResult result) { + String logDatasetID = userPrefs.getLogDatasetID(); + String logPublishingErrorMessage = null; + SocrataConnectionInfo connectionInfo = userPrefs.getConnectionInfo(); + + if (logDatasetID != null && !logDatasetID.equals("")) { + String logDatasetUrl = userPrefs.getDomain() + "/d/" + userPrefs.getLogDatasetID(); + System.out.println("Publishing results to logging dataset (" + logDatasetUrl + ")..."); + logPublishingErrorMessage = addLogEntry( + logDatasetID, connectionInfo, this, runStatus, result); + if (logPublishingErrorMessage != null) { + System.out.println("Error publishing results to logging dataset (" + logDatasetUrl + "): " + + logPublishingErrorMessage); + } + } + return logPublishingErrorMessage; + } + + private void emailAdmin(JobStatus status, String logPublishingErrorMessage) { + String adminEmail = userPrefs.getAdminEmail(); + String logDatasetID = userPrefs.getLogDatasetID(); + SocrataConnectionInfo connectionInfo = userPrefs.getConnectionInfo(); + + if(userPrefs.emailUponError() && adminEmail != null && !adminEmail.equals("")) { + sendErrorNotificationEmail( + adminEmail, connectionInfo, status, status.getMessage(), logDatasetID, logPublishingErrorMessage); + } + } + + private JobStatus deserializeControlFile() { + if (controlFile != null) + return JobStatus.VALID; + + JobStatus controlDeserialization = null; + if (controlFileContent != null && !controlFileContent.equals("")) + controlDeserialization = deserializeControlFile(controlFileContent); + + if (pathToControlFile != null && !pathToControlFile.equals("")) + controlDeserialization = deserializeControlFile(new File(pathToControlFile)); + + if (controlDeserialization == null) { + JobStatus noControl = JobStatus.PUBLISH_ERROR; + noControl.setMessage("You must generate or select a Control file if publishing via FTP SmartUpdate or delta-importer-2 over HTTP"); + return noControl; + } else if (controlDeserialization.isError()) { + return controlDeserialization; + } + return JobStatus.VALID; + } + + + private JobStatus deserializeControlFile(String contents) { + try { + controlFile = controlFileMapper.readValue(contents, ControlFile.class); + return JobStatus.SUCCESS; + } catch (Exception e) { + JobStatus status = JobStatus.PUBLISH_ERROR; + status.setMessage("Unable to interpret control file contents: " + e); + return status; + } + } + + private JobStatus deserializeControlFile(File controlFilePath) { + try { + controlFile = controlFileMapper.readValue(controlFilePath, ControlFile.class); + return JobStatus.SUCCESS; + } catch (Exception e) { + JobStatus status = JobStatus.PUBLISH_ERROR; + status.setMessage("Unable to read in and interpret control file contents: " + e); + return status; + } + } + + public class ControlDisagreementException extends Exception { + public ControlDisagreementException(String msg) { + super(msg); + } + } + + /** + * Sets up the control file from the two possible sources in an sij file. + * @param controlFilePath the path to the control file + * @param controlFileContent the content of the control file + * @throw ControlDisagreementException the content of the two sources disagrees + */ + private void setControlFile(String controlFilePath, String controlFileContent) throws IOException, ControlDisagreementException { + ControlFile controlFileFromFile = null; + ControlFile controlFileFromContents = null; + + if (!Utils.nullOrEmpty(controlFileContent)) { + controlFileFromContents = controlFileMapper.readValue(controlFileContent, ControlFile.class); + controlFile = controlFileFromContents; + } + if (!Utils.nullOrEmpty(controlFilePath)) { + controlFileFromFile = controlFileMapper.readValue(new File(controlFilePath), ControlFile.class); + controlFile = controlFileFromFile; + } + + if (controlFileFromFile != null && controlFileFromContents != null) { + String controlTextFromFile = controlFileMapper.writeValueAsString(controlFileFromFile); + String controlTextFromContents = controlFileMapper.writeValueAsString(controlFileFromContents); + if(!controlTextFromFile.equals(controlTextFromContents)) { + throw new ControlDisagreementException("The contents of control file \n'" + controlFilePath + + "' differ from the contents in the .sij file"); + } + } + + } + + /** + * This allows backward compatability with DataSync 0.1 .sij file format + * + * @param pathToFile .sij file that uses old serialization format (Java native) + * @throws IOException + */ + private void loadOldSijFile(String pathToFile) throws IOException { + try { + InputStream file = new FileInputStream(pathToFile); + InputStream buffer = new BufferedInputStream(file); + ObjectInput input = new ObjectInputStream (buffer); + try{ + com.socrata.datasync.IntegrationJob loadedJobOld = (com.socrata.datasync.IntegrationJob) input.readObject(); + setDatasetID(loadedJobOld.getDatasetID()); + setFileToPublish(loadedJobOld.getFileToPublish()); + setPublishMethod(loadedJobOld.getPublishMethod()); + setPathToSavedFile(pathToFile); + setFileToPublishHasHeaderRow(true); + setPublishViaFTP(false); + setPathToControlFile(null); + setControlFileContent(null); + } + finally{ + input.close(); + } + } catch(Exception e) { + // TODO add log entry? + throw new IOException(e.toString()); + } + } + +} diff --git a/src/main/java/com/socrata/datasync/job/Job.java b/src/main/java/com/socrata/datasync/job/Job.java index 982b78df..376c2891 100644 --- a/src/main/java/com/socrata/datasync/job/Job.java +++ b/src/main/java/com/socrata/datasync/job/Job.java @@ -3,7 +3,7 @@ import com.socrata.datasync.Utils; import org.apache.commons.cli.CommandLine; import org.apache.commons.lang3.StringUtils; -import org.codehaus.jackson.map.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; import java.io.IOException; diff --git a/src/main/java/com/socrata/datasync/job/LoadPreferencesJob.java b/src/main/java/com/socrata/datasync/job/LoadPreferencesJob.java index b95aa1f1..99247488 100644 --- a/src/main/java/com/socrata/datasync/job/LoadPreferencesJob.java +++ b/src/main/java/com/socrata/datasync/job/LoadPreferencesJob.java @@ -5,7 +5,7 @@ import com.socrata.datasync.config.userpreferences.UserPreferencesFile; import com.socrata.datasync.config.userpreferences.UserPreferencesJava; import org.apache.commons.cli.CommandLine; -import org.codehaus.jackson.map.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; import java.io.IOException; @@ -58,7 +58,6 @@ public JobStatus run() { newUserPrefs.saveDomain(userPrefs.getDomain()); newUserPrefs.saveUsername(userPrefs.getUsername()); newUserPrefs.savePassword(userPrefs.getPassword()); - newUserPrefs.saveAPIKey(userPrefs.getAPIKey()); newUserPrefs.saveAdminEmail(userPrefs.getAdminEmail()); newUserPrefs.saveEmailUponError(userPrefs.emailUponError()); newUserPrefs.saveLogDatasetID(userPrefs.getLogDatasetID()); @@ -69,6 +68,7 @@ public JobStatus run() { newUserPrefs.saveSMTPPassword(userPrefs.getSmtpPassword()); newUserPrefs.saveProxyHost(userPrefs.getProxyHost()); newUserPrefs.saveProxyPort(userPrefs.getProxyPort()); + newUserPrefs.saveDefaultTimeFormats(userPrefs.getDefaultTimeFormats()); if (userPrefs.getFilesizeChunkingCutoffMB() != null) newUserPrefs.saveFilesizeChunkingCutoffMB(Integer.parseInt(userPrefs.getFilesizeChunkingCutoffMB())); if (userPrefs.getNumRowsPerChunk() != null) diff --git a/src/main/java/com/socrata/datasync/job/MetadataJob.java b/src/main/java/com/socrata/datasync/job/MetadataJob.java index 748a9ced..707d72b7 100644 --- a/src/main/java/com/socrata/datasync/job/MetadataJob.java +++ b/src/main/java/com/socrata/datasync/job/MetadataJob.java @@ -24,10 +24,10 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.annotate.JsonSerialize; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; @JsonIgnoreProperties(ignoreUnknown=true) @JsonSerialize(include= JsonSerialize.Inclusion.NON_NULL) diff --git a/src/main/java/com/socrata/datasync/job/PortJob.java b/src/main/java/com/socrata/datasync/job/PortJob.java index 466f669f..2bf7d6ab 100644 --- a/src/main/java/com/socrata/datasync/job/PortJob.java +++ b/src/main/java/com/socrata/datasync/job/PortJob.java @@ -17,10 +17,10 @@ import com.socrata.datasync.validation.PortJobValidity; import org.apache.commons.cli.CommandLine; import org.apache.http.HttpException; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.annotate.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.io.File; import java.io.IOException; @@ -41,7 +41,6 @@ public class PortJob extends Job { private PortMethod portMethod = PortMethod.copy_all; private String sourceSiteDomain ="https://"; private String sourceSetID = ""; - private String sinkSiteDomain= "https://"; private String sinkSetID = ""; private PublishMethod publishMethod = PublishMethod.upsert; private PublishDataset publishDataset = PublishDataset.working_copy; @@ -102,16 +101,6 @@ public void setSourceSetID(String sourceSetID) { this.sourceSetID = sourceSetID; } - @JsonProperty("sinkSiteDomain") - public String getSinkSiteDomain() { - return sinkSiteDomain; - } - - @JsonProperty("sinkSiteDomain") - public void setSinkSiteDomain(String sinkSiteDomain) { - this.sinkSiteDomain = sinkSiteDomain; - } - @JsonProperty("publishMethod") public PublishMethod getPublishMethod() { return publishMethod; @@ -162,6 +151,10 @@ public void setPortResult(String portResult) { this.portResult = portResult; } + public String getSinkSiteDomain() { + return userPrefs.getDomain(); + } + public String getDefaultJobName() { return defaultJobName; } public boolean validateArgs(CommandLine cmd) { @@ -184,7 +177,6 @@ public PortJob(String pathToFile) throws IOException { setPathToSavedFile(loadedJob.getPathToSavedFile()); setSourceSiteDomain(loadedJob.getSourceSiteDomain()); setSourceSetID(loadedJob.getSourceSetID()); - setSinkSiteDomain(loadedJob.getSinkSiteDomain()); setSinkSetID(loadedJob.getSinkSetID()); setPortMethod(loadedJob.getPortMethod()); setPublishMethod(loadedJob.getPublishMethod()); @@ -200,7 +192,6 @@ public void configure(CommandLine cmd) { setPortMethod(PortMethod.valueOf(cmd.getOptionValue("pm"))); setSourceSiteDomain(cmd.getOptionValue("pd1")); setSourceSetID(cmd.getOptionValue("pi1")); - setSinkSiteDomain(cmd.getOptionValue("pd2")); if (cmd.getOptionValue("pi2") != null) setSinkSetID(cmd.getOptionValue("pi2")); @@ -229,46 +220,45 @@ public JobStatus run() { } else { boolean useOldCodePath; try { - useOldCodePath = !Utils.regionOfDomain(userPrefs, sourceSiteDomain).equals(Utils.regionOfDomain(userPrefs, sinkSiteDomain)); + useOldCodePath = !Utils.regionOfDomain(userPrefs, sourceSiteDomain).equals(Utils.regionOfDomain(userPrefs, userPrefs.getDomain())); } catch(URISyntaxException | IOException e) { runStatus = JobStatus.PORT_ERROR; runStatus.setMessage(e.getMessage()); return runStatus; } - if(useOldCodePath) { - // loader "loads" the source dataset metadata and schema - final SodaDdl loader = SodaDdl.newDdl(sourceSiteDomain, - connectionInfo.getUser(), connectionInfo.getPassword(), - connectionInfo.getToken()); - - // special feature to enable porting datasets to Staging (where app token is different) - String portDestinationDomainAppToken = connectionInfo.getToken(); - if(userPrefs.getPortDestinationDomainAppToken() != null && !userPrefs.getPortDestinationDomainAppToken().equals("")) { - portDestinationDomainAppToken = userPrefs.getPortDestinationDomainAppToken(); - } - // creator "creates" a new dataset on the sink site (and publishes if applicable) - final SodaDdl creator = SodaDdl.newDdl(sinkSiteDomain, - connectionInfo.getUser(), connectionInfo.getPassword(), - portDestinationDomainAppToken); + // loader "loads" the source dataset metadata and schema + final SodaDdl loader = SodaDdl.newDdl(sourceSiteDomain, + connectionInfo.getUser(), connectionInfo.getPassword(), + connectionInfo.getToken()); + + // creator "creates" a new dataset on the sink site (and publishes if applicable) + final SodaDdl creator = SodaDdl.newDdl(userPrefs.getDomain(), + connectionInfo.getUser(), connectionInfo.getPassword(), + connectionInfo.getToken()); + + + if(useOldCodePath) { // streamExporter "exports" the source dataset rows final Soda2Consumer streamExporter = Soda2Consumer.newConsumer( sourceSiteDomain, connectionInfo.getUser(), connectionInfo.getPassword(), connectionInfo.getToken()); // streamUpserter "upserts" the rows exported to the created dataset final Soda2Producer streamUpserter = Soda2Producer.newProducer( - sinkSiteDomain, connectionInfo.getUser(), - connectionInfo.getPassword(), portDestinationDomainAppToken); + userPrefs.getDomain(), connectionInfo.getUser(), + connectionInfo.getPassword(), connectionInfo.getToken()); String errorMessage = ""; boolean noPortExceptions = false; try { if (portMethod.equals(PortMethod.copy_schema)) { sinkSetID = PortUtility.portSchema(loader, creator, - sourceSetID, destinationDatasetTitle, userPrefs.getUseNewBackend()); + sourceSetID, destinationDatasetTitle, + true); noPortExceptions = true; } else if (portMethod.equals(PortMethod.copy_all)) { sinkSetID = PortUtility.portSchema(loader, creator, - sourceSetID, destinationDatasetTitle, userPrefs.getUseNewBackend()); + sourceSetID, destinationDatasetTitle, + true); PortUtility.portContents(streamExporter, streamUpserter, sourceSetID, sinkSetID, PublishMethod.upsert); noPortExceptions = true; @@ -305,10 +295,30 @@ public JobStatus run() { } } else { try { - PortControlFile control = new PortControlFile(new URI("https://" + DatasetUtils.getDomainWithoutScheme(sinkSiteDomain)).getHost(), + if (portMethod.equals(PortMethod.copy_schema)) { + sinkSetID = PortUtility.portSchema(loader, creator, + sourceSetID, destinationDatasetTitle, + false); + } else if (portMethod.equals(PortMethod.copy_all)) { + sinkSetID = PortUtility.portSchema(loader, creator, + sourceSetID, destinationDatasetTitle, + false); + } else if (portMethod.equals(PortMethod.copy_data)) { + JobStatus schemaCheck = PortUtility.assertSchemasAreAlike(loader, creator, sourceSetID, sinkSetID); + if (schemaCheck.isError()) { + runStatus = JobStatus.PORT_ERROR; + runStatus.setMessage(schemaCheck.getMessage()); + return runStatus; + } + } else { + runStatus = JobStatus.PORT_ERROR; + runStatus.setMessage(JobStatus.INVALID_PORT_METHOD.toString()); + return runStatus; + } + + PortControlFile control = new PortControlFile(new URI("https://" + DatasetUtils.getDomainWithoutScheme(sourceSiteDomain)).getHost(), + sourceSetID, destinationDatasetTitle, - sinkSetID, - userPrefs.getUseNewBackend(), portMethod, publishDataset.equals(PublishDataset.publish)); @@ -317,7 +327,7 @@ public JobStatus run() { // exactly the manner of all other di2 jobs DeltaImporter2Publisher publisher = new DeltaImporter2Publisher(userPrefs, "fixme"); - runStatus = publisher.copyWithDi2(sourceSetID, control); + runStatus = publisher.copyWithDi2(sinkSetID, control); if(runStatus == JobStatus.SUCCESS) { // Urrrrghghghgh Pattern p = Pattern.compile("The new dataset id is (....-....)"); @@ -329,7 +339,7 @@ public JobStatus run() { runStatus.setMessage("Unable to find newly-created dataset"); } } - } catch(IOException | URISyntaxException e) { + } catch(Exception e) { runStatus = JobStatus.PORT_ERROR; runStatus.setMessage(e.getMessage()); } diff --git a/src/main/java/com/socrata/datasync/model/ControlFileModel.java b/src/main/java/com/socrata/datasync/model/ControlFileModel.java index 415b0a6c..a996094c 100644 --- a/src/main/java/com/socrata/datasync/model/ControlFileModel.java +++ b/src/main/java/com/socrata/datasync/model/ControlFileModel.java @@ -2,14 +2,21 @@ import com.socrata.datasync.config.controlfile.ControlFile; import com.socrata.datasync.config.controlfile.LocationColumn; +import com.socrata.datasync.config.controlfile.SyntheticPointColumn; import com.socrata.datasync.job.JobStatus; import com.socrata.datasync.validation.IntegrationJobValidity; +import com.socrata.datasync.Utils; import com.socrata.model.importer.Column; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.SerializationConfig; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationConfig; +import com.fasterxml.jackson.databind.SerializationFeature; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; import org.joda.time.format.ISODateTimeFormat; +import info.debatty.java.stringsimilarity.Levenshtein; +import info.debatty.java.stringsimilarity.WeightedLevenshtein; +import info.debatty.java.stringsimilarity.CharacterInsDelInterface; +import info.debatty.java.stringsimilarity.CharacterSubstitutionInterface; import java.io.File; import java.io.IOException; @@ -42,8 +49,6 @@ public class ControlFileModel extends Observable { private CSVModel csvModel; private DatasetModel datasetModel; private String path; - final String constructingFieldSeparator = ","; - final String deconstructingFieldSeparator = "\\s*,\\s*"; public ControlFileModel (ControlFile file, DatasetModel dataset) throws IOException{ controlFile = file; @@ -59,6 +64,7 @@ public ControlFileModel (ControlFile file, DatasetModel dataset) throws IOExcept if (!file.getFileTypeControl().hasColumns()){ initializeColumns(); } + // Now attempt to match those in the dataset to those in the CSV matchColumns(); } @@ -91,17 +97,239 @@ public ControlFile getControlFile(){ return controlFile; } - private void matchColumns(){ + private static final CharacterSubstitutionInterface insDel = + new CharacterSubstitutionInterface() { + public double cost(char c1, char c2) { + // Where we minify the cost of a change, we return + // something non-zero because we don't want to ignore + // it entirely, because we want to prefer solutions + // with a minimal number of even expected changes. + if(!Character.isAlphabetic(c1) && !Character.isDigit(c1) && c2 == '_') return 0.1; + if(Character.isUpperCase(c1) && Character.isLowerCase(c2)) return 0.1; + return 1.0; + } + }; + + private static final CharacterInsDelInterface subst = + new CharacterInsDelInterface() { + public double deletionCost(char c) { + if(Character.isAlphabetic(c) || Character.isDigit(c) || c == '_') { + return 1.0; + } + return 0.1; + } + public double insertionCost(char c) { + return 1.0; + } + }; + + private static final WeightedLevenshtein csvToFieldName = new WeightedLevenshtein(insDel, subst); + + private static final CharacterSubstitutionInterface penalizeChanges = + new CharacterSubstitutionInterface() { + public double cost(char c1, char c2) { + if(c1 == '_' && !Character.isAlphabetic(c2) && !Character.isDigit(c2)) return 0; + return 1.0; + } + }; + + private static final CharacterInsDelInterface penalizeDeletions = + new CharacterInsDelInterface() { + public double deletionCost(char c) { + return 1.0; + } + public double insertionCost(char c) { + return 0; + } + }; + + private static final WeightedLevenshtein fieldNameToCsv = new WeightedLevenshtein(penalizeChanges, penalizeDeletions); + + public static double fieldNameEditDistance(String csvHeader, String fieldName) { + double editDistance = csvToFieldName.distance(csvHeader, fieldName); + // ok, if we stop and produce a solution here, we'll get + // badly confused if the CSV has extra columns in it + // (because if we encounter one such early, we'll + // basically pick a random existing column to pair it + // with). So instead we'll look at the edit distance from + // dataset column to the CSV field name, heavily + // penalizing deletions. If the result is a significant + // fraction of the size of the dataset column name, we'll + // consider things unmatched. + if(fieldNameToCsv.distance(fieldName, csvHeader) < fieldName.length() * 0.25) { + return editDistance; + } else { + // Nope; too many deletions / unexpected changes in the + // field name -> csv name direction, reject this + // possibility. + return Double.POSITIVE_INFINITY; + } + } + + private static final Levenshtein csvToHumanName = new Levenshtein(); + + public static double humanNameEditDistance(String csvHeader, String humanName) { + double editDistance = csvToHumanName.distance(csvHeader, humanName); + if(editDistance < Math.max(csvHeader.length(), humanName.length()) * 0.25) { + return editDistance; + } else { + // too much change to the human name; just assume they're different + return Double.POSITIVE_INFINITY; + } + } + + private static class Guess implements Comparable { + public final Column column; + public final double badness; + public final int index; + + public Guess(Column column, double badness, int index) { + this.column = column; + this.badness = badness; + this.index = index; + } + + public int compareTo(Guess that) { + int badnessOrd = Double.compare(this.badness, that.badness); + if(badnessOrd == 0) return Integer.compare(this.index, that.index); + else return badnessOrd; + } + + @Override public boolean equals(Object that) { + if(that instanceof Guess) return compareTo((Guess) that) == 0; + return false; + } + } + + private static class Candidate implements Comparable { + public final int sourceColumnIndex; + public final PriorityQueue preferences; + + public Candidate(int sourceColumnIndex, PriorityQueue preferences) { + this.sourceColumnIndex = sourceColumnIndex; + this.preferences = preferences; + } + + public int compareTo(Candidate that) { + boolean thisIsInfinitelyBad = this.preferences.isEmpty() || this.preferences.peek().badness == Double.POSITIVE_INFINITY; + boolean thatIsInfinitelyBad = that.preferences.isEmpty() || that.preferences.peek().badness == Double.POSITIVE_INFINITY; + + if(thisIsInfinitelyBad) { + if(thatIsInfinitelyBad) { + return Integer.compare(this.sourceColumnIndex, that.sourceColumnIndex); + } else { + return 1; // that is not infinitely bad, so put it first + } + } else { + if(thatIsInfinitelyBad) { + return -1; + } else { + int badnessOrd = Double.compare(this.preferences.peek().badness, that.preferences.peek().badness); + if(badnessOrd == 0) return Integer.compare(this.sourceColumnIndex, that.sourceColumnIndex); + else return badnessOrd; + } + } + } + + @Override public boolean equals(Object that) { + if(that instanceof Candidate) return compareTo((Candidate) that) == 0; + return false; + } + } + + private static int bestGuess(SortedMap> allPreferences) { + // Returns the leftmost column with the best (== lowest) + // badness at the head. Precondition: allPreferences is + // non-empty. + int best = -1; + double bestBadness = Double.POSITIVE_INFINITY; + for(Map.Entry> ent : allPreferences.entrySet()) { + if(best == -1) { + best = ent.getKey(); + if(ent.getValue().isEmpty()) bestBadness = Double.POSITIVE_INFINITY; + else bestBadness = ent.getValue().peek().badness; + } else if(ent.getValue().isEmpty()) { + // ok, we're definitely not better than our best + // guess. + } else if(ent.getValue().peek().badness < bestBadness) { + best = ent.getKey(); + bestBadness = ent.getValue().peek().badness; + } + if(bestBadness == 0) return best; // short circuit: we're not going to get any better + } + return best; + } + + static final boolean DEBUG = false; + private static void debug(Object... items) { + if(DEBUG) { + StringBuilder sb = new StringBuilder(); + for(Object item : items) { + sb.append(item); + } + System.out.println(sb.toString()); + } + } + + private void matchColumns() { + debug("Starting match columns"); + + List columnNames = new ArrayList<>(); + for(Column column : datasetModel.getColumns()) { + columnNames.add(column.getName()); + } + + PriorityQueue candidates = new PriorityQueue<>(); + for (int i = 0; i < csvModel.getColumnCount(); i++) { String csvHeader = csvModel.getColumnName(i); - //TODO: I'm running over the dataset column list probably an unnecessary number of times. Consider fixing later - Column col = datasetModel.getColumnByFieldName(csvHeader); - //If we can't match on field name, check for a match on the friendly name - if (col == null){ - col = datasetModel.getColumnByFriendlyName(csvHeader); + debug("Looking at CSV column: ", csvHeader); + + PriorityQueue preferences = new PriorityQueue<>(); + int idx = 0; + for(Column column : datasetModel.getColumns()) { + double editDistanceToName = humanNameEditDistance(csvHeader, column.getName()); + double editDistanceToFieldName = fieldNameEditDistance(csvHeader, column.getFieldName()); + double badness = Math.min(editDistanceToName, editDistanceToFieldName); + debug(" Badness to ", column.getName(), " (", column.getFieldName(), ") : ", badness); + preferences.add(new Guess(column, badness, idx)); + idx += 1; } - if (col != null) - updateColumnAtPosition(col.getFieldName(), i); + + candidates.add(new Candidate(i, preferences)); + } + + Set usedFieldNames = new HashSet<>(); + while(!candidates.isEmpty()) { + Candidate candidate = candidates.poll(); + debug("Leftmost csv column with the least bad preference is ", csvModel.getColumnName(candidate.sourceColumnIndex)); + PriorityQueue preferences = candidate.preferences; + if(preferences.isEmpty()) { + debug(" It has no preferences. Bailing."); + ignoreColumnInCSVAtPosition(candidate.sourceColumnIndex); + continue; + } + + Guess guess = preferences.poll(); + if(guess.badness == Double.POSITIVE_INFINITY) { + debug(" Its most-prefered choice is infinitely bad. Bailing."); + ignoreColumnInCSVAtPosition(candidate.sourceColumnIndex); + continue; + } + + if(usedFieldNames.contains(guess.column.getFieldName())) { + // Its best choice is something we've already + // selected; put the candidate back in the pool, minus + // that guess because mutability, and try again. + debug(" Its most-prefered choice (", guess.column.getFieldName(), ") has already been picked. Removing that option and trying again."); + candidates.add(candidate); + continue; + } + + Column col = guess.column; + debug(" That best matches column ", col.getName(), " (", col.getFieldName(), ") with badness ", guess.badness); + updateColumnAtPosition(col.getFieldName(), candidate.sourceColumnIndex); + usedFieldNames.add(col.getFieldName()); } } @@ -295,18 +523,8 @@ public String getDisplayName(int i){ return getColumnAtPosition(i); } - public String getStringFromArray(String[] array){ - StringBuffer strbuf = new StringBuffer(); - for (int i = 0; i < array.length; i++){ - strbuf.append(array[i]); - if (i+1 != array.length) - strbuf.append(constructingFieldSeparator); - } - return strbuf.toString(); - } - public String getFloatingDateTime(){ - return getStringFromArray(controlFile.getFileTypeControl().floatingTimestampFormat); + return Utils.commaJoin(controlFile.getFileTypeControl().floatingTimestampFormat); } public String getTimezone(){ @@ -314,13 +532,13 @@ public String getTimezone(){ } public void setFixedDateTime(String fixed){ - String[] newDateTime = fixed.split(deconstructingFieldSeparator); + String[] newDateTime = Utils.commaSplit(fixed); controlFile.getFileTypeControl().fixedTimestampFormat(newDateTime); updateListeners(); } public void setFloatingDateTime(String floating){ - String[] newDateTime = floating.split(deconstructingFieldSeparator); + String[] newDateTime = Utils.commaSplit(floating); controlFile.getFileTypeControl().floatingTimestampFormat(newDateTime); updateListeners(); } @@ -340,11 +558,28 @@ public void setSyntheticLocation(String fieldName, LocationColumn locationField) if (columnsMap != null) { controlFile.getFileTypeControl().syntheticLocations.put(fieldName, locationField); } else { - HashMap map = new HashMap<>(); + TreeMap map = new TreeMap<>(); map.put(fieldName, locationField); controlFile.getFileTypeControl().syntheticLocations = map; } + syncSynth(fieldName); + } + + public void setSyntheticPoint(String fieldName, SyntheticPointColumn locationField) { + Map columnsMap = controlFile.getFileTypeControl().syntheticPoints; + if (columnsMap != null) { + controlFile.getFileTypeControl().syntheticPoints.put(fieldName, locationField); + } else { + TreeMap map = new TreeMap<>(); + map.put(fieldName, locationField); + controlFile.getFileTypeControl().syntheticPoints = map; + } + + syncSynth(fieldName); + } + + private void syncSynth(String fieldName) { //Reset the location column int locationIndex = getIndexOfColumnName(fieldName); if (locationIndex != -1) { @@ -358,10 +593,17 @@ public void setSyntheticLocation(String fieldName, LocationColumn locationField) public Map getSyntheticLocations(){ Map locations = controlFile.getFileTypeControl().syntheticLocations; if (locations == null) - locations = new HashMap<>(); + locations = new TreeMap<>(); return locations; } + public Map getSyntheticPoints() { + Map points = controlFile.getFileTypeControl().syntheticPoints; + if (points == null) + points = new TreeMap<>(); + return points; + } + public ArrayList getUnmappedDatasetColumns(){ ArrayList unmappedColumns = new ArrayList(); for (Column datasetColumn : datasetModel.getColumns()){ @@ -376,7 +618,7 @@ public ArrayList getUnmappedDatasetColumns(){ } public String getControlFileContents() { - ObjectMapper mapper = new ObjectMapper().configure(SerializationConfig.Feature.INDENT_OUTPUT, true); + ObjectMapper mapper = new ObjectMapper().configure(SerializationFeature.INDENT_OUTPUT, true); try { return mapper.writeValueAsString(controlFile); } catch (IOException e) { diff --git a/src/main/java/com/socrata/datasync/model/DatasetModel.java b/src/main/java/com/socrata/datasync/model/DatasetModel.java index 135bce44..924d7812 100644 --- a/src/main/java/com/socrata/datasync/model/DatasetModel.java +++ b/src/main/java/com/socrata/datasync/model/DatasetModel.java @@ -1,6 +1,5 @@ package com.socrata.datasync.model; -import au.com.bytecode.opencsv.CSVReader; import com.socrata.api.Soda2Consumer; import com.socrata.datasync.DatasetUtils; import com.socrata.datasync.config.userpreferences.UserPreferences; @@ -19,6 +18,7 @@ import org.apache.http.HttpException; import java.util.ArrayList; import java.util.Vector; +import java.util.List; /** @@ -48,19 +48,14 @@ public String getDomain() { private boolean initializeDataset(UserPreferences prefs, String fourbyfour) throws LongRunningQueryException, InterruptedException, HttpException, IOException, URISyntaxException { - datasetInfo = DatasetUtils.getDatasetInfo(prefs, fourbyfour, Dataset.class); + datasetInfo = DatasetUtils.getDatasetInfo(prefs, fourbyfour); columns = (ArrayList) datasetInfo.getColumns(); - String csv = DatasetUtils.getDatasetSample(prefs, fourbyfour, rowsToSample); + List> csv = DatasetUtils.getDatasetSample(prefs, datasetInfo, rowsToSample); - CSVReader reader = new CSVReader(new StringReader(csv)); - - String[] lines = reader.readNext(); - - while (lines != null) { - insertData(lines); - lines = reader.readNext(); + for(List row : csv) { + insertData(row.toArray(new Object[0])); } return true; @@ -78,6 +73,15 @@ public int getLocationCount() { return locationCount; } + public int getPointCount() { + int pointCount = 0; + for (Column c : columns){ + if (c.getDataTypeName().equals("point")) + pointCount++; + } + return pointCount; + } + public Dataset getDatasetInfo(){ return datasetInfo; } diff --git a/src/main/java/com/socrata/datasync/publishers/DeltaImporter2Publisher.java b/src/main/java/com/socrata/datasync/publishers/DeltaImporter2Publisher.java index 7b560ed6..30a3990a 100644 --- a/src/main/java/com/socrata/datasync/publishers/DeltaImporter2Publisher.java +++ b/src/main/java/com/socrata/datasync/publishers/DeltaImporter2Publisher.java @@ -7,6 +7,7 @@ import com.socrata.datasync.config.controlfile.PortControlFile; import com.socrata.datasync.config.controlfile.FileTypeControl; import com.socrata.datasync.config.userpreferences.UserPreferences; +import com.socrata.datasync.ui.SimpleIntegrationWizard; import com.socrata.datasync.deltaimporter2.*; import com.socrata.datasync.job.JobStatus; import com.socrata.ssync.PatchComputer; @@ -21,10 +22,12 @@ import org.apache.http.client.utils.URIBuilder; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; -import org.codehaus.jackson.JsonProcessingException; -import org.codehaus.jackson.map.DeserializationConfig; -import org.codehaus.jackson.map.ObjectMapper; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationConfig; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import javax.swing.*; import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.File; @@ -45,7 +48,7 @@ public class DeltaImporter2Publisher implements AutoCloseable { private static final String datasyncPath = datasyncBasePath + "/id"; private static final String statusPath = "/status"; private static final String commitPath = "/commit"; - private static final String portPath = "/copy"; + private static final String portPath = "/copy_from"; private static final String logPath = "/log"; private static final String ssigContentType = "application/x-socrata-ssig"; private static final String patchExtenstion = ".sdiff"; @@ -61,7 +64,7 @@ private static class CompletelyRestartJob extends Exception {} private static String domain; private static HttpUtility http; private static URIBuilder baseUri; - private static ObjectMapper mapper = new ObjectMapper().enable(DeserializationConfig.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY); + private static ObjectMapper mapper = new ObjectMapper().enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY); private String pathToSignature = null; CloseableHttpResponse signatureResponse = null; @@ -117,6 +120,8 @@ public JobStatus publishWithDi2OverHttp(String datasetId, final File csvOrTsvFil @Override protected void progress(long count) { System.out.println("\tRead " + count + " of " + fileSize + " bytes of " + csvOrTsvFile.getName()); + int pct = (int) (count*100/fileSize); + updateStatus("Reading File", pct, true, ""); } }; // compute the patch between the csv/tsv file and its previous signature @@ -263,7 +268,8 @@ private int fetchDatasyncChunkSize() { */ private List postPatchBlobs(InputStream patchStream, String datasetId, int chunkSize) throws IOException, URISyntaxException, HttpException { - System.out.println("Chunking and posting the diff"); + updateStatus("Chunking and posting the diff", 0, false, ""); + System.out.println("Creating the diff..."); URI postingPath = baseUri.setPath(datasyncPath + "/" + datasetId).build(); List blobIds = new LinkedList<>(); @@ -291,6 +297,7 @@ private List postPatchBlobs(InputStream patchStream, String datasetId, i } while (status != HttpStatus.SC_CREATED && retries < httpRetries); //We hit the max number of retries without success and should throw an exception accordingly. if (retries >= httpRetries) throw new HttpException(statusLine.toString()); + updateStatus("Uploading file", 0, false, bytesRead + " bytes"); System.out.println("\tUploaded " + bytesRead + " bytes"); } return blobIds; @@ -304,13 +311,14 @@ private List postPatchBlobs(InputStream patchStream, String datasetId, i * @return the jobId of the job applying the diff */ private String commitStandardJob(final CommitMessage msg, final String datasetId, final String uuid) throws URISyntaxException, IOException, CompletelyRestartJob { - System.out.println("Commiting the chunked diffs to apply the patch"); + updateStatus("Commiting the job", 0, false, ""); + System.out.println("Committing the job"); final URI committingPath = baseUri.setPath(datasyncPath + "/" + datasetId + commitPath).build(); return commitGenericJob(msg, committingPath, datasetId, uuid); } private String commitPortJob(final CommitMessage msg, final String datasetId, final String uuid) throws URISyntaxException, IOException, CompletelyRestartJob { - System.out.println("Commiting the port job"); + System.out.println("Committing the port job"); final URI committingPath = baseUri.setPath(datasyncPath + "/" + datasetId + portPath).build(); return commitGenericJob(msg, committingPath, datasetId, uuid); } @@ -430,9 +438,9 @@ void handleHttpResponsePath(CloseableHttpResponse response) throws IOException, private JobStatus getJobStatus(String datasetId, String jobId) throws URISyntaxException, IOException, InterruptedException, HttpException { JobStatus jobStatus = null; - String status = null; + StatusResponse status = null; StatusLine statusLine = null; - URI statusUri = baseUri.setPath(datasyncPath + "/" + datasetId + statusPath + "/" + jobId).build(); + URI statusUri = baseUri.setPath(datasyncPath + "/" + datasetId + statusPath + "/" + jobId + ".json").build(); URI logUri = baseUri.setPath(datasyncPath + "/" + datasetId + logPath + "/" + jobId + ".json").build(); int retries = 0; while (jobStatus == null && retries < httpRetries) { @@ -441,15 +449,49 @@ private JobStatus getJobStatus(String datasetId, String jobId) throws int statusCode = statusLine.getStatusCode(); if (statusCode == HttpStatus.SC_OK) { retries = 0; // we got one, so reset the retry count. - status = IOUtils.toString(response.getEntity().getContent()); - System.out.print("Polling the job status: " + status); - if (status.startsWith("SUCCESS")) { + status = mapper.readValue(IOUtils.toString(response.getEntity().getContent()), StatusResponse.class); + System.out.println("Polling the job status: " + status.english); + if(status.type.equals("read-input-rows")) { + long rows = ((Number)status.data.get("rows")).longValue(); + long total = ((Number)status.data.get("total")).longValue(); + int percent = (int)(rows * 100 / total); + updateStatus("Reading Rows from File...", percent, true, ""); + } else if(status.type.equals("read-dataset-rows")) { + Object rows = status.data.get("rows"); + updateStatus("Reading Rows from Dataset...", 0, false, rows + " rows"); + } else if(status.type.equals("applying-diff")) { + Object bytes = status.data.get("bytesWritten"); + updateStatus("Applying diff...", 0, false, bytes + " bytes"); + } else if(status.type.equals("counting-records")) { + Object records = status.data.get("recordsFound"); + updateStatus("Counting records...", 0, false, records + " records"); + } else if(status.type.equals("computing-upsert")) { + Object inserts = status.data.get("inserts"); + Object updates = status.data.get("updates"); + Object deletes = status.data.get("deletes"); + updateStatus("Counting upsert...", 0, false, inserts + " inserts, " + updates + " updates, " + deletes + " deletes"); + } else if(status.type.equals("uploading-upsert")) { + long bytes = ((Number)status.data.get("sentBytes")).longValue(); + long total = ((Number)status.data.get("totalBytes")).longValue(); + int percent = (int)(bytes * 100 / total); + updateStatus("Uploading Upsert Script...", percent, true, ""); + } else if(status.type.equals("storing-completed")) { + long bytes = ((Number)status.data.get("bytesWritten")).longValue(); + long total = ((Number)status.data.get("totalBytes")).longValue(); + int percent = (int)(bytes * 100 / total); + updateStatus("Storing File...", percent, true, ""); + } else if(status.english.startsWith("SUCCESS")) { + updateStatus("Processing...", 0, false, ""); jobStatus = JobStatus.SUCCESS; - } else if (status.startsWith("FAILURE")) { + break; + } else if(status.english.startsWith("FAILURE")) { + updateStatus("Processing...", 0, false, ""); jobStatus = JobStatus.PUBLISH_ERROR; + break; } else { - Thread.sleep(1000); + updateStatus(status.english, 0, false, ""); } + Thread.sleep(1000); } else if (statusCode == HttpStatus.SC_BAD_GATEWAY || statusCode == HttpStatus.SC_SERVICE_UNAVAILABLE) { // No-penalty retry; we're willing to keep doing this forever Thread.sleep(1000); @@ -462,11 +504,15 @@ private JobStatus getJobStatus(String datasetId, String jobId) throws if (jobStatus == null) { throw new HttpException(statusLine.toString()); } - jobStatus.setMessage(status + "(jobId:" + jobId + ")"); + jobStatus.setMessage(status.english + "(jobId:" + jobId + ")"); if(!jobStatus.isError()) loadStatusWithCRUD(jobStatus, logUri); return jobStatus; } + private void updateStatus(String loadingLabel, int progressPercent, boolean showProgress, String message) { + SimpleIntegrationWizard.updateStatus(loadingLabel, progressPercent, showProgress, message); + } + private Commital getJobCommitment(String datasetId, String uuid) throws URISyntaxException { URI logUri = baseUri.setPath(datasyncPath + "/" + datasetId + logPath + "/index.json").build(); try(CloseableHttpResponse response = http.get(logUri, ContentType.APPLICATION_JSON.getMimeType())) { @@ -521,7 +567,7 @@ private void loadStatusWithCRUD(JobStatus status, URI logUri) { if (statusCode == HttpStatus.SC_OK) { // The payload that we get back from DI2 may change over time. Since we are only looking at the delta // section, disable the strict parsing. - mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); LogItem[] deltaLog = mapper.readValue(response.getEntity().getContent(), LogItem[].class); LogItem deltas = getLogItem(deltaLog, finishedLogKey); if (deltas != null) { diff --git a/src/main/java/com/socrata/datasync/publishers/GISPublisher.java b/src/main/java/com/socrata/datasync/publishers/GISPublisher.java index 508589bc..c7b0363e 100644 --- a/src/main/java/com/socrata/datasync/publishers/GISPublisher.java +++ b/src/main/java/com/socrata/datasync/publishers/GISPublisher.java @@ -12,7 +12,7 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.mime.MultipartEntityBuilder; import org.apache.http.util.EntityUtils; -import org.codehaus.jackson.map.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; diff --git a/src/main/java/com/socrata/datasync/publishers/StatusResponse.java b/src/main/java/com/socrata/datasync/publishers/StatusResponse.java new file mode 100644 index 00000000..483ae9e2 --- /dev/null +++ b/src/main/java/com/socrata/datasync/publishers/StatusResponse.java @@ -0,0 +1,11 @@ +package com.socrata.datasync.publishers; + +import java.util.Map; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class StatusResponse { + public String type; + public Map data; + public String english; +} diff --git a/src/main/java/com/socrata/datasync/ui/AdvancedOptionsPanel.java b/src/main/java/com/socrata/datasync/ui/AdvancedOptionsPanel.java index 8327ac3f..9e63dafa 100644 --- a/src/main/java/com/socrata/datasync/ui/AdvancedOptionsPanel.java +++ b/src/main/java/com/socrata/datasync/ui/AdvancedOptionsPanel.java @@ -1,8 +1,9 @@ package com.socrata.datasync.ui; import com.socrata.datasync.model.ControlFileModel; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.SerializationConfig; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationConfig; +import com.fasterxml.jackson.databind.SerializationFeature; import javax.swing.*; import java.awt.*; @@ -108,7 +109,8 @@ public void actionPerformed(ActionEvent e) { } }); - showSynthetic = UIUtility.getButtonAsLink("Manage Synthetic Columns (" + model.getSyntheticLocations().size() +")"); + int size = isNBE() ? model.getSyntheticPoints().size() : model.getSyntheticLocations().size(); + showSynthetic = UIUtility.getButtonAsLink("Manage Synthetic Columns (" + size +")"); showSynthetic.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent e) { @@ -116,12 +118,20 @@ public void actionPerformed(ActionEvent e) { renderCollapsed(); else { //If there are no locations to show, take me to the dialog where I can create one - if (model.getSyntheticLocations().isEmpty()) { - JDialog dialog = new SyntheticLocationDialog(model, (JFrame) ((JDialog) SwingUtilities.getRoot((JButton) e.getSource())).getParent(), model.getSyntheticLocations(), null, "Manage synthetic columns"); - + if (isNBE()) { + if (model.getSyntheticPoints().isEmpty()) { + JDialog dialog = SyntheticPointDialog.create(model, (JFrame) ((JDialog) SwingUtilities.getRoot((JButton) e.getSource())).getParent(), model.getSyntheticPoints()); + } else { + renderSyntheticPointsExpanded(); + } + } else { + if (model.getSyntheticLocations().isEmpty()) { + JDialog dialog = new SyntheticLocationDialog(model, (JFrame) ((JDialog) SwingUtilities.getRoot((JButton) e.getSource())).getParent(), model.getSyntheticLocations(), null, "Manage synthetic columns"); + + } + else + renderSyntheticLocationsExpanded(); } - else - renderSyntheticExpanded(); } } }); @@ -143,11 +153,13 @@ private JPanel getBottomPanel(){ else advancedOptions.setText("Advanced Import Options"); - if (model.getDatasetModel().getLocationCount() > 0) { - if (syntheticExpanded) + if (model.getDatasetModel().getLocationCount() > 0 || model.getDatasetModel().getPointCount() > 0) { + if (syntheticExpanded) { showSynthetic.setText("Hide Synthetic Columns"); - else - showSynthetic.setText("Manage Synthetic Columns (" + model.getSyntheticLocations().size() + ")"); + } else { + int size = isNBE() ? model.getSyntheticPoints().size() : model.getSyntheticLocations().size(); + showSynthetic.setText("Manage Synthetic Columns (" + size + ")"); + } showSynthetic.setAlignmentX(LEFT_ALIGNMENT); showSynthetic.setHorizontalAlignment(SwingConstants.LEFT); @@ -159,6 +171,10 @@ private JPanel getBottomPanel(){ return panel; } + private boolean isNBE() { + return model.getDatasetModel().getDatasetInfo().isNewBackend(); + } + private void renderCollapsed(){ this.removeAll(); advancedExpanded = false; @@ -170,7 +186,16 @@ private void renderCollapsed(){ } - private void renderSyntheticExpanded(){ + private void renderSyntheticPointsExpanded(){ + this.removeAll(); + syntheticExpanded = true; + setLayout(new BorderLayout()); + add(new SyntheticPointsContainer(model), BorderLayout.CENTER); + add(getBottomPanel(),BorderLayout.SOUTH); + this.revalidate(); + } + + private void renderSyntheticLocationsExpanded(){ this.removeAll(); syntheticExpanded = true; setLayout(new BorderLayout()); @@ -286,7 +311,7 @@ private JTextField generateGenericInput(String label, String defaultValue, Actio private void showEditControlFileDialog() { try { - ObjectMapper mapper = new ObjectMapper().configure(SerializationConfig.Feature.INDENT_OUTPUT, true); + ObjectMapper mapper = new ObjectMapper().configure(SerializationFeature.INDENT_OUTPUT, true); String textAreaContent = mapper.writeValueAsString(model.getControlFile()); JTextArea controlFileContentTextArea = new JTextArea(); diff --git a/src/main/java/com/socrata/datasync/ui/ControlFileEditDialog.java b/src/main/java/com/socrata/datasync/ui/ControlFileEditDialog.java index b1849933..a57bd1df 100644 --- a/src/main/java/com/socrata/datasync/ui/ControlFileEditDialog.java +++ b/src/main/java/com/socrata/datasync/ui/ControlFileEditDialog.java @@ -13,7 +13,7 @@ public class ControlFileEditDialog extends JDialog { ControlFileModel controlFileModel; - private static final Dimension CONTROL_FILE_DIALOG_DIMENSIONS = new Dimension(550, 640); + private static final Dimension CONTROL_FILE_DIALOG_DIMENSIONS = new Dimension(750, 640); public ControlFileEditDialog(ControlFileModel controlFileModel, JFrame parent){ super(parent,"Map Fields"); diff --git a/src/main/java/com/socrata/datasync/ui/GISJobTab.java b/src/main/java/com/socrata/datasync/ui/GISJobTab.java index b74a9898..00c4dfaa 100644 --- a/src/main/java/com/socrata/datasync/ui/GISJobTab.java +++ b/src/main/java/com/socrata/datasync/ui/GISJobTab.java @@ -104,6 +104,7 @@ private void addFileToPublishFieldToJobPanel() { JOB_FILE_TEXTFIELD_WIDTH, JOB_TEXTFIELD_HEIGHT)); fileSelectorContainer.add(fileToPublishTextField); JFileChooser fileToPublishChooser = new JFileChooser(); + fileToPublishChooser.setCurrentDirectory(new File(".")); JButton openButton = new JButton(BROWSE_BUTTON_TEXT); FileToPublishSelectorListener chooserListener = new FileToPublishSelectorListener( fileToPublishChooser, fileToPublishTextField); @@ -165,6 +166,7 @@ public void saveJob() { String selectedJobFileLocation = jobFileLocation; if (selectedJobFileLocation.equals("")) { JFileChooser savedJobFileChooser = new JFileChooser(); + savedJobFileChooser.setCurrentDirectory(new File(".")); FileNameExtensionFilter filter = new FileNameExtensionFilter( JOB_FILE_NAME + " (*." + JOB_FILE_EXTENSION + ")", JOB_FILE_EXTENSION); savedJobFileChooser.setFileFilter(filter); @@ -216,10 +218,12 @@ public String getJobFileLocation() { private class FileToPublishSelectorListener implements ActionListener { JFileChooser fileChooser; + File base; JTextField filePathTextField; public FileToPublishSelectorListener(JFileChooser chooser, JTextField textField) { fileChooser = chooser; + base = fileChooser.getCurrentDirectory().getAbsoluteFile(); filePathTextField = textField; fileChooser.setFileFilter( UIUtility.getFileChooserFilter(GISJobValidity.allowedGeoFileToPublishExtensions)); @@ -228,7 +232,7 @@ public FileToPublishSelectorListener(JFileChooser chooser, JTextField textField) public void actionPerformed(ActionEvent e) { if (fileChooser.showOpenDialog(mainFrame) == JFileChooser.APPROVE_OPTION) { File file = fileChooser.getSelectedFile(); - filePathTextField.setText(file.getAbsolutePath()); + filePathTextField.setText(UIUtility.relativize(base, file)); } // If open command was cancelled by user: do nothing diff --git a/src/main/java/com/socrata/datasync/ui/IntegrationJobTab.java b/src/main/java/com/socrata/datasync/ui/IntegrationJobTab.java index 68dae605..bcde3bbd 100644 --- a/src/main/java/com/socrata/datasync/ui/IntegrationJobTab.java +++ b/src/main/java/com/socrata/datasync/ui/IntegrationJobTab.java @@ -17,10 +17,11 @@ import com.socrata.exceptions.SodaError; import com.socrata.model.importer.Dataset; import org.apache.http.HttpException; -import org.codehaus.jackson.map.DeserializationConfig; -import org.codehaus.jackson.map.DeserializationConfig; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.SerializationConfig; +import com.fasterxml.jackson.databind.DeserializationConfig; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationConfig; +import com.fasterxml.jackson.databind.SerializationFeature; import javax.swing.*; import javax.swing.filechooser.FileNameExtensionFilter; @@ -99,7 +100,9 @@ public class IntegrationJobTab implements JobTab { private JLabel jobTabTitleLabel = new JLabel("Untitled"); private JTextField datasetIDTextField; + private String lastDatasetId; // used to decide when to regen the control file private JTextField fileToPublishTextField; + private String lastFileToPublish; // used to decide when to regen the control file private JComboBox publishMethodComboBox; private ButtonGroup publishMethodRadioButtonGroup; private JRadioButton soda2Button; @@ -118,8 +121,12 @@ public class IntegrationJobTab implements JobTab { private boolean usingControlFile; + private UserPreferences userPrefs; + // build Container with all tab components populated with given job data - public IntegrationJobTab(IntegrationJob job, JFrame containingFrame) { + public IntegrationJobTab(IntegrationJob job, JFrame containingFrame, UserPreferences userPrefs) { + this.userPrefs = userPrefs; + mainFrame = containingFrame; // build tab panel form @@ -133,10 +140,11 @@ public IntegrationJobTab(IntegrationJob job, JFrame containingFrame) { loadJobDataIntoUIFields(job); + lastDatasetId = datasetIDTextField.getText(); + lastFileToPublish = fileToPublishTextField.getText(); if(job.getPublishMethod() == null) publishMethodComboBox.setSelectedItem(PublishMethod.replace); - } @@ -195,9 +203,6 @@ private void addDatasetIdFieldToJobPanel() { datasetIDTextField = new JTextField(); datasetIDTextField.setPreferredSize(new Dimension( DATASET_ID_TEXTFIELD_WIDTH, JOB_TEXTFIELD_HEIGHT)); - RegenerateControlFileListener regenerateListener = new RegenerateControlFileListener(); - datasetIDTextField.addActionListener(regenerateListener); - datasetIDTextField.addFocusListener(regenerateListener); datasetIDTextFieldContainer.add(datasetIDTextField); jobPanel.add(datasetIDTextFieldContainer); } @@ -211,14 +216,12 @@ private void addFileToPublishFieldToJobPanel() { JOB_FILE_TEXTFIELD_WIDTH, JOB_TEXTFIELD_HEIGHT)); fileSelectorContainer.add(fileToPublishTextField); JFileChooser fileToPublishChooser = new JFileChooser(); + fileToPublishChooser.setCurrentDirectory(new File(".")); JButton openButton = new JButton(BROWSE_BUTTON_TEXT); FileToPublishSelectorListener chooserListener = new FileToPublishSelectorListener( fileToPublishChooser, fileToPublishTextField); openButton.addActionListener(chooserListener); fileSelectorContainer.add(openButton); - RegenerateControlFileListener regenerateListener = new RegenerateControlFileListener(); - fileToPublishTextField.addActionListener(regenerateListener); - fileToPublishTextField.addFocusListener(regenerateListener); jobPanel.add(fileSelectorContainer); } @@ -234,7 +237,7 @@ private void setReplaceRadioButtons(IntegrationJob job) { private void loadJobDataIntoUIFields(IntegrationJob job) { try { if (job.getControlFileContent() != null) { - ObjectMapper mapper = new ObjectMapper().enable(DeserializationConfig.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY); + ObjectMapper mapper = new ObjectMapper().enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY); ControlFile controlFile = mapper.readValue(job.getControlFileContent(), ControlFile.class); //Ideally this could be saved with the control file or factored out. However, we're stuck with this redundant call // because of DI2's strict enforcement of control files and the current factoring of what CSVTableModel knows about @@ -345,6 +348,7 @@ public void saveJob() { String selectedJobFileLocation = jobFileLocation; if(selectedJobFileLocation.equals("")) { JFileChooser savedJobFileChooser = new JFileChooser(); + savedJobFileChooser.setCurrentDirectory(new File(".")); FileNameExtensionFilter filter = new FileNameExtensionFilter( JOB_FILE_NAME + " (*." + JOB_FILE_EXTENSION + ")", JOB_FILE_EXTENSION); savedJobFileChooser.setFileFilter(filter); @@ -394,10 +398,12 @@ public String getJobFileLocation() { private class FileToPublishSelectorListener implements ActionListener { JFileChooser fileChooser; + File base; JTextField filePathTextField; public FileToPublishSelectorListener(JFileChooser chooser, JTextField textField) { fileChooser = chooser; + base = fileChooser.getCurrentDirectory().getAbsoluteFile(); filePathTextField = textField; fileChooser.setFileFilter( UIUtility.getFileChooserFilter(IntegrationJobValidity.allowedFileToPublishExtensions)); @@ -407,7 +413,7 @@ public void actionPerformed(ActionEvent e) { int returnVal = fileChooser.showOpenDialog(mainFrame); if (returnVal == JFileChooser.APPROVE_OPTION) { File file = fileChooser.getSelectedFile(); - filePathTextField.setText(file.getAbsolutePath()); + filePathTextField.setText(UIUtility.relativize(base, file)); } else { // Open command cancelled by user: do nothing } @@ -476,6 +482,10 @@ public void actionPerformed(ActionEvent e) { } } + private boolean isDirty() { + return controlFileModel == null || !datasetIDTextField.getText().equals(lastDatasetId) || !fileToPublishTextField.getText().equals(lastFileToPublish); + } + private class EditControlFileListener implements ActionListener { public void actionPerformed(ActionEvent evnt) { String generateControlFileErrorMessage; @@ -485,7 +495,7 @@ public void actionPerformed(ActionEvent evnt) { JOptionPane.showMessageDialog(mainFrame, generateControlFileErrorMessage); } else { try { - if (controlFileModel == null) { + if (isDirty()) { ControlFile controlFile = generateControlFile( new UserPreferencesJava(), fileToPublishTextField.getText(), @@ -494,6 +504,8 @@ public void actionPerformed(ActionEvent evnt) { true); updateControlFileModel(controlFile,datasetIDTextField.getText()); + lastDatasetId = datasetIDTextField.getText(); + lastFileToPublish = fileToPublishTextField.getText(); } ControlFileEditDialog editorFrame = new ControlFileEditDialog(controlFileModel,mainFrame); @@ -530,14 +542,14 @@ private boolean fileToPublishIsSelected() { private String generateControlFileContent(UserPreferences prefs, String fileToPublish, PublishMethod publishMethod, String datasetId, boolean containsHeaderRow) throws HttpException, URISyntaxException, InterruptedException, IOException { ControlFile control = generateControlFile(prefs,fileToPublish,publishMethod,datasetId,containsHeaderRow); - ObjectMapper mapper = new ObjectMapper().configure(SerializationConfig.Feature.INDENT_OUTPUT, true); + ObjectMapper mapper = new ObjectMapper().configure(SerializationFeature.INDENT_OUTPUT, true); return mapper.writeValueAsString(control); } private ControlFile generateControlFile(UserPreferences prefs, String fileToPublish, PublishMethod publishMethod, String datasetId, boolean containsHeaderRow) throws HttpException, URISyntaxException, InterruptedException, IOException { - Dataset datasetInfo = DatasetUtils.getDatasetInfo(prefs, datasetId, Dataset.class); + Dataset datasetInfo = DatasetUtils.getDatasetInfo(prefs, datasetId); boolean useGeocoding = DatasetUtils.hasLocationColumn(datasetInfo); String[] columns = null; @@ -548,7 +560,7 @@ private ControlFile generateControlFile(UserPreferences prefs, String fileToPubl columns = DatasetUtils.getFieldNamesArray(datasetInfo); } - return ControlFile.generateControlFile(fileToPublish, publishMethod, columns, useGeocoding, containsHeaderRow); + return ControlFile.generateControlFile(fileToPublish, publishMethod, columns, useGeocoding, containsHeaderRow, userPrefs.getDefaultTimeFormats()); } } diff --git a/src/main/java/com/socrata/datasync/ui/MappingPanel.java b/src/main/java/com/socrata/datasync/ui/MappingPanel.java index 511e5bbc..ec061536 100644 --- a/src/main/java/com/socrata/datasync/ui/MappingPanel.java +++ b/src/main/java/com/socrata/datasync/ui/MappingPanel.java @@ -10,6 +10,8 @@ import java.awt.event.ItemEvent; import java.awt.event.ItemListener; import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; /** * Provides UI to allow the customer to map a single column in the CSV to a single field name in the dataset @@ -23,12 +25,34 @@ public class MappingPanel extends JPanel { JLabel CSVTitle = new JLabel(); JLabel CSVPreview = new JLabel(); int index; - JComboBox columnNamesComboBox; + + private static class FieldSelector { + public final String humanName; + public final String fieldName; + + public FieldSelector(String humanName, String fieldName) { + this.humanName = humanName; + this.fieldName = fieldName; + } + + @Override + public String toString() { + return humanName + " (" + fieldName + ")"; + } + } + + JComboBox columnNamesComboBox; JLabel arrow = new JLabel("" + '\u2192'); String lastSelection; ControlFileModel model; + Map fieldNameOrder; final int IGNORE_INDEX = 0; - final String ignoreField = "- Ignore this field -"; + final FieldSelector ignoreField = new FieldSelector("", "") { + @Override + public String toString() { + return "- Ignore this field -"; + } + }; public MappingPanel(int index, ControlFileModel model, DatasetModel datasetModel){ initializeValues(index, model, datasetModel); @@ -62,7 +86,7 @@ private void updateCombobox(){ if (model.isIgnored(selection)) columnNamesComboBox.setSelectedIndex(IGNORE_INDEX); else - columnNamesComboBox.setSelectedItem(selection); + columnNamesComboBox.setSelectedIndex(fieldNameOrder.get(selection) + 1); lastSelection = selection; } @@ -75,8 +99,10 @@ private void initializeValues(int index, ControlFileModel model, DatasetModel da columnNamesComboBox.addItem(ignoreField); ArrayList columns = datasetModel.getColumns(); + fieldNameOrder = new HashMap<>(); for (Column column : columns) { - columnNamesComboBox.addItem(column.getFieldName()); + fieldNameOrder.put(column.getFieldName(), fieldNameOrder.size()); + columnNamesComboBox.addItem(new FieldSelector(column.getName(), column.getFieldName())); } update(); @@ -98,8 +124,8 @@ private void styleComponents(){ private void layoutComponents(){ // this.setLayout(new BoxLayout(this,BoxLayout.X_AXIS)); this.setLayout(new BorderLayout()); - this.setMinimumSize(new Dimension(800,48)); - this.setMaximumSize(new Dimension(800,48)); + this.setMinimumSize(new Dimension(1000,48)); + this.setMaximumSize(new Dimension(1000,48)); CSVValuePreview.setLayout(new BoxLayout(CSVValuePreview,BoxLayout.Y_AXIS)); @@ -113,8 +139,8 @@ private void layoutComponents(){ CSVValuePreview.setMaximumSize(new Dimension(300, 48)); columnNamesComboBox.setBorder(BorderFactory.createEmptyBorder(5,5,10,5)); - columnNamesComboBox.setPreferredSize(new Dimension(200,48)); - columnNamesComboBox.setMaximumSize(new Dimension(200, 48)); + columnNamesComboBox.setPreferredSize(new Dimension(400, 48)); + columnNamesComboBox.setMaximumSize(new Dimension(400, 48)); this.add(CSVValuePreview,BorderLayout.WEST); this.add(arrow, BorderLayout.CENTER); @@ -138,8 +164,8 @@ public static JPanel getHeaderPanel(){ fieldLabel.setBorder(BorderFactory.createEmptyBorder(5,7,0,0)); - fieldLabel.setPreferredSize(new Dimension(200,16)); - fieldLabel.setMaximumSize(new Dimension(200, 16)); + fieldLabel.setPreferredSize(new Dimension(400,16)); + fieldLabel.setMaximumSize(new Dimension(400, 16)); header.add(csvLabel,BorderLayout.WEST); //header.add() header.add(fieldLabel, BorderLayout.EAST); @@ -154,13 +180,13 @@ private class MappingComboboxListener implements ItemListener public void itemStateChanged(ItemEvent e) { JComboBox box = (JComboBox) e.getSource(); if (e.getStateChange() == ItemEvent.SELECTED) { - String selectedItem = (String) box.getSelectedItem(); + FieldSelector selectedItem = (FieldSelector) box.getSelectedItem(); //Apparently there is a way to select a null item in Java? if (selectedItem != null) { if (selectedItem.equals(ignoreField)) { model.ignoreColumnInCSVAtPosition(index); } else { - model.updateColumnAtPosition(selectedItem, index); + model.updateColumnAtPosition(selectedItem.fieldName, index); } } } diff --git a/src/main/java/com/socrata/datasync/ui/MetadataJobTab.java b/src/main/java/com/socrata/datasync/ui/MetadataJobTab.java index 5a88dade..2d96c926 100644 --- a/src/main/java/com/socrata/datasync/ui/MetadataJobTab.java +++ b/src/main/java/com/socrata/datasync/ui/MetadataJobTab.java @@ -240,6 +240,7 @@ public void saveJob() { String selectedJobFileLocation = jobFileLocation; if(selectedJobFileLocation.equals("")) { JFileChooser savedJobFileChooser = new JFileChooser(); + savedJobFileChooser.setCurrentDirectory(new File(".")); FileNameExtensionFilter filter = new FileNameExtensionFilter( JOB_FILE_NAME + " (*." + JOB_FILE_EXTENSION + ")", JOB_FILE_EXTENSION); savedJobFileChooser.setFileFilter(filter); @@ -334,7 +335,7 @@ private void populateJobFromFields() { metadataJob.setDescription(descriptionTextArea.getText()); metadataJob.setCategory(categoryTextField.getText()); if (!StringUtils.isBlank(keywordsTextField.getText())) { - metadataJob.setKeywords(Arrays.asList(keywordsTextField.getText().split("\\s*,\\s*"))); + metadataJob.setKeywords(Arrays.asList(Utils.commaSplit(keywordsTextField.getText()))); } else { metadataJob.setKeywords(null); diff --git a/src/main/java/com/socrata/datasync/ui/PortJobTab.java b/src/main/java/com/socrata/datasync/ui/PortJobTab.java index afa688fd..bdd83176 100644 --- a/src/main/java/com/socrata/datasync/ui/PortJobTab.java +++ b/src/main/java/com/socrata/datasync/ui/PortJobTab.java @@ -43,9 +43,8 @@ public class PortJobTab implements JobTab { "Copy schema only: copies only the metadata and columns to a new dataset
" + "Copy schema and data: makes an identical copy to a new dataset
" + "Copy data only: copies only the rows to an existing dataset"; - private final String SOURCE_SITE_TIP_TEXT = "Domain where the source dataset is located."; + private final String SOURCE_SITE_TIP_TEXT = "Domain where the source dataset is located. The destination domain will be the one configured for authentication."; private final String SOURCE_SET_TIP_TEXT = "The xxxx-xxxx ID of the source dataset (e.g. n38h-y5wp)"; - private final String SINK_SITE_TIP_TEXT = "Domain where the destination dataset will be [or is] located."; private final String SINK_SET_TIP_TEXT = "If Port Method is 'copy schema' or 'copy schema and data' " + "this field will be populated with the xxxx-xxxx ID of the newly created dataset.
" + "If Port method is 'copy data only' enter the xxxx-xxxx ID of the existing dataset you wish to copy data to (e.g. n38h-y5wp)."; @@ -53,6 +52,8 @@ public class PortJobTab implements JobTab { private final String PUBLISH_DATASET_TIP_TEXT = "If Yes, publish the newly created destination dataset.
" + "If No, create it as an unpublished working copy."; + private final String NBE_TIP_TEXT = "Select the backend to create the target dataset on."; + private JFrame mainFrame; private JPanel jobPanel; @@ -62,7 +63,6 @@ public class PortJobTab implements JobTab { private JComboBox portMethodComboBox; private JTextField sourceSiteDomainTextField; private JTextField sourceSetIDTextField; - private JTextField sinkSiteDomainTextField; private JTextField sinkSetIDTextField; // Need to expose more of the JComponents locally in order to toggle between PublishMethod and PublishDataset @@ -72,10 +72,13 @@ public class PortJobTab implements JobTab { private JPanel publishDatasetContainerLeft; private JComboBox publishDatasetComboBox; private JPanel publishDatasetContainerRight; + private UserPreferences userPrefs; // build Container with all tab components and load data into form public PortJobTab(PortJob job, JFrame containingFrame) { + userPrefs = new UserPreferencesJava(); + mainFrame = containingFrame; // build tab panel form @@ -118,16 +121,6 @@ public PortJobTab(PortJob job, JFrame containingFrame) { sourceSetIDTextFieldContainer.add(sourceSetIDTextField); jobPanel.add(sourceSetIDTextFieldContainer); - // Sink Site - jobPanel.add(UIUtility.generateLabelWithHelpBubble( - "Destination Domain", SINK_SITE_TIP_TEXT, HELP_ICON_TOP_PADDING)); - JPanel sinkSiteTextFieldContainer = new JPanel(flowRight); - sinkSiteDomainTextField = new JTextField(); - sinkSiteDomainTextField.setPreferredSize(new Dimension( - JOB_TEXTFIELD_WIDTH, JOB_TEXTFIELD_HEIGHT)); - sinkSiteTextFieldContainer.add(sinkSiteDomainTextField); - jobPanel.add(sinkSiteTextFieldContainer); - // Sink Site Dataset ID jobPanel.add(UIUtility.generateLabelWithHelpBubble( "Destination Dataset ID", SINK_SET_TIP_TEXT, HELP_ICON_TOP_PADDING)); @@ -196,7 +189,6 @@ public PortJobTab(PortJob job, JFrame containingFrame) { jobPanel.add(publishMethodContainerRight); publishMethodComboBox.setEnabled(true); } - UserPreferences userPrefs = new UserPreferencesJava(); SocrataConnectionInfo connectionInfo = userPrefs.getConnectionInfo(); if (job.getSourceSiteDomain().equals("https://") && !connectionInfo.getUrl().equals("https://")) { @@ -205,12 +197,6 @@ public PortJobTab(PortJob job, JFrame containingFrame) { sourceSiteDomainTextField.setText(job.getSourceSiteDomain()); } sourceSetIDTextField.setText(job.getSourceSetID()); - if (job.getSinkSiteDomain().equals("https://") && - !connectionInfo.getUrl().equals("https://")) { - sinkSiteDomainTextField.setText(connectionInfo.getUrl()); - } else { - sinkSiteDomainTextField.setText(job.getSinkSiteDomain()); - } if (job.getSinkSetID().equals("") && !sinkSetIDTextField.isEditable()){ sinkSetIDTextField.setText(DEFAULT_DESTINATION_SET_ID); } else { @@ -235,7 +221,6 @@ public JobStatus runJobNow() { .getSelectedItem()); jobToRun.setSourceSiteDomain(sourceSiteDomainTextField.getText()); jobToRun.setSourceSetID(sourceSetIDTextField.getText()); - jobToRun.setSinkSiteDomain(sinkSiteDomainTextField.getText()); if (publishMethodComboBox.isEnabled()) { jobToRun.setPublishMethod((PublishMethod) publishMethodComboBox .getSelectedItem()); @@ -262,7 +247,6 @@ public void saveJob() { .getSelectedItem()); newPortJob.setSourceSiteDomain(sourceSiteDomainTextField.getText()); newPortJob.setSourceSetID(sourceSetIDTextField.getText()); - newPortJob.setSinkSiteDomain(sinkSiteDomainTextField.getText()); newPortJob.setSinkSetID(sinkSetIDTextField.getText()); newPortJob.setPublishMethod((PublishMethod) publishMethodComboBox .getSelectedItem()); @@ -278,6 +262,7 @@ public void saveJob() { String selectedJobFileLocation = jobFileLocation; if (selectedJobFileLocation.equals("")) { JFileChooser savedJobFileChooser = new JFileChooser(); + savedJobFileChooser.setCurrentDirectory(new File(".")); FileNameExtensionFilter filter = new FileNameExtensionFilter( JOB_FILE_NAME + " (*." + JOB_FILE_EXTENSION + ")", JOB_FILE_EXTENSION); @@ -319,7 +304,7 @@ public void saveJob() { public URI getURIToSinkDataset() { URI sinkDatasetURI = null; try { - sinkDatasetURI = new URI("https://" + DatasetUtils.getDomainWithoutScheme(sinkSiteDomainTextField.getText()) + "/d/" + sinkDatasetURI = new URI(userPrefs.getDomain() + "/d/" + sinkSetIDTextField.getText()); } catch (URISyntaxException uriE) { diff --git a/src/main/java/com/socrata/datasync/ui/SimpleIntegrationWizard.java b/src/main/java/com/socrata/datasync/ui/SimpleIntegrationWizard.java index 67374b46..e9f45832 100644 --- a/src/main/java/com/socrata/datasync/ui/SimpleIntegrationWizard.java +++ b/src/main/java/com/socrata/datasync/ui/SimpleIntegrationWizard.java @@ -1,907 +1,958 @@ -package com.socrata.datasync.ui; - -import java.awt.*; -import java.awt.event.*; - -import javax.swing.*; -import javax.swing.border.Border; -import javax.swing.filechooser.FileNameExtensionFilter; - -import com.socrata.datasync.*; -import com.socrata.datasync.job.IntegrationJob; -import com.socrata.datasync.job.Job; -import com.socrata.datasync.job.JobStatus; -import com.socrata.datasync.job.MetadataJob; -import com.socrata.datasync.job.PortJob; -import com.socrata.datasync.job.GISJob; -import com.socrata.datasync.config.userpreferences.UserPreferencesJava; - -import java.io.File; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; - -public class SimpleIntegrationWizard { - /** - * @author Adrian Laurenzi - * - * GUI interface to DataSync - */ - private static final String TITLE = "Socrata DataSync " + VersionProvider.getThisVersion(); - private static final String LOGO_FILE_PATH = "/datasync_logo.png"; - private static final String LOADING_SPINNER_FILE_PATH = "/loading_spinner.gif"; - private static final int FRAME_WIDTH = 800; - private static final int FRAME_HEIGHT = 550; - private static final Dimension JOB_PANEL_DIMENSION = new Dimension(780, 350); - private static final Dimension BUTTON_PANEL_DIMENSION = new Dimension(780, 40); - private static final int SSL_PORT_TEXTFIELD_HEIGHT = 26; - private static final int DEFAULT_TEXTFIELD_COLS = 25; - private static final Dimension AUTH_DETAILS_DIMENSION = new Dimension(465, 100); - private static final int PREFERENCES_FRAME_WIDTH = 475; - private static final int PREFERENCES_FRAME_HEIGHT = 675; - - private static UserPreferencesJava userPrefs; - - // TODO remove these declarations from this file (duplicates...) - private static final String STANDARD_JOB_FILE_EXTENSION = "sij"; - private static final String PORT_JOB_FILE_EXTENSION = "spj"; - private static final String METADATA_JOB_FILE_EXTENSION = "smj"; - private static final String GIS_JOB_FILE_EXTENSION = "gij"; - - // help icon balloon tip text - private static final String FILE_CHUNKING_THRESHOLD_TIP_TEXT = "If using the upsert, append, or " + - "delete methods (over HTTP) and the CSV/TSV file to be published is larger than this value (in megabytes), " + - "the file is automatically split up and published in chunks (because it is problematic to publish large files all at once). " + - "Usually chunking is necessary when a file is larger than about 64 MB."; - private static final String CHUNK_SIZE_THRESHOLD_TIP_TEXT = "The number of rows to publish in each chunk " + - "(in cases where filesize exceeds above filesize threshold). Higher values usually means faster upload time but setting the value too " + - "high could crash the program, depending on your computer's memory limits."; - private static final String DOMAIN_TIP_TEXT = "The domain of the Socrata data site you wish to publish data to (e.g. https://explore.data.gov/)"; - private static final String USERNAME_TIP_TEXT = "Socrata account username (account must have Publisher or Administrator permissions)"; - private static final String PASSWORD_TIP_TEXT = "Socrata account password"; - private static final String APP_TOKEN_TIP_TEXT = "You can create an app token free at http://dev.socrata.com/register"; - private static final String RUN_JOB_NOW_TIP_TEXT = "" + - "To view detailed logging information run the job by copying the" + - " 'Command to execute with scheduler' and running it in your Terminal/Command Prompt (instead of clicking 'Run Job Now' button)"; - - private static final String QUICK_START_GUIDE = "http://socrata.github.io/datasync/guides/quick-start.html"; - private static final String PORTING_GUIDE = "http://socrata.github.io/datasync/guides/setup-port-job.html"; - private static final String HEADLESS_GUIDE_URL = "http://socrata.github.io/datasync/guides/setup-standard-job-headless.html"; - private static final String CONTROL_GUIDE_URL = "http://socrata.github.io/datasync/resources/control-config.html"; - private static final String SCHEDULING_GUIDE_URL = "http://socrata.github.io/datasync/resources/schedule-job.html"; - private static final String FAQ_URL = "http://socrata.github.io/datasync/resources/faq-common-problems.html"; - - private JTextField domainTextField, usernameTextField, apiKeyTextField; - private JPasswordField passwordField; - private JTextField filesizeChunkingCutoffTextField, numRowsPerChunkTextField; - private JTextField logDatasetIDTextField, adminEmailTextField; - private JTextField outgoingMailServerTextField, smtpPortTextField, sslPortTextField, smtpUsernameTextField; - private JTextField proxyHostTextField, proxyPortTextField, proxyUsernameTextField, proxyPasswordTextField; - private JPasswordField smtpPasswordField; - private JCheckBox useSSLCheckBox; - private JCheckBox emailUponErrorCheckBox; - - /** - * Stores a list of open JobTabs. Each JobTab object contains - * the UI content of a single job tab. The indices of the - * tabs within jobTabsPane correspond to the indices of the - * JobTab objects (holding the UI for each tab) within this - * list. - * - * *IMPORTANT*: only modify this list in the 'addJobTab' and - * 'closeJobTab' methods - */ - private List jobTabs; - - private JTabbedPane jobTabsPane; - private JFrame frame; - private JFrame prefsFrame; - private JPanel loadingNoticePanel; - private JButton runJobNowButton; - - /* - * Constructs the GUI and displays it on the screen. - */ - public SimpleIntegrationWizard() { - // load user preferences (saved locally) - userPrefs = new UserPreferencesJava(); - - // Build GUI - frame = new JFrame(TITLE); - frame.setSize(FRAME_WIDTH, FRAME_HEIGHT); - - jobTabs = new ArrayList<>(); - // save tabs on close - frame.setDefaultCloseOperation(JFrame.DO_NOTHING_ON_CLOSE); - frame.addWindowListener(new WindowAdapter() { - @Override - public void windowClosing(WindowEvent we) { - saveAuthenticationInfoFromForm(); - // TODO save open tabs to userPrefs - System.exit(0); - } - }); - - JMenuBar menuBar = generateMenuBar(); - frame.setJMenuBar(menuBar); - - JPanel mainPanel = generateMainPanel(); - loadAuthenticationInfoIntoForm(); - generatePreferencesFrame(); - - frame.add(mainPanel); - - frame.pack(); - // centers the window - frame.setLocationRelativeTo(null); - frame.setVisible(true); - - // Alert user if new version is available - try { - checkVersion(); - } catch (Exception e) { - // do nothing upon failure - } - } - - private void generatePreferencesFrame() { - prefsFrame = new JFrame("Preferences"); - prefsFrame.setSize(PREFERENCES_FRAME_WIDTH, PREFERENCES_FRAME_HEIGHT); - prefsFrame.setVisible(false); - JPanel preferencesPanel = generatePreferencesPanel(); - prefsFrame.add(preferencesPanel); - prefsFrame.setDefaultCloseOperation(JFrame.DO_NOTHING_ON_CLOSE); - prefsFrame.addWindowListener(new WindowAdapter() { - @Override - public void windowClosing(WindowEvent we) { - prefsFrame.setVisible(false); - } - }); - } - - /** - * Queries github for the most recent release. If query is successful - * and the major version of the user's datasync is outdated, alerts - * that a new version is available - * - * @throws URISyntaxException - */ - private void checkVersion() throws URISyntaxException { - if(VersionProvider.isLatestMajorVersion() == VersionProvider.VersionStatus.NOT_LATEST) { - String currentVersion = VersionProvider.getLatestVersion(); - if (currentVersion != null) { - Object[] options = {"Download Now", "No Thanks"}; - int n = JOptionPane.showOptionDialog(frame, - "A new version of DataSync is available (version " + currentVersion + ").\n" + - "Do you want to download it now?\n", - "Alert: New Version Available", - JOptionPane.YES_NO_OPTION, - JOptionPane.QUESTION_MESSAGE, - null, options, options[0] - ); - if (n == JOptionPane.YES_OPTION) { - URI currentVersionDownloadURI = new URI(VersionProvider.getDownloadUrlForLatestVersion()); - if (currentVersionDownloadURI != null) - Utils.openWebpage(currentVersionDownloadURI); - } - } - } - } - - private void addJobTab(Job job) throws IllegalArgumentException { - JobTab newJobTab; - if(job.getClass().equals(IntegrationJob.class)) { - newJobTab = new IntegrationJobTab((IntegrationJob) job, frame); - } else if(job.getClass().equals(PortJob.class)) { - newJobTab = new PortJobTab((PortJob) job, frame); - } else if(job.getClass().equals(GISJob.class)) { - newJobTab = new GISJobTab((GISJob) job, frame); - } else if(job.getClass().equals(MetadataJob.class)) { - newJobTab = new MetadataJobTab((MetadataJob) job, frame); - } else { - throw new IllegalArgumentException("Given job is invalid: unrecognized class '" + job.getClass() + "'"); - } - JPanel newJobPanel = newJobTab.getTabPanel(); - - // Build the tab with close button - FlowLayout tabLayout = new FlowLayout(FlowLayout.CENTER, 5, 0); - JPanel tabPanel = new JPanel(tabLayout); - tabPanel.setOpaque(false); - - // Create a JButton for the close tab button - JButton closeTabButton = new AwesomeButton("cross41"); - closeTabButton.setBorder(null); - closeTabButton.setFocusable(false); - - tabPanel.add(newJobTab.getJobTabTitleLabel()); - tabPanel.add(closeTabButton); - // Add a thin border to keep the image below the top edge of the tab when the tab is selected - tabPanel.setBorder(BorderFactory.createEmptyBorder(2, 0, 0, 0)); - - // Put tab with close button into tabbed pane - //TODO: BW: Possibly implement way to keep other tabs from being scrollable? - JScrollPane newJobScrollPanel = new JScrollPane(newJobPanel); - newJobScrollPanel.setHorizontalScrollBarPolicy(JScrollPane.HORIZONTAL_SCROLLBAR_NEVER); - jobTabsPane.addTab(null, newJobScrollPanel); - int pos = jobTabsPane.indexOfComponent(newJobScrollPanel); - - // Now assign the component for the tab - jobTabsPane.setTabComponentAt(pos, tabPanel); - - closeTabButton.addActionListener(new CloseJobFromTabListener()); - - jobTabs.add(newJobTab); - assert(jobTabsValid()); - } - - private void closeJobTab(int tabIndex) throws IllegalArgumentException { - if(tabIndex >= jobTabsPane.getTabCount()) { - throw new IllegalArgumentException("Tab index is invalid"); - } - jobTabsPane.remove(tabIndex); - jobTabs.remove(tabIndex); - assert(jobTabsValid()); - } - - private class RunJobNowListener implements ActionListener { - public void actionPerformed(ActionEvent e) { - saveAuthenticationInfoFromForm(); - - // run integration job with data from form - int selectedJobTabIndex = jobTabsPane.getSelectedIndex(); - JobTab selectedJobTab = jobTabs.get(selectedJobTabIndex); - - SwingWorker jobWorker = new RunJobWorker(selectedJobTab); - jobWorker.execute(); - } - } - - private class RunJobWorker extends SwingWorker { - private JobTab jobTabToRun; - private JobStatus jobStatus; - - public RunJobWorker(JobTab jobTabToRun){ - loadingNoticePanel.setVisible(true); - runJobNowButton.setEnabled(false); - this.jobTabToRun = jobTabToRun; - } - - @Override - protected Void doInBackground() { - try { - jobStatus = jobTabToRun.runJobNow(); - } catch (OutOfMemoryError err) { - jobStatus = JobStatus.PUBLISH_ERROR; - jobStatus.setMessage("Error: ran out of memory " + - "(try decreasing the chunking size and/or threshold by going to Edit -> Preferences)"); - } catch (Exception e) { - e.printStackTrace(); - jobStatus = JobStatus.PUBLISH_ERROR; - jobStatus.setMessage("Unexpected error: " + e); - } - return null; - } - - //Executed on the Event Dispatch Thread after the doInBackground method is finished - @Override - protected void done() { - loadingNoticePanel.setVisible(false); - runJobNowButton.setEnabled(true); - - // show popup with returned status - if(jobStatus.isError()) { - JOptionPane.showMessageDialog(frame, "Job completed with errors: " + jobStatus.getMessage()); - } else { - if (jobTabToRun.getClass().equals(PortJobTab.class)) { - PortJobTab selectedPortJobTab = (PortJobTab) jobTabToRun; - Object[] options = {"Yes", "No"}; - int n = JOptionPane.showOptionDialog(frame, - "Port job completed successfully. Would you like to open the destination dataset?\n", - "Port Job Successful", - JOptionPane.YES_NO_OPTION, - JOptionPane.QUESTION_MESSAGE, - null, options, options[0]); - if (n == JOptionPane.YES_OPTION) { - Utils.openWebpage(selectedPortJobTab.getURIToSinkDataset()); - } - } else { - JOptionPane.showMessageDialog(frame, "Job completed successfully. " + jobStatus.getMessage()); - } - } - } - } - - private class SaveJobListener implements ActionListener { - public void actionPerformed(ActionEvent e) { - saveAuthenticationInfoFromForm(); - int selectedJobTabIndex = jobTabsPane.getSelectedIndex(); - JobTab selectedJobTab = jobTabs.get(selectedJobTabIndex); - selectedJobTab.saveJob(); - } - } - - private File openToDirectory = new File("."); - private class OpenJobListener implements ActionListener { - public void actionPerformed(ActionEvent e) { - JFileChooser savedJobFileChooser = new JFileChooser(); - savedJobFileChooser.setCurrentDirectory(openToDirectory); - String fileExtensionsAllowed = "*." + STANDARD_JOB_FILE_EXTENSION + ", *." + PORT_JOB_FILE_EXTENSION + ", *."+ GIS_JOB_FILE_EXTENSION + ", *." + METADATA_JOB_FILE_EXTENSION; - FileNameExtensionFilter filter = new FileNameExtensionFilter( - "Socrata Job File (" + fileExtensionsAllowed + ")", - STANDARD_JOB_FILE_EXTENSION, PORT_JOB_FILE_EXTENSION, GIS_JOB_FILE_EXTENSION, METADATA_JOB_FILE_EXTENSION); - savedJobFileChooser.setFileFilter(filter); - int returnVal = savedJobFileChooser.showOpenDialog(frame); - if (returnVal == JFileChooser.APPROVE_OPTION) { - File openedFile = savedJobFileChooser.getSelectedFile(); - // Ensure file exists - if(openedFile.exists()) { - openToDirectory = savedJobFileChooser.getCurrentDirectory(); - // ensure this job is not already open - String openedFileLocation = openedFile.getAbsolutePath(); - int indexOfAlreadyOpenFile = -1; - for(int curTabIndex = 0; curTabIndex < jobTabs.size(); curTabIndex++) { - String curTabJobFileLocation = jobTabs.get(curTabIndex).getJobFileLocation(); - if(curTabJobFileLocation.equals(openedFileLocation)) { - indexOfAlreadyOpenFile = curTabIndex; - break; - } - } - if(indexOfAlreadyOpenFile == -1) { - try { - int i = openedFileLocation.lastIndexOf('.'); - if (i > 0) { - String openedFileExtension = openedFileLocation.substring(i+1); - if(openedFileExtension.equals(STANDARD_JOB_FILE_EXTENSION)) { - addJobTab(new IntegrationJob(openedFileLocation)); - } else if(openedFileExtension.equals(GIS_JOB_FILE_EXTENSION)){ - addJobTab(new GISJob(openedFileLocation)); - } else if(openedFileExtension.equals(PORT_JOB_FILE_EXTENSION)) { - addJobTab(new PortJob(openedFileLocation)); - } else if (openedFileExtension.equals(METADATA_JOB_FILE_EXTENSION)) { - addJobTab(new MetadataJob(openedFileLocation)); - } else { - throw new Exception("unrecognized file extension (" + openedFileExtension + ")"); - } - } - // Switch to opened file's tab - jobTabsPane.setSelectedIndex(jobTabsPane.getTabCount() - 1); - } catch(IntegrationJob.ControlDisagreementException ex) { - JOptionPane.showMessageDialog(frame, "Warning: \n" + ex.getMessage() + " found in \n'" + - openedFileLocation + "'. \nLoading job, but please confirm the contents of your control file are accurate."); - try { - addJobTab(new IntegrationJob(openedFileLocation, true)); - jobTabsPane.setSelectedIndex(jobTabsPane.getTabCount() - 1); - } catch(Exception e2) { - JOptionPane.showMessageDialog(frame, "Error opening " + openedFileLocation + ": " + e2.toString()); - } - } catch(Exception e2) { - JOptionPane.showMessageDialog(frame, "Error opening " + openedFileLocation + ": " + e2.toString()); - } - } else { - // file already open, select that tab - jobTabsPane.setSelectedIndex(indexOfAlreadyOpenFile); - } - } - } - } - } - - private class AuthenticationDetailsFocusListener implements FocusListener { - @Override - public void focusGained(FocusEvent e) { } - @Override - public void focusLost(FocusEvent e) { - saveAuthenticationInfoFromForm(); - } - } - - private class NewStandardJobListener implements ActionListener { - public void actionPerformed(ActionEvent e) { - addJobTab(getNewIntegrationJob()); - jobTabsPane.setSelectedIndex(jobTabsPane.getTabCount() - 1); - } - } - - private IntegrationJob getNewIntegrationJob() { - IntegrationJob newJob = new IntegrationJob(); - // set publishViaDi2Http to true as default (ONLY for GUI mode) - newJob.setPublishViaDi2Http(true); - return newJob; - } - - private class NewPortJobListener implements ActionListener { - public void actionPerformed(ActionEvent e) { - addJobTab(new PortJob()); - jobTabsPane.setSelectedIndex(jobTabsPane.getTabCount() - 1); - } - } - - private class NewGISJobListener implements ActionListener { - public void actionPerformed(ActionEvent e) { - addJobTab(new GISJob()); - jobTabsPane.setSelectedIndex(jobTabsPane.getTabCount() - 1); - } - } - - private class NewMetadataJobListener implements ActionListener { - public void actionPerformed(ActionEvent e) { - addJobTab(new MetadataJob()); - jobTabsPane.setSelectedIndex(jobTabsPane.getTabCount() - 1); - } - } - - /** - * Listen for action to close currently selected tab - */ - private class CloseJobFromTabListener implements ActionListener { - public void actionPerformed(ActionEvent e) { - JComponent source = (JComponent) e.getSource(); - Container tabComponent = source.getParent(); - int jobTabIndex = jobTabsPane.indexOfTabComponent(tabComponent); - closeJobTab(jobTabIndex); - } - } - - private JMenuBar generateMenuBar() { - JMenuBar menuBar = new JMenuBar(); - - JMenu fileMenu = new JMenu("File"); - menuBar.add(fileMenu); - - JMenu newJobMenu = new JMenu("New..."); - JMenuItem newStandardJobItem = new JMenuItem("Standard Job"); - newJobMenu.add(newStandardJobItem); - JMenuItem newPortJobItem = new JMenuItem("Port Job"); - newJobMenu.add(newPortJobItem); - JMenuItem newGISJobItem = new JMenuItem("GIS Job"); - newJobMenu.add(newGISJobItem); - JMenuItem newMetadataJobItem = new JMenuItem("Metadata Job (beta)"); - newJobMenu.add(newMetadataJobItem); - fileMenu.add(newJobMenu); - - JMenuItem openJobItem = new JMenuItem("Open Job"); - openJobItem.setAccelerator(KeyStroke.getKeyStroke( - KeyEvent.VK_O, ActionEvent.CTRL_MASK)); - fileMenu.add(openJobItem); - - JMenuItem saveJobItem = new JMenuItem("Save Job"); - saveJobItem.setAccelerator(KeyStroke.getKeyStroke( - KeyEvent.VK_S, ActionEvent.CTRL_MASK)); - fileMenu.add(saveJobItem); - - JMenuItem runJobItem = new JMenuItem("Run Job Now"); - runJobItem.setAccelerator(KeyStroke.getKeyStroke( - KeyEvent.VK_R, ActionEvent.CTRL_MASK)); - fileMenu.add(runJobItem); - JMenuItem prefsItem = new JMenuItem("Preferences"); - fileMenu.add(prefsItem); - - JMenu helpMenu = new JMenu("Help"); - menuBar.add(helpMenu); - JMenuItem gettingStartedGuideItem = new JMenuItem("Quick start guide"); - JMenuItem portingGuideItem = new JMenuItem("Port job guide"); - JMenuItem headlessDocumentationItem = new JMenuItem("Running in headless mode"); - JMenuItem controlDocumentationItem = new JMenuItem("Control file configuration"); - JMenuItem schedulingItem = new JMenuItem("Scheduling a job"); - JMenuItem faqItem = new JMenuItem("FAQ"); - helpMenu.add(gettingStartedGuideItem); - helpMenu.add(portingGuideItem); - helpMenu.add(headlessDocumentationItem); - helpMenu.add(controlDocumentationItem); - helpMenu.add(schedulingItem); - helpMenu.add(faqItem); - - newStandardJobItem.addActionListener(new NewStandardJobListener()); - newPortJobItem.addActionListener(new NewPortJobListener()); - newGISJobItem.addActionListener(new NewGISJobListener()); - newMetadataJobItem.addActionListener(new NewMetadataJobListener()); - openJobItem.addActionListener(new OpenJobListener()); - saveJobItem.addActionListener(new SaveJobListener()); - runJobItem.addActionListener(new RunJobNowListener()); - prefsItem.addActionListener(new OpenPreferencesListener()); - gettingStartedGuideItem.addActionListener(new ActionListener() { - @Override - public void actionPerformed(ActionEvent e) { - try { - Utils.openWebpage(new URI(QUICK_START_GUIDE)); - } catch (URISyntaxException e1) { e1.printStackTrace(); } - } - }); - portingGuideItem.addActionListener(new ActionListener() { - @Override - public void actionPerformed(ActionEvent e) { - try { - Utils.openWebpage(new URI(PORTING_GUIDE)); - } catch (URISyntaxException e1) { e1.printStackTrace(); } - } - }); - headlessDocumentationItem.addActionListener(new ActionListener() { - @Override - public void actionPerformed(ActionEvent e) { - try { - Utils.openWebpage(new URI(HEADLESS_GUIDE_URL)); - } catch (URISyntaxException e1) { e1.printStackTrace(); } - } - }); - controlDocumentationItem.addActionListener(new ActionListener() { - @Override - public void actionPerformed(ActionEvent e) { - try { - Utils.openWebpage(new URI(CONTROL_GUIDE_URL)); - } catch (URISyntaxException e1) { e1.printStackTrace(); } - } - }); - schedulingItem.addActionListener(new ActionListener() { - @Override - public void actionPerformed(ActionEvent e) { - try { - Utils.openWebpage(new URI(SCHEDULING_GUIDE_URL)); - } catch (URISyntaxException e1) { e1.printStackTrace(); } - } - }); - faqItem.addActionListener(new ActionListener() { - @Override - public void actionPerformed(ActionEvent e) { - try { - Utils.openWebpage(new URI(FAQ_URL)); - } catch (URISyntaxException e1) { e1.printStackTrace(); } - } - }); - - return menuBar; - } - - private JPanel generateMainPanel() { - JPanel mainContainer = new JPanel(new FlowLayout(FlowLayout.LEFT)); - mainContainer.setPreferredSize(new Dimension(FRAME_WIDTH, FRAME_HEIGHT)); - - // Build empty job tabbed pane - JPanel jobTabsContainer = new JPanel(new GridLayout(1, 1)); - jobTabsContainer.setPreferredSize(JOB_PANEL_DIMENSION); - jobTabsPane = new JTabbedPane(); - jobTabsContainer.add(jobTabsPane); - jobTabsPane.setTabLayoutPolicy(JTabbedPane.SCROLL_TAB_LAYOUT); - - JPanel jobButtonContainer = new JPanel(new GridLayout(1,2)); - JPanel leftButtonPanel = new JPanel(new FlowLayout(FlowLayout.LEFT)); - runJobNowButton = new JButton("Run Job Now"); - runJobNowButton.addActionListener(new RunJobNowListener()); - leftButtonPanel.add(runJobNowButton); - leftButtonPanel.add(UIUtility.generateHelpBubble(RUN_JOB_NOW_TIP_TEXT)); - - generateLoadingNotice(); - leftButtonPanel.add(loadingNoticePanel); - - JPanel rightButtonPanel = new JPanel(new FlowLayout(FlowLayout.RIGHT)); - JButton saveJobButton = new JButton("Save Job"); - saveJobButton.addActionListener(new SaveJobListener()); - rightButtonPanel.add(saveJobButton); - jobButtonContainer.add(leftButtonPanel); - jobButtonContainer.add(rightButtonPanel); - - jobButtonContainer.setPreferredSize(BUTTON_PANEL_DIMENSION); - mainContainer.add(jobTabsContainer); - mainContainer.add(jobButtonContainer); - - JPanel authenticationDetailsPanel = generateAuthenticationDetailsFormPanel(); - mainContainer.add(authenticationDetailsPanel); - - URL logoImageURL = getClass().getResource(LOGO_FILE_PATH); - if(logoImageURL != null) { - JLabel logoLabel = new JLabel(new ImageIcon(logoImageURL)); - Border paddingBorder = BorderFactory.createEmptyBorder(15,15,15,15); - logoLabel.setBorder(paddingBorder); - mainContainer.add(logoLabel); - } - - // TODO populate job tabs w/ previously opened tabs or [if none] a new job tab - - addJobTab(getNewIntegrationJob()); - return mainContainer; - } - - private void generateLoadingNotice() { - loadingNoticePanel = new JPanel(new FlowLayout(FlowLayout.LEFT)); - URL spinnerImageURL = getClass().getResource(LOADING_SPINNER_FILE_PATH); - if(spinnerImageURL != null) { - JLabel loadingImageLabel = new JLabel(new ImageIcon(spinnerImageURL)); - loadingNoticePanel.add(loadingImageLabel); - } - JLabel loadingTextLabel = new JLabel(" Job is in progress..."); - loadingNoticePanel.add(loadingTextLabel); - loadingNoticePanel.setVisible(false); - } - - private JPanel generatePreferencesPanel() { - JPanel prefsPanel = new JPanel(new GridLayout(0,2)); - - // File chunking settings - JLabel fileChunkingSettingsLabel = new JLabel(" File Chunking Settings"); - Font boldFont = new Font(fileChunkingSettingsLabel.getFont().getFontName(), - Font.BOLD, fileChunkingSettingsLabel.getFont().getSize()); - fileChunkingSettingsLabel.setFont(boldFont); - prefsPanel.add(fileChunkingSettingsLabel); - prefsPanel.add(new JLabel("")); - - prefsPanel.add( - UIUtility.generateLabelWithHelpBubble(" Chunking filesize threshold", FILE_CHUNKING_THRESHOLD_TIP_TEXT)); - JPanel filesizeChuckingContainer = new JPanel(new FlowLayout(FlowLayout.LEFT)); - filesizeChunkingCutoffTextField = new JTextField(); - filesizeChunkingCutoffTextField.setPreferredSize(new Dimension( - 140, 20)); - filesizeChuckingContainer.add(filesizeChunkingCutoffTextField); - filesizeChuckingContainer.add(new JLabel(" MB")); - prefsPanel.add(filesizeChuckingContainer); - - prefsPanel.add( - UIUtility.generateLabelWithHelpBubble(" Chunk size", CHUNK_SIZE_THRESHOLD_TIP_TEXT)); - JPanel chunkSizeContainer = new JPanel(new FlowLayout(FlowLayout.LEFT)); - numRowsPerChunkTextField = new JTextField(); - numRowsPerChunkTextField.setPreferredSize(new Dimension( - 140, 20)); - chunkSizeContainer.add(numRowsPerChunkTextField); - chunkSizeContainer.add(new JLabel(" rows")); - prefsPanel.add(chunkSizeContainer); - - // Logging and auto-email settings - JLabel loggingAutoEmailSettingsLabel = new JLabel(" Logging and Auto-Email Settings"); - loggingAutoEmailSettingsLabel.setFont(boldFont); - prefsPanel.add(loggingAutoEmailSettingsLabel); - prefsPanel.add(new JLabel("")); - - prefsPanel.add(new JLabel(" Log Dataset ID (e.g., n38h-y5wp)")); - logDatasetIDTextField = new JTextField(DEFAULT_TEXTFIELD_COLS); - prefsPanel.add(logDatasetIDTextField); - - prefsPanel.add(new JLabel(" Admin Email")); - adminEmailTextField = new JTextField(DEFAULT_TEXTFIELD_COLS); - prefsPanel.add(adminEmailTextField); - - emailUponErrorCheckBox = new JCheckBox(" Auto-email admin upon error"); - prefsPanel.add(emailUponErrorCheckBox); - prefsPanel.add(new JLabel("*must fill in SMTP Settings below")); - - // Auto-email SMTP settings - JLabel smtpSettingsLabel = new JLabel(" SMTP Settings"); - smtpSettingsLabel.setFont(boldFont); - prefsPanel.add(smtpSettingsLabel); - prefsPanel.add(new JLabel("")); - - prefsPanel.add(new JLabel(" Outgoing Mail Server")); - outgoingMailServerTextField = new JTextField(DEFAULT_TEXTFIELD_COLS); - prefsPanel.add(outgoingMailServerTextField); - prefsPanel.add(new JLabel(" SMTP Port")); - smtpPortTextField = new JTextField(DEFAULT_TEXTFIELD_COLS); - prefsPanel.add(smtpPortTextField); - final JPanel sslPortContainer = new JPanel( - new FlowLayout(FlowLayout.LEFT, 0, 0)); - sslPortContainer.setVisible(false); - useSSLCheckBox = new JCheckBox("Use SSL"); - prefsPanel.add(useSSLCheckBox); - sslPortContainer.add(new JLabel(" SSL Port ")); - sslPortTextField = new JTextField(); - sslPortTextField.setPreferredSize(new Dimension( - 50, SSL_PORT_TEXTFIELD_HEIGHT)); - sslPortContainer.add(sslPortTextField); - useSSLCheckBox.addItemListener(new ItemListener() { - public void itemStateChanged(ItemEvent e) { - if(useSSLCheckBox.isSelected()) { - sslPortContainer.setVisible(true); - } else { - sslPortContainer.setVisible(false); - } - } - }); - prefsPanel.add(sslPortContainer); - prefsPanel.add(new JLabel(" SMTP Username")); - smtpUsernameTextField = new JTextField(DEFAULT_TEXTFIELD_COLS); - prefsPanel.add(smtpUsernameTextField); - prefsPanel.add(new JLabel(" SMTP Password")); - smtpPasswordField = new JPasswordField(DEFAULT_TEXTFIELD_COLS); - prefsPanel.add(smtpPasswordField); - - - // Proxy settings - JLabel proxySettingsLabel = new JLabel(" Proxy Settings"); - proxySettingsLabel.setFont(boldFont); - prefsPanel.add(proxySettingsLabel); - prefsPanel.add(new JLabel("")); - - prefsPanel.add(new JLabel(" Proxy Host")); - proxyHostTextField = new JTextField(DEFAULT_TEXTFIELD_COLS); - prefsPanel.add(proxyHostTextField); - prefsPanel.add(new JLabel(" Proxy Port")); - proxyPortTextField = new JTextField(DEFAULT_TEXTFIELD_COLS); - prefsPanel.add(proxyPortTextField); - prefsPanel.add(new JLabel(" Proxy Username")); - proxyUsernameTextField = new JTextField(DEFAULT_TEXTFIELD_COLS); - prefsPanel.add(proxyUsernameTextField); - prefsPanel.add(new JLabel(" Proxy Password")); - proxyPasswordTextField = new JPasswordField(DEFAULT_TEXTFIELD_COLS); - prefsPanel.add(proxyPasswordTextField); - - JPanel testSMTPSettingsContainer = new JPanel(new FlowLayout(FlowLayout.LEFT)); - JButton testSMTPSettingsButton = new JButton("Test SMTP Settings"); - testSMTPSettingsButton.addActionListener(new TestSMTPSettingsListener()); - testSMTPSettingsContainer.add(testSMTPSettingsButton); - prefsPanel.add(testSMTPSettingsContainer); - - JPanel prefsButtonContainer = new JPanel(new FlowLayout(FlowLayout.RIGHT)); - JButton cancelPrefsButton = new JButton("Cancel"); - cancelPrefsButton.addActionListener(new CancelPreferencesListener()); - prefsButtonContainer.add(cancelPrefsButton); - JButton savePrefsButton = new JButton("Save"); - savePrefsButton.addActionListener(new SavePreferencesListener()); - prefsButtonContainer.add(savePrefsButton); - prefsPanel.add(prefsButtonContainer); - - loadPreferencesIntoForm(); - - return prefsPanel; - } - - private void loadPreferencesIntoForm() { - filesizeChunkingCutoffTextField.setText(userPrefs.getFilesizeChunkingCutoffMB()); - numRowsPerChunkTextField.setText(userPrefs.getNumRowsPerChunk()); - - adminEmailTextField.setText(userPrefs.getAdminEmail()); - logDatasetIDTextField.setText(userPrefs.getLogDatasetID()); - emailUponErrorCheckBox.setSelected(userPrefs.emailUponError()); - - outgoingMailServerTextField.setText(userPrefs.getOutgoingMailServer()); - smtpPortTextField.setText(userPrefs.getSmtpPort()); - String sslPort = userPrefs.getSslPort(); - sslPortTextField.setText(sslPort); - if(sslPort.equals("")) { - useSSLCheckBox.setSelected(false); - } else { - useSSLCheckBox.setSelected(true); - } - smtpUsernameTextField.setText(userPrefs.getSmtpUsername()); - smtpPasswordField.setText(userPrefs.getSmtpPassword()); - - proxyHostTextField.setText(userPrefs.getProxyHost()); - proxyPortTextField.setText(userPrefs.getProxyPort()); - proxyUsernameTextField.setText(userPrefs.getProxyUsername()); - proxyPasswordTextField.setText(userPrefs.getProxyPassword()); - - } - - private void savePreferences() { - try { - userPrefs.saveFilesizeChunkingCutoffMB( - Integer.parseInt(filesizeChunkingCutoffTextField.getText())); - } catch(NumberFormatException e) { - JOptionPane.showMessageDialog(prefsFrame, "Invalid chunking filesize threshold: must be an integer"); - } - try { - userPrefs.saveNumRowsPerChunk( - Integer.parseInt(numRowsPerChunkTextField.getText())); - } catch(NumberFormatException e) { - JOptionPane.showMessageDialog(prefsFrame, "Invalid chunk size: must be an integer"); - } - - userPrefs.saveAdminEmail(adminEmailTextField.getText()); - userPrefs.saveLogDatasetID(logDatasetIDTextField.getText()); - userPrefs.saveEmailUponError(emailUponErrorCheckBox.isSelected()); - - userPrefs.saveOutgoingMailServer(outgoingMailServerTextField.getText()); - userPrefs.saveSMTPPort(smtpPortTextField.getText()); - if(useSSLCheckBox.isSelected()) { - userPrefs.saveSSLPort(sslPortTextField.getText()); - } else { - userPrefs.saveSSLPort(""); - } - userPrefs.saveSMTPUsername(smtpUsernameTextField.getText()); - String smtpPassword = new String(smtpPasswordField.getPassword()); - userPrefs.saveSMTPPassword(smtpPassword); - - userPrefs.saveProxyHost(proxyHostTextField.getText()); - userPrefs.saveProxyPort(proxyPortTextField.getText()); - userPrefs.saveProxyUsername(proxyUsernameTextField.getText()); - userPrefs.saveProxyPassword(proxyPasswordTextField.getText()); - - } - - private class SavePreferencesListener implements ActionListener { - public void actionPerformed(ActionEvent e) { - // TODO validation of email and log dataset ID - savePreferences(); - prefsFrame.setVisible(false); - } - } - - private class CancelPreferencesListener implements ActionListener { - public void actionPerformed(ActionEvent e) { - loadPreferencesIntoForm(); - prefsFrame.setVisible(false); - } - } - - private class TestSMTPSettingsListener implements ActionListener { - public void actionPerformed(ActionEvent e) { - savePreferences(); - String adminEmail = userPrefs.getAdminEmail(); - String message; - try { - SMTPMailer.send(adminEmail, "Socrata DataSync Test Email", - "This email confirms that your SMTP Settings are valid."); - message = "Sent test email to " + adminEmail + ". Please ensure the " - + "email was delievered successfully (it may take a few minutes)."; - } catch (Exception emailE) { - message = "Error sending email to " + adminEmail + ":\n" + emailE.getMessage(); - } - JOptionPane.showMessageDialog(prefsFrame, message); - } - } - - private class OpenPreferencesListener implements ActionListener { - public void actionPerformed(ActionEvent e) { - // centers the window - prefsFrame.setLocationRelativeTo(null); - prefsFrame.setVisible(true); - } - } - - private JPanel generateAuthenticationDetailsFormPanel() { - JPanel authenticationDetailsPanel = new JPanel(new GridLayout(0,2)); - - authenticationDetailsPanel.add( - UIUtility.generateLabelWithHelpBubble("Domain", DOMAIN_TIP_TEXT)); - domainTextField = new JTextField(DEFAULT_TEXTFIELD_COLS); - authenticationDetailsPanel.add(domainTextField); - authenticationDetailsPanel.add( - UIUtility.generateLabelWithHelpBubble("Username", USERNAME_TIP_TEXT)); - usernameTextField = new JTextField(DEFAULT_TEXTFIELD_COLS); - authenticationDetailsPanel.add(usernameTextField); - authenticationDetailsPanel.add( - UIUtility.generateLabelWithHelpBubble("Password", PASSWORD_TIP_TEXT)); - passwordField = new JPasswordField(DEFAULT_TEXTFIELD_COLS); - authenticationDetailsPanel.add(passwordField); - authenticationDetailsPanel.add( - UIUtility.generateLabelWithHelpBubble("App Token", APP_TOKEN_TIP_TEXT)); - apiKeyTextField = new JTextField(DEFAULT_TEXTFIELD_COLS); - authenticationDetailsPanel.add(apiKeyTextField); - authenticationDetailsPanel.setPreferredSize(AUTH_DETAILS_DIMENSION); - - AuthenticationDetailsFocusListener focusListener = new AuthenticationDetailsFocusListener(); - domainTextField.addFocusListener(focusListener); - usernameTextField.addFocusListener(focusListener); - passwordField.addFocusListener(focusListener); - apiKeyTextField.addFocusListener(focusListener); - - return authenticationDetailsPanel; - } - - /** - * Ensures consistency of fields within job tabs - * @return true if no issues were found, false otherwise - */ - private boolean jobTabsValid() { - return (jobTabsPane.getTabCount() == jobTabs.size()); - } - - /** - * Saves user authentication data input into form - */ - private void saveAuthenticationInfoFromForm() { - userPrefs.saveDomain(domainTextField.getText()); - userPrefs.saveUsername(usernameTextField.getText()); - String password = new String(passwordField.getPassword()); - userPrefs.savePassword(password); - userPrefs.saveAPIKey(apiKeyTextField.getText()); - } - - /** - * Loads user authentication data from userPrefs - */ - private void loadAuthenticationInfoIntoForm() { - domainTextField.setText(userPrefs.getDomain()); - usernameTextField.setText(userPrefs.getUsername()); - passwordField.setText(userPrefs.getPassword()); - apiKeyTextField.setText(userPrefs.getAPIKey()); - } - -} +package com.socrata.datasync.ui; + +import java.awt.*; +import java.awt.event.*; + +import javax.swing.*; +import javax.swing.border.Border; +import javax.swing.filechooser.FileNameExtensionFilter; + +import com.socrata.datasync.*; +import com.socrata.datasync.job.IntegrationJob; +import com.socrata.datasync.job.Job; +import com.socrata.datasync.job.JobStatus; +import com.socrata.datasync.job.MetadataJob; +import com.socrata.datasync.job.PortJob; +import com.socrata.datasync.job.GISJob; +import com.socrata.datasync.config.userpreferences.UserPreferences; +import com.socrata.datasync.config.userpreferences.UserPreferencesJava; + +import java.io.File; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class SimpleIntegrationWizard { + /** + * @author Adrian Laurenzi + * + * GUI interface to DataSync + */ + private static final String TITLE = "Socrata DataSync " + VersionProvider.getThisVersion(); + private static final String LOGO_FILE_PATH = "/datasync_logo.png"; + private static final String LOADING_SPINNER_FILE_PATH = "/loading_spinner.gif"; + private static final int FRAME_WIDTH = 800; + private static final int FRAME_HEIGHT = 550; + private static final Dimension JOB_PANEL_DIMENSION = new Dimension(780, 350); + private static final Dimension BUTTON_PANEL_DIMENSION = new Dimension(780, 40); + private static final int SSL_PORT_TEXTFIELD_HEIGHT = 26; + private static final int DEFAULT_TEXTFIELD_COLS = 25; + private static final Dimension AUTH_DETAILS_DIMENSION = new Dimension(465, 100); + private static final int PREFERENCES_FRAME_WIDTH = 475; + private static final int PREFERENCES_FRAME_HEIGHT = 675; + + private static UserPreferencesJava userPrefs; + + // TODO remove these declarations from this file (duplicates...) + private static final String STANDARD_JOB_FILE_EXTENSION = "sij"; + private static final String PORT_JOB_FILE_EXTENSION = "spj"; + private static final String METADATA_JOB_FILE_EXTENSION = "smj"; + private static final String GIS_JOB_FILE_EXTENSION = "gij"; + + // help icon balloon tip text + private static final String FILE_CHUNKING_THRESHOLD_TIP_TEXT = "If using the upsert, append, or " + + "delete methods (over HTTP) and the CSV/TSV file to be published is larger than this value (in megabytes), " + + "the file is automatically split up and published in chunks (because it is problematic to publish large files all at once). " + + "Usually chunking is necessary when a file is larger than about 64 MB."; + private static final String CHUNK_SIZE_THRESHOLD_TIP_TEXT = "The number of rows to publish in each chunk " + + "(in cases where filesize exceeds above filesize threshold). Higher values usually means faster upload time but setting the value too " + + "high could crash the program, depending on your computer's memory limits."; + private static final String DOMAIN_TIP_TEXT = "The domain of the Socrata data site you wish to publish data to (e.g. https://explore.data.gov/)"; + private static final String USERNAME_TIP_TEXT = "Socrata account username (account must have Publisher or Administrator permissions)"; + private static final String PASSWORD_TIP_TEXT = "Socrata account password"; + private static final String RUN_JOB_NOW_TIP_TEXT = "" + + "To view detailed logging information run the job by copying the" + + " 'Command to execute with scheduler' and running it in your Terminal/Command Prompt (instead of clicking 'Run Job Now' button)"; + + private static final String QUICK_START_GUIDE = "http://socrata.github.io/datasync/guides/quick-start.html"; + private static final String PORTING_GUIDE = "http://socrata.github.io/datasync/guides/setup-port-job.html"; + private static final String HEADLESS_GUIDE_URL = "http://socrata.github.io/datasync/guides/setup-standard-job-headless.html"; + private static final String CONTROL_GUIDE_URL = "http://socrata.github.io/datasync/resources/control-config.html"; + private static final String SCHEDULING_GUIDE_URL = "http://socrata.github.io/datasync/resources/schedule-job.html"; + private static final String FAQ_URL = "http://socrata.github.io/datasync/resources/faq-common-problems.html"; + + private JTextField domainTextField, usernameTextField; + private JPasswordField passwordField; + private JTextField filesizeChunkingCutoffTextField, numRowsPerChunkTextField; + private JTextField logDatasetIDTextField, adminEmailTextField; + private JTextField outgoingMailServerTextField, smtpPortTextField, sslPortTextField, smtpUsernameTextField; + private JTextField proxyHostTextField, proxyPortTextField, proxyUsernameTextField, proxyPasswordTextField; + private JPasswordField smtpPasswordField; + private JCheckBox useSSLCheckBox; + private JCheckBox emailUponErrorCheckBox; + private JTextField timeFormatsTextField; + + /** + * Stores a list of open JobTabs. Each JobTab object contains + * the UI content of a single job tab. The indices of the + * tabs within jobTabsPane correspond to the indices of the + * JobTab objects (holding the UI for each tab) within this + * list. + * + * *IMPORTANT*: only modify this list in the 'addJobTab' and + * 'closeJobTab' methods + */ + private List jobTabs; + + private JTabbedPane jobTabsPane; + private JFrame frame; + private JFrame prefsFrame; + private JPanel loadingNoticePanel; + private JPanel progressPanel; + private JButton runJobNowButton; + private JLabel loadingTextLabel = new JLabel("Processing..."); + private JProgressBar progress = new JProgressBar(0, 100); + private JLabel progressText = new JLabel(""); + + private static SimpleIntegrationWizard instance; + + /* + * Constructs the GUI and displays it on the screen. + */ + private SimpleIntegrationWizard() { + // load user preferences (saved locally) + userPrefs = new UserPreferencesJava(); + + // Build GUI + frame = new JFrame(TITLE); + frame.setSize(FRAME_WIDTH, FRAME_HEIGHT); + + jobTabs = new ArrayList<>(); + // save tabs on close + frame.setDefaultCloseOperation(JFrame.DO_NOTHING_ON_CLOSE); + frame.addWindowListener(new WindowAdapter() { + @Override + public void windowClosing(WindowEvent we) { + saveAuthenticationInfoFromForm(); + // TODO save open tabs to userPrefs + System.exit(0); + } + }); + + JMenuBar menuBar = generateMenuBar(); + frame.setJMenuBar(menuBar); + + JPanel mainPanel = generateMainPanel(); + loadAuthenticationInfoIntoForm(); + generatePreferencesFrame(); + + frame.add(mainPanel); + + frame.pack(); + // centers the window + frame.setLocationRelativeTo(null); + frame.setVisible(true); + + // Alert user if new version is available + try { + checkVersion(); + } catch (Exception e) { + // do nothing upon failure + } + } + + public static SimpleIntegrationWizard get() { + if(instance == null) instance = new SimpleIntegrationWizard(); + return instance; + } + + private void generatePreferencesFrame() { + prefsFrame = new JFrame("Preferences"); + prefsFrame.setSize(PREFERENCES_FRAME_WIDTH, PREFERENCES_FRAME_HEIGHT); + prefsFrame.setVisible(false); + JPanel preferencesPanel = generatePreferencesPanel(); + prefsFrame.add(preferencesPanel); + prefsFrame.setDefaultCloseOperation(JFrame.DO_NOTHING_ON_CLOSE); + prefsFrame.addWindowListener(new WindowAdapter() { + @Override + public void windowClosing(WindowEvent we) { + prefsFrame.setVisible(false); + } + }); + } + + /** + * Queries github for the most recent release. If query is successful + * and the major version of the user's datasync is outdated, alerts + * that a new version is available + * + * @throws URISyntaxException + */ + private void checkVersion() throws URISyntaxException { + if(VersionProvider.isLatestMajorVersion() == VersionProvider.VersionStatus.NOT_LATEST) { + String currentVersion = VersionProvider.getLatestVersion(); + if (currentVersion != null) { + Object[] options = {"Download Now", "No Thanks"}; + int n = JOptionPane.showOptionDialog(frame, + "A new version of DataSync is available (version " + currentVersion + ").\n" + + "Do you want to download it now?\n", + "Alert: New Version Available", + JOptionPane.YES_NO_OPTION, + JOptionPane.QUESTION_MESSAGE, + null, options, options[0] + ); + if (n == JOptionPane.YES_OPTION) { + URI currentVersionDownloadURI = new URI(VersionProvider.getDownloadUrlForLatestVersion()); + if (currentVersionDownloadURI != null) + Utils.openWebpage(currentVersionDownloadURI); + } + } + } + } + + private void addJobTab(Job job) throws IllegalArgumentException { + JobTab newJobTab; + if(job.getClass().equals(IntegrationJob.class)) { + newJobTab = new IntegrationJobTab((IntegrationJob) job, frame, userPrefs); + } else if(job.getClass().equals(PortJob.class)) { + newJobTab = new PortJobTab((PortJob) job, frame); + } else if(job.getClass().equals(GISJob.class)) { + newJobTab = new GISJobTab((GISJob) job, frame); + } else if(job.getClass().equals(MetadataJob.class)) { + newJobTab = new MetadataJobTab((MetadataJob) job, frame); + } else { + throw new IllegalArgumentException("Given job is invalid: unrecognized class '" + job.getClass() + "'"); + } + JPanel newJobPanel = newJobTab.getTabPanel(); + + // Build the tab with close button + FlowLayout tabLayout = new FlowLayout(FlowLayout.CENTER, 5, 0); + JPanel tabPanel = new JPanel(tabLayout); + tabPanel.setOpaque(false); + + // Create a JButton for the close tab button + JButton closeTabButton = new AwesomeButton("cross41"); + closeTabButton.setBorder(null); + closeTabButton.setFocusable(false); + + tabPanel.add(newJobTab.getJobTabTitleLabel()); + tabPanel.add(closeTabButton); + // Add a thin border to keep the image below the top edge of the tab when the tab is selected + tabPanel.setBorder(BorderFactory.createEmptyBorder(2, 0, 0, 0)); + + // Put tab with close button into tabbed pane + //TODO: BW: Possibly implement way to keep other tabs from being scrollable? + JScrollPane newJobScrollPanel = new JScrollPane(newJobPanel); + newJobScrollPanel.setHorizontalScrollBarPolicy(JScrollPane.HORIZONTAL_SCROLLBAR_NEVER); + jobTabsPane.addTab(null, newJobScrollPanel); + int pos = jobTabsPane.indexOfComponent(newJobScrollPanel); + + // Now assign the component for the tab + jobTabsPane.setTabComponentAt(pos, tabPanel); + + closeTabButton.addActionListener(new CloseJobFromTabListener()); + + jobTabs.add(newJobTab); + assert(jobTabsValid()); + } + + private void closeJobTab(int tabIndex) throws IllegalArgumentException { + if(tabIndex >= jobTabsPane.getTabCount()) { + throw new IllegalArgumentException("Tab index is invalid"); + } + jobTabsPane.remove(tabIndex); + jobTabs.remove(tabIndex); + assert(jobTabsValid()); + } + + private class RunJobNowListener implements ActionListener { + public void actionPerformed(ActionEvent e) { + saveAuthenticationInfoFromForm(); + + // run integration job with data from form + int selectedJobTabIndex = jobTabsPane.getSelectedIndex(); + JobTab selectedJobTab = jobTabs.get(selectedJobTabIndex); + + SwingWorker jobWorker = new RunJobWorker(selectedJobTab); + jobWorker.execute(); + } + } + + private class RunJobWorker extends SwingWorker { + private JobTab jobTabToRun; + private JobStatus jobStatus; + + public RunJobWorker(JobTab jobTabToRun){ + loadingNoticePanel.setVisible(true); + progressPanel.setVisible(true); + loadingNoticePanel.add(loadingTextLabel); + progressPanel.add(progress); + progressPanel.add(progressText); + runJobNowButton.setEnabled(false); + this.jobTabToRun = jobTabToRun; + } + + @Override + protected Void doInBackground() { + try { + jobStatus = jobTabToRun.runJobNow(); + } catch (OutOfMemoryError err) { + jobStatus = JobStatus.PUBLISH_ERROR; + jobStatus.setMessage("Error: ran out of memory " + + "(try decreasing the chunking size and/or threshold by going to Edit -> Preferences)"); + } catch (Exception e) { + e.printStackTrace(); + jobStatus = JobStatus.PUBLISH_ERROR; + jobStatus.setMessage("Unexpected error: " + e); + } catch (Error e) { + e.printStackTrace(); + throw e; + } + return null; + } + + //Executed on the Event Dispatch Thread after the doInBackground method is finished + @Override + protected void done() { + loadingNoticePanel.setVisible(false); + progressPanel.setVisible(false); + runJobNowButton.setEnabled(true); + + // show popup with returned status + if(jobStatus == null) { + System.out.println("null jobStatus?!"); + } else if(jobStatus.isError()) { + JOptionPane.showMessageDialog(frame, "Job completed with errors: " + jobStatus.getMessage()); + } else { + if (jobTabToRun.getClass().equals(PortJobTab.class)) { + PortJobTab selectedPortJobTab = (PortJobTab) jobTabToRun; + Object[] options = {"Yes", "No"}; + int n = JOptionPane.showOptionDialog(frame, + "Port job completed successfully. Would you like to open the destination dataset?\n", + "Port Job Successful", + JOptionPane.YES_NO_OPTION, + JOptionPane.QUESTION_MESSAGE, + null, options, options[0]); + if (n == JOptionPane.YES_OPTION) { + Utils.openWebpage(selectedPortJobTab.getURIToSinkDataset()); + } + } else { + JOptionPane.showMessageDialog(frame, "Job completed successfully. " + jobStatus.getMessage()); + } + } + } + } + + private class SaveJobListener implements ActionListener { + public void actionPerformed(ActionEvent e) { + saveAuthenticationInfoFromForm(); + int selectedJobTabIndex = jobTabsPane.getSelectedIndex(); + JobTab selectedJobTab = jobTabs.get(selectedJobTabIndex); + selectedJobTab.saveJob(); + } + } + + private File openToDirectory = new File("."); + private class OpenJobListener implements ActionListener { + public void actionPerformed(ActionEvent e) { + JFileChooser savedJobFileChooser = new JFileChooser(); + savedJobFileChooser.setCurrentDirectory(openToDirectory); + String fileExtensionsAllowed = "*." + STANDARD_JOB_FILE_EXTENSION + ", *." + PORT_JOB_FILE_EXTENSION + ", *."+ GIS_JOB_FILE_EXTENSION + ", *." + METADATA_JOB_FILE_EXTENSION; + FileNameExtensionFilter filter = new FileNameExtensionFilter( + "Socrata Job File (" + fileExtensionsAllowed + ")", + STANDARD_JOB_FILE_EXTENSION, PORT_JOB_FILE_EXTENSION, GIS_JOB_FILE_EXTENSION, METADATA_JOB_FILE_EXTENSION); + savedJobFileChooser.setFileFilter(filter); + int returnVal = savedJobFileChooser.showOpenDialog(frame); + if (returnVal == JFileChooser.APPROVE_OPTION) { + File openedFile = savedJobFileChooser.getSelectedFile(); + // Ensure file exists + if(openedFile.exists()) { + openToDirectory = savedJobFileChooser.getCurrentDirectory(); + // ensure this job is not already open + String openedFileLocation = openedFile.getAbsolutePath(); + int indexOfAlreadyOpenFile = -1; + for(int curTabIndex = 0; curTabIndex < jobTabs.size(); curTabIndex++) { + String curTabJobFileLocation = jobTabs.get(curTabIndex).getJobFileLocation(); + if(curTabJobFileLocation.equals(openedFileLocation)) { + indexOfAlreadyOpenFile = curTabIndex; + break; + } + } + if(indexOfAlreadyOpenFile == -1) { + try { + int i = openedFileLocation.lastIndexOf('.'); + if (i > 0) { + String openedFileExtension = openedFileLocation.substring(i+1); + if(openedFileExtension.equals(STANDARD_JOB_FILE_EXTENSION)) { + addJobTab(new IntegrationJob(openedFileLocation)); + } else if(openedFileExtension.equals(GIS_JOB_FILE_EXTENSION)){ + addJobTab(new GISJob(openedFileLocation)); + } else if(openedFileExtension.equals(PORT_JOB_FILE_EXTENSION)) { + addJobTab(new PortJob(openedFileLocation)); + } else if (openedFileExtension.equals(METADATA_JOB_FILE_EXTENSION)) { + addJobTab(new MetadataJob(openedFileLocation)); + } else { + throw new Exception("unrecognized file extension (" + openedFileExtension + ")"); + } + } + // Switch to opened file's tab + jobTabsPane.setSelectedIndex(jobTabsPane.getTabCount() - 1); + } catch(IntegrationJob.ControlDisagreementException ex) { + JOptionPane.showMessageDialog(frame, "Warning: \n" + ex.getMessage() + " found in \n'" + + openedFileLocation + "'. \nLoading job, but please confirm the contents of your control file are accurate."); + try { + addJobTab(new IntegrationJob(openedFileLocation, true)); + jobTabsPane.setSelectedIndex(jobTabsPane.getTabCount() - 1); + } catch(Exception e2) { + JOptionPane.showMessageDialog(frame, "Error opening " + openedFileLocation + ": " + e2.toString()); + } + } catch(Exception e2) { + JOptionPane.showMessageDialog(frame, "Error opening " + openedFileLocation + ": " + e2.toString()); + } + } else { + // file already open, select that tab + jobTabsPane.setSelectedIndex(indexOfAlreadyOpenFile); + } + } + } + } + } + + private class AuthenticationDetailsFocusListener implements FocusListener { + @Override + public void focusGained(FocusEvent e) { } + @Override + public void focusLost(FocusEvent e) { + saveAuthenticationInfoFromForm(); + } + } + + private class NewStandardJobListener implements ActionListener { + public void actionPerformed(ActionEvent e) { + addJobTab(getNewIntegrationJob()); + jobTabsPane.setSelectedIndex(jobTabsPane.getTabCount() - 1); + } + } + + private IntegrationJob getNewIntegrationJob() { + IntegrationJob newJob = new IntegrationJob(); + // set publishViaDi2Http to true as default (ONLY for GUI mode) + newJob.setPublishViaDi2Http(true); + return newJob; + } + + private class NewPortJobListener implements ActionListener { + public void actionPerformed(ActionEvent e) { + addJobTab(new PortJob()); + jobTabsPane.setSelectedIndex(jobTabsPane.getTabCount() - 1); + } + } + + private class NewGISJobListener implements ActionListener { + public void actionPerformed(ActionEvent e) { + addJobTab(new GISJob()); + jobTabsPane.setSelectedIndex(jobTabsPane.getTabCount() - 1); + } + } + + private class NewMetadataJobListener implements ActionListener { + public void actionPerformed(ActionEvent e) { + addJobTab(new MetadataJob()); + jobTabsPane.setSelectedIndex(jobTabsPane.getTabCount() - 1); + } + } + + /** + * Listen for action to close currently selected tab + */ + private class CloseJobFromTabListener implements ActionListener { + public void actionPerformed(ActionEvent e) { + JComponent source = (JComponent) e.getSource(); + Container tabComponent = source.getParent(); + int jobTabIndex = jobTabsPane.indexOfTabComponent(tabComponent); + closeJobTab(jobTabIndex); + } + } + + private JMenuBar generateMenuBar() { + JMenuBar menuBar = new JMenuBar(); + + JMenu fileMenu = new JMenu("File"); + menuBar.add(fileMenu); + + JMenu newJobMenu = new JMenu("New..."); + JMenuItem newStandardJobItem = new JMenuItem("Standard Job"); + newJobMenu.add(newStandardJobItem); + JMenuItem newPortJobItem = new JMenuItem("Port Job"); + newJobMenu.add(newPortJobItem); + JMenuItem newGISJobItem = new JMenuItem("GIS Job"); + newJobMenu.add(newGISJobItem); + JMenuItem newMetadataJobItem = new JMenuItem("Metadata Job (beta)"); + newJobMenu.add(newMetadataJobItem); + fileMenu.add(newJobMenu); + + JMenuItem openJobItem = new JMenuItem("Open Job"); + openJobItem.setAccelerator(KeyStroke.getKeyStroke( + KeyEvent.VK_O, ActionEvent.CTRL_MASK)); + fileMenu.add(openJobItem); + + JMenuItem saveJobItem = new JMenuItem("Save Job"); + saveJobItem.setAccelerator(KeyStroke.getKeyStroke( + KeyEvent.VK_S, ActionEvent.CTRL_MASK)); + fileMenu.add(saveJobItem); + + JMenuItem runJobItem = new JMenuItem("Run Job Now"); + runJobItem.setAccelerator(KeyStroke.getKeyStroke( + KeyEvent.VK_R, ActionEvent.CTRL_MASK)); + fileMenu.add(runJobItem); + JMenuItem prefsItem = new JMenuItem("Preferences"); + fileMenu.add(prefsItem); + + JMenu helpMenu = new JMenu("Help"); + menuBar.add(helpMenu); + JMenuItem gettingStartedGuideItem = new JMenuItem("Quick start guide"); + JMenuItem portingGuideItem = new JMenuItem("Port job guide"); + JMenuItem headlessDocumentationItem = new JMenuItem("Running in headless mode"); + JMenuItem controlDocumentationItem = new JMenuItem("Control file configuration"); + JMenuItem schedulingItem = new JMenuItem("Scheduling a job"); + JMenuItem faqItem = new JMenuItem("FAQ"); + helpMenu.add(gettingStartedGuideItem); + helpMenu.add(portingGuideItem); + helpMenu.add(headlessDocumentationItem); + helpMenu.add(controlDocumentationItem); + helpMenu.add(schedulingItem); + helpMenu.add(faqItem); + + newStandardJobItem.addActionListener(new NewStandardJobListener()); + newPortJobItem.addActionListener(new NewPortJobListener()); + newGISJobItem.addActionListener(new NewGISJobListener()); + newMetadataJobItem.addActionListener(new NewMetadataJobListener()); + openJobItem.addActionListener(new OpenJobListener()); + saveJobItem.addActionListener(new SaveJobListener()); + runJobItem.addActionListener(new RunJobNowListener()); + prefsItem.addActionListener(new OpenPreferencesListener()); + gettingStartedGuideItem.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + try { + Utils.openWebpage(new URI(QUICK_START_GUIDE)); + } catch (URISyntaxException e1) { e1.printStackTrace(); } + } + }); + portingGuideItem.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + try { + Utils.openWebpage(new URI(PORTING_GUIDE)); + } catch (URISyntaxException e1) { e1.printStackTrace(); } + } + }); + headlessDocumentationItem.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + try { + Utils.openWebpage(new URI(HEADLESS_GUIDE_URL)); + } catch (URISyntaxException e1) { e1.printStackTrace(); } + } + }); + controlDocumentationItem.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + try { + Utils.openWebpage(new URI(CONTROL_GUIDE_URL)); + } catch (URISyntaxException e1) { e1.printStackTrace(); } + } + }); + schedulingItem.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + try { + Utils.openWebpage(new URI(SCHEDULING_GUIDE_URL)); + } catch (URISyntaxException e1) { e1.printStackTrace(); } + } + }); + faqItem.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + try { + Utils.openWebpage(new URI(FAQ_URL)); + } catch (URISyntaxException e1) { e1.printStackTrace(); } + } + }); + + return menuBar; + } + + private JPanel generateMainPanel() { + JPanel mainContainer = new JPanel(new FlowLayout(FlowLayout.LEFT)); + mainContainer.setPreferredSize(new Dimension(FRAME_WIDTH, FRAME_HEIGHT)); + + // Build empty job tabbed pane + JPanel jobTabsContainer = new JPanel(new GridLayout(1, 1)); + jobTabsContainer.setPreferredSize(JOB_PANEL_DIMENSION); + jobTabsPane = new JTabbedPane(); + jobTabsContainer.add(jobTabsPane); + jobTabsPane.setTabLayoutPolicy(JTabbedPane.SCROLL_TAB_LAYOUT); + + JPanel jobButtonContainer = new JPanel(new GridLayout(1,3)); + JPanel leftButtonPanel = new JPanel(new FlowLayout(FlowLayout.LEFT)); + runJobNowButton = new JButton("Run Job Now"); + runJobNowButton.addActionListener(new RunJobNowListener()); + leftButtonPanel.add(runJobNowButton); + leftButtonPanel.add(UIUtility.generateHelpBubble(RUN_JOB_NOW_TIP_TEXT)); + + JPanel noticesContainer = new JPanel(new GridLayout(2, 1)); + generateLoadingNotice(); + noticesContainer.add(loadingNoticePanel); + generateProgressBar(); + noticesContainer.add(progressPanel); + + JPanel rightButtonPanel = new JPanel(new FlowLayout(FlowLayout.RIGHT)); + JButton saveJobButton = new JButton("Save Job"); + saveJobButton.addActionListener(new SaveJobListener()); + rightButtonPanel.add(saveJobButton); + jobButtonContainer.add(leftButtonPanel); + jobButtonContainer.add(noticesContainer); + jobButtonContainer.add(rightButtonPanel); + + jobButtonContainer.setPreferredSize(BUTTON_PANEL_DIMENSION); + mainContainer.add(jobTabsContainer); + mainContainer.add(jobButtonContainer); + + JPanel authenticationDetailsPanel = generateAuthenticationDetailsFormPanel(); + mainContainer.add(authenticationDetailsPanel); + + URL logoImageURL = getClass().getResource(LOGO_FILE_PATH); + if(logoImageURL != null) { + JLabel logoLabel = new JLabel(new ImageIcon(logoImageURL)); + Border paddingBorder = BorderFactory.createEmptyBorder(15,15,15,15); + logoLabel.setBorder(paddingBorder); + mainContainer.add(logoLabel); + } + + // TODO populate job tabs w/ previously opened tabs or [if none] a new job tab + + addJobTab(getNewIntegrationJob()); + return mainContainer; + } + + private void generateLoadingNotice() { + loadingNoticePanel = new JPanel(new FlowLayout(FlowLayout.LEFT)); + loadingNoticePanel.setVisible(false); + } + + private void generateProgressBar() { + progressPanel = new JPanel(new FlowLayout(FlowLayout.CENTER)); + + progressPanel.setVisible(false); + } + + private JPanel generatePreferencesPanel() { + JPanel prefsPanel = new JPanel(new GridLayout(0,2)); + + // File chunking settings + JLabel fileChunkingSettingsLabel = new JLabel(" File Chunking Settings"); + Font boldFont = new Font(fileChunkingSettingsLabel.getFont().getFontName(), + Font.BOLD, fileChunkingSettingsLabel.getFont().getSize()); + fileChunkingSettingsLabel.setFont(boldFont); + prefsPanel.add(fileChunkingSettingsLabel); + prefsPanel.add(new JLabel("")); + + prefsPanel.add( + UIUtility.generateLabelWithHelpBubble(" Chunking filesize threshold", FILE_CHUNKING_THRESHOLD_TIP_TEXT)); + JPanel filesizeChuckingContainer = new JPanel(new FlowLayout(FlowLayout.LEFT)); + filesizeChunkingCutoffTextField = new JTextField(); + filesizeChunkingCutoffTextField.setPreferredSize(new Dimension( + 140, 20)); + filesizeChuckingContainer.add(filesizeChunkingCutoffTextField); + filesizeChuckingContainer.add(new JLabel(" MB")); + prefsPanel.add(filesizeChuckingContainer); + + prefsPanel.add( + UIUtility.generateLabelWithHelpBubble(" Chunk size", CHUNK_SIZE_THRESHOLD_TIP_TEXT)); + JPanel chunkSizeContainer = new JPanel(new FlowLayout(FlowLayout.LEFT)); + numRowsPerChunkTextField = new JTextField(); + numRowsPerChunkTextField.setPreferredSize(new Dimension( + 140, 20)); + chunkSizeContainer.add(numRowsPerChunkTextField); + chunkSizeContainer.add(new JLabel(" rows")); + prefsPanel.add(chunkSizeContainer); + + // Logging and auto-email settings + JLabel loggingAutoEmailSettingsLabel = new JLabel(" Logging and Auto-Email Settings"); + loggingAutoEmailSettingsLabel.setFont(boldFont); + prefsPanel.add(loggingAutoEmailSettingsLabel); + prefsPanel.add(new JLabel("")); + + prefsPanel.add(new JLabel(" Log Dataset ID (e.g., n38h-y5wp)")); + logDatasetIDTextField = new JTextField(DEFAULT_TEXTFIELD_COLS); + prefsPanel.add(logDatasetIDTextField); + + prefsPanel.add(new JLabel(" Admin Email")); + adminEmailTextField = new JTextField(DEFAULT_TEXTFIELD_COLS); + prefsPanel.add(adminEmailTextField); + + emailUponErrorCheckBox = new JCheckBox(" Auto-email admin upon error"); + prefsPanel.add(emailUponErrorCheckBox); + prefsPanel.add(new JLabel("*must fill in SMTP Settings below")); + + // Auto-email SMTP settings + JLabel smtpSettingsLabel = new JLabel(" SMTP Settings"); + smtpSettingsLabel.setFont(boldFont); + prefsPanel.add(smtpSettingsLabel); + prefsPanel.add(new JLabel("")); + + prefsPanel.add(new JLabel(" Outgoing Mail Server")); + outgoingMailServerTextField = new JTextField(DEFAULT_TEXTFIELD_COLS); + prefsPanel.add(outgoingMailServerTextField); + prefsPanel.add(new JLabel(" SMTP Port")); + smtpPortTextField = new JTextField(DEFAULT_TEXTFIELD_COLS); + prefsPanel.add(smtpPortTextField); + final JPanel sslPortContainer = new JPanel( + new FlowLayout(FlowLayout.LEFT, 0, 0)); + sslPortContainer.setVisible(false); + useSSLCheckBox = new JCheckBox("Use SSL"); + prefsPanel.add(useSSLCheckBox); + sslPortContainer.add(new JLabel(" SSL Port ")); + sslPortTextField = new JTextField(); + sslPortTextField.setPreferredSize(new Dimension( + 50, SSL_PORT_TEXTFIELD_HEIGHT)); + sslPortContainer.add(sslPortTextField); + useSSLCheckBox.addItemListener(new ItemListener() { + public void itemStateChanged(ItemEvent e) { + if(useSSLCheckBox.isSelected()) { + sslPortContainer.setVisible(true); + } else { + sslPortContainer.setVisible(false); + } + } + }); + prefsPanel.add(sslPortContainer); + prefsPanel.add(new JLabel(" SMTP Username")); + smtpUsernameTextField = new JTextField(DEFAULT_TEXTFIELD_COLS); + prefsPanel.add(smtpUsernameTextField); + prefsPanel.add(new JLabel(" SMTP Password")); + smtpPasswordField = new JPasswordField(DEFAULT_TEXTFIELD_COLS); + prefsPanel.add(smtpPasswordField); + + + // Proxy settings + JLabel proxySettingsLabel = new JLabel(" Proxy Settings"); + proxySettingsLabel.setFont(boldFont); + prefsPanel.add(proxySettingsLabel); + prefsPanel.add(new JLabel("")); + + prefsPanel.add(new JLabel(" Proxy Host")); + proxyHostTextField = new JTextField(DEFAULT_TEXTFIELD_COLS); + prefsPanel.add(proxyHostTextField); + prefsPanel.add(new JLabel(" Proxy Port")); + proxyPortTextField = new JTextField(DEFAULT_TEXTFIELD_COLS); + prefsPanel.add(proxyPortTextField); + prefsPanel.add(new JLabel(" Proxy Username")); + proxyUsernameTextField = new JTextField(DEFAULT_TEXTFIELD_COLS); + prefsPanel.add(proxyUsernameTextField); + prefsPanel.add(new JLabel(" Proxy Password")); + proxyPasswordTextField = new JPasswordField(DEFAULT_TEXTFIELD_COLS); + prefsPanel.add(proxyPasswordTextField); + + JPanel testSMTPSettingsContainer = new JPanel(new FlowLayout(FlowLayout.LEFT)); + JButton testSMTPSettingsButton = new JButton("Test SMTP Settings"); + testSMTPSettingsButton.addActionListener(new TestSMTPSettingsListener()); + testSMTPSettingsContainer.add(testSMTPSettingsButton); + prefsPanel.add(testSMTPSettingsContainer); + prefsPanel.add(new JLabel("")); + + // Parsing settings + JLabel parsingSettingsLabel = new JLabel(" Parsing Settings"); + parsingSettingsLabel.setFont(boldFont); + prefsPanel.add(parsingSettingsLabel); + prefsPanel.add(new JLabel("")); + + prefsPanel.add(new JLabel(" Default time formats")); + timeFormatsTextField = new JTextField(DEFAULT_TEXTFIELD_COLS); + prefsPanel.add(timeFormatsTextField); + + JPanel prefsButtonContainer = new JPanel(new FlowLayout(FlowLayout.RIGHT)); + JButton cancelPrefsButton = new JButton("Cancel"); + cancelPrefsButton.addActionListener(new CancelPreferencesListener()); + prefsButtonContainer.add(cancelPrefsButton); + JButton savePrefsButton = new JButton("Save"); + savePrefsButton.addActionListener(new SavePreferencesListener()); + prefsButtonContainer.add(savePrefsButton); + prefsPanel.add(prefsButtonContainer); + + loadPreferencesIntoForm(); + + return prefsPanel; + } + + private void loadPreferencesIntoForm() { + filesizeChunkingCutoffTextField.setText(userPrefs.getFilesizeChunkingCutoffMB()); + numRowsPerChunkTextField.setText(userPrefs.getNumRowsPerChunk()); + + adminEmailTextField.setText(userPrefs.getAdminEmail()); + logDatasetIDTextField.setText(userPrefs.getLogDatasetID()); + emailUponErrorCheckBox.setSelected(userPrefs.emailUponError()); + + outgoingMailServerTextField.setText(userPrefs.getOutgoingMailServer()); + smtpPortTextField.setText(userPrefs.getSmtpPort()); + String sslPort = userPrefs.getSslPort(); + sslPortTextField.setText(sslPort); + if(sslPort.equals("")) { + useSSLCheckBox.setSelected(false); + } else { + useSSLCheckBox.setSelected(true); + } + smtpUsernameTextField.setText(userPrefs.getSmtpUsername()); + smtpPasswordField.setText(userPrefs.getSmtpPassword()); + + proxyHostTextField.setText(userPrefs.getProxyHost()); + proxyPortTextField.setText(userPrefs.getProxyPort()); + proxyUsernameTextField.setText(userPrefs.getProxyUsername()); + proxyPasswordTextField.setText(userPrefs.getProxyPassword()); + timeFormatsTextField.setText(Utils.commaJoin(userPrefs.getDefaultTimeFormats())); + } + + private void savePreferences() { + try { + userPrefs.saveFilesizeChunkingCutoffMB( + Integer.parseInt(filesizeChunkingCutoffTextField.getText())); + } catch(NumberFormatException e) { + JOptionPane.showMessageDialog(prefsFrame, "Invalid chunking filesize threshold: must be an integer"); + } + try { + userPrefs.saveNumRowsPerChunk( + Integer.parseInt(numRowsPerChunkTextField.getText())); + } catch(NumberFormatException e) { + JOptionPane.showMessageDialog(prefsFrame, "Invalid chunk size: must be an integer"); + } + + userPrefs.saveAdminEmail(adminEmailTextField.getText()); + userPrefs.saveLogDatasetID(logDatasetIDTextField.getText()); + userPrefs.saveEmailUponError(emailUponErrorCheckBox.isSelected()); + + userPrefs.saveOutgoingMailServer(outgoingMailServerTextField.getText()); + userPrefs.saveSMTPPort(smtpPortTextField.getText()); + if(useSSLCheckBox.isSelected()) { + userPrefs.saveSSLPort(sslPortTextField.getText()); + } else { + userPrefs.saveSSLPort(""); + } + userPrefs.saveSMTPUsername(smtpUsernameTextField.getText()); + String smtpPassword = new String(smtpPasswordField.getPassword()); + userPrefs.saveSMTPPassword(smtpPassword); + + userPrefs.saveProxyHost(proxyHostTextField.getText()); + userPrefs.saveProxyPort(proxyPortTextField.getText()); + userPrefs.saveProxyUsername(proxyUsernameTextField.getText()); + userPrefs.saveProxyPassword(proxyPasswordTextField.getText()); + + String newTimeFormats = timeFormatsTextField.getText(); + if(newTimeFormats.trim().isEmpty()) { + userPrefs.saveDefaultTimeFormats(Arrays.asList(UserPreferences.DEFAULT_TIME_FORMATS)); + timeFormatsTextField.setText(Utils.commaJoin(UserPreferences.DEFAULT_TIME_FORMATS)); + } else { + userPrefs.saveDefaultTimeFormats(Arrays.asList(Utils.commaSplit(newTimeFormats))); + } + } + + private class SavePreferencesListener implements ActionListener { + public void actionPerformed(ActionEvent e) { + // TODO validation of email and log dataset ID + savePreferences(); + prefsFrame.setVisible(false); + } + } + + private class CancelPreferencesListener implements ActionListener { + public void actionPerformed(ActionEvent e) { + loadPreferencesIntoForm(); + prefsFrame.setVisible(false); + } + } + + private class TestSMTPSettingsListener implements ActionListener { + public void actionPerformed(ActionEvent e) { + savePreferences(); + String adminEmail = userPrefs.getAdminEmail(); + String message; + try { + SMTPMailer.send(adminEmail, "Socrata DataSync Test Email", + "This email confirms that your SMTP Settings are valid."); + message = "Sent test email to " + adminEmail + ". Please ensure the " + + "email was delievered successfully (it may take a few minutes)."; + } catch (Exception emailE) { + message = "Error sending email to " + adminEmail + ":\n" + emailE.getMessage(); + } + JOptionPane.showMessageDialog(prefsFrame, message); + } + } + + private class OpenPreferencesListener implements ActionListener { + public void actionPerformed(ActionEvent e) { + // centers the window + prefsFrame.setLocationRelativeTo(null); + prefsFrame.setVisible(true); + } + } + + private JPanel generateAuthenticationDetailsFormPanel() { + JPanel authenticationDetailsPanel = new JPanel(new GridLayout(0,2)); + + authenticationDetailsPanel.add( + UIUtility.generateLabelWithHelpBubble("Domain", DOMAIN_TIP_TEXT)); + domainTextField = new JTextField(DEFAULT_TEXTFIELD_COLS); + authenticationDetailsPanel.add(domainTextField); + authenticationDetailsPanel.add( + UIUtility.generateLabelWithHelpBubble("Username", USERNAME_TIP_TEXT)); + usernameTextField = new JTextField(DEFAULT_TEXTFIELD_COLS); + authenticationDetailsPanel.add(usernameTextField); + authenticationDetailsPanel.add( + UIUtility.generateLabelWithHelpBubble("Password", PASSWORD_TIP_TEXT)); + passwordField = new JPasswordField(DEFAULT_TEXTFIELD_COLS); + authenticationDetailsPanel.add(passwordField); + authenticationDetailsPanel.setPreferredSize(AUTH_DETAILS_DIMENSION); + + AuthenticationDetailsFocusListener focusListener = new AuthenticationDetailsFocusListener(); + domainTextField.addFocusListener(focusListener); + usernameTextField.addFocusListener(focusListener); + passwordField.addFocusListener(focusListener); + + return authenticationDetailsPanel; + } + + /** + * Ensures consistency of fields within job tabs + * @return true if no issues were found, false otherwise + */ + private boolean jobTabsValid() { + return (jobTabsPane.getTabCount() == jobTabs.size()); + } + + /** + * Saves user authentication data input into form + */ + private void saveAuthenticationInfoFromForm() { + userPrefs.saveDomain(domainTextField.getText()); + userPrefs.saveUsername(usernameTextField.getText()); + String password = new String(passwordField.getPassword()); + userPrefs.savePassword(password); + } + + /** + * Loads user authentication data from userPrefs + */ + private void loadAuthenticationInfoIntoForm() { + domainTextField.setText(userPrefs.getDomain()); + usernameTextField.setText(userPrefs.getUsername()); + passwordField.setText(userPrefs.getPassword()); + } + + public static void updateStatus(String loadingLabel, int progressPercent, boolean showProgress, String message) { + if(instance != null) { + instance.loadingTextLabel.setText(loadingLabel); + instance.progress.setValue(0); + if(showProgress) { + instance.progress.setVisible(true); + instance.progressText.setText(""); + instance.progress.setValue(progressPercent); + } else { + instance.progress.setVisible(false); + instance.progressText.setText(message); + } + } + } +} diff --git a/src/main/java/com/socrata/datasync/ui/SyntheticLocationDialog.java b/src/main/java/com/socrata/datasync/ui/SyntheticLocationDialog.java index 655e9590..6b1c7e3b 100644 --- a/src/main/java/com/socrata/datasync/ui/SyntheticLocationDialog.java +++ b/src/main/java/com/socrata/datasync/ui/SyntheticLocationDialog.java @@ -274,7 +274,7 @@ private LocationColumn getActiveLocation() { LocationColumn locationColumn = syntheticColumnsCopy.get(fieldName); if (locationColumn == null) { locationColumn = new LocationColumn(); - syntheticColumnsCopy.put(fieldName,locationColumn); + syntheticColumnsCopy.put(fieldName, locationColumn.clone()); } return locationColumn; } diff --git a/src/main/java/com/socrata/datasync/ui/SyntheticPointDialog.java b/src/main/java/com/socrata/datasync/ui/SyntheticPointDialog.java new file mode 100644 index 00000000..920c937e --- /dev/null +++ b/src/main/java/com/socrata/datasync/ui/SyntheticPointDialog.java @@ -0,0 +1,438 @@ +package com.socrata.datasync.ui; + +import com.socrata.datasync.config.controlfile.SyntheticPointColumn; +import com.socrata.datasync.config.controlfile.GeocodedPointColumn; +import com.socrata.datasync.config.controlfile.ProvidedPointColumn; +import com.socrata.datasync.model.CSVModel; +import com.socrata.datasync.model.ControlFileModel; +import com.socrata.model.importer.Column; + +import javax.swing.*; +import javax.swing.event.ChangeEvent; +import javax.swing.event.ChangeListener; +import java.awt.*; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.awt.event.ItemEvent; +import java.awt.event.ItemListener; +import java.util.ArrayList; +import java.util.TreeMap; +import java.util.HashMap; +import java.util.Map; + +/** + * A dialog box which allows the customer to pick from columns in a CSV and then map them into + * a location column + */ +public class SyntheticPointDialog extends JDialog { + static class PointSpec { + enum Active { + GEOCODED, PROVIDED + } + + Active active; + GeocodedPointColumn geocoded; + ProvidedPointColumn provided; + + PointSpec(SyntheticPointColumn pt) { + if(pt == null) { + active = Active.GEOCODED; + geocoded = new GeocodedPointColumn(); + provided = new ProvidedPointColumn(); + } else if(pt instanceof GeocodedPointColumn) { + active = Active.GEOCODED; + geocoded = (GeocodedPointColumn) pt; + provided = new ProvidedPointColumn(); + } else { + active = Active.PROVIDED; + geocoded = new GeocodedPointColumn(); + provided = (ProvidedPointColumn) pt; + } + } + + public SyntheticPointColumn getPointColumn() { + return active == Active.GEOCODED ? geocoded : provided; + } + + @Override + public String toString() { + return "#<" + active + " " + geocoded + " " + provided + ">"; + } + } + Map syntheticColumnsCopy; + Map columnIndexes; + ControlFileModel model; + JComboBox addressCombo; + JComboBox cityCombo; + JComboBox stateCombo; + JComboBox zipCombo; + JComboBox countryCombo; + JComboBox latCombo; + JComboBox lonCombo; + JButton buttonOK; + JButton buttonCancel; + JLabel headerLabel; + JComboBox activeLocation; + JTabbedPane tabs; + + private static final int GEOCODED_PANE = 0; + private static final int PROVIDED_PANE = 1; + + public static JDialog create(ControlFileModel model, JFrame parent, Map syntheticColumns) { + return create(model, parent, syntheticColumns, null); + } + + public static JDialog create(ControlFileModel model, JFrame parent, Map syntheticColumns, String initFieldName) { + return new SyntheticPointDialog(model, parent, syntheticColumns, initFieldName, "Manage synthetic columns"); + } + + private SyntheticPointDialog(ControlFileModel model, JFrame parent, Map syntheticColumns, String initFieldName, String title) { + super(parent, title); + this.model = model; + //Copy the columns so that playing around with them in the dialog doesn't accidentially change the underlying model. + this.syntheticColumnsCopy = copySyntheticColumns(syntheticColumns); + + columnIndexes = new HashMap<>(); + for (int i = 0; i < model.getColumnCount();i++){ + columnIndexes.put(model.getColumnAtPosition(i), i); + } + + setLocationRelativeTo(null); + setModal(true); + initComponents(); + layoutComponents(); + if (initFieldName != null) + activeLocation.setSelectedItem(initFieldName); + updatePane(); + + // this has to happen AFTER we've selected our initial pane + tabs.addChangeListener(new ChangeListener() { + @Override + public void stateChanged(ChangeEvent e) { + if(tabs.getSelectedIndex() == GEOCODED_PANE) { + ensureActiveIsGeocoded(); + } else { + ensureActiveIsProvided(); + } + } + }); + + addFieldListeners(); + + pack(); + setVisible(true); + } + + //Make a copy of the columns so that we don't change the control file just by launching the dialog. + private Map copySyntheticColumns(Map original){ + Map result = new TreeMap<>(); + for(Map.Entry ent : original.entrySet()) { + result.put(ent.getKey(), new PointSpec(ent.getValue().clone())); + } + return result; + } + + private void initComponents() { + + headerLabel = new JLabel(); + headerLabel.setText("Select the fields that make up the column"); + buttonOK = new JButton(); + buttonOK.setText("OK"); + + buttonCancel = new JButton(); + buttonCancel.setText("Cancel"); + + buttonCancel.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + setVisible(false); + dispose(); + } + }); + + buttonOK.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + commitUpdates(); + setVisible(false); + dispose(); + } + }); + + addressCombo = getComboBox(); + cityCombo = getComboBox(); + stateCombo = getComboBox(); + zipCombo = getComboBox(); + countryCombo = getComboBox(); + latCombo = getComboBox(); + lonCombo = getComboBox(); + + activeLocation = getLocationsCombobox(); + activeLocation.addItemListener(new ItemListener() { + @Override + public void itemStateChanged(ItemEvent e) { + if (e.getStateChange() == ItemEvent.SELECTED) { + updatePane(); + updateSyntheticComponentComboboxes(); + } + } + }); + + tabs = new JTabbedPane(); + + updateSyntheticComponentComboboxes(); + } + + private void addFieldListeners() { + abstract class Listener implements ItemListener { + @Override + public void itemStateChanged(ItemEvent e) { + if(e.getStateChange() == ItemEvent.SELECTED) { + T col = extractPoint(getActiveLocation()); + SyntheticComboBoxItem item = (SyntheticComboBoxItem) e.getItem(); + setField(col, model.getColumnAtPosition(item.getIndex())); + } + } + + abstract T extractPoint(PointSpec spec); + abstract void setField(T col, String value); + } + + abstract class GeocodedListener extends Listener { + GeocodedPointColumn extractPoint(PointSpec spec) { + return spec.geocoded; + } + } + + abstract class ProvidedListener extends Listener { + ProvidedPointColumn extractPoint(PointSpec spec) { + return spec.provided; + } + } + + addressCombo.addItemListener(new GeocodedListener() { + void setField(GeocodedPointColumn col, String value) { + col.address = value; + } + }); + + cityCombo.addItemListener(new GeocodedListener() { + void setField(GeocodedPointColumn col, String value) { + col.city = value; + } + }); + + stateCombo.addItemListener(new GeocodedListener() { + void setField(GeocodedPointColumn col, String value) { + col.state = value; + } + }); + + zipCombo.addItemListener(new GeocodedListener() { + void setField(GeocodedPointColumn col, String value) { + col.zip = value; + } + }); + + countryCombo.addItemListener(new GeocodedListener() { + void setField(GeocodedPointColumn col, String value) { + col.country = value; + } + }); + + latCombo.addItemListener(new ProvidedListener() { + void setField(ProvidedPointColumn col, String value) { + col.latitude = value; + } + }); + + lonCombo.addItemListener(new ProvidedListener() { + void setField(ProvidedPointColumn col, String value) { + col.longitude = value; + } + }); + } + + private void updateSyntheticComponentComboboxes(){ + PointSpec locationColumn = getActiveLocation(); + updateGeocodedComponentComboboxes(locationColumn.geocoded); + updateProvidedComponentComboboxes(locationColumn.provided); + } + + private int indexOf(String name) { + Integer r = columnIndexes.get(name); + if(r == null) return -1; + else return r; + } + + private void updatePane() { + if(getActiveLocation().active == PointSpec.Active.GEOCODED) { + tabs.setSelectedIndex(GEOCODED_PANE); + } else { + tabs.setSelectedIndex(PROVIDED_PANE); + } + } + + private void updateGeocodedComponentComboboxes(GeocodedPointColumn locationColumn) { + if (locationColumn.address != null) + addressCombo.setSelectedIndex(indexOf(locationColumn.address)); + else + addressCombo.setSelectedIndex(-1); + + if (locationColumn.city != null) + cityCombo.setSelectedIndex(indexOf(locationColumn.city)); + else + cityCombo.setSelectedIndex(-1); + + if (locationColumn.state != null) + stateCombo.setSelectedIndex(indexOf(locationColumn.state)); + else + stateCombo.setSelectedIndex(-1); + + if (locationColumn.zip != null) + zipCombo.setSelectedIndex(indexOf(locationColumn.zip)); + else + zipCombo.setSelectedIndex(-1); + + if (locationColumn.country != null) + countryCombo.setSelectedIndex(indexOf(locationColumn.country)); + else + countryCombo.setSelectedIndex(-1); + } + + private void updateProvidedComponentComboboxes(ProvidedPointColumn locationColumn) { + if (locationColumn.latitude != null) + latCombo.setSelectedIndex(indexOf(locationColumn.latitude)); + else + latCombo.setSelectedIndex(-1); + + if (locationColumn.longitude != null) + lonCombo.setSelectedIndex(indexOf(locationColumn.longitude)); + else + lonCombo.setSelectedIndex(-1); + } + + private void layoutComponents() { + JPanel contentPane = new JPanel(); + contentPane.setLayout(new com.intellij.uiDesigner.core.GridLayoutManager(4, 1, new Insets(10, 10, 10, 10), -1, -1)); + contentPane.setMaximumSize(new Dimension(315, 291)); + final JPanel panel1 = new JPanel(); + panel1.setLayout(new com.intellij.uiDesigner.core.GridLayoutManager(1, 2, new Insets(0, 0, 0, 0), -1, -1)); + contentPane.add(panel1, new com.intellij.uiDesigner.core.GridConstraints(3, 0, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_CENTER, com.intellij.uiDesigner.core.GridConstraints.FILL_BOTH, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_CAN_SHRINK | com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_CAN_GROW, 1, null, null, null, 0, false)); + final com.intellij.uiDesigner.core.Spacer spacer1 = new com.intellij.uiDesigner.core.Spacer(); + panel1.add(spacer1, new com.intellij.uiDesigner.core.GridConstraints(0, 0, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_CENTER, com.intellij.uiDesigner.core.GridConstraints.FILL_HORIZONTAL, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_WANT_GROW, 1, null, null, null, 0, false)); + final JPanel panel2 = new JPanel(); + panel2.setLayout(new com.intellij.uiDesigner.core.GridLayoutManager(1, 2, new Insets(0, 0, 0, 0), -1, -1, true, false)); + panel1.add(panel2, new com.intellij.uiDesigner.core.GridConstraints(0, 1, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_CENTER, com.intellij.uiDesigner.core.GridConstraints.FILL_BOTH, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_CAN_SHRINK | com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_CAN_GROW, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_CAN_SHRINK | com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_CAN_GROW, null, null, null, 0, false)); + + panel2.add(buttonOK, new com.intellij.uiDesigner.core.GridConstraints(0, 1, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_CENTER, com.intellij.uiDesigner.core.GridConstraints.FILL_HORIZONTAL, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_CAN_SHRINK | com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_CAN_GROW, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, null, null, null, 0, false)); + + panel2.add(buttonCancel, new com.intellij.uiDesigner.core.GridConstraints(0, 0, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_CENTER, com.intellij.uiDesigner.core.GridConstraints.FILL_HORIZONTAL, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_CAN_SHRINK | com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_CAN_GROW, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, null, null, null, 0, false)); + final JPanel panel3 = new JPanel(); + panel3.setLayout(new com.intellij.uiDesigner.core.GridLayoutManager(1, 2, new Insets(0, 0, 0, 0), -1, -1)); + contentPane.add(panel3, new com.intellij.uiDesigner.core.GridConstraints(2, 0, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_CENTER, com.intellij.uiDesigner.core.GridConstraints.FILL_BOTH, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_CAN_SHRINK | com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_CAN_GROW, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_CAN_SHRINK | com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_CAN_GROW, null, null, null, 0, false)); + panel3.setBorder(BorderFactory.createTitledBorder(BorderFactory.createEtchedBorder(), null)); + + final JPanel panel4 = new JPanel(); + panel3.add(panel4, new com.intellij.uiDesigner.core.GridConstraints(0, 0, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_CENTER, com.intellij.uiDesigner.core.GridConstraints.FILL_BOTH, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_CAN_SHRINK | com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_CAN_GROW, null, new Dimension(50, -1), null, 1, false)); + panel4.setLayout(new com.intellij.uiDesigner.core.GridLayoutManager(2, 1, new Insets(0, 0, 0, 0), -1, -1)); + headerLabel.setText("Select the point column to configure"); + + panel4.add(headerLabel, new com.intellij.uiDesigner.core.GridConstraints(0, 0, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_NORTH, com.intellij.uiDesigner.core.GridConstraints.FILL_NONE, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, null, null, null, 0, false)); + panel4.add(activeLocation, new com.intellij.uiDesigner.core.GridConstraints(1, 0, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_NORTH, com.intellij.uiDesigner.core.GridConstraints.FILL_HORIZONTAL, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, null, null, null, 0, false)); + + panel3.add(tabs, new com.intellij.uiDesigner.core.GridConstraints(0, 1, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_CENTER, com.intellij.uiDesigner.core.GridConstraints.FILL_BOTH, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_CAN_SHRINK | com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_CAN_GROW, null, new Dimension(50, -1), null, 1, false)); + + final JPanel panel5 = new JPanel(); + tabs.addTab("Geocoded", panel5); + panel5.setLayout(new com.intellij.uiDesigner.core.GridLayoutManager(5, 2, new Insets(0, 0, 0, 0), -1, -1)); + final JLabel label1 = new JLabel(); + label1.setText("Address"); + panel5.add(label1, new com.intellij.uiDesigner.core.GridConstraints(0, 0, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_EAST, com.intellij.uiDesigner.core.GridConstraints.FILL_NONE, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, null, null, null, 0, false)); + panel5.add(addressCombo, new com.intellij.uiDesigner.core.GridConstraints(0, 1, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_WEST, com.intellij.uiDesigner.core.GridConstraints.FILL_HORIZONTAL, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, null, new Dimension(220, -1), new Dimension(220, -1), 0, false)); + final JLabel label2 = new JLabel(); + label2.setText("City"); + panel5.add(label2, new com.intellij.uiDesigner.core.GridConstraints(1, 0, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_EAST, com.intellij.uiDesigner.core.GridConstraints.FILL_NONE, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, null, null, null, 0, false)); + panel5.add(cityCombo, new com.intellij.uiDesigner.core.GridConstraints(1, 1, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_WEST, com.intellij.uiDesigner.core.GridConstraints.FILL_HORIZONTAL, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, null, new Dimension(220, -1), new Dimension(220, -1), 0, false)); + final JLabel label3 = new JLabel(); + label3.setText("State"); + panel5.add(label3, new com.intellij.uiDesigner.core.GridConstraints(2, 0, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_EAST, com.intellij.uiDesigner.core.GridConstraints.FILL_NONE, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, null, null, null, 0, false)); + panel5.add(stateCombo, new com.intellij.uiDesigner.core.GridConstraints(2, 1, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_WEST, com.intellij.uiDesigner.core.GridConstraints.FILL_HORIZONTAL, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_CAN_GROW, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, null, new Dimension(220, -1), new Dimension(220, -1), 0, false)); + final JLabel label4 = new JLabel(); + label4.setText("Zip"); + panel5.add(label4, new com.intellij.uiDesigner.core.GridConstraints(3, 0, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_EAST, com.intellij.uiDesigner.core.GridConstraints.FILL_NONE, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, null, null, null, 0, false)); + panel5.add(zipCombo, new com.intellij.uiDesigner.core.GridConstraints(3, 1, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_WEST, com.intellij.uiDesigner.core.GridConstraints.FILL_HORIZONTAL, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_CAN_GROW, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, null, new Dimension(220, -1), new Dimension(220, -1), 0, false)); + final JLabel label5 = new JLabel(); + label5.setText("Country"); + panel5.add(label5, new com.intellij.uiDesigner.core.GridConstraints(4, 0, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_EAST, com.intellij.uiDesigner.core.GridConstraints.FILL_NONE, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, null, null, null, 0, false)); + panel5.add(countryCombo, new com.intellij.uiDesigner.core.GridConstraints(4, 1, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_WEST, com.intellij.uiDesigner.core.GridConstraints.FILL_HORIZONTAL, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, null, new Dimension(220, -1), new Dimension(220, -1), 0, false)); + + final JPanel panel6 = new JPanel(); + tabs.addTab("Lat/Lon", panel6); + panel6.setLayout(new com.intellij.uiDesigner.core.GridLayoutManager(2, 2, new Insets(0, 0, 0, 0), -1, -1)); + final JLabel label6 = new JLabel(); + label6.setText("Latitude"); + panel6.add(label6, new com.intellij.uiDesigner.core.GridConstraints(0, 0, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_EAST, com.intellij.uiDesigner.core.GridConstraints.FILL_NONE, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, null, null, null, 0, false)); + panel6.add(latCombo, new com.intellij.uiDesigner.core.GridConstraints(0, 1, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_WEST, com.intellij.uiDesigner.core.GridConstraints.FILL_HORIZONTAL, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, null, new Dimension(220, -1), new Dimension(220, -1), 0, false)); + final JLabel label7 = new JLabel(); + label7.setText("Longitude"); + panel6.add(label7, new com.intellij.uiDesigner.core.GridConstraints(1, 0, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_EAST, com.intellij.uiDesigner.core.GridConstraints.FILL_NONE, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, null, null, null, 0, false)); + panel6.add(lonCombo, new com.intellij.uiDesigner.core.GridConstraints(1, 1, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_WEST, com.intellij.uiDesigner.core.GridConstraints.FILL_HORIZONTAL, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, null, new Dimension(220, -1), new Dimension(220, -1), 0, false)); + + final com.intellij.uiDesigner.core.Spacer spacer2 = new com.intellij.uiDesigner.core.Spacer(); + contentPane.add(spacer2, new com.intellij.uiDesigner.core.GridConstraints(1, 0, 1, 1, com.intellij.uiDesigner.core.GridConstraints.ANCHOR_CENTER, com.intellij.uiDesigner.core.GridConstraints.FILL_HORIZONTAL, com.intellij.uiDesigner.core.GridConstraints.SIZEPOLICY_FIXED, 1, new Dimension(-1, 5), new Dimension(-1, 5), new Dimension(-1, 5), 3, false)); + getContentPane().add(contentPane); + } + + // Get a combobox whose values are all of the possible fields in the control file that could be mapped to this column + private JComboBox getComboBox() { + JComboBox columnNamesComboBox = new JComboBox<>(); + for (int i = 0; i < model.getColumnCount();i++){ + String friendlyName = model.getDisplayName(i); + columnNamesComboBox.addItem(new SyntheticComboBoxItem(friendlyName,i)); + } + columnNamesComboBox.setSelectedIndex(-1); + return columnNamesComboBox; + } + + private JComboBox getLocationsCombobox(){ + ArrayList columns = model.getDatasetModel().getColumns(); + JComboBox locationComboBox = new JComboBox(); + + for (Column column : columns) { + if (column.getDataTypeName().equals("point")) { + locationComboBox.addItem(column.getFieldName()); + } + } + locationComboBox.setSelectedIndex(0); + return locationComboBox; + } + + private String getActiveFieldName() { + return (String) activeLocation.getSelectedItem(); + } + + private PointSpec getActiveLocation() { + String fieldName = getActiveFieldName(); + PointSpec locationColumn = syntheticColumnsCopy.get(fieldName); + if (locationColumn == null) { + locationColumn = new PointSpec(null); + syntheticColumnsCopy.put(fieldName,locationColumn); + } + return locationColumn; + } + + private void ensureActiveIsGeocoded() { + getActiveLocation().active = PointSpec.Active.GEOCODED; + } + + private void ensureActiveIsProvided() { + getActiveLocation().active = PointSpec.Active.PROVIDED; + } + + private void commitUpdates() { + for (String field : syntheticColumnsCopy.keySet()) { + model.setSyntheticPoint(field, syntheticColumnsCopy.get(field).getPointColumn()); + } + } +} diff --git a/src/main/java/com/socrata/datasync/ui/SyntheticPointPanel.java b/src/main/java/com/socrata/datasync/ui/SyntheticPointPanel.java new file mode 100644 index 00000000..b14ac3c5 --- /dev/null +++ b/src/main/java/com/socrata/datasync/ui/SyntheticPointPanel.java @@ -0,0 +1,89 @@ +package com.socrata.datasync.ui; + +import com.socrata.datasync.config.controlfile.SyntheticPointColumn; +import com.socrata.datasync.config.controlfile.GeocodedPointColumn; +import com.socrata.datasync.model.ControlFileModel; + +import javax.swing.*; +import java.awt.*; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.util.Collection; +import java.util.Iterator; +import java.util.Map; + +/** + * Created by franklinwilliams on 5/17/15. + */ +public class SyntheticPointPanel extends JPanel { + + String fieldName; + SyntheticPointColumn locationColumn; + ControlFileModel model; + + public SyntheticPointPanel(ControlFileModel model, String fieldName, SyntheticPointColumn column){ + this.model = model; + this.fieldName = fieldName; + this.locationColumn = column; + initComponents(); + } + + private void initComponents(){ + setLayout(new BoxLayout(this, BoxLayout.X_AXIS)); + JLabel label = new JLabel(); + label.setText("" + getSyntheticMappedColumns() + " " + '\u2192' + " " + fieldName); + JButton manage = UIUtility.getButtonAsLink("Manage"); + JButton remove = UIUtility.getButtonAsLink("Remove"); + + + manage.setPreferredSize(new Dimension(100,16)); + manage.setMaximumSize(new Dimension(100, 16)); + remove.setPreferredSize(new Dimension(100, 16)); + remove.setMaximumSize(new Dimension(100, 16)); + + manage.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + JDialog dialog = SyntheticPointDialog.create(model, (JFrame) ((JDialog)SwingUtilities.getRoot((JButton) e.getSource())).getParent(),model.getSyntheticPoints(), fieldName); + } + }); + + remove.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + model.removeSyntheticColumn(fieldName); + } + }); + label.setAlignmentX(LEFT_ALIGNMENT); + manage.setAlignmentX(LEFT_ALIGNMENT); + remove.setAlignmentX(LEFT_ALIGNMENT); + + label.setHorizontalAlignment(SwingConstants.LEFT); + manage.setHorizontalAlignment(SwingConstants.LEFT); + remove.setHorizontalAlignment(SwingConstants.LEFT); + + this.add(label); + this.add(manage); + this.add(remove); + } + + private String getSyntheticMappedColumns(){ + StringBuffer buffer = new StringBuffer(); + // buffer.append("Creating \"" + fieldName + "\" from "); + Map components = locationColumn.findComponentColumns(); + Collection values = components.values(); + Iterator it = values.iterator(); + while (it.hasNext()){ + + String component = it.next(); + //TODO: Ugh. Have the underlying name. Going to get the index and then the friendly name. Clean this up. + int index = model.getIndexOfColumnName(component); + String displayName = model.getDisplayName(index); + buffer.append(displayName); + if (it.hasNext()) + buffer.append(", "); + } + return buffer.toString(); + } + +} diff --git a/src/main/java/com/socrata/datasync/ui/SyntheticPointsContainer.java b/src/main/java/com/socrata/datasync/ui/SyntheticPointsContainer.java new file mode 100644 index 00000000..456591c7 --- /dev/null +++ b/src/main/java/com/socrata/datasync/ui/SyntheticPointsContainer.java @@ -0,0 +1,90 @@ +package com.socrata.datasync.ui; + +import com.socrata.datasync.config.controlfile.SyntheticPointColumn; +import com.socrata.datasync.config.controlfile.GeocodedPointColumn; +import com.socrata.datasync.model.ControlFileModel; + +import javax.swing.*; +import java.awt.*; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.util.Map; +import java.util.Observable; +import java.util.Observer; + +/** + * Container for holding the synthetic points in the control file. Available when the customer clicks + * "manage synthetic points" from the Control File Editor dialog box on an NBE dataset. + */ +public class SyntheticPointsContainer extends JPanel implements Observer { + ControlFileModel model; + + public SyntheticPointsContainer(ControlFileModel model){ + this.model = model; + model.addObserver(this); + styleComponents(); + updateComponents(); + } + + private void styleComponents(){ + this.setBorder(BorderFactory.createTitledBorder("Synthetic Points")); + this.setLayout(new BoxLayout(this, BoxLayout.Y_AXIS)); + } + + private JPanel getEmptyPanel(){ + JPanel panel = new JPanel(); + panel.setLayout(new BoxLayout(panel,BoxLayout.X_AXIS)); + panel.setPreferredSize(new Dimension(-1, 16)); + JLabel label = new JLabel(); + label.setText("No synthetic points configured"); + label.setHorizontalAlignment(SwingConstants.LEFT); + label.setAlignmentX(LEFT_ALIGNMENT); + panel.add(Box.createRigidArea(new Dimension(10,10))); + panel.add(label); + JButton button = getAddButton(); + panel.add(button); + panel.setAlignmentX(LEFT_ALIGNMENT); + return panel; + } + + private JButton getAddButton(){ + JButton button = UIUtility.getButtonAsLink("Add"); + button.setAlignmentX(LEFT_ALIGNMENT); + button.setHorizontalAlignment(SwingConstants.LEFT); + button.setHorizontalTextPosition(SwingConstants.LEFT); + button.setPreferredSize(new Dimension(16, -1)); + button.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + JDialog dialog = SyntheticPointDialog.create(model, (JFrame) ((JDialog) SwingUtilities.getRoot((JButton) e.getSource())).getParent(), model.getSyntheticPoints()); + } + }); + return button; + } + + private void updateComponents(){ + this.removeAll(); + Map locations = model.getSyntheticPoints(); + if (locations != null && !locations.isEmpty()){ + for (String key : locations.keySet()){ + JPanel panel = new SyntheticPointPanel(model, key, locations.get(key)); + panel.setAlignmentX(LEFT_ALIGNMENT); + this.add(Box.createRigidArea(new Dimension(10,5))); + this.add(panel); + } + + if (model.getDatasetModel().getLocationCount() > model.getSyntheticLocations().size()) + this.add(Box.createRigidArea(new Dimension(10, 5))); + this.add(getAddButton()); + } + else + this.add(getEmptyPanel()); + this.revalidate(); + } + + + @Override + public void update(Observable o, Object arg) { + updateComponents(); + } +} diff --git a/src/main/java/com/socrata/datasync/ui/UIUtility.java b/src/main/java/com/socrata/datasync/ui/UIUtility.java index bf30c933..f9895f92 100644 --- a/src/main/java/com/socrata/datasync/ui/UIUtility.java +++ b/src/main/java/com/socrata/datasync/ui/UIUtility.java @@ -10,6 +10,11 @@ import java.awt.*; import java.awt.datatransfer.Clipboard; import java.awt.datatransfer.StringSelection; +import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; /** * Author: Adrian Laurenzi @@ -101,4 +106,32 @@ static void copyToClipboard(String textToCopy) { Clipboard clpbrd = Toolkit.getDefaultToolkit().getSystemClipboard(); clpbrd.setContents(stringSelection, null); } + + public static String relativize(File base, File file) { + file = file.getAbsoluteFile(); + List segments = new ArrayList<>(); + + File pointer = file; + while(pointer != null && !pointer.equals(base)) { + segments.add(pointer.getName()); + pointer = pointer.getParentFile(); + } + + if(pointer == null) { + return file.getAbsolutePath(); + } else if(segments.isEmpty()) { + // we selected the current directory? I suppose this + // is theoretically possible in the presence of + // filesystem changes during selection. + return file.getAbsolutePath(); + } else { + Collections.reverse(segments); + Iterator it = segments.iterator(); + File result = new File(it.next()); + while(it.hasNext()) { + result = new File(result, it.next()); + } + return result.getPath(); + } + } } diff --git a/src/main/java/com/socrata/datasync/validation/IntegrationJobValidity.java b/src/main/java/com/socrata/datasync/validation/IntegrationJobValidity.java index 72cbf4ca..9bfe5ce4 100644 --- a/src/main/java/com/socrata/datasync/validation/IntegrationJobValidity.java +++ b/src/main/java/com/socrata/datasync/validation/IntegrationJobValidity.java @@ -1,6 +1,5 @@ package com.socrata.datasync.validation; -import com.socrata.api.SodaImporter; import com.socrata.datasync.DatasetUtils; import com.socrata.datasync.HttpUtility; import com.socrata.datasync.PublishMethod; @@ -17,7 +16,7 @@ import org.apache.commons.cli.CommandLine; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.entity.ContentType; -import org.codehaus.jackson.map.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; @@ -60,8 +59,8 @@ public static boolean validateArgs(CommandLine cmd) { /** * @return an error JobStatus if any input is invalid, otherwise JobStatus.VALID */ - public static JobStatus validateJobParams(SocrataConnectionInfo connectionInfo, IntegrationJob job) { - if(connectionInfo.getUrl().equals("") || connectionInfo.getUrl().equals("https://")) + public static JobStatus validateJobParams(UserPreferences userPrefs, IntegrationJob job) { + if(userPrefs.getConnectionInfo().getUrl().equals("") || userPrefs.getConnectionInfo().getUrl().equals("https://")) return JobStatus.INVALID_DOMAIN; if(!Utils.uidIsValid(job.getDatasetID())) @@ -82,10 +81,9 @@ public static JobStatus validateJobParams(SocrataConnectionInfo connectionInfo, if(!allowedFileToPublishExtensions.contains(fileExtension)) return JobStatus.FILE_TO_PUBLISH_INVALID_TABULAR_FORMAT; - final SodaImporter importer = SodaImporter.newImporter(connectionInfo.getUrl(), connectionInfo.getUser(), connectionInfo.getPassword(), connectionInfo.getToken()); Dataset schema; try { - schema = (Dataset) importer.loadDatasetInfo(job.getDatasetID()); + schema = DatasetUtils.getDatasetInfo(userPrefs, job.getDatasetID()); if(job.getPublishViaDi2Http() || job.getPublishViaFTP()) { @@ -103,7 +101,7 @@ public static JobStatus validateJobParams(SocrataConnectionInfo connectionInfo, if (actionOkay.isError()) return actionOkay; - JobStatus controlOkay = checkControl(control,fileControl,schema,publishFile,connectionInfo.getUrl()); + JobStatus controlOkay = checkControl(control,fileControl,schema,publishFile,userPrefs.getConnectionInfo().getUrl()); if (controlOkay.isError()) return controlOkay; @@ -618,15 +616,18 @@ private static JobStatus checkColumnAgreement(FileTypeControl fileControl, Publi } - Map syntheticColumnsMap = fileControl.syntheticLocations; + Map syntheticColumnsMap = fileControl.syntheticLocations; if (syntheticColumnsMap != null) { Set syntheticColumns = syntheticColumnsMap.keySet(); - for (String synthetic : syntheticColumns) { - if (columnNames.contains(synthetic)) - columnNames.remove(synthetic); - } + columnNames.removeAll(syntheticColumnsMap.keySet()); } + + syntheticColumnsMap = fileControl.syntheticPoints; + if (syntheticColumnsMap != null) { + columnNames.removeAll(syntheticColumnsMap.keySet()); + } + if (columnNames.size() > 0 && method.equals(PublishMethod.replace)) { if (rowIdentifier == null) { JobStatus status = JobStatus.MISSING_COLUMNS; diff --git a/src/test/java/com/socrata/datasync/TestBase.java b/src/test/java/com/socrata/datasync/TestBase.java index 373843c5..611983d7 100644 --- a/src/test/java/com/socrata/datasync/TestBase.java +++ b/src/test/java/com/socrata/datasync/TestBase.java @@ -9,7 +9,7 @@ import com.socrata.exceptions.LongRunningQueryException; import com.socrata.exceptions.SodaError; import com.sun.jersey.api.client.ClientResponse; -import org.codehaus.jackson.map.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; import java.io.IOException; diff --git a/src/test/java/com/socrata/datasync/UtilsTest.java b/src/test/java/com/socrata/datasync/UtilsTest.java index 3e60e293..a9491261 100644 --- a/src/test/java/com/socrata/datasync/UtilsTest.java +++ b/src/test/java/com/socrata/datasync/UtilsTest.java @@ -2,8 +2,10 @@ import com.socrata.datasync.config.controlfile.ControlFile; import junit.framework.TestCase; -import org.codehaus.jackson.map.DeserializationConfig; -import org.codehaus.jackson.map.ObjectMapper; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import com.fasterxml.jackson.databind.DeserializationConfig; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Test; import java.io.ByteArrayInputStream; @@ -80,4 +82,49 @@ public void testReadHeadersFromFile() throws IOException { TestCase.assertEquals(expectedHeaders4[i], actualHeaders4[i]); } + + @Test + public void testSplitEmpty() { + assertArrayEquals(new String[0], Utils.commaSplit("")); + } + + @Test + public void testSplitEscaped() { + assertArrayEquals(new String[] { "a,b" }, Utils.commaSplit("a\\,b")); + } + + @Test + public void testSplitUnecaped() { + assertArrayEquals(new String[] { "a", "b" }, Utils.commaSplit("a,b")); + } + + @Test + public void testSplitWhitespace() { + assertArrayEquals(new String[] { "a", "b" }, Utils.commaSplit(" a , b ")); + } + + @Test + public void testSplitOne() { + assertArrayEquals(new String[] { "a" }, Utils.commaSplit(" a ")); + } + + @Test + public void testJoinNone() { + assertEquals("", Utils.commaJoin(new String[0])); + } + + @Test + public void testJoinOne() { + assertEquals("abc", Utils.commaJoin(new String[] { "abc" })); + } + + @Test + public void testJoinTrim() { + assertEquals("abc, def", Utils.commaJoin(new String[] { " abc ", " def " })); + } + + @Test + public void testJoinEscape() { + assertEquals("abc, def\\,ghi", Utils.commaJoin(new String[] { " abc ", " def,ghi " })); + } } diff --git a/src/test/java/com/socrata/datasync/config/controlfile/ControlFileTest.java b/src/test/java/com/socrata/datasync/config/controlfile/ControlFileTest.java index a77c62b8..c33514ba 100644 --- a/src/test/java/com/socrata/datasync/config/controlfile/ControlFileTest.java +++ b/src/test/java/com/socrata/datasync/config/controlfile/ControlFileTest.java @@ -3,8 +3,8 @@ import com.socrata.datasync.PublishMethod; import com.socrata.exceptions.SodaError; import junit.framework.TestCase; -import org.codehaus.jackson.map.DeserializationConfig; -import org.codehaus.jackson.map.ObjectMapper; +import com.fasterxml.jackson.databind.DeserializationConfig; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Test; import java.io.File; diff --git a/src/test/java/com/socrata/datasync/config/userpreferences/UserPreferencesTest.java b/src/test/java/com/socrata/datasync/config/userpreferences/UserPreferencesTest.java index 85bd6e2a..98c99f82 100644 --- a/src/test/java/com/socrata/datasync/config/userpreferences/UserPreferencesTest.java +++ b/src/test/java/com/socrata/datasync/config/userpreferences/UserPreferencesTest.java @@ -2,7 +2,7 @@ import com.socrata.datasync.TestBase; import junit.framework.TestCase; -import org.codehaus.jackson.map.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Test; import java.io.File; diff --git a/src/test/java/com/socrata/datasync/model/CSVModelTest.java b/src/test/java/com/socrata/datasync/model/CSVModelTest.java index 1abf3833..aae1154d 100644 --- a/src/test/java/com/socrata/datasync/model/CSVModelTest.java +++ b/src/test/java/com/socrata/datasync/model/CSVModelTest.java @@ -2,9 +2,9 @@ import com.socrata.datasync.config.controlfile.ControlFile; import junit.framework.TestCase; -import org.codehaus.jackson.JsonParseException; -import org.codehaus.jackson.map.DeserializationConfig; -import org.codehaus.jackson.map.ObjectMapper; +import com.fasterxml.jackson.JsonParseException; +import com.fasterxml.jackson.databind.DeserializationConfig; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Test; import java.io.File; diff --git a/src/test/java/com/socrata/datasync/model/ControlFileModelTest.java b/src/test/java/com/socrata/datasync/model/ControlFileModelTest.java index 29623685..907b3a4c 100644 --- a/src/test/java/com/socrata/datasync/model/ControlFileModelTest.java +++ b/src/test/java/com/socrata/datasync/model/ControlFileModelTest.java @@ -8,9 +8,9 @@ import com.socrata.exceptions.LongRunningQueryException; import org.apache.http.HttpException; import junit.framework.TestCase; -import org.codehaus.jackson.map.DeserializationConfig; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.SerializationConfig; +import com.fasterxml.jackson.databind.DeserializationConfig; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationConfig; import org.junit.Test; import java.io.File; diff --git a/src/test/java/com/socrata/datasync/utilities/DeltaImporter2PublisherTest.java b/src/test/java/com/socrata/datasync/utilities/DeltaImporter2PublisherTest.java index 3f015077..4200493b 100644 --- a/src/test/java/com/socrata/datasync/utilities/DeltaImporter2PublisherTest.java +++ b/src/test/java/com/socrata/datasync/utilities/DeltaImporter2PublisherTest.java @@ -8,8 +8,8 @@ import com.socrata.datasync.deltaimporter2.JobId; import com.socrata.datasync.deltaimporter2.LogItem; import junit.framework.TestCase; -import org.codehaus.jackson.map.DeserializationConfig; -import org.codehaus.jackson.map.ObjectMapper; +import com.fasterxml.jackson.databind.DeserializationConfig; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Test; import java.io.File; diff --git a/src/test/java/com/socrata/datasync/utilities/HttpUtilityTest.java b/src/test/java/com/socrata/datasync/utilities/HttpUtilityTest.java index 98eda782..e70163e5 100644 --- a/src/test/java/com/socrata/datasync/utilities/HttpUtilityTest.java +++ b/src/test/java/com/socrata/datasync/utilities/HttpUtilityTest.java @@ -10,7 +10,7 @@ import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.utils.URIBuilder; import org.apache.http.entity.ContentType; -import org.codehaus.jackson.map.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.After; import org.junit.Before; import org.junit.Test;