Merge pull request #5 from conveyal/mtc-fixes

Mtc fixes

.travis.yml

@@ -16,7 +16,7 @@ notifications:
 before_deploy:
 - mkdir deploy
 - cp target/dt-*.jar deploy/
-- cp target/dt-*.jar deploy/dt-latest.jar
+- cp "target/dt-*.jar" "deploy/dt-latest-$(git rev-parse --abbrev-ref HEAD).jar"
 deploy:
   skip_cleanup: true
   provider: s3
@@ -28,4 +28,4 @@ deploy:
   acl: public_read
   on:
     repo: conveyal/datatools-server
-    branch: dev
+    all_branches: true

pom.xml

@@ -164,7 +164,7 @@
         <dependency>
             <groupId>com.conveyal</groupId>
             <artifactId>gtfs-lib</artifactId>
-            <version>1.4.0</version>
+            <version>2.0.0</version>
         </dependency>
 
         <dependency>
@@ -200,9 +200,13 @@
         <dependency>
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-java-sdk-s3</artifactId>
-            <version>1.10.66</version>
+            <version>1.11.103</version>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk</artifactId>
+            <version>1.11.103</version>
         </dependency>
-
         <dependency>
             <groupId>com.fasterxml.jackson.dataformat</groupId>
             <artifactId>jackson-dataformat-yaml</artifactId>

src/main/java/com/conveyal/datatools/manager/DataManager.java

@@ -67,6 +67,9 @@ public class DataManager {
     public static boolean useS3;
     public static final String apiPrefix = "/api/manager/";
 
+    public static final String DEFAULT_ENV = "configurations/default/env.yml";
+    public static final String DEFAULT_CONFIG = "configurations/default/server.yml";
+
     private static List<String> apiFeedSources = new ArrayList<>();
 
     public static void main(String[] args) throws IOException {
@@ -293,10 +296,14 @@ private static void loadConfig (String[] args) throws IOException {
         FileInputStream serverConfigStream;
 
         if (args.length == 0) {
-            configStream = new FileInputStream(new File("configurations/default/env.yml"));
-            serverConfigStream = new FileInputStream(new File("configurations/default/server.yml"));
+            LOG.warn("Using default env.yml: {}", DEFAULT_ENV);
+            LOG.warn("Using default server.yml: {}", DEFAULT_CONFIG);
+            configStream = new FileInputStream(new File(DEFAULT_ENV));
+            serverConfigStream = new FileInputStream(new File(DEFAULT_CONFIG));
         }
         else {
+            LOG.info("Loading env.yml: {}", args[0]);
+            LOG.info("Loading server.yml: {}", args[1]);
             configStream = new FileInputStream(new File(args[0]));
             serverConfigStream = new FileInputStream(new File(args[1]));
         }

src/main/java/com/conveyal/datatools/manager/auth/Auth0UserProfile.java

@@ -283,6 +283,9 @@ public String getOrganizationId() {
 
     public boolean canAdministerOrganization(String organizationId) {
 //      TODO: adapt for specific org
+        if (organizationId == null) {
+            return false;
+        }
         Organization org = getAuth0Organization();
         if (org != null && org.organizationId.equals(organizationId)) {
             for(Permission permission : org.permissions) {
@@ -327,7 +330,6 @@ public boolean canManageFeed(String organizationId, String projectID, String fee
         }
         Project[] projectList = app_metadata.getDatatoolsInfo().projects;
         for(Project project : projectList) {
-            System.out.println("project_id: " + project.project_id);
             if (project.project_id.equals(projectID)) {
                 return checkFeedPermission(project, feedID, "manage-feed");
             }
@@ -341,7 +343,6 @@ public boolean canEditGTFS(String organizationId, String projectID, String feedI
         }
         Project[] projectList = app_metadata.getDatatoolsInfo().projects;
         for(Project project : projectList) {
-            System.out.println("project_id: " + project.project_id);
             if (project.project_id.equals(projectID)) {
                 return checkFeedPermission(project, feedID, "edit-gtfs");
             }
@@ -355,7 +356,6 @@ public boolean canApproveGTFS(String organizationId, String projectID, String fe
         }
         Project[] projectList = app_metadata.getDatatoolsInfo().projects;
         for(Project project : projectList) {
-            System.out.println("project_id: " + project.project_id);
             if (project.project_id.equals(projectID)) {
                 return checkFeedPermission(project, feedID, "approve-gtfs");
             }

src/main/java/com/conveyal/datatools/manager/controllers/api/GtfsPlusController.java

@@ -1,11 +1,13 @@
 package com.conveyal.datatools.manager.controllers.api;
 
+import com.conveyal.datatools.common.utils.SparkUtils;
 import com.conveyal.datatools.manager.DataManager;
 import com.conveyal.datatools.manager.models.FeedVersion;
 import com.conveyal.datatools.manager.persistence.FeedStore;
 import com.conveyal.datatools.manager.utils.json.JsonUtil;
 import com.conveyal.gtfs.GTFSFeed;
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import spark.Request;
@@ -52,7 +54,6 @@ public static Boolean uploadGtfsPlusFile (Request req, Response res) throws IOEx
         InputStream uploadStream;
         try {
             uploadStream = part.getInputStream();
-            //v.newGtfsFile(uploadStream);
             gtfsPlusStore.newFeed(feedVersionId, uploadStream, null);
         } catch (Exception e) {
             LOG.error("Unable to open input stream from upload");
@@ -152,6 +153,9 @@ private static Long getGtfsPlusFileTimestamp(Request req, Response res) {
         File file = gtfsPlusStore.getFeed(feedVersionId);
         if(file == null) {
             FeedVersion feedVersion = FeedVersion.get(feedVersionId);
+            if (feedVersion == null) {
+                halt(400, SparkUtils.formatJSON("Feed version ID is not valid", 400));
+            }
             file = feedVersion.getGtfsFile();
         }
 
@@ -262,10 +266,11 @@ private static Collection<ValidationIssue> getGtfsPlusValidation(Request req, Re
         GTFSFeed gtfsFeed = feedVersion.getGtfsFeed();
         // check for saved GTFS+ data
         File file = gtfsPlusStore.getFeed(feedVersionId);
-        if(file == null) {
+        if (file == null) {
+            LOG.warn("GTFS+ file not found, loading from main version GTFS.");
             file = feedVersion.getGtfsFile();
         }
-
+        int gtfsPlusTableCount = 0;
         try {
             ZipFile zipFile = new ZipFile(file);
             final Enumeration<? extends ZipEntry> entries = zipFile.entries();
@@ -275,6 +280,7 @@ private static Collection<ValidationIssue> getGtfsPlusValidation(Request req, Re
                     JsonNode tableNode = DataManager.gtfsPlusConfig.get(i);
                     if(tableNode.get("name").asText().equals(entry.getName())) {
                         LOG.info("Validating GTFS+ table: " + entry.getName());
+                        gtfsPlusTableCount++;
                         validateTable(issues, tableNode, zipFile.getInputStream(entry), gtfsFeed);
                     }
                 }
@@ -284,7 +290,7 @@ private static Collection<ValidationIssue> getGtfsPlusValidation(Request req, Re
             e.printStackTrace();
             halt(500);
         }
-
+        LOG.info("GTFS+ tables found: {}/{}", gtfsPlusTableCount, DataManager.gtfsPlusConfig.size());
         return issues;
     }
 
@@ -310,7 +316,7 @@ private static void validateTable(Collection<ValidationIssue> issues, JsonNode t
 
         int rowIndex = 0;
         while((line = in.readLine()) != null) {
-            String[] values = line.split(",", -1);
+            String[] values = line.split(",(?=([^\"]*\"[^\"]*\")*[^\"]*$)", -1);
             for(int v=0; v < values.length; v++) {
                 validateTableValue(issues, tableId, rowIndex, values[v], fieldNodes[v], gtfsFeed);
             }
@@ -329,6 +335,23 @@ private static void validateTableValue(Collection<ValidationIssue> issues, Strin
         }
 
         switch(fieldNode.get("inputType").asText()) {
+            case "DROPDOWN":
+                boolean invalid = true;
+                ArrayNode options = (ArrayNode) fieldNode.get("options");
+                for (JsonNode option : options) {
+                    String optionValue = option.get("value").asText();
+
+                    // if value is found in list of options, break out of loop
+                    if (optionValue.equals(value) || !fieldNode.get("required").asBoolean() && value.equals("")) {
+                        invalid = false;
+                        break;
+                    }
+                }
+                if (invalid) {
+                    System.out.println("invalid: " + " " + value);
+                    issues.add(new ValidationIssue(tableId, fieldName, rowIndex, "Value: " + value + " is not a valid option."));
+                }
+                break;
             case "TEXT":
                 if(fieldNode.get("maxLength") != null) {
                     int maxLength = fieldNode.get("maxLength").asInt();
@@ -367,6 +390,7 @@ private static void validateTableValue(Collection<ValidationIssue> issues, Strin
     }
 
     public static class ValidationIssue implements Serializable {
+        private static final long serialVersionUID = 1L;
         public String tableId;
         public String fieldName;
         public int rowIndex;

src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java

@@ -98,7 +98,7 @@ public void importFeedsForProject(Project project, String authHeader) {
                 for (FeedSource existingSource : project.getProjectFeedSources()) {
                     ExternalFeedSourceProperty agencyIdProp =
                             ExternalFeedSourceProperty.find(existingSource, this.getResourceType(), "AgencyId");
-                    if (agencyIdProp != null && agencyIdProp.value.equals(car.AgencyId)) {
+                    if (agencyIdProp != null && agencyIdProp.value != null && agencyIdProp.value.equals(car.AgencyId)) {
                         //System.out.println("already exists: " + car.AgencyId);
                         source = existingSource;
                     }

src/main/java/com/conveyal/datatools/manager/jobs/ValidateFeedJob.java

@@ -41,6 +41,7 @@ public void run() {
             status.message = "Running validation...";
             status.percentComplete = 30;
         }
+        feedVersion.setUserById(owner);
         feedVersion.validate(eventBus);
         feedVersion.save();
         if (!status.error)

src/main/java/com/conveyal/datatools/manager/models/Organization.java

@@ -2,6 +2,7 @@
 
 import com.conveyal.datatools.manager.persistence.DataStore;
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonView;
 
 import java.io.Serializable;
@@ -15,6 +16,7 @@
 /**
  * Created by landon on 1/30/17.
  */
+@JsonIgnoreProperties(ignoreUnknown = true)
 public class Organization extends Model implements Serializable {
     private static final long serialVersionUID = 1L;
     private static DataStore<Organization> organizationStore = new DataStore<>("organizations");

src/main/java/com/conveyal/datatools/manager/persistence/FeedStore.java

@@ -178,19 +178,24 @@ public File newFeed (String id, InputStream inputStream, FeedSource feedSource)
         // For s3 storage (store locally and let gtfsCache handle loading feed to s3)
         return storeFeedLocally(id, inputStream, feedSource);
     }
-
     private File storeFeedLocally(String id, InputStream inputStream, FeedSource feedSource) {
-        // store latest as feed-source-id.zip
+        File feed = null;
+        try {
+            // write feed to specified ID.
+            // NOTE: depending on the feed store, there may not be a feedSource provided (e.g., gtfsplus)
+            feed = writeFileUsingInputStream(id, inputStream);
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
         if (feedSource != null) {
             try {
-                File version = writeFileUsingInputStream(id, inputStream);
-                copyVersionToLatest(version, feedSource);
-                return version;
+                // store latest as feed-source-id.zip if feedSource provided
+                copyVersionToLatest(feed, feedSource);
             } catch (Exception e) {
                 e.printStackTrace();
             }
         }
-        return null;
+        return feed;
     }
 
     private void copyVersionToLatest(File version, FeedSource feedSource) {
@@ -208,6 +213,7 @@ private File writeFileUsingInputStream(String filename, InputStream inputStream)
         OutputStream output = null;
         File out = new File(path, filename);
         try {
+            LOG.info("Writing file to {}/{}", path, filename);
             output = new FileOutputStream(out);
             byte[] buf = new byte[1024];
             int bytesRead;