Merge branch 'dev' into add-gtfs-extended-route-types

Author: binh-dam-ibigroup

.github/issue_template.md

@@ -1,4 +1,4 @@
-_**NOTE:** This issue system is intended for reporting bugs and tracking progress in software development. Although this software is licensed with an open-source license, any issue opened here may not be responded to in a timely manner. [Conveyal](https://www.conveyal.com) is unable to provide technical support for custom deployments of this software unless your company has a support contract with us. Please remove this note when creating the issue._
+_**NOTE:** This issue system is intended for reporting bugs and tracking progress in software development. Although this software is licensed with an open-source license, any issue opened here may not be dealt with in a timely manner. [IBI Group](https://www.ibigroup.com/) is able to provide technical support for custom deployments of this software. Please contact [Ritesh Warade](mailto:[email protected]?subject=Data%20Tools%20inquiry%20via%20GitHub&body=Name:%20%0D%0AAgency/Company:%20%0D%0ABest%20date/time%20for%20a%20demo/discussion:%20%0D%0ADescription%20of%20needs:%20) if your company or organization is interested in opening a support contract with us. Please remove this note when creating the issue._
 
 ## Observed behavior (please include a screenshot if possible)
 

.github/pull_request_template.md

@@ -7,7 +7,6 @@
 - [ ] All tests and CI builds passing
 - [ ] The description lists all relevant PRs included in this release _(remove this if not merging to master)_
 - [ ] e2e tests are all passing _(remove this if not merging to master)_
-- [ ] Code coverage does not significantly worsen (ideally it improves) _(remove this if not merging to master)_
 
 ### Description
 

.github/workflows/node-ci.yml

@@ -50,12 +50,8 @@ jobs:
         run: yarn lint-messages
       - name: Run flow check
         run: yarn flow
-      - name: Run tests with coverage
-        run: yarn cover-client
-      # Upload coverage results from unit tests and then delete coverage reports to
-      # avoid uploading the same coverage results twice.
-      - name: Upload coverage results from unit tests
-        run: bash <(curl -s https://codecov.io/bash) -c -F unit_tests
+      - name: Run tests
+        run: yarn test-client
       - name: Build with minification
         run: yarn run build -- --minify
       - name: Build docs
@@ -72,9 +68,12 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_REGION: ${{ secrets.AWS_REGION }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      - name: Install otp-runner
+        if: env.SHOULD_RUN_E2E == 'true'
+        run: yarn global add https://github.com/ibi-group/otp-runner.git
       - name: Run e2e tests
         if: env.SHOULD_RUN_E2E == 'true'
-        run: yarn cover-end-to-end
+        run: yarn test-end-to-end
         env:
           AUTH0_API_CLIENT: ${{ secrets.AUTH0_API_CLIENT }}
           AUTH0_API_SECRET: ${{ secrets.AUTH0_API_SECRET }}
@@ -101,12 +100,6 @@ jobs:
           SPARKPOST_KEY: ${{ secrets.SPARKPOST_KEY }}
           TRANSITFEEDS_KEY: ${{ secrets.TRANSITFEEDS_KEY }}
       # At this point, the build is successful.
-      # Upload coverage results from e2e tests and then delete coverage reports to
-      # avoid uploading the same coverage results twice.
-      - name: Upload coverage results from e2e
-        if: env.SHOULD_RUN_E2E == 'true'
-        run: bash <(curl -s https://codecov.io/bash) -c -F end_to_end_tests
-        continue-on-error: true
       - name: Semantic Release
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.husky/.gitignore

@@ -0,0 +1 @@
+_

.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/bin/sh
+. "$(dirname "$0")/_/husky.sh"
+
+npx lint-staged

__tests__/end-to-end.js

@@ -259,6 +259,20 @@ async function expectSelectorToNotContainHtml (selector: string, html: string) {
   expect(innerHTML).not.toContain(html)
 }
 
+/**
+ * Checks that the expected feed version validity dates are displayed.
+ */
+async function expectFeedVersionValidityDates (startDate: string, endDate: string) {
+  await expectSelectorToContainHtml(
+    '[data-test-id="active-feed-version-validity-start"]',
+    startDate
+  )
+  await expectSelectorToContainHtml(
+    '[data-test-id="active-feed-version-validity-end"]',
+    endDate
+  )
+}
+
 /**
  * Create a new project.  Assumes that this is called while the browser is on
  * the home page.
@@ -301,6 +315,7 @@ async function deleteProject (projectId: string) {
   // verify deletion
   await goto(`http://localhost:9966/project/${projectId}`)
   await waitForSelector('.project-not-found')
+  await wait(5000, 'for previously rendered project markup to be removed')
   await expectSelectorToContainHtml('.project-not-found', projectId)
   log.info(`confirmed successful deletion of project with id ${projectId}`)
 }
@@ -1085,13 +1100,10 @@ describe('end-to-end', () => {
       await uploadGtfs()
 
       // wait for main tab to show up with version validity info
-      await waitForSelector('[data-test-id="feed-version-validity"]')
+      await waitForSelector('[data-test-id="active-feed-version-validity-start"]')
 
       // verify feed was uploaded
-      await expectSelectorToContainHtml(
-        '[data-test-id="feed-version-validity"]',
-        'Valid from Jan. 01, 2014 to Dec. 31, 2018'
-      )
+      await expectFeedVersionValidityDates('Jan 1, 2014', 'Dec 31, 2018')
     }, defaultTestTimeout)
 
     // this test also sets the feed source as deployable
@@ -1132,10 +1144,7 @@ describe('end-to-end', () => {
       await wait(2000, 'for feed source to update')
 
       // verify that feed was fetched and processed
-      await expectSelectorToContainHtml(
-        '[data-test-id="feed-version-validity"]',
-        'Valid from Apr. 08, 2018 to Jun. 30, 2018'
-      )
+      await expectFeedVersionValidityDates('Apr 8, 2018', 'Jun 30, 2018')
     }, defaultTestTimeout)
 
     if (doNonEssentialSteps) {
@@ -1245,10 +1254,7 @@ describe('end-to-end', () => {
         await wait(2000, 'for data to refresh')
         await waitForSelector('#feed-source-viewer-tabs')
         // verify that the previous feed is now the displayed feed
-        await expectSelectorToContainHtml(
-          '[data-test-id="feed-version-validity"]',
-          'Valid from Apr. 08, 2018 to Jun. 30, 2018'
-        )
+        await expectFeedVersionValidityDates('Apr 8, 2018', 'Jun 30, 2018')
       }, defaultTestTimeout)
     }
   })
@@ -2623,6 +2629,9 @@ describe('end-to-end', () => {
         // wait for dialog to appear
         await waitForSelector('[data-test-id="snapshot-dialog-name"]')
 
+        // confirm that we want to publish with unapproved routes
+        await click('[data-test-id="confirmPublishWithUnapproved"]')
+
         // enter name
         await type('[data-test-id="snapshot-dialog-name"]', 'test-snapshot')
 
@@ -2657,13 +2666,10 @@ describe('end-to-end', () => {
       await click('#feed-source-viewer-tabs-tab-')
 
       // wait for main tab to show up with version validity info
-      await waitForSelector('[data-test-id="feed-version-validity"]')
+      await waitForSelector('[data-test-id="active-feed-version-validity-start"]')
 
       // verify that snapshot was made active version
-      await expectSelectorToContainHtml(
-        '[data-test-id="feed-version-validity"]',
-        'Valid from May. 29, 2018 to May. 29, 2028'
-      )
+      await expectFeedVersionValidityDates('May 29, 2018', 'May 29, 2028')
     }, defaultTestTimeout, 'should create snapshot')
 
     // TODO: download and validate gtfs??
@@ -2692,7 +2698,10 @@ describe('end-to-end', () => {
         '[data-test-id="deployment-router-id"]'
       )
       // get rid of router id text and react tags
-      routerId = innerHTML.replace('Router ID: ', '')
+      // (remove any square brackets too)
+      routerId = innerHTML
+        .replace('Router ID: ', '')
+        .replace(/[[\]]/g, '')
 
       // confirm deployment
       await click('[data-test-id="confirm-deploy-server-button"]')
@@ -2702,9 +2711,11 @@ describe('end-to-end', () => {
     }, defaultTestTimeout + 30000) // Add thirty seconds for deployment job
 
     makeEditorEntityTest('should be able to do a trip plan on otp', async () => {
+      await wait(15000, 'for OTP to pick up the newly-built graph')
       // hit the otp endpoint
+      const url = `${OTP_ROOT}${routerId}/plan?fromPlace=37.04532992924222%2C-122.07542181015015&toPlace=37.04899494106061%2C-122.07432746887208&time=00%3A32&date=2018-07-24&mode=TRANSIT%2CWALK&maxWalkDistance=804.672&arriveBy=false&wheelchair=false&locale=en`
       const response = await fetch(
-        `${OTP_ROOT}${routerId}/plan?fromPlace=37.04532992924222%2C-122.07542181015015&toPlace=37.04899494106061%2C-122.07432746887208&time=12%3A32am&date=07-24-2018&mode=TRANSIT%2CWALK&maxWalkDistance=804.672&arriveBy=false&wheelchair=false&locale=en`,
+        url,
         {
           headers: {
             'Content-Type': 'application/json; charset=utf-8'

__tests__/test-utils/mock-data/manager.js

@@ -4,7 +4,6 @@ import clone from 'lodash/cloneDeep'
 
 import UserPermissions from '../../../lib/common/user/UserPermissions'
 import UserSubscriptions from '../../../lib/common/user/UserSubscriptions'
-
 import type {
   Deployment,
   FeedVersion,
@@ -55,8 +54,12 @@ export function makeMockDeployment (
     osmExtractUrl: null,
     otpCommit: null,
     otpVersion: null,
-    projectId: project.id,
+    peliasCsvFiles: [],
+    peliasResetDb: null,
+    peliasUpdate: null,
+    pinnedfeedVersionIds: [],
     projectBounds: {east: 0, west: 0, north: 0, south: 0},
+    projectId: project.id,
     routerId: null,
     skipOsmExtract: false,
     tripPlannerVersion: 'OTP_1',
@@ -69,6 +72,9 @@ export const mockProject = {
   autoFetchFeeds: true,
   autoFetchHour: 0,
   autoFetchMinute: 0,
+  autoDeploy: false,
+  autoDeployTypes: [],
+  autoDeployWithCriticalErrors: false,
   bounds: null,
   buildConfig: {
     fares: null,
@@ -82,10 +88,12 @@ export const mockProject = {
   feedSources: [],
   id: 'mock-project-id',
   lastUpdated: 1553236399556,
+  labels: [],
   name: 'mock-project',
   organizationId: null,
   otpServers: [],
   pinnedDeploymentId: null,
+  peliasWebhookUrl: null,
   routerConfig: {
     carDropoffTime: null,
     numItineraries: null,
@@ -130,6 +138,7 @@ export const mockFeedWithVersion = {
     tripCount: 415
   },
   latestVersionId: 'mock-feed-version-id',
+  labelIds: [],
   name: 'test feed with a version',
   noteCount: 0,
   organizationId: null,
@@ -140,7 +149,8 @@ export const mockFeedWithVersion = {
   snapshotVersion: null,
   transformRules: [],
   url: 'http://mdtrip.org/googletransit/AnnapolisTransit/google_transit.zip',
-  user: null
+  user: null,
+  versionCount: 1
 }
 
 // a mock feed with no versions
@@ -153,6 +163,7 @@ export const mockFeedWithoutVersion = {
   isPublic: false,
   lastFetched: null,
   name: 'test feed with no version',
+  labelIds: [],
   noteCount: 0,
   organizationId: null,
   projectId: mockProject.id,
@@ -162,7 +173,8 @@ export const mockFeedWithoutVersion = {
   snapshotVersion: null,
   transformRules: [],
   url: null,
-  user: null
+  user: null,
+  versionCount: 0
 }
 
 // a mock feedversion that has validation data

__tests__/test-utils/setup-e2e.js

@@ -339,6 +339,10 @@ async function startOtp () {
   )
 
   console.log('starting otp')
+  // Ensure default folder for graphs exists.
+  // (OTP 1.4.0 autoscan() does a directory listing without checking directory existence.)
+  const otpBasePath = '/tmp/otp'
+  await fs.mkdirp(`${otpBasePath}/graphs`)
 
   // start otp
   try {
@@ -349,6 +353,9 @@ async function startOtp () {
         '-jar',
         otpJarFilename,
         '--server',
+        '--autoScan',
+        '--basePath',
+        otpBasePath,
         '--insecure',
         '--router',
         'default'

configurations/default/env.yml.tmp

@@ -7,5 +7,7 @@ MAPBOX_ATTRIBUTION: <a href="https://www.mapbox.com/about/maps/" target="_blank"
 SLACK_CHANNEL: optional-slack-channel
 SLACK_WEBHOOK: optional-slack-webhook
 GRAPH_HOPPER_KEY: your-graph-hopper-key
+# Optional override to use a custom service instead of the graphhopper.com hosted service.
+# GRAPH_HOPPER_URL: http://localhost:8989/
 GOOGLE_ANALYTICS_TRACKING_ID: optional-ga-key
 # GRAPH_HOPPER_POINT_LIMIT: 10 # Defaults to 30

docs/user/deploying-feeds.md

@@ -22,6 +22,14 @@ To deploy or update GTFS feeds to OTP:
 7. If you select `Custom` under `Build configuration` or `Router configuration`, enter the desired configuration settings.
 8. Click the `Deploy` dropdown at the top of the main pane, then pick the server on which to perform the deployment. Existing deployments on that server will be discarded.
 
+## Updating the Custom Places Index
+
+A GTFS feed's stops can be sent to a Custom Places Index, should one be set up. You'll need the secret Webhook URL of your Custom Places Index server.
+
+Only a pinned deployment's feed can be sent to a Custom Places Index. When opening a pinned deployment, a `Custom Geocoder Settings` pane appears below the `OTP Configuration` pane. In the text field, paste in the secret Webhook URL for your Custom Places Index server. Once it's entered, the `Update Custom Geocoder` checkbox will be clickable. If it is checked, your Custom Places Index will be updated when deploying the feed.
+
+The pane also has an option to upload Custom POI CSV files. These files contain special landmarks and coordinates which are prioritized when returning geocoder results. You can upload as many custom CSV files as you like. They will all be added to the Custom Places Index.
+
 ## Watching deployments take place
 
 After click Deploy, you can watch the deployment progress from the right-hand panel:

docs/user/managing-projects-feeds.md

@@ -82,6 +82,7 @@ Once the Feed Transformations have all been configured, you can import a new GTF
 
 ![screenshot](../img/feed-transformation-summary.png)
 
+A more in depth summary is available under the Transformation Results tab on the left sidebar. This panel will show you the modifications made to each file in the feed, along with how many rows were added, deleted, or modified. 
 
 ## Viewing and Managing Feed Versions