Merge pull request #12 from TransitApp/Fix-csv-parser

Fix csv parser
TransitApp · Oct 19, 2018 · c58cdeb · c58cdeb
2 parents 3f12da7 + febe4dd
commit c58cdeb
Show file tree

Hide file tree

Showing 6 changed files with 24 additions and 17 deletions.
diff --git a/helpers/csv.js b/helpers/csv.js
@@ -73,7 +73,10 @@ function fromCsvStringToArray(string, tableName) {
   }
 
   if (!reValid.test(string)) {
-    if (string.match(/,"""/)) { // Handle the case with 3 " at start/end of string
+    if (string.match(/""""/)) {
+      string = string.replace(/""""/g, '\\"\\"');
+      return fromCsvStringToArray(string, tableName);
+    } else if (string.match(/,"""/)) { // Handle the case with 3 " at start/end of string
       string = string.replace(/,"""/g, ',"\\"');
       return fromCsvStringToArray(string, tableName);
     } else if (string.match(/""",/)) {

diff --git a/tests/samples/1/routes.txt b/tests/samples/1/routes.txt
@@ -1,4 +1,5 @@
 agency_id,route_id,route_short_name,route_long_name,route_type
 agency_0,route_0,R0,Route 0,3
 agency_0,route_x,RX,"""Route X""",3
-agency_0,route_utf8,RÛTF8,route_😎êωn → ∞⠁⠧⠑ ⠼éöÿΚαλημέρα'´`,3
+agency_0,route_utf8,RÛTF8,route_😎êωn → ∞⠁⠧⠑ ⠼éöÿΚαλημέρα'´`,3
+agency_0, route_y, RY,"{""routeLongName"":""""}",3
diff --git a/tests/test_generic_table_functions.js b/tests/test_generic_table_functions.js
@@ -31,21 +31,21 @@ describe('Tests on GTFS generic table functions', () => {
     const uftRouteId =
 
     gtfs.addItemInTable({ route_id: 'route_1', route_long_name: 'Route 1' }, ROUTE_TABLE_NAME);
-    expect(sortedKeys(gtfs.getIndexedRoutes())).to.deep.equal(['route_0', 'route_1', 'route_utf8', 'route_x']);
+    expect(sortedKeys(gtfs.getIndexedRoutes())).to.deep.equal(['route_0', 'route_1', 'route_utf8', 'route_x', 'route_y']);
 
     gtfs.addItemsInTable([
       { route_id: 'route_2', route_long_name: 'Route 2' },
       { route_id: 'route_3', route_long_name: 'Route 3' },
     ], ROUTE_TABLE_NAME);
     expect(sortedKeys(gtfs.getIndexedRoutes())).to.deep.equal(['route_0', 'route_1', 'route_2',
-      'route_3', 'route_utf8', 'route_x']);
+      'route_3', 'route_utf8', 'route_x', 'route_y']);
 
     gtfs.removeItemInTable(gtfs.getRouteWithId('route_2'), ROUTE_TABLE_NAME);
     expect(sortedKeys(gtfs.getIndexedRoutes())).to.deep.equal(['route_0', 'route_1', 'route_3',
-      'route_utf8', 'route_x']);
+      'route_utf8', 'route_x', 'route_y']);
 
     gtfs.removeItemsInTable([gtfs.getRouteWithId('route_0'), gtfs.getRouteWithId('route_3')], ROUTE_TABLE_NAME);
-    expect(sortedKeys(gtfs.getIndexedRoutes())).to.deep.equal(['route_1', 'route_utf8', 'route_x']);
+    expect(sortedKeys(gtfs.getIndexedRoutes())).to.deep.equal(['route_1', 'route_utf8', 'route_x', 'route_y']);
 
     gtfs.setIndexedItemsAsTable(new Map([['route_0', route0]]), ROUTE_TABLE_NAME);
     expect(sortedKeys(gtfs.getIndexedRoutes())).to.deep.equal(['route_0']);
@@ -78,7 +78,7 @@ describe('Tests on GTFS generic table functions', () => {
   it('Test on gtfs.getNumberOfItemsInTable(tableName)', (done) => {
     const gtfs = new Gtfs(`${__dirname}/samples/1`);
 
-    expect(gtfs.getNumberOfItemsInTable('routes')).to.equal(3);
+    expect(gtfs.getNumberOfItemsInTable('routes')).to.equal(4);
 
     gtfs.resetRoutes();
     expect(gtfs.getNumberOfItemsInTable('routes')).to.equal(0);

diff --git a/tests/test_gtfs_options.js b/tests/test_gtfs_options.js
@@ -56,7 +56,8 @@ describe('Tests on GTFS constructor options', () => {
           'route_text_color,route_sort_order,temp\n' +
           'route_0,agency_0,R0,Route 0,,3,,,,,some value\n' +
           'route_x,agency_0,RX,"""Route X""",,3,,,,,some value\n' +
-          'route_utf8,agency_0,RÛTF8,route_😎êωn → ∞⠁⠧⠑ ⠼éöÿΚαλημέρα\'´`,,3,,,,,some value\n'
+          'route_utf8,agency_0,RÛTF8,route_😎êωn → ∞⠁⠧⠑ ⠼éöÿΚαλημέρα\'´`,,3,,,,,some value\n' +
+          'route_y,agency_0,RY,"{""routeLongName"":""""}",,3,,,,,some value\n'
         );
 
         fs.remove(outputPath, (removeError) => {
@@ -94,7 +95,8 @@ describe('Tests on GTFS constructor options', () => {
           'route_text_color,route_sort_order,temp\n' +
           'route_0,agency_0,R0,Route 0,,3,,,,,"{""key"":""value""}"\n' +
           'route_x,agency_0,RX,"""Route X""",,3,,,,,"{""key"":""value""}"\n' +
-          'route_utf8,agency_0,RÛTF8,route_😎êωn → ∞⠁⠧⠑ ⠼éöÿΚαλημέρα\'´`,,3,,,,,"{""key"":""value""}"\n'
+          'route_utf8,agency_0,RÛTF8,route_😎êωn → ∞⠁⠧⠑ ⠼éöÿΚαλημέρα\'´`,,3,,,,,"{""key"":""value""}"\n' +
+          'route_y,agency_0,RY,"{""routeLongName"":""""}",,3,,,,,"{""key"":""value""}"\n'
         );
 
         fs.remove(outputPath, (removeError) => {

diff --git a/tests/test_routes.js b/tests/test_routes.js
@@ -8,7 +8,7 @@ describe('Tests on GTFS routes', () => {
     const path = `${__dirname}/samples/1`;
     const gtfs = new Gtfs(path);
 
-    expect(sortedKeys(gtfs.getIndexedRoutes())).to.deep.equal(['route_0', 'route_utf8', 'route_x']);
+    expect(sortedKeys(gtfs.getIndexedRoutes())).to.deep.equal(['route_0', 'route_utf8', 'route_x', 'route_y']);
 
     const route0 = gtfs.getRouteWithId('route_0');
     expect(route0.route_long_name).to.equal('Route 0');
@@ -17,21 +17,21 @@ describe('Tests on GTFS routes', () => {
     expect(routeX.route_long_name).to.equal('"Route X"');
 
     gtfs.addRoute({ route_id: 'route_1', route_long_name: 'Route 1' });
-    expect(sortedKeys(gtfs.getIndexedRoutes())).to.deep.equal(['route_0', 'route_1', 'route_utf8', 'route_x']);
+    expect(sortedKeys(gtfs.getIndexedRoutes())).to.deep.equal(['route_0', 'route_1', 'route_utf8', 'route_x', 'route_y']);
 
     gtfs.addRoutes([
       { route_id: 'route_2', route_long_name: 'Route 2' },
       { route_id: 'route_3', route_long_name: 'Route 3' },
     ]);
     expect(sortedKeys(gtfs.getIndexedRoutes())).to.deep.equal(['route_0', 'route_1', 'route_2',
-      'route_3', 'route_utf8', 'route_x']);
+      'route_3', 'route_utf8', 'route_x', 'route_y']);
 
     gtfs.removeRoute(gtfs.getRouteWithId('route_2'));
     expect(sortedKeys(gtfs.getIndexedRoutes())).to.deep.equal(['route_0', 'route_1', 'route_3',
-      'route_utf8', 'route_x']);
+      'route_utf8', 'route_x', 'route_y']);
 
     gtfs.removeRoutes([gtfs.getRouteWithId('route_0'), gtfs.getRouteWithId('route_3')]);
-    expect(sortedKeys(gtfs.getIndexedRoutes())).to.deep.equal(['route_1', 'route_utf8', 'route_x']);
+    expect(sortedKeys(gtfs.getIndexedRoutes())).to.deep.equal(['route_1', 'route_utf8', 'route_x', 'route_y']);
 
     gtfs.setIndexedRoutes(new Map([['route_0', route0]]));
     expect(sortedKeys(gtfs.getIndexedRoutes())).to.deep.equal(['route_0']);
@@ -56,7 +56,7 @@ describe('Tests on GTFS routes', () => {
   it('Tests on gtfs.getNumberOfRoutes()', (done) => {
     const gtfs = new Gtfs(`${__dirname}/samples/1`);
 
-    expect(gtfs.getNumberOfRoutes()).to.equal(3);
+    expect(gtfs.getNumberOfRoutes()).to.equal(4);
 
     gtfs.resetRoutes();
 
@@ -79,7 +79,7 @@ describe('Tests on GTFS routes', () => {
   it('Tests on gtfs.resetRoutes()', (done) => {
     const gtfs = new Gtfs(`${__dirname}/samples/1`);
 
-    expect(gtfs.getIndexedRoutes().size).to.equal(3);
+    expect(gtfs.getIndexedRoutes().size).to.equal(4);
 
     gtfs.resetRoutes();
 

diff --git a/tests/tests.js b/tests/tests.js
@@ -44,7 +44,8 @@ describe('Tests on GTFS', () => {
           'route_0,agency_0,R0,Route 0,Some new description,3,,,,,some_extra_route_value\n' +
           'route_x,agency_0,RX,"""Route X""",Some new description,3,,,,,some_extra_route_value\n' +
           'route_utf8,agency_0,RÛTF8,route_😎êωn → ∞⠁⠧⠑ ⠼éöÿΚαλημέρα\'´`,' +
-          'Some new description,3,,,,,some_extra_route_value\n'
+          'Some new description,3,,,,,some_extra_route_value\n' +
+          'route_y,agency_0,RY,"{""routeLongName"":""""}",Some new description,3,,,,,some_extra_route_value\n'
         );
 
         fs.readFile(`${outputPath}feed_info.txt`, (readFeedInfoError, feedInfoTxt) => {