chore: replace ex_csv with nimble_csv (#742)

* chore: replace ex_csv with nimble_csv * fix: skip headers but apply trim * chore: address credo complaints * fix: wickford junction name change
mbta · Jan 23, 2024 · b174083 · b174083
1 parent 1671ba0
commit b174083
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 4 deletions.
diff --git a/apps/parse/lib/parse/simple.ex b/apps/parse/lib/parse/simple.ex
@@ -12,6 +12,8 @@ defmodule Parse.Simple do
 
   """
 
+  alias NimbleCSV.RFC4180, as: CSV
+
   @callback parse_row(%{String.t() => String.t()}) :: any
 
   defmacro __using__([]) do
@@ -36,7 +38,13 @@ defmodule Parse.Simple do
 
   def parse(blob, row_callback) do
     blob
-    |> ExCsv.parse!(headings: true)
+    |> String.trim()
+    |> CSV.parse_string(skip_headers: false)
+    |> Stream.transform(nil, fn
+      headers, nil -> {[], headers}
+      row, headers -> {[headers |> Enum.zip(row) |> Map.new()], headers}
+    end)
+    |> Enum.to_list()
     |> Enum.map(row_callback)
   end
 end
diff --git a/apps/parse/mix.exs b/apps/parse/mix.exs
@@ -46,7 +46,7 @@ defmodule Parse.Mixfile do
   # Type "mix help deps" for more examples and options
   defp deps do
     [
-      {:ex_csv, "~> 0.1.5"},
+      {:nimble_csv, "~> 1.2"},
       {:exprotobuf, "~> 1.2"},
       {:gpb, "~> 4.19"},
       {:timex, "~> 3.7"},

diff --git a/apps/state_mediator/test/state_mediator/integration/gtfs_test.exs b/apps/state_mediator/test/state_mediator/integration/gtfs_test.exs
@@ -199,7 +199,7 @@ defmodule StateMediator.Integration.GtfsTest do
       [shapes_0, shapes_1] = shapes_in_both_directions("CR-Providence")
 
       assert [
-               %{name: "South Station - Wickford Junction via Back Bay"},
+               %{name: "South Station - Wickford Junction"},
                %{id: "9890004"},
                %{id: "SouthStationToStoughtonViaFairmount"}
              ] = shapes_0

diff --git a/mix.lock b/mix.lock
@@ -27,7 +27,6 @@
   "ex_aws_elastic_load_balancing": {:hex, :ex_aws_elastic_load_balancing, "2.1.1", "541f513d68040c5a97e7d02cb664f338c099232422f8a9dfd75256c88385d0cf", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "e9942d2bd4dd0b8dc6e5af70bde3e07f94ebc80782a68a95773fba83eec6289c"},
   "ex_aws_s3": {:hex, :ex_aws_s3, "2.4.0", "ce8decb6b523381812798396bc0e3aaa62282e1b40520125d1f4eff4abdff0f4", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "85dda6e27754d94582869d39cba3241d9ea60b6aa4167f9c88e309dc687e56bb"},
   "ex_aws_ses": {:hex, :ex_aws_ses, "2.1.1", "7324f2d0038203c70f8b9f5d0f0473a1f473b94e8eca57c84bf4b6aac04d584b", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}], "hexpm", "86c8a7b139533649c21c4ace6b625f7f2dd2a9190285dd1dc8e0cb9b4002dd19"},
-  "ex_csv": {:hex, :ex_csv, "0.1.5", "517222efbc87673c30c20ea8fff2cb2f6c39d7ce4ca85631cfe02293c93b3ca0", [:mix], [], "hexpm", "52a284295bab0ee1d7a74f66ffc46c4b0086028a981e25fda0ec2146b335b656"},
   "ex_doc": {:hex, :ex_doc, "0.29.1", "b1c652fa5f92ee9cf15c75271168027f92039b3877094290a75abcaac82a9f77", [:mix], [{:earmark_parser, "~> 1.4.19", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "b7745fa6374a36daf484e2a2012274950e084815b936b1319aeebcf7809574f6"},
   "ex_json_schema": {:hex, :ex_json_schema, "0.6.2", "de23d80478215987469c81688208fe0ff440ee0e0e6ae2268fcadbb2ff35df9d", [:mix], [], "hexpm", "2f25c57e919ffc5d6b02f2f130548284342dd6c3e99555ee0beeb9f2d2366a96"},
   "ex_maybe": {:hex, :ex_maybe, "1.1.1", "95c0188191b43bd278e876ae4f0a688922e3ca016a9efd97ee7a0b741a61b899", [:mix], [], "hexpm", "1af8c78c915c7f119a513b300a1702fc5cc9fed42d54fd85995265e4c4b763d2"},
@@ -62,6 +61,7 @@
   "mint": {:hex, :mint, "1.4.2", "50330223429a6e1260b2ca5415f69b0ab086141bc76dc2fbf34d7c389a6675b2", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "ce75a5bbcc59b4d7d8d70f8b2fc284b1751ffb35c7b6a6302b5192f8ab4ddd80"},
   "mix_audit": {:hex, :mix_audit, "2.1.0", "3c0dafb29114dffcdb508164a3d35311a9ac2c5baeba6495c9cd5315c25902b9", [:make, :mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:yaml_elixir, "~> 2.9", [hex: :yaml_elixir, repo: "hexpm", optional: false]}], "hexpm", "14c57a23e0a5f652c1e7f6e8dab93f166f66d63bd0c85f97278f5972b14e2be0"},
   "mox": {:hex, :mox, "1.0.2", "dc2057289ac478b35760ba74165b4b3f402f68803dd5aecd3bfd19c183815d64", [:mix], [], "hexpm", "f9864921b3aaf763c8741b5b8e6f908f44566f1e427b2630e89e9a73b981fef2"},
+  "nimble_csv": {:hex, :nimble_csv, "1.2.0", "4e26385d260c61eba9d4412c71cea34421f296d5353f914afe3f2e71cce97722", [:mix], [], "hexpm", "d0628117fcc2148178b034044c55359b26966c6eaa8e2ce15777be3bbc91b12a"},
   "nimble_options": {:hex, :nimble_options, "0.5.2", "42703307b924880f8c08d97719da7472673391905f528259915782bb346e0a1b", [:mix], [], "hexpm", "4da7f904b915fd71db549bcdc25f8d56f378ef7ae07dc1d372cbe72ba950dce0"},
   "nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"},
   "nimble_pool": {:hex, :nimble_pool, "0.2.6", "91f2f4c357da4c4a0a548286c84a3a28004f68f05609b4534526871a22053cde", [:mix], [], "hexpm", "1c715055095d3f2705c4e236c18b618420a35490da94149ff8b580a2144f653f"},