VAN Bulk Import Additions (move-coop#1078)

* van bulk import - add apply canvass results & custom contact fields / docstring fixes / clarification / contact mapping

* van bulk import additions - fix E501

Author: codygordon

parsons/ngpvan/bulk_import.py

@@ -189,6 +189,9 @@ def bulk_apply_activist_codes(self, tbl, url_type, **url_kwargs):
             * - ``datecanvassed``
               - No
               - An ISO formatted date
+            * - ``canvassedby``
+              - No
+              - A valid User ID; Required when DateCanvassed is provided
             * - ``contacttypeid``
               - No
               - The method of contact.
@@ -249,7 +252,7 @@ def bulk_upsert_contacts(self, tbl, url_type, result_fields=None, **url_kwargs):
               -
               -
             * - First Name
-              - ``fn``, ``firstname``, ``last``
+              - ``fn``, ``firstname``, ``first``
               -
             * - Middle Name
               - ``mn``, ``middlename``, ``middle``
@@ -258,8 +261,8 @@ def bulk_upsert_contacts(self, tbl, url_type, result_fields=None, **url_kwargs):
               - ``ln``, ``lastname``, ``last``
               -
             * - Date of Birth
-              - ``dob``, ``dateofbirth`` ``birthdate``
-              - What type of thing does this need?
+              - ``dob``, ``dateofbirth``, ``birthdate``
+              - An ISO formatted date
             * - Sex
               - ``sex``, ``gender``
               -
@@ -281,6 +284,9 @@ def bulk_upsert_contacts(self, tbl, url_type, result_fields=None, **url_kwargs):
             * - State Or Province
               - ``state``, ``st``, ``stateorprovince``
               -
+            * - Zip or Postal Code
+              - ``ziporpostal``, ``postal``, ``postalcode``, ``zip``, ``zipcode``
+              -
             * - Country Code
               - ``countrycode``, ``country``
               - A valid two character country code (e.g. ``US``)
@@ -309,6 +315,9 @@ def bulk_upsert_contacts(self, tbl, url_type, result_fields=None, **url_kwargs):
             * - Email
               - ``email``, ``emailaddress``
               -
+            * - Other Email
+              - ``otheremail``, ``email2``, ``emailaddress2``
+              -
 
         `Args:`
             table: Parsons table
@@ -325,7 +334,7 @@ def bulk_upsert_contacts(self, tbl, url_type, result_fields=None, **url_kwargs):
                 The bulk import job id
         """
 
-        tbl = tbl.map_columns(COLUMN_MAP, exact_match=False)
+        tbl = tbl.map_columns(CONTACTS_COLUMN_MAP, exact_match=False)
 
         return self.post_bulk_import(
             tbl,
@@ -339,13 +348,13 @@ def bulk_upsert_contacts(self, tbl, url_type, result_fields=None, **url_kwargs):
 
     def bulk_apply_suppressions(self, tbl, url_type, **url_kwargs):
         """
-        Bulk apply contact suppressions codes.
+        Bulk apply contact suppression codes.
 
         The table may include the following columns. The first column
         must be ``vanid``.
 
         .. list-table::
-            :widths: 25 25
+            :widths: 25 25 50
             :header-rows: 1
 
             * - Column Name
@@ -381,11 +390,140 @@ def bulk_apply_suppressions(self, tbl, url_type, **url_kwargs):
             **url_kwargs,
         )
 
+    def bulk_apply_canvass_results(self, tbl, url_type, **url_kwargs):
+        """
+        Bulk apply contact canvass results.
+
+        The table may include the following columns. The first column
+        must be ``vanid``.
+
+        .. list-table::
+            :widths: 25 25 50
+            :header-rows: 1
+
+            * - Column Name
+              - Required
+              - Description
+            * - ``vanid``
+              - Yes
+              - A valid VANID primary key
+            * - ``contacttypeid``
+              - Yes
+              - Valid Contact Type ID
+            * - ``resultid``
+              - Yes
+              - Valid Contact Result ID
+            * - ``datecanvassed``
+              - Yes
+              - ISO Date Format
+            * - ``canvassedby``
+              - Yes
+              - Valid User ID
+            * - ``phone``
+              - No
+              - Attempted Phone Number
+            * - ``countrycode``
+              - No
+              - Country Code (ISO 3166-1 alpha-2)
+            * - ``phonetypeid``
+              - No
+              - Phone Type
+            * - ``phoneoptinstatusid``
+              - No
+              - SMS Opt-In Status
+            * - ``addressid``
+              - No
+              - The Contact Address ID of the address that was canvassed
+
+        `Args:`
+            table: Parsons table
+                A Parsons table.
+            url_type: str
+                The cloud file storage to use to post the file (``S3`` or ``GCS``).
+                See :ref:`Cloud Storage <cloud-storage>` for more details.
+            **url_kwargs: kwargs
+                Arguments to configure your cloud storage url type. See
+                :ref:`Cloud Storage <cloud-storage>` for more details.
+        `Returns:`
+            int
+                The bulk import job id
+        """
+
+        return self.post_bulk_import(
+            tbl,
+            url_type,
+            "Contacts",
+            [{"name": "CanvassResults"}],
+            "Apply Canvass Results",
+            **url_kwargs,
+        )
+
+    def bulk_apply_contact_custom_fields(self, custom_field_group_id, tbl, url_type, **url_kwargs):
+        """
+        Bulk apply contact custom fields.
+
+        The table may include the following columns. The first column
+        must be ``vanid``.
+
+        .. list-table::
+            :widths: 25 25 60
+            :header-rows: 1
+
+            * - Column Name
+              - Required
+              - Description
+            * - ``vanid``
+              - Yes
+              - A valid VANID primary key
+            * - ***``CF{CustomFieldID}``
+              - Yes
+              - At least one custom field column to be loaded associated with the provided
+                custom_field_group_id. The column name should be a valid Custom Field ID
+                prefixed with ``CF``, i.e. CF123.
+
+        `Args:`
+            custom_field_group_id: int
+                Valid Custom Contact Field Group ID; must be the parent of
+                the provided Custom Field IDs in the file.
+            table: Parsons table
+                A Parsons table.
+            url_type: str
+                The cloud file storage to use to post the file (``S3`` or ``GCS``).
+                See :ref:`Cloud Storage <cloud-storage>` for more details.
+            **url_kwargs: kwargs
+                Arguments to configure your cloud storage url type. See
+                :ref:`Cloud Storage <cloud-storage>` for more details.
+        `Returns:`
+            int
+                The bulk import job id
+        """
+
+        mapping_types = [
+            {
+                "name": "ApplyContactCustomFields",
+                "fieldValueMappings": [
+                    {
+                        "fieldName": "CustomFieldGroupID",
+                        "staticValue": custom_field_group_id,
+                    },
+                ],
+            }
+        ]
+
+        return self.post_bulk_import(
+            tbl,
+            url_type,
+            "Contacts",
+            mapping_types,
+            "Apply Contact Custom Fields",
+            **url_kwargs,
+        )
+
 
 # This is a column mapper that is used to accept additional column names and provide
 # flexibility for the user.
 
-COLUMN_MAP = {
+CONTACTS_COLUMN_MAP = {
     "firstname": ["fn", "first"],
     "middlename": ["mn", "middle"],
     "lastname": ["ln", "last"],
@@ -396,11 +534,13 @@ def bulk_apply_suppressions(self, tbl, url_type, **url_kwargs):
     "addressline3": ["addressline3", "address3"],
     "city": [],
     "stateorprovince": ["state", "st"],
+    "ziporpostal": ["postal", "postalcode", "zip", "zipcode"],
     "countrycode": ["country"],
     "displayasentered": [],
     "cellphone": ["cell"],
     "cellphonecountrycode": ["cellcountrycode"],
     "phone": ["home", "homephone"],
     "phonecountrycode": ["phonecountrycode"],
     "email": ["emailaddress"],
+    "otheremail": ["email2", "emailaddress2"],
 }

test/test_van/test_bulkimport.py

@@ -164,6 +164,45 @@ def test_bulk_upsert_contacts(self, m):
 
         self.assertEqual(job_id, 54679)
 
+    @requests_mock.Mocker()
+    def test_bulk_apply_canvass_results(self, m):
+
+        # Mock Cloud Storage
+        cloud_storage.post_file = mock.MagicMock()
+        cloud_storage.post_file.return_value = "https://s3.com/my_file.zip"
+
+        tbl = Table(
+            [
+                ["vanid", "contacttypeid", "resultid", "datecanvassed", "canvassedby", "phone"],
+                [1234, 1, 1, "2020-01-01", 987, "5554443210"],
+            ]
+        )
+
+        m.post(self.van.connection.uri + "bulkImportJobs", json={"jobId": 54679})
+
+        job_id = self.van.bulk_apply_canvass_results(tbl, url_type="S3", bucket="my-bucket")
+
+        self.assertEqual(job_id, 54679)
+
+    @requests_mock.Mocker()
+    def test_bulk_apply_contact_custom_fields(self, m):
+
+        # Mock Cloud Storage
+        cloud_storage.post_file = mock.MagicMock()
+        cloud_storage.post_file.return_value = "https://s3.com/my_file.zip"
+
+        tbl = Table([["vanid", "CF123", "CF124"], [1234, "Test String Value", 999]])
+
+        m.post(self.van.connection.uri + "bulkImportJobs", json={"jobId": 54679})
+
+        custom_field_group_id = 1234
+
+        job_id = self.van.bulk_apply_contact_custom_fields(
+            custom_field_group_id, tbl, url_type="S3", bucket="my-bucket"
+        )
+
+        self.assertEqual(job_id, 54679)
+
 
 mapping_type = {
     "name": "ActivistCode",