YDA-5512: Import CSV Tests, linting

groups.py

@@ -535,10 +535,10 @@ def parse_data(ctx, csv_header_and_data):
             # EVERY row should have all the headers that were listed at the top of the file
             if header_cols[x] not in line_dict:
                 line_dict[header_cols[x]] = []
-            
+
             if len(data[x]):
-                line_dict[header_cols[x]].append(data[x]) 
-            
+                line_dict[header_cols[x]].append(data[x])
+
         lines.append(line_dict)
 
     for line in lines:
@@ -704,7 +704,7 @@ def parse_csv_file(ctx):
         if label not in reader.fieldnames:
             _exit_with_error(
                 'CSV header is missing compulsory field "{}"'.format(label))
-    
+
     # Check that all header names are valid
     possible_labels = _get_csv_possible_labels()
     for label in header:
@@ -764,8 +764,8 @@ def _process_csv_line(ctx, line):
        and optionally schema id and expiration date.
     """
     if (not len(line['category'])
-        or not len(line['subcategory'])
-        or not len(line['groupname'])):
+            or not len(line['subcategory'])
+            or not len(line['groupname'])):
         return None, "Row has a missing group name, category or subcategory"
 
     category = line['category'][0].strip().lower().replace('.', '')
@@ -810,7 +810,7 @@ def _process_csv_line(ctx, line):
 
     if not yoda_names.is_valid_groupname("research-" + groupname):
         return None, '"{}" is not a valid group name.'.format(groupname)
-    
+
     if not yoda_names.is_valid_schema_id(schema_id):
         return None, '"{}" is not a valid schema id.'.format(schema_id)
 

tests/features/api/api_group.feature

@@ -159,7 +159,7 @@ Feature: Group API
 
     Scenario Outline: Group import CSV
         Given user technicaladmin is authenticated
-        And the Yoda API for processing csv group data API is queried
+        And the Yoda API for processing csv group data API is queried for data "csvtestgroup"
         Then the response status code is "200"
         And user "[email protected]" is now a member of the group "research-csvtestgroup"
         And user "[email protected]" is now a member of the group "research-csvtestgroup"
@@ -168,6 +168,24 @@ Feature: Group API
         And user "[email protected]" is now a member of the group "research-csvtestgroup"
 
 
+    Scenario Outline: Group import CSV schema id and expiration date
+        Given user technicaladmin is authenticated
+        And the Yoda API for processing csv group data API is queried for data "csvtestgroup1"
+        Then the response status code is "200"
+        And user "[email protected]" is now a member of the group "research-csvtestgroup1"
+
+
+    Scenario Outline: Group import CSV errors
+        Given user technicaladmin is authenticated
+        And the Yoda API for processing csv group data API is queried for data "<group_name>"
+        Then the response status code is "400"
+
+        Examples:
+            | group_name         |
+            | csv-missing-header |
+            | csv-missing-entry  |
+
+
     Scenario Outline: Group delete
         Given user <user> is authenticated
         And the group "<group_name>" exists
@@ -182,6 +200,7 @@ Feature: Group API
             | functionaladminpriv | research-api-test1-group |
             | technicaladmin      | datamanager-api-test1    |
             | technicaladmin      | research-csvtestgroup    |
+            | technicaladmin      | research-csvtestgroup1   |
             | technicaladmin      | not-a-yoda-group         |
 
 

tests/features/ui/ui_group.feature

@@ -76,10 +76,12 @@ Feature: Group UI
         Given user functionaladminpriv is logged in
         And module "group_manager" is shown
         When user opens group import dialog
-        And user clicks upload button
-        And user clicks allow updates checkbox
+        And user clicks upload button and uploads csv "csv-import-test.csv"
+        Then there are 4 groups presented
+        When user clicks allow updates checkbox
         And user clicks allow deletions checkbox
-        Then process csv and check number of rows
+        Then process csv
+        And check number of rows is 4
         And click on imported row 0 and check group properties
         And find group member "[email protected]"
         And user opens group import dialog
@@ -93,6 +95,26 @@ Feature: Group UI
         And find group member "[email protected]"
 
 
+    Scenario: Imports group CSV schema id and expiration date
+        Given user functionaladminpriv is logged in
+        And module "group_manager" is shown
+        When user opens group import dialog
+        And user clicks upload button and uploads csv "csv-import-test-exp-schema.csv"
+        Then there are 2 groups presented
+        When user clicks allow updates checkbox
+        And user clicks allow deletions checkbox
+        Then process csv
+        And check number of rows is 2
+        And click on imported row 0 and check group properties
+        And find group member "[email protected]"
+        And find group member "[email protected]"
+        And user opens group import dialog
+        And click on imported row 1 and check group properties
+        And schema id is "default-3"
+        And expiration date is "2027-01-01"
+        And find group member "[email protected]"
+
+
     Scenario Outline: Group research create with default schema id
         Given user <user> is logged in
         And module "group_manager" is shown
@@ -192,6 +214,8 @@ Feature: Group UI
             | functionaladminpriv | test-automation   | csv-test          | research-csv-test-group2      |
             | functionaladminpriv | test-automation   | csv-test          | research-csv-test-group3      |
             | functionaladminpriv | test-automation   | csv-test          | research-csv-test-group4      |
+            | functionaladminpriv | test-automation   | csv-test          | research-csv-test-group5      |
+            | functionaladminpriv | test-automation   | csv-test          | research-csv-test-group6      |
             | functionaladminpriv | test-datamanager  | test-datamanager  | datamanager-test-datamanager  |
             | functionaladminpriv | test-datamanager  | test-datamanager  | research-test-datamanager     |
             | technicaladmin      | test-datamanager1 | test-datamanager1 | datamanager-test-datamanager1 |

tests/files/csv-import-test-exp-schema.csv

@@ -0,0 +1,3 @@
+category,subcategory,groupname,manager,member,schema_id,expiration_date
+test-automation,csv-test,csv-test-group5,[email protected],[email protected],,
+test-automation,csv-test,csv-test-group6,[email protected],,default-3,2027-01-01

tests/files/csv-import-test.csv

@@ -1,4 +1,4 @@
-category,subcategory,groupname,manager:manager,member:member1,member:member2,viewer:viewer1
+category,subcategory,groupname,manager,member,member,viewer
 test-automation,csv-test,csv-test-group1,[email protected],[email protected],[email protected],[email protected]
 test-automation,csv-test,csv-test-group2,[email protected],[email protected],[email protected],[email protected]
 test-automation,csv-test,csv-test-group3,[email protected],[email protected],[email protected],[email protected]

tests/step_defs/api/test_api_group.py

@@ -294,13 +294,18 @@ def then_user_update_persisted(user, new_user, group_name):
     assert role == "manager"
 
 
-@given('the Yoda API for processing csv group data API is queried', target_fixture="api_response")
-def api_group_import_csv_data(user):
-    header_and_data = "category,subcategory,groupname,manager:manager,member:member1,member:member2,viewer:viewer1,member:member3\rdefault-2,default-2,csvtestgroup,[email protected],[email protected],[email protected],[email protected],[email protected]"
+@given(parsers.parse('the Yoda API for processing csv group data API is queried for data "{data_id}"'), target_fixture="api_response")
+def api_group_import_csv_data(user, data_id):
+    headers_and_data = {
+        "csvtestgroup": "category,subcategory,groupname,manager,member,member,viewer,member\rdefault-2,default-2,csvtestgroup,[email protected],[email protected],[email protected],[email protected],[email protected]",
+        "csvtestgroup1": "category,subcategory,groupname,manager,expiration_date,schema_id\rdefault-2,default-2,csvtestgroup1,[email protected],2030-01-01,default-2",
+        "csv-missing-header": "category,,groupname,manager,expiration_date,schema_id\rdefault-2,default-2,csvtestgroup1,[email protected]",
+        "csv-missing-entry": "category,subcategory,groupname,manager,expiration_date,schema_id\rdefault-2,,csvtestgroup1,[email protected]",
+    }
     return api_request(
         user,
         "group_process_csv",
-        {"csv_header_and_data": header_and_data,
+        {"csv_header_and_data": headers_and_data[data_id],
          "allow_update": True,
          "delete_users": True}
     )

tests/step_defs/ui/test_ui_group.py

@@ -276,16 +276,19 @@ def ui_group_click_group_import_dlg_button(browser):
     browser.find_by_css('.import-groups-csv').click()
 
 
-@when("user clicks upload button")
-def ui_group_click_upload_button(browser):
+@when(parsers.parse('user clicks upload button and uploads csv "{csv_file}"'))
+def ui_group_click_upload_button(browser, csv_file):
     cwd = os.getcwd()
     if os.name == 'nt':
-        browser.find_by_css('.csv-import-file')[0].fill("{}\\files\\csv-import-test.csv".format(cwd))
+        browser.find_by_css('.csv-import-file')[0].fill("{}\\files\\{}".format(cwd, csv_file))
     else:
-        browser.find_by_css('.csv-import-file')[0].fill("{}/files/csv-import-test.csv".format(cwd))
+        browser.find_by_css('.csv-import-file')[0].fill("{}/files/{}".format(cwd, csv_file))
 
+
+@then(parsers.parse("there are {num_groups} groups presented"))
+def ui_group_csv_groups_presented(browser, num_groups):
     # File contains 4 groups - check the number of rows presented.
-    assert len(browser.find_by_css('.import-groupname')) == 4
+    assert len(browser.find_by_css('.import-groupname')) == int(num_groups)
 
 
 @when("user clicks allow updates checkbox")
@@ -308,19 +311,22 @@ def ui_group_confirms_group_removal(browser):
     browser.find_by_id('f-group-delete').click()
 
 
-@then("process csv and check number of rows")
+@then("process csv")
 def ui_group_process_csv(browser):
     # Start processing the uploaded file.
     browser.find_by_css('.process-csv').click()
 
     # Take enough time so processing is complete.
     time.sleep(5)
 
+
+@then(parsers.parse("check number of rows is {num_rows}"))
+def ui_group_csv_check_rows(browser, num_rows):
     # Check whether 4 checkmarks are present so each row was processed.
-    assert len(browser.find_by_css('.import-groupname-done')) == 4
+    assert len(browser.find_by_css('.import-groupname-done')) == int(num_rows)
 
     # Check whether each row was processed correctly.
-    assert len(browser.find_by_css('.import-csv-group-ok')) == 4
+    assert len(browser.find_by_css('.import-csv-group-ok')) == int(num_rows)
 
 
 @then(parsers.parse("click on imported row {row} and check group properties"))
@@ -335,6 +341,16 @@ def ui_group_csv_click_row(browser, row):
     assert browser.find_by_id('f-group-update-name').value == groupname
 
 
+@then(parsers.parse('schema id is "{schema}"'))
+def ui_group_check_schema_id(browser, schema):
+    assert browser.find_by_id('f-group-update-schema-id').value == schema
+
+
+@then(parsers.parse('expiration date is "{expir}"'))
+def ui_group_check_schema_id_expiration(browser, expir):
+    assert browser.find_by_id('f-group-update-expiration-date').value == expir
+
+
 @then(parsers.parse('find group member "{group_member}"'))
 def ui_group_csv_find_group_member(browser, group_member):
     # Find the groupmember in the group member list.