Merge pull request #686 from ElectionDataAnalysis/issue683-tests

Test new juris files (#683, in part)
ElectionDataAnalysis · Jun 17, 2021 · 9cabc4c · 9cabc4c
2 parents 02b9f2a + 8a1ddab
commit 9cabc4c
Show file tree

Hide file tree

Showing 28 changed files with 237 additions and 32 deletions.
diff --git a/.gitignore b/.gitignore
@@ -21,4 +21,5 @@ src/ini_files_for_results/Delaware/de_20g_nist_export.ini
 /tests/Passed_Tests/
 /tests/TestingData/
 /tests/Staging/
+/tests/000_data_for_pytest/Temp/*
 
diff --git a/src/election_data_analysis/__init__.py b/src/election_data_analysis/__init__.py
@@ -761,7 +761,7 @@ def add_totals_if_missing(self, election, jurisdiction) -> Optional[dict]:
                 )
         return err
 
-    def load_data_from_db_dump(self, dbname, dump_file: str, param_file="run_time.ini") -> Optional[str]:
+    def load_data_from_db_dump(self, dbname, dump_file: str) -> Optional[str]:
         """Create a database from a file dumped from another database (but only if db does not
         already exist). Return error string"""
         connection = self.session.bind.raw_connection()
@@ -865,12 +865,16 @@ def check_and_init_singledataloader(
 
 
 class JurisdictionPrepper:
-    def __new__(cls):
+    def __new__(cls,
+                prep_param_file: str = "jurisdiction_prep.ini",
+                run_time_param_file: str = "run_time.ini",
+                target_dir: Optional[str] = None
+                ):
         """Checks if parameter file exists and is correct. If not, does
         not create JurisdictionPrepper object."""
         for param_file, required in [
-            ("jurisdiction_prep.ini", prep_pars),
-            ("run_time.ini", ["repository_content_root", "reports_and_plots_dir"]),
+            (prep_param_file, prep_pars),
+            (run_time_param_file, ["repository_content_root", "reports_and_plots_dir"]),
         ]:
             try:
                 d, parameter_err = ui.get_parameters(
@@ -892,23 +896,26 @@ def __new__(cls):
                 return None
         return super().__new__(cls)
 
-    def new_juris_files(self):
-        """<juris_path> identifies the directory where the files will live.
-        <abbr> is the two-letter abbreviation for state/district/territory.
-        <state_house>, etc., gives the number of districts;
-        <other_districts> is a dictionary of other district names, types & counts, e.g.,
-        {'Circuit Court':{'ReportingUnitType':'judicial','count':5}}
+    def new_juris_files(
+            self,
+            target_dir: Optional[str] = None,
+            templates: Optional[str] = None,
+    ):
+        """Create starter files in <target_dir>. If no <target_dir> is given, put the standard
+        jurisdiction files into a subdirectory of the jurisdictions directory in the repo, and put
+        the starter dictionary in the current directory.
         """
-
-        ## create and fill jurisdiction directory
+        # create and fill jurisdiction directory
         # TODO Feature: allow other districts to be set in paramfile
         print(f"\nStarting {inspect.currentframe().f_code.co_name}")
         error = jm.ensure_jurisdiction_dir(self.d["jurisdiction_path"])
         # add default entries
         project_root = Path(__file__).absolute().parents[1]
-        templates = os.path.join(
-            project_root, "juris_and_munger", "jurisdiction_templates"
-        )
+        # default templates are from repo
+        if not templates:
+            templates = os.path.join(
+                project_root, "election_data_analysis", "juris_and_munger", "jurisdiction_templates"
+            )
         for element in ["Party", "Election"]:
             new_err = prep.add_defaults(self.d["jurisdiction_path"], templates, element)
             if new_err:
@@ -919,7 +926,7 @@ def new_juris_files(self):
 
         # Feature create starter dictionary.txt with cdf_internal name
         #  used as placeholder for raw_identifier_value
-        dict_err = self.starter_dictionary()
+        dict_err = self.starter_dictionary(target_dir=target_dir)
 
         error = ui.consolidate_errors([error, asc_err, dict_err])
         ui.report(
@@ -1151,7 +1158,7 @@ def add_primaries_to_candidate_contest(self) -> Optional[str]:
         err_str = ui.consolidate_errors([err_str, new_err])
         return err_str
 
-    def starter_dictionary(self, include_existing=True) -> dict:
+    def starter_dictionary(self, include_existing=True, target_dir: Optional[str] = None) -> dict:
         """Creates a starter file for dictionary.txt, assuming raw_identifiers are the same as cdf_internal names.
         Puts file in the current directory. Returns error dictionary"""
         w = dict()
@@ -1180,11 +1187,16 @@ def starter_dictionary(self, include_existing=True) -> dict:
                 for element in elements
             ]
         ).drop_duplicates()
+        if target_dir:
+            ssd_str = starter_dict_dir = target_dir
+        else:
+            ssd_str = "current directory (not in jurisdiction directory)"
+            starter_dict_dir = "."
         err = prep.write_element(
-            ".", "dictionary", starter, file_name=starter_file_name
+            starter_dict_dir, "dictionary", starter, file_name=starter_file_name
         )
         print(
-            f"Starter dictionary created in current directory (not in jurisdiction directory):\n{starter_file_name}"
+            f"Starter dictionary created in {ssd_str}:\n{starter_file_name}"
         )
         return err
 
@@ -1384,12 +1396,16 @@ def make_munger_file(self, munger_name: str):
             )
         return
 
-    def __init__(self):
+    def __init__(self,
+                 prep_param_file: str = "jurisdiction_prep.ini",
+                 run_time_param_file: str = "run_time.ini",
+                 target_dir: Optional[str] = None
+                 ):
         self.d = dict()
         # get parameters from jurisdiction_prep.ini and run_time.ini
         for param_file, required in [
-            ("jurisdiction_prep.ini", prep_pars),
-            ("run_time.ini", ["repository_content_root", "reports_and_plots_dir"]),
+            (prep_param_file, prep_pars),
+            (run_time_param_file, ["repository_content_root", "reports_and_plots_dir"]),
         ]:
             d, parameter_err = ui.get_parameters(
                 required_keys=required,
@@ -1399,14 +1415,18 @@ def __init__(self):
             self.d.update(d)
 
         # add dictionary attributes derived from other parameters
-        derived = {
-            "system_name": jm.system_name_from_true_name(self.d["name"]),
-            "mungers_dir": os.path.join(self.d["repository_content_root"], "mungers"),
-            "jurisdiction_path": os.path.join(
+        if target_dir:
+            juris_path = target_dir
+        else:
+            juris_path = os.path.join(
                 self.d["repository_content_root"],
                 "jurisdictions",
                 jm.system_name_from_true_name(self.d["name"]),
-            ),
+            )
+        derived = {
+            "system_name": jm.system_name_from_true_name(self.d["name"]),
+            "mungers_dir": os.path.join(self.d["repository_content_root"], "mungers"),
+            "jurisdiction_path": juris_path,
         }
         self.d.update(derived)
 

diff --git a/src/election_data_analysis/juris_and_munger/jurisdiction_templates/Election.txt b/src/election_data_analysis/juris_and_munger/jurisdiction_templates/Election.txt
@@ -1,4 +1,5 @@
 Name	ElectionType
+2020 General Runoff	runoff
 2020 General	general
 2020 Primary	primary
 2018 General	general

diff --git a/src/election_data_analysis/preparation/__init__.py b/src/election_data_analysis/preparation/__init__.py
@@ -46,7 +46,7 @@ def remove_empty_lines(df: pd.DataFrame, element: str) -> pd.DataFrame:
 def write_element(
     juris_path: str, element: str, df: pd.DataFrame, file_name=None
 ) -> dict:
-    """<juris> is path to jurisdiction directory. Info taken
+    """<juris> is path to target directory. Info taken
     from <element>.txt file in that directory.
     <element>.txt is overwritten with info in <df>"""
     err = None

diff --git a/src/jurisdictions/Northern-Mariana-Islands/CandidateContest.txt b/src/jurisdictions/Northern-Mariana-Islands/CandidateContest.txt
@@ -12,7 +12,26 @@ MP Senate District 3	1	MP Senate District 3
 US House MP District 1	1	US House MP District 1	
 US President (MP)	1	US President (MP)	
 MP Governor	1	MP Governor	
-US House MP Delegate	1	US House MP Delegate
+US House MP Delegate	1	US House MP Delegate	
+MP Attorney General	1	MP Attorney General	
+MP Lieutenant Governor	1	MP Lieutenant Governor	
+MP Treasurer	1	MP Treasurer	
+MP Secretary of State	1	MP Secretary of State	
+US Senate MP	1	US Senate MP	
+MP House District 1	1	MP House District 1	
+MP House District 2	1	MP House District 2	
+MP House District 3	1	MP House District 3	
+MP House District 4	1	MP House District 4	
+MP House District 5	1	MP House District 5	
+MP House District 6	1	MP House District 6	
+MP House District 7	1	MP House District 7	
+MP Senate District 1	1	MP Senate District 1	
+MP Senate District 2	1	MP Senate District 2	
+MP Senate District 3	1	MP Senate District 3	
+US House MP District 1	1	US House MP District 1	
+US President (MP)	1	US President (MP)	
+MP Governor	1	MP Governor	
+US Senate MP	1	US Senate MP	
 MP Attorney General	1	MP Attorney General	
 MP Lieutenant Governor	1	MP Lieutenant Governor	
 MP Treasurer	1	MP Treasurer	

diff --git a/src/jurisdictions/Northern-Mariana-Islands/Office.txt b/src/jurisdictions/Northern-Mariana-Islands/Office.txt
@@ -17,3 +17,4 @@ MP Attorney General	Northern Mariana Islands
 MP Lieutenant Governor	Northern Mariana Islands
 MP Treasurer	Northern Mariana Islands
 MP Secretary of State	Northern Mariana Islands
+US Senate MP	Northern Mariana Islands
diff --git a/src/jurisdictions/Northern-Mariana-Islands/Party.txt b/src/jurisdictions/Northern-Mariana-Islands/Party.txt
@@ -4,4 +4,4 @@ Republican Party
 Green Party
 Libertarian Party
 Independent
-Nonpartisan
+Nonpartisan
diff --git a/src/jurisdictions/Northern-Mariana-Islands/ReportingUnit.txt b/src/jurisdictions/Northern-Mariana-Islands/ReportingUnit.txt
@@ -17,4 +17,4 @@ Northern Mariana Islands;Election District 3	district
 Northern Mariana Islands;Election District 4	district
 Northern Mariana Islands;Election District 5	district
 Northern Mariana Islands;Election District 6	district
-Northern Mariana Islands;Election District 7	district
+Northern Mariana Islands;Election District 7	district
diff --git a/tests/000_data_for_pytest/JurisdictionPrepper_reference_files/BallotMeasureContest.txt b/tests/000_data_for_pytest/JurisdictionPrepper_reference_files/BallotMeasureContest.txt
@@ -0,0 +1 @@
+Name	ElectionDistrict	Election
diff --git a/tests/000_data_for_pytest/JurisdictionPrepper_reference_files/Candidate.txt b/tests/000_data_for_pytest/JurisdictionPrepper_reference_files/Candidate.txt
@@ -0,0 +1 @@
+BallotName
diff --git a/tests/000_data_for_pytest/JurisdictionPrepper_reference_files/CandidateContest.txt b/tests/000_data_for_pytest/JurisdictionPrepper_reference_files/CandidateContest.txt
@@ -0,0 +1,19 @@
+Name	NumberElected	Office	PrimaryParty
+MP House District 1	1	MP House District 1	
+MP House District 2	1	MP House District 2	
+MP House District 3	1	MP House District 3	
+MP House District 4	1	MP House District 4	
+MP House District 5	1	MP House District 5	
+MP House District 6	1	MP House District 6	
+MP House District 7	1	MP House District 7	
+MP Senate District 1	1	MP Senate District 1	
+MP Senate District 2	1	MP Senate District 2	
+MP Senate District 3	1	MP Senate District 3	
+US House MP District 1	1	US House MP District 1	
+US President (MP)	1	US President (MP)	
+MP Governor	1	MP Governor	
+US Senate MP	1	US Senate MP	
+MP Attorney General	1	MP Attorney General	
+MP Lieutenant Governor	1	MP Lieutenant Governor	
+MP Treasurer	1	MP Treasurer	
+MP Secretary of State	1	MP Secretary of State	
diff --git a/tests/000_data_for_pytest/JurisdictionPrepper_reference_files/Election.txt b/tests/000_data_for_pytest/JurisdictionPrepper_reference_files/Election.txt
@@ -0,0 +1,6 @@
+Name	ElectionType
+2020 General Runoff	runoff
+2020 General	general
+2020 Primary	primary
+2018 General	general
+2016 General	general
diff --git a/tests/000_data_for_pytest/JurisdictionPrepper_reference_files/MP_starter_dictionary.txt b/tests/000_data_for_pytest/JurisdictionPrepper_reference_files/MP_starter_dictionary.txt
@@ -0,0 +1,40 @@
+cdf_element	cdf_internal_name	raw_identifier_value
+CandidateContest	MP House District 1	MP House District 1
+CandidateContest	MP House District 2	MP House District 2
+CandidateContest	MP House District 3	MP House District 3
+CandidateContest	MP House District 4	MP House District 4
+CandidateContest	MP House District 5	MP House District 5
+CandidateContest	MP House District 6	MP House District 6
+CandidateContest	MP House District 7	MP House District 7
+CandidateContest	MP Senate District 1	MP Senate District 1
+CandidateContest	MP Senate District 2	MP Senate District 2
+CandidateContest	MP Senate District 3	MP Senate District 3
+CandidateContest	US House MP District 1	US House MP District 1
+CandidateContest	US President (MP)	US President (MP)
+CandidateContest	MP Governor	MP Governor
+CandidateContest	US Senate MP	US Senate MP
+CandidateContest	MP Attorney General	MP Attorney General
+CandidateContest	MP Lieutenant Governor	MP Lieutenant Governor
+CandidateContest	MP Treasurer	MP Treasurer
+CandidateContest	MP Secretary of State	MP Secretary of State
+Election	2020 General Runoff	2020 General Runoff
+Election	2020 General	2020 General
+Election	2020 Primary	2020 Primary
+Election	2018 General	2018 General
+Election	2016 General	2016 General
+Party	Democratic Party	Democratic Party
+Party	Republican Party	Republican Party
+Party	Green Party	Green Party
+Party	Libertarian Party	Libertarian Party
+ReportingUnit	Northern Mariana Islands;MP House District 1	Northern Mariana Islands;MP House District 1
+ReportingUnit	Northern Mariana Islands;MP House District 2	Northern Mariana Islands;MP House District 2
+ReportingUnit	Northern Mariana Islands;MP House District 3	Northern Mariana Islands;MP House District 3
+ReportingUnit	Northern Mariana Islands;MP House District 4	Northern Mariana Islands;MP House District 4
+ReportingUnit	Northern Mariana Islands;MP House District 5	Northern Mariana Islands;MP House District 5
+ReportingUnit	Northern Mariana Islands;MP House District 6	Northern Mariana Islands;MP House District 6
+ReportingUnit	Northern Mariana Islands;MP House District 7	Northern Mariana Islands;MP House District 7
+ReportingUnit	Northern Mariana Islands;MP Senate District 1	Northern Mariana Islands;MP Senate District 1
+ReportingUnit	Northern Mariana Islands;MP Senate District 2	Northern Mariana Islands;MP Senate District 2
+ReportingUnit	Northern Mariana Islands;MP Senate District 3	Northern Mariana Islands;MP Senate District 3
+ReportingUnit	Northern Mariana Islands;US House MP District 1	Northern Mariana Islands;US House MP District 1
+ReportingUnit	Northern Mariana Islands	Northern Mariana Islands
diff --git a/tests/000_data_for_pytest/JurisdictionPrepper_reference_files/Office.txt b/tests/000_data_for_pytest/JurisdictionPrepper_reference_files/Office.txt
@@ -0,0 +1,19 @@
+Name	ElectionDistrict
+MP House District 1	Northern Mariana Islands;MP House District 1
+MP House District 2	Northern Mariana Islands;MP House District 2
+MP House District 3	Northern Mariana Islands;MP House District 3
+MP House District 4	Northern Mariana Islands;MP House District 4
+MP House District 5	Northern Mariana Islands;MP House District 5
+MP House District 6	Northern Mariana Islands;MP House District 6
+MP House District 7	Northern Mariana Islands;MP House District 7
+MP Senate District 1	Northern Mariana Islands;MP Senate District 1
+MP Senate District 2	Northern Mariana Islands;MP Senate District 2
+MP Senate District 3	Northern Mariana Islands;MP Senate District 3
+US House MP District 1	Northern Mariana Islands;US House MP District 1
+US President (MP)	Northern Mariana Islands
+MP Governor	Northern Mariana Islands
+US Senate MP	Northern Mariana Islands
+MP Attorney General	Northern Mariana Islands
+MP Lieutenant Governor	Northern Mariana Islands
+MP Treasurer	Northern Mariana Islands
+MP Secretary of State	Northern Mariana Islands
diff --git a/tests/000_data_for_pytest/JurisdictionPrepper_reference_files/Party.txt b/tests/000_data_for_pytest/JurisdictionPrepper_reference_files/Party.txt
@@ -0,0 +1,5 @@
+Name
+Democratic Party
+Republican Party
+Green Party
+Libertarian Party
diff --git a/tests/000_data_for_pytest/JurisdictionPrepper_reference_files/ReportingUnit.txt b/tests/000_data_for_pytest/JurisdictionPrepper_reference_files/ReportingUnit.txt
@@ -0,0 +1,13 @@
+Name	ReportingUnitType
+Northern Mariana Islands;MP House District 1	state-house
+Northern Mariana Islands;MP House District 2	state-house
+Northern Mariana Islands;MP House District 3	state-house
+Northern Mariana Islands;MP House District 4	state-house
+Northern Mariana Islands;MP House District 5	state-house
+Northern Mariana Islands;MP House District 6	state-house
+Northern Mariana Islands;MP House District 7	state-house
+Northern Mariana Islands;MP Senate District 1	state-senate
+Northern Mariana Islands;MP Senate District 2	state-senate
+Northern Mariana Islands;MP Senate District 3	state-senate
+Northern Mariana Islands;US House MP District 1	congressional
+Northern Mariana Islands	territory
diff --git a/tests/000_data_for_pytest/JurisdictionPrepper_reference_files/dictionary.txt b/tests/000_data_for_pytest/JurisdictionPrepper_reference_files/dictionary.txt
@@ -0,0 +1 @@
+cdf_element	cdf_internal_name	raw_identifier_value
diff --git a/tests/000_data_for_pytest/jurisdiction_templates/BallotMeasureContest.txt b/tests/000_data_for_pytest/jurisdiction_templates/BallotMeasureContest.txt
@@ -0,0 +1 @@
+Name	ElectionDistrict	Election
diff --git a/tests/000_data_for_pytest/jurisdiction_templates/Candidate.txt b/tests/000_data_for_pytest/jurisdiction_templates/Candidate.txt
@@ -0,0 +1 @@
+BallotName
diff --git a/tests/000_data_for_pytest/jurisdiction_templates/CandidateContest.txt b/tests/000_data_for_pytest/jurisdiction_templates/CandidateContest.txt
@@ -0,0 +1 @@
+Name	NumberElected	Office	PrimaryParty
diff --git a/tests/000_data_for_pytest/jurisdiction_templates/Election.txt b/tests/000_data_for_pytest/jurisdiction_templates/Election.txt
@@ -0,0 +1,6 @@
+Name	ElectionType
+2020 General Runoff	runoff
+2020 General	general
+2020 Primary	primary
+2018 General	general
+2016 General	general
diff --git a/tests/000_data_for_pytest/jurisdiction_templates/Office.txt b/tests/000_data_for_pytest/jurisdiction_templates/Office.txt
@@ -0,0 +1 @@
+Name	ElectionDistrict
diff --git a/tests/000_data_for_pytest/jurisdiction_templates/Party.txt b/tests/000_data_for_pytest/jurisdiction_templates/Party.txt
@@ -0,0 +1,5 @@
+Name
+Democratic Party
+Republican Party
+Green Party
+Libertarian Party
diff --git a/tests/000_data_for_pytest/jurisdiction_templates/ReportingUnit.txt b/tests/000_data_for_pytest/jurisdiction_templates/ReportingUnit.txt
@@ -0,0 +1 @@
+Name	ReportingUnitType
diff --git a/tests/000_data_for_pytest/jurisdiction_templates/dictionary.txt b/tests/000_data_for_pytest/jurisdiction_templates/dictionary.txt
@@ -0,0 +1 @@
+cdf_element	cdf_internal_name	raw_identifier_value
diff --git a/tests/analyzer_tests/test_exports.py b/tests/analyzer_tests/test_exports.py
@@ -18,7 +18,7 @@ def test_nist_v2_and_v1(runtime):
 
     # load test data to the test db
     dl = DataLoader(param_file=runtime)
-    err_str = dl.load_data_from_db_dump(param_file=runtime, dbname=test_db_name, dump_file=db_dump)
+    err_str = dl.load_data_from_db_dump(dbname=test_db_name, dump_file=db_dump)
 
     # create Analyzer object
     an = Analyzer(dbname=test_db_name, param_file=runtime)

diff --git a/tests/jurisdiction_prepper_tests/jurisdiction_prep.ini_for_test b/tests/jurisdiction_prepper_tests/jurisdiction_prep.ini_for_test
@@ -0,0 +1,8 @@
+[election_data_analysis]
+name=Northern Mariana Islands
+reporting_unit_type=territory
+abbreviated_name=MP
+count_of_state_house_districts=7
+count_of_state_senate_districts=3
+count_of_us_house_districts=1
+
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		cdf_element cdf_internal_name raw_identifier_value