diff --git a/README.md b/README.md index ba60486e..a19b28a8 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ optional arguments: --pgservice PGSERVICE name of the pgservice to use to connect to the database (default: pg_qgep) --log saves a log file next to the input/output file (default: False) + --export_sia405 export the model SIA405_ABWASSER_2015_LV95 (instead of default VSA_KEK_2019_LV95) (default: False) ``` ### Import/export QWAT diff --git a/qgepqwat2ili/__init__.py b/qgepqwat2ili/__init__.py index fd48866c..4064e841 100644 --- a/qgepqwat2ili/__init__.py +++ b/qgepqwat2ili/__init__.py @@ -61,6 +61,11 @@ def main(args): action="store_true", help="saves the log files next to the input/output file", ) + parser_qgep.add_argument( + "--export_sia405", + action="store_true", + help="export the model SIA405_ABWASSER_2015_LV95 (instead of default VSA_KEK_2019_LV95)", + ) parser_qwat = subparsers.add_parser( "qwat", @@ -129,13 +134,23 @@ def main(args): config.PGSERVICE = args.pgservice SCHEMA = config.ABWASSER_SCHEMA ILI_MODEL = config.ABWASSER_ILI_MODEL - ILI_MODEL_NAME = config.ABWASSER_ILI_MODEL_NAME + if args.export_sia405: + ILI_MODEL_NAME = config.ABWASSER_ILI_MODEL_NAME_SIA405 + ILI_EXPORT_MODEL_NAME = config.ABWASSER_ILI_MODEL_NAME_SIA405 + else: + ILI_MODEL_NAME = config.ABWASSER_ILI_MODEL_NAME + ILI_EXPORT_MODEL_NAME = None + if args.direction == "export": utils.ili2db.create_ili_schema( SCHEMA, ILI_MODEL, make_log_path(log_path, "ilicreate"), recreate_schema=args.recreate_schema ) qgep_export(selection=args.selection.split(",") if args.selection else None, labels_file=args.labels_file) - utils.ili2db.export_xtf_data(SCHEMA, ILI_MODEL_NAME, args.path, make_log_path(log_path, "iliexport")) + + utils.ili2db.export_xtf_data( + SCHEMA, ILI_MODEL_NAME, ILI_EXPORT_MODEL_NAME, args.path, make_log_path(log_path, "iliexport") + ) + if not args.skip_validation: try: utils.ili2db.validate_xtf_data(args.path, make_log_path(log_path, "ilivalidate")) @@ -164,12 +179,16 @@ def main(args): SCHEMA = config.WASSER_SCHEMA ILI_MODEL = config.WASSER_ILI_MODEL ILI_MODEL_NAME = config.WASSER_ILI_MODEL_NAME + ILI_EXPORT_MODEL_NAME = None + if args.direction == "export": utils.ili2db.create_ili_schema( SCHEMA, ILI_MODEL, make_log_path(log_path, "ilicreate"), recreate_schema=args.recreate_schema ) qwat_export(include_hydraulics=args.include_hydraulics) - utils.ili2db.export_xtf_data(SCHEMA, ILI_MODEL_NAME, args.path, make_log_path(log_path, "iliexport")) + utils.ili2db.export_xtf_data( + SCHEMA, ILI_MODEL_NAME, ILI_EXPORT_MODEL_NAME, args.path, make_log_path(log_path, "iliexport") + ) if not args.skip_validation: try: utils.ili2db.validate_xtf_data(args.path, make_log_path(log_path, "ilivalidate")) diff --git a/qgepqwat2ili/config.py b/qgepqwat2ili/config.py index 8da3f987..3284af59 100644 --- a/qgepqwat2ili/config.py +++ b/qgepqwat2ili/config.py @@ -19,6 +19,8 @@ ABWASSER_SCHEMA = "pg2ili_abwasser" ABWASSER_ILI_MODEL = os.path.join(ILI_FOLDER, "VSA_KEK_2019_2_d_LV95-20210120.ili") ABWASSER_ILI_MODEL_NAME = "VSA_KEK_2019_LV95" +ABWASSER_ILI_MODEL_NAME_SIA405 = "SIA405_ABWASSER_2015_LV95" +ABWASSER_ILI_EXPORT_MODEL = os.path.join(ILI_FOLDER, "SIA405_Abwasser_2015_2_d-20180417.ili") QWAT_DEFAULT_PGSERVICE = "qwat" QWAT_SCHEMA = "qwat_od" diff --git a/qgepqwat2ili/gui/__init__.py b/qgepqwat2ili/gui/__init__.py index d7d48818..ae954998 100644 --- a/qgepqwat2ili/gui/__init__.py +++ b/qgepqwat2ili/gui/__init__.py @@ -175,6 +175,9 @@ def action_do_export(): return QgsSettings().setValue("qgep_pluging/last_interlis_path", os.path.dirname(file_name)) + # File name without extension (used later for export) + file_name_base, _ = os.path.splitext(file_name) + # Configure logging if export_dialog.logs_next_to_file: base_log_path = file_name @@ -254,48 +257,59 @@ def action_do_export(): # Cleanup tempdir.cleanup() - # Export from ili2pg model to file - progress_dialog.setLabelText("Saving XTF file...") - QApplication.processEvents() - log_path = make_log_path(base_log_path, "ili2pg-export") - try: - export_xtf_data( - config.ABWASSER_SCHEMA, - config.ABWASSER_ILI_MODEL_NAME, - file_name, - log_path, - ) - except CmdException: - progress_dialog.close() - show_failure( - "Could not export the ili2pg schema", - "Open the logs for more details on the error.", - log_path, - ) - return - progress_dialog.setValue(75) + for model_name, export_model_name, progress in [ + (config.ABWASSER_ILI_MODEL_NAME, None, 50), + (config.ABWASSER_ILI_MODEL_NAME_SIA405, config.ABWASSER_ILI_MODEL_NAME_SIA405, 70), + ]: + + export_file_name = f"{file_name_base}_{model_name}.xtf" + + # Export from ili2pg model to file + progress_dialog.setLabelText(f"Saving XTF file [{model_name}]...") + QApplication.processEvents() + log_path = make_log_path(base_log_path, f"ili2pg-export-{model_name}") + try: + export_xtf_data( + config.ABWASSER_SCHEMA, + model_name, + export_model_name, + export_file_name, + log_path, + ) + except CmdException: + progress_dialog.close() + show_failure( + "Could not export the ili2pg schema", + "Open the logs for more details on the error.", + log_path, + ) + continue + progress_dialog.setValue(progress + 10) + + progress_dialog.setLabelText(f"Validating the network output file [{model_name}]...") + QApplication.processEvents() + log_path = make_log_path(base_log_path, f"ilivalidator-{model_name}") + try: + validate_xtf_data( + export_file_name, + log_path, + ) + except CmdException: + progress_dialog.close() + show_failure( + "Invalid file", + f"The created file is not a valid {model_name} XTF file.", + log_path, + ) + continue + + progress_dialog.setValue(progress + 20) - progress_dialog.setLabelText("Validating the output file...") - QApplication.processEvents() - log_path = make_log_path(base_log_path, "ilivalidator") - try: - validate_xtf_data( - file_name, - log_path, - ) - except CmdException: - progress_dialog.close() - show_failure( - "Invalid file", - "The created file is not a valid XTF file.", - log_path, - ) - return progress_dialog.setValue(100) show_success( "Sucess", - f"Data successfully exported to {file_name}", + f"Data successfully exported to {file_name_base}", os.path.dirname(log_path), ) diff --git a/qgepqwat2ili/tests/test_qgep.py b/qgepqwat2ili/tests/test_qgep.py index 1dfda008..1e6bf32b 100644 --- a/qgepqwat2ili/tests/test_qgep.py +++ b/qgepqwat2ili/tests/test_qgep.py @@ -18,7 +18,12 @@ logger.addHandler(handler) -def findall_in_xml(root, tag, basket="VSA_KEK_2019_LV95.KEK"): +def findall_in_xml_kek_2019(root, tag, basket="VSA_KEK_2019_LV95.KEK"): + ns = {"ili": "http://www.interlis.ch/INTERLIS2.3"} + return root.findall(f"ili:DATASECTION/ili:{basket}/ili:{tag}", ns) + + +def findall_in_xml_sia_abwasser_2015(root, tag, basket="SIA405_ABWASSER_2015_LV95.SIA405_Abwasser"): ns = {"ili": "http://www.interlis.ch/INTERLIS2.3"} return root.findall(f"ili:DATASECTION/ili:{basket}/ili:{tag}", ns) @@ -86,7 +91,6 @@ def test_case_b_export_complete_qgep_to_xtf(self): path = os.path.join(tempfile.mkdtemp(), "export.xtf") main(["qgep", "export", path, "--recreate_schema"]) - def test_case_d_import_complete_xtf_to_qgep(self): """ # D. import a whole valid interlis transfer file into QGEP @@ -94,7 +98,9 @@ def test_case_d_import_complete_xtf_to_qgep(self): # Incomming XTF case_c_import_all_without_errors.xtf # THIS INPUT FILE IS VALID ! - path = os.path.join(os.path.dirname(__file__), "..", "data", "test_data", "case_d_import_all_without_errors.xtf") + path = os.path.join( + os.path.dirname(__file__), "..", "data", "test_data", "case_d_import_all_without_errors.xtf" + ) # Prepare subset db (we import in an empty schema) main(["setupdb", "empty"]) @@ -116,7 +122,7 @@ def test_case_d_import_complete_xtf_to_qgep(self): # checking some properties # TODO : add some more... self.assertEqual(session.query(QGEP.manhole).get("ch080qwzNS000113").year_of_construction, 1950) session.close() - + def test_case_e_export_selection(self): """ # E. export a selection @@ -125,7 +131,52 @@ def test_case_e_export_selection(self): # Prepare db main(["setupdb", "full"]) - path = os.path.join(tempfile.mkdtemp(), "export.xtf") + # 1. Export VSA_KEK_2019_LV95 + path = os.path.join(tempfile.mkdtemp(), "export_VSA_KEK_2019_LV95.xtf") + selection = [ + # reach_id + "ch13p7mzRE001221", + # node_a_id + "ch13p7mzWN003445", + # node_b_id + "ch13p7mzWN008122", + ] + labels_file = os.path.join(os.path.dirname(__file__), "data", "labels.geojson") + main( + [ + "qgep", + "export", + path, + "--recreate_schema", + "--selection", + ",".join(selection), + "--labels_file", + labels_file, + ] + ) + + # Perform various checks + logger.warning("Perform various checks VSA_KEK_2019_LV95 ...") + + root = ET.parse(path) + + self.assertEqual(len(findall_in_xml_kek_2019(root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Kanal")), 1) + self.assertEqual( + len(findall_in_xml_kek_2019(root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Normschacht")), 2 + ) + self.assertEqual( + len(findall_in_xml_kek_2019(root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Haltung_Text")), 3 + ) + self.assertEqual( + len(findall_in_xml_kek_2019(root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Abwasserbauwerk_Text")), 6 + ) + + def test_case_f_export_selection_sia405(self): + """ + # F. export a selection + """ + + path = os.path.join(tempfile.mkdtemp(), "export_VSA_KEK_2019_LV95.xtf") selection = [ # reach_id "ch13p7mzRE001221", @@ -140,6 +191,7 @@ def test_case_e_export_selection(self): "qgep", "export", path, + "--export_sia405", "--recreate_schema", "--selection", ",".join(selection), @@ -147,13 +199,28 @@ def test_case_e_export_selection(self): labels_file, ] ) + # Perform various checks + logger.warning("Perform various checks SIA405_ABWASSER_2015_LV95 ...") + root = ET.parse(path) - self.assertEquals(len(findall_in_xml(root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Kanal")), 1) - self.assertEquals(len(findall_in_xml(root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Normschacht")), 2) - self.assertEquals(len(findall_in_xml(root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Haltung_Text")), 3) - self.assertEquals( - len(findall_in_xml(root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Abwasserbauwerk_Text")), 6 + + self.assertEqual( + len(findall_in_xml_sia_abwasser_2015(root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Kanal")), 1 + ) + self.assertEqual( + len(findall_in_xml_sia_abwasser_2015(root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Normschacht")), 2 + ) + self.assertEqual( + len(findall_in_xml_sia_abwasser_2015(root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Haltung_Text")), 3 + ) + self.assertEqual( + len( + findall_in_xml_sia_abwasser_2015( + root, "SIA405_ABWASSER_2015_LV95.SIA405_Abwasser.Abwasserbauwerk_Text" + ) + ), + 6, ) diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index b2e535aa..33bdbe4c 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -97,8 +97,15 @@ def import_xtf_data(schema, xtf_file, log_path): ) -def export_xtf_data(schema, model_name, xtf_file, log_path): +def export_xtf_data(schema, model_name, export_model_name, xtf_file, log_path): logger.info("EXPORT ILIDB...") + + # if optional export_model_name is set, add it to the args + if export_model_name: + export_model_name_args = ["--exportModels", export_model_name] + else: + export_model_name_args = [] + exec_( " ".join( [ @@ -108,6 +115,7 @@ def export_xtf_data(schema, model_name, xtf_file, log_path): "--export", "--models", f"{model_name}", + *export_model_name_args, *get_pgconf_as_ili_args(), "--dbschema", f"{schema}",