Enceladus ECS rollback script: first version for evaluation

scripts/migration/dataset_paths_ecs_rollback.py

@@ -54,12 +54,15 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument('-f', '--fields-to-map', dest='fieldstomap', choices=[MAPPING_FIELD_HDFS_PATH, MAPPING_FIELD_HDFS_PUBLISH_PATH, MAPPING_FIELD_HDFS_ALL],
                         default=MAPPING_FIELD_HDFS_ALL, help="Rollback either item's 'hdfsPath', 'hdfsPublishPath' or 'all'.")
 
-    parser.add_argument('-d', '--datasets', dest='datasets', metavar="DATASET_NAME", default=[],
-                        nargs="+", help='list datasets names to rollback path changes in (hint: MTs are also datasets).')
-
     parser.add_argument('-o', '--only-datasets', dest='onlydatasets', action='store_true', default=DEFAULT_DATASETS_ONLY,
                         help="if specified, only dataset rollback path changes will be done (not MTs).")
 
+    input_options_group = parser.add_mutually_exclusive_group(required=True)
+    input_options_group.add_argument('-d', '--datasets', dest='datasets', metavar="DATASET_NAME", default=[],
+                                     nargs="+", help='list datasets names to rollback path changes in')
+    input_options_group.add_argument('-m', '--mapping-tables', dest="mtables", metavar="MTABLE_NAME", default=[],
+                                     nargs="+", help='list mapping tables names to rollback path changes in')
+
 
     return parser.parse_args()
 
@@ -134,7 +137,7 @@ def data_update_to_nice_string(data_update: dict) -> str:
 def rollback_pathchange_entities(target_db: MenasDb, collection_name: str, entity_name: str, entity_names_list: List[str],
                                  mapping_settings: RollbackSettings, dryrun: bool) -> None:
 
-    assert entity_name == "dataset" or entity_name == "mapping table" , "this method supports datasets and MTs only!"
+    assert entity_name == "dataset" or entity_name == "mapping table", "this method supports datasets and MTs only!"
 
     if not entity_names_list:
         print("No {}s to rollback path-changes in {}, skipping.".format(entity_name, collection_name))
@@ -143,7 +146,7 @@ def rollback_pathchange_entities(target_db: MenasDb, collection_name: str, entit
     print("Rollbacking path-change of collection {} started".format(collection_name))
     dataset_collection = target_db.mongodb[collection_name]
 
-    query =  {"$and": [
+    query = {"$and": [
         {"name": {"$in": entity_names_list}}  # dataset/MT name
     ]}
 
@@ -187,7 +190,8 @@ def rollback_pathchange_entities(target_db: MenasDb, collection_name: str, entit
                     patched += 1
         else:
             if verbose:
-                print("Nothing left to rollback for {} '{}' v{} (_id={}).".format(entity_name, item["name"], item["version"], item["_id"]))
+                print("  Nothing left to rollback for {} '{}' v{} (_id={}).".format(entity_name, item["name"], item["version"], item["_id"]))
+                print("")
 
     print("Successfully rollbacked {} of {} {} entries, failed: {}".format(patched, docs_count, entity_name, failed_count))
     print("")
@@ -218,28 +222,57 @@ def rollback_pathchange_collections_by_ds_names(target_db: MenasDb,
         rollback_pathchange_entities(target_db, MAPPING_TABLE_COLLECTION, "mapping table", mapping_table_found_for_dss, mapping_settings, dryrun)
 
 
+
+def rollback_pathchange_collections_by_mt_names(target_db: MenasDb,
+                                                supplied_mt_names: List[str],
+                                                mapping_settings: RollbackSettings,
+                                                dryrun: bool) -> None:
+
+    if verbose:
+        print("Mapping table names given: {}".format(supplied_mt_names))
+
+    mt_names_found = target_db.get_distinct_mt_names_from_mt_names(supplied_mt_names, migration_free_only=False)
+    print('Mapping table names to rollback path-changes (actually found db): {}'.format(mt_names_found))
+
+    print("")
+    rollback_pathchange_entities(target_db, MAPPING_TABLE_COLLECTION, "mapping table", mt_names_found, mapping_settings, dryrun)
+
+
 def run(parsed_args: argparse.Namespace):
     target_conn_string = parsed_args.target
     target_db_name = parsed_args.targetdb
 
     dryrun = args.dryrun  # if set, only path change rollback description will be printed, no actual patching will run
 
     skip_prefixes = args.skipprefixes
-    fields_to_map = args.fieldstomap # argparse allow only one of HDFS_MAPPING_FIELD_HDFS_*
-    rollback_settings  = RollbackSettings(skip_prefixes, fields_to_map)
+    fields_to_map = args.fieldstomap  # argparse allow only one of HDFS_MAPPING_FIELD_HDFS_*
+    rollback_settings = RollbackSettings(skip_prefixes, fields_to_map)
 
     print('Menas mongo ECS paths mapping ROLLBACK')
     print('Running with settings: dryrun={}, verbose={}'.format(dryrun, verbose))
     print('Skipping prefixes: {}'.format(skip_prefixes))
     print('  target connection-string: {}'.format(target_conn_string))
     print('  target DB: {}'.format(target_db_name))
-
-
     target_db = MenasDb.from_connection_string(target_conn_string, target_db_name, alias="target db", verbose=verbose)
 
     dataset_names = parsed_args.datasets
     only_datasets = parsed_args.onlydatasets
-    rollback_pathchange_collections_by_ds_names(target_db, dataset_names, rollback_settings, only_datasets, dryrun=dryrun)
+    mt_names = parsed_args.mtables
+
+    if dataset_names:
+        print('Dataset names supplied: {}'.format(dataset_names))
+        rollback_pathchange_collections_by_ds_names(target_db, dataset_names, rollback_settings, only_datasets, dryrun=dryrun)
+
+    elif mt_names:
+        if only_datasets:
+            raise Exception("Invalid run options: -o/--only-datasets cannot be used with -m/--mapping-tables, only for -d/--datasets")
+
+        print('Mapping table names supplied: {}'.format(mt_names))
+        rollback_pathchange_collections_by_mt_names(target_db, mt_names, rollback_settings, dryrun=dryrun)
+
+    else:
+        # should not happen (-d/-m is exclusive and required)
+        raise Exception("Invalid run options: DS names (-d ds1 ds2 ...)..,  MT names (-m mt1 mt2 ... ) must be given.")
 
     print("Done.")
     print("")

scripts/migration/dataset_paths_to_ecs.py

@@ -64,12 +64,15 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument('-f', '--fields-to-map', dest='fieldstomap', choices=[MAPPING_FIELD_HDFS_PATH, MAPPING_FIELD_HDFS_PUBLISH_PATH, MAPPING_FIELD_HDFS_ALL],
                         default=MAPPING_FIELD_HDFS_ALL, help="Map either item's 'hdfsPath', 'hdfsPublishPath' or 'all'.")
 
-    parser.add_argument('-d', '--datasets', dest='datasets', metavar="DATASET_NAME", default=[],
-                        nargs="+", help='list datasets names to change paths in (hint: MTs are also datasets).')
-
     parser.add_argument('-o', '--only-datasets', dest='onlydatasets', action='store_true', default=DEFAULT_DATASETS_ONLY,
-                        help="if specified, only dataset path changes will be done (not MTs).")
+                        help="if specified, only dataset path changes will be done (not MTs). Cannot be used with -m/--mapping-tables")
+
 
+    input_options_group = parser.add_mutually_exclusive_group(required=True)
+    input_options_group.add_argument('-d', '--datasets', dest='datasets', metavar="DATASET_NAME", default=[],
+                                     nargs="+", help='list datasets names to change paths in')
+    input_options_group.add_argument('-m', '--mapping-tables', dest="mtables", metavar="MTABLE_NAME", default=[],
+                                     nargs="+", help='list mapping tables names to change paths in')
 
     return parser.parse_args()
 
@@ -232,6 +235,21 @@ def pathchange_collections_by_ds_names(target_db: MenasDb,
     if not onlydatasets:
         pathchange_entities(target_db, MAPPING_TABLE_COLLECTION, "mapping table", mapping_table_found_for_dss, mapping_settings, dryrun)
 
+def pathchange_collections_by_mt_names(target_db: MenasDb,
+                                       supplied_mt_names: List[str],
+                                       mapping_settings: MappingSettings,
+                                       dryrun: bool) -> None:
+
+    if verbose:
+        print("Mapping table names given: {}".format(supplied_mt_names))
+
+    mt_names_found = target_db.get_distinct_mt_names_from_mt_names(supplied_mt_names, migration_free_only=False)
+    print('Mapping table names to path change (actually found db): {}'.format(mt_names_found))
+
+    print("")
+    pathchange_entities(target_db, MAPPING_TABLE_COLLECTION, "mapping table", mt_names_found, mapping_settings, dryrun)
+
+
 def pre_run_mapping_service_check(svc_url: str, path_prefix_to_add: str):
     test_path = "/bigdatahdfs/datalake/publish/pcub/just/a/path/to/test/the/service/"
 
@@ -265,7 +283,22 @@ def run(parsed_args: argparse.Namespace):
 
     dataset_names = parsed_args.datasets
     only_datasets = parsed_args.onlydatasets
-    pathchange_collections_by_ds_names(target_db, dataset_names, mapping_settings, only_datasets, dryrun=dryrun)
+    mt_names = parsed_args.mtables
+
+    if dataset_names:
+        print('Dataset names supplied: {}'.format(dataset_names))
+        pathchange_collections_by_ds_names(target_db, dataset_names, mapping_settings, only_datasets, dryrun=dryrun)
+
+    elif mt_names:
+        if only_datasets:
+            raise Exception("Invalid run options: -o/--only-datasets cannot be used with -m/--mapping-tables, only for -d/--datasets")
+
+        print('Mapping table names supplied: {}'.format(mt_names))
+        pathchange_collections_by_mt_names(target_db, mt_names, mapping_settings, dryrun=dryrun)
+
+    else:
+        # should not happen (-d/-m is exclusive and required)
+        raise Exception("Invalid run options: DS names (-d ds1 ds2 ...)..,  MT names (-m mt1 mt2 ... ) must be given.")
 
     print("Done.")
     print("")

scripts/migration/menas_db.py

@@ -101,6 +101,9 @@ def ensure_collections_exist(collection_names: List[str]) -> None:
     def get_distinct_ds_names_from_ds_names(self, ds_names: List[str], migration_free_only: bool) -> List[str]:
         return self.get_distinct_entities_ids(ds_names, DATASET_COLLECTION, migration_free_only)
 
+    def get_distinct_mt_names_from_mt_names(self, mt_names: List[str], migration_free_only: bool) -> List[str]:
+        return self.get_distinct_entities_ids(mt_names, MAPPING_TABLE_COLLECTION, migration_free_only)
+
     def get_distinct_schema_names_from_schema_names(self, schema_names: List[str],
                                                     migration_free_only: bool) -> List[str]:
         return self.get_distinct_entities_ids(schema_names, SCHEMA_COLLECTION, migration_free_only)

scripts/migration/migrate_menas.py

@@ -349,7 +349,7 @@ def run(parsed_args: argparse.Namespace):
         print('Property definition names supplied: {}'.format(propdef_names))
         migrate_propdefs_by_propdef_names(source_db, target_db, propdef_names, dryrun=dryrun)
     else:
-        # should not happen (-d/-m is exclusive and required)
+        # should not happen (-d/-m/-p is exclusive and required)
         raise Exception("Invalid run options: DS names (-d ds1 ds2 ...)..,  MT names (-m mt1 mt2 ... ), "
                         "or prop defs (-p prop1 prop2 ...) must be given.")