update migrations (#667)

* add WCMP2 migration script (#663) * fix refs * address PR comments * fix * fix for station migration in case station_list.csv has no stations * flake8 * fix * dotpath, extra logging, update identifier * also update properties id * Delete wis2box-management/migrations/v1_0b6_to_v1_0b7/update_wcmp2_identifiers.py * Update Dockerfile --------- Co-authored-by: Maaike Limper <[email protected]> Co-authored-by: Maaike <[email protected]>
wmo-im · Apr 30, 2024 · 262ffd3 · 262ffd3
1 parent 2025fab
commit 262ffd3
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 21 deletions.
diff --git a/wis2box-management/migrations/v1_0b6_to_v1_0b7/update_station_definition_v1.0b7.py b/wis2box-management/migrations/v1_0b6_to_v1_0b7/update_station_definition_v1.0b7.py
@@ -29,7 +29,7 @@
 
 from wis2box.log import LOGGER, setup_logger
 
-LOG_LEVEL = os.getenv("LOG_LEVEL", "DEBUG").upper()
+LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()
 setup_logger(loglevel=LOG_LEVEL)
 
 DATADIR = os.getenv("WIS2BOX_DATADIR")
@@ -107,27 +107,29 @@ def migrate(dryrun: bool = False):
                 else:
                     pass
             stations.append(row)
-
-    if dryrun:
-        LOGGER.info(
-            f"dryrun == True, writing updated {station_file} to stdout")
-        print(','.join(map(str, stations[0].keys())))
-        for station in stations:
-            print(','.join(map(str, station.values())))
+    if len(stations) > 0:
+        if dryrun:
+            LOGGER.info(
+                f"dryrun == True, writing updated {station_file} to stdout")
+            print(','.join(map(str, stations[0].keys())))
+            for station in stations:
+                print(','.join(map(str, station.values())))
+        else:
+            # now write data to file
+            LOGGER.info(
+                f"Writing updated {station_file} to {station_file}.v1.0b7")
+            try:
+                with open(f"{station_file}.v1.0b7", 'w') as fh:
+                    columns = list(stations[0].keys())
+                    writer = csv.DictWriter(fh, fieldnames=columns)
+                    writer.writeheader()
+                    for station in stations:
+                        writer.writerow(station)
+            except Exception as e:
+                LOGGER.error("Error writing updated station file")
+                raise e
     else:
-        # now write data to file
-        LOGGER.info(
-            f"Writing updated {station_file} to {station_file}.v1.0b7")
-        try:
-            with open(f"{station_file}.v1.0b7", 'w') as fh:
-                columns = list(stations[0].keys())
-                writer = csv.DictWriter(fh, fieldnames=columns)
-                writer.writeheader()
-                for station in stations:
-                    writer.writerow(station)
-        except Exception as e:
-            LOGGER.error("Error writing updated station file")
-            raise e
+        LOGGER.info("No stations in station_list.csv to be updated")
 
     # now migrate ES data
     # Get elastic search connection

diff --git a/wis2box-management/wis2box/metadata/discovery.py b/wis2box-management/wis2box/metadata/discovery.py
@@ -252,6 +252,11 @@ def publish_discovery_metadata(metadata: Union[dict, str]):
             record_mcf = dm.parse_record(metadata)
             record = dm.generate(record_mcf)
 
+        if 'x-wmo' in record['id']:
+            msg = 'Change x-wmo to wmo in metadata identifier'
+            LOGGER.error(msg)
+            raise RuntimeError(msg)
+
         if 'data_mappings' not in record['wis2box']:
             msg = 'Missing wis2box.data_mappings definition'
             LOGGER.error(msg)