feat: improved workflows

.github/build_registry.py

@@ -1,56 +1,15 @@
 import logging
-import os
+import sys
 
 import yaml
 
-from validate import validate_against_schema
+from validate import read_and_merge_configs
 
 # Set up basic logging
 logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
 
 
-def read_and_merge_configs(base_dir="."):
-    """
-    Reads all config.yml files in the directory structure and merges them.
-    """
-    merged_config = {}
-
-    for root, _, files in os.walk(base_dir):
-        if "config.yml" in files:
-            file_path = os.path.join(root, "config.yml")
-
-            try:
-                with open(file_path, 'r') as stream:
-                    config = yaml.safe_load(stream)
-
-                # Store and remove global properties
-                global_properties = config.pop("properties", {})
-
-                # Merge global properties into network-specific properties
-                for _, network_config in config.get("networks", {}).items():
-                    local_properties = network_config.get("properties", {})
-                    network_config["properties"] = {**global_properties, **local_properties}
-
-                # validate configuration
-                validate_against_schema(config)
-
-                # get source id
-                source_id = config["source-id"]
-                # check if source id already exists
-                if source_id in merged_config:
-                    raise Exception(f"Duplicate source_id: {source_id}")
-
-                logging.info(f"Successfully validated source: {source_id}")
-
-                merged_config[source_id] = config
-
-            except Exception as e:
-                logging.error(f"Error validating file {file_path}: {e}")
-
-    return merged_config
-
-
-def write_yaml_file(data, file_path=".github/registry.yml"):
+def write_yaml_file(data, file_path="registry.yml"):
     """
     Writes data to a YAML file.
     """
@@ -62,10 +21,15 @@ def write_yaml_file(data, file_path=".github/registry.yml"):
         logging.error(f"Error writing to file {file_path}: {e}")
 
 
-def main():
-    merged_config = read_and_merge_configs()
-    write_yaml_file(merged_config)
+if __name__ == "__main__":
+    args = sys.argv[1:]
+    if len(args) > 1:
+        raise ValueError(f"Unexpected arguments: {args}")
 
+    if len(args) == 0:
+        file_path = "registry.yml"
+    else:
+        file_path = args[0]
 
-if __name__ == "__main__":
-    main()
+    merged_config = read_and_merge_configs()
+    write_yaml_file(merged_config, file_path=file_path)

.github/validate.py

@@ -1,7 +1,9 @@
 import json
 import logging
+import os
 from pathlib import Path
 
+import yaml
 from jsonschema.validators import validator_for
 from referencing import Registry, Resource
 from referencing.exceptions import NoSuchResource
@@ -49,8 +51,45 @@ def validate_against_schema(data):
     Validator = validator_for(schema)
     validator = Validator(schema, registry=registry)
 
-    try:
-        validator.validate(data)
-    except Exception as e:
-        logging.error(f"Validation error: {e}")
-        raise
+    validator.validate(data)
+
+
+def read_and_merge_configs(base_dir="."):
+    """
+    Reads all config.yml files in the directory structure and merges them.
+    """
+    merged_config = {}
+
+    for root, _, files in os.walk(base_dir):
+        if "config.yml" in files:
+            file_path = os.path.join(root, "config.yml")
+
+            try:
+                with open(file_path, 'r') as stream:
+                    config = yaml.safe_load(stream)
+
+                # Store and remove global properties
+                global_properties = config.pop("properties", {})
+
+                # Merge global properties into network-specific properties
+                for _, network_config in config.get("networks", {}).items():
+                    local_properties = network_config.get("properties", {})
+                    network_config["properties"] = {**global_properties, **local_properties}
+
+                # validate configuration
+                validate_against_schema(config)
+
+                # get source id
+                source_id = config["source-id"]
+                # check if source id already exists
+                if source_id in merged_config:
+                    raise Exception(f"Duplicate source_id: {source_id}")
+
+                logging.info(f"Successfully validated source: {source_id}")
+
+                merged_config[source_id] = config
+
+            except Exception:
+                raise
+
+    return merged_config

.github/validate_registry.py

@@ -1,70 +1,9 @@
 import logging
-import os
 
-import yaml
-
-from validate import validate_against_schema
+from validate import read_and_merge_configs
 
 # Set up basic logging
 logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
 
-
-def read_and_merge_configs(base_dir="."):
-    """
-    Reads all config.yml files in the directory structure and merges them.
-    """
-    merged_config = {}
-
-    for root, _, files in os.walk(base_dir):
-        if "config.yml" in files:
-            file_path = os.path.join(root, "config.yml")
-
-            try:
-                with open(file_path, 'r') as stream:
-                    config = yaml.safe_load(stream)
-
-                # Store and remove global properties
-                global_properties = config.pop("properties", {})
-
-                # Merge global properties into network-specific properties
-                for _, network_config in config.get("networks", {}).items():
-                    local_properties = network_config.get("properties", {})
-                    network_config["properties"] = {**global_properties, **local_properties}
-
-                # validate configuration
-                validate_against_schema(config)
-
-                # get source id
-                source_id = config["source-id"]
-                # check if source id already exists
-                if source_id in merged_config:
-                    raise Exception(f"Duplicate source_id: {source_id}")
-
-                logging.info(f"Successfully validated source: {source_id}")
-
-                merged_config[source_id] = config
-
-            except Exception as e:
-                logging.error(f"Error validating file {file_path}: {e}")
-
-    return merged_config
-
-
-def write_yaml_file(data, file_path="registry.yml"):
-    """
-    Writes data to a YAML file.
-    """
-    try:
-        with open(file_path, "w") as output_file:
-            yaml.dump(data, output_file, default_flow_style=False, indent=2)
-        logging.info(f"Successfully wrote to {file_path}")
-    except Exception as e:
-        logging.error(f"Error writing to file {file_path}: {e}")
-
-
-def main():
-    merged_config = read_and_merge_configs()
-
-
 if __name__ == "__main__":
-    main()
+    read_and_merge_configs()

.github/workflows/build_registry.yml

@@ -10,7 +10,7 @@ on:
       - '**.yml'
 
 jobs:
-  merge_yaml:
+  build_registry:
     runs-on: ubuntu-latest
 
     steps:
@@ -22,7 +22,7 @@ jobs:
         working-directory: ${{ github.workspace }}
 
       - name: Merge Source configs and write Source-Registry
-        run: python .github/build_registry.py
+        run: python .github/build_registry.py .github/registry.yml
         working-directory: ${{ github.workspace }}
 
       - name: Commit and push changes

.github/workflows/validate_registry.yml

@@ -1,12 +1,12 @@
-name: Merge Source configs and write Source-Registry
+name: Merge Source configs and validate them
 
 on:
   push:
     paths:
       - '**.yml'
 
 jobs:
-  merge_yaml:
+  validate_registry:
     runs-on: ubuntu-latest
 
     steps:

.gitignore

@@ -2,4 +2,7 @@
 /workspace.xml
 # Editor-based HTTP Client requests
 /httpRequests/
-/.idea
+/.idea
+# For local build the registry.yml will be built here
+registry.yml
+!.github/registry.yml