update: validate & build workflows.

.github/build_registry.py

@@ -0,0 +1,71 @@
+import logging
+import os
+
+import yaml
+
+from validate import validate_against_schema
+
+# Set up basic logging
+logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
+
+
+def read_and_merge_configs(base_dir="."):
+    """
+    Reads all config.yml files in the directory structure and merges them.
+    """
+    merged_config = {}
+
+    for root, _, files in os.walk(base_dir):
+        if "config.yml" in files:
+            file_path = os.path.join(root, "config.yml")
+
+            try:
+                with open(file_path, 'r') as stream:
+                    config = yaml.safe_load(stream)
+
+                # Store and remove global properties
+                global_properties = config.pop("properties", {})
+
+                # Merge global properties into network-specific properties
+                for _, network_config in config.get("networks", {}).items():
+                    local_properties = network_config.get("properties", {})
+                    network_config["properties"] = {**global_properties, **local_properties}
+
+                # validate configuration
+                validate_against_schema(config)
+
+                # get source id
+                source_id = config["source-id"]
+                # check if source id already exists
+                if source_id in merged_config:
+                    raise Exception(f"Duplicate source_id: {source_id}")
+
+                logging.info(f"Successfully validated source: {source_id}")
+
+                merged_config[source_id] = config
+
+            except Exception as e:
+                logging.error(f"Error validating file {file_path}: {e}")
+
+    return merged_config
+
+
+def write_yaml_file(data, file_path=".github/registry.yml"):
+    """
+    Writes data to a YAML file.
+    """
+    try:
+        with open(file_path, "w") as output_file:
+            yaml.dump(data, output_file, default_flow_style=False, indent=2)
+        logging.info(f"Successfully wrote to {file_path}")
+    except Exception as e:
+        logging.error(f"Error writing to file {file_path}: {e}")
+
+
+def main():
+    merged_config = read_and_merge_configs()
+    write_yaml_file(merged_config)
+
+
+if __name__ == "__main__":
+    main()

validate.py → .github/validate.py

@@ -49,4 +49,8 @@ def validate_against_schema(data):
     Validator = validator_for(schema)
     validator = Validator(schema, registry=registry)
 
-    validator.validate(data)
+    try:
+        validator.validate(data)
+    except Exception as e:
+        logging.error(f"Validation error: {e}")
+        raise

main.py → .github/validate_registry.py

@@ -45,7 +45,7 @@ def read_and_merge_configs(base_dir="."):
                 merged_config[source_id] = config
 
             except Exception as e:
-                raise
+                logging.error(f"Error validating file {file_path}: {e}")
 
     return merged_config
 
@@ -64,7 +64,6 @@ def write_yaml_file(data, file_path="registry.yml"):
 
 def main():
     merged_config = read_and_merge_configs()
-    write_yaml_file(merged_config)
 
 
 if __name__ == "__main__":

.github/workflows/build_registry.yml

@@ -0,0 +1,35 @@
+name: Merge Source configs and write Source-Registry
+
+on:
+  pull_request:
+    branches:
+      - main
+    types:
+      - closed
+    paths:
+      - '**.yml'
+
+jobs:
+  merge_yaml:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out the code
+        uses: actions/checkout@v2
+
+      - name: Install required Python packages
+        run: pip install -r .github/requirements.txt
+        working-directory: ${{ github.workspace }}
+
+      - name: Merge Source configs and write Source-Registry
+        run: python .github/build_registry.py
+        working-directory: ${{ github.workspace }}
+
+      - name: Commit and push changes
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "GitHub Actions"
+          git add .github/registry.yml
+          git commit -m "Update registry.yml"
+          git push
+        working-directory: ${{ github.workspace }}

.github/workflows/validate_registry.yml

@@ -0,0 +1,22 @@
+name: Merge Source configs and write Source-Registry
+
+on:
+  push:
+    paths:
+      - '**.yml'
+
+jobs:
+  merge_yaml:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out the code
+        uses: actions/checkout@v2
+
+      - name: Install required Python packages
+        run: pip install -r .github/requirements.txt
+        working-directory: ${{ github.workspace }}
+
+      - name: Merge Source configs and write Source-Registry
+        run: python .github/validate_registry.py
+        working-directory: ${{ github.workspace }}

.gitignore

@@ -2,6 +2,4 @@
 /workspace.xml
 # Editor-based HTTP Client requests
 /httpRequests/
-/.idea
-# ignoring built registry to avoid a push
-registry.yml
+/.idea