From 943b19423d3a8a14632683a8421cbf38d2859157 Mon Sep 17 00:00:00 2001 From: fabianriewe Date: Tue, 28 Nov 2023 12:11:03 +0100 Subject: [PATCH] feat: improved workflows --- .github/build_registry.py | 62 +++++------------------ .github/validate.py | 49 +++++++++++++++++-- .github/validate_registry.py | 65 +------------------------ .github/workflows/build_registry.yml | 4 +- .github/workflows/validate_registry.yml | 4 +- .gitignore | 5 +- 6 files changed, 67 insertions(+), 122 deletions(-) diff --git a/.github/build_registry.py b/.github/build_registry.py index 7539aca..ab87c44 100644 --- a/.github/build_registry.py +++ b/.github/build_registry.py @@ -1,56 +1,15 @@ import logging -import os +import sys import yaml -from validate import validate_against_schema +from validate import read_and_merge_configs # Set up basic logging logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') -def read_and_merge_configs(base_dir="."): - """ - Reads all config.yml files in the directory structure and merges them. - """ - merged_config = {} - - for root, _, files in os.walk(base_dir): - if "config.yml" in files: - file_path = os.path.join(root, "config.yml") - - try: - with open(file_path, 'r') as stream: - config = yaml.safe_load(stream) - - # Store and remove global properties - global_properties = config.pop("properties", {}) - - # Merge global properties into network-specific properties - for _, network_config in config.get("networks", {}).items(): - local_properties = network_config.get("properties", {}) - network_config["properties"] = {**global_properties, **local_properties} - - # validate configuration - validate_against_schema(config) - - # get source id - source_id = config["source-id"] - # check if source id already exists - if source_id in merged_config: - raise Exception(f"Duplicate source_id: {source_id}") - - logging.info(f"Successfully validated source: {source_id}") - - merged_config[source_id] = config - - except Exception as e: - logging.error(f"Error validating file {file_path}: {e}") - - return merged_config - - -def write_yaml_file(data, file_path=".github/registry.yml"): +def write_yaml_file(data, file_path="registry.yml"): """ Writes data to a YAML file. """ @@ -62,10 +21,15 @@ def write_yaml_file(data, file_path=".github/registry.yml"): logging.error(f"Error writing to file {file_path}: {e}") -def main(): - merged_config = read_and_merge_configs() - write_yaml_file(merged_config) +if __name__ == "__main__": + args = sys.argv[1:] + if len(args) > 1: + raise ValueError(f"Unexpected arguments: {args}") + if len(args) == 0: + file_path = "registry.yml" + else: + file_path = args[0] -if __name__ == "__main__": - main() + merged_config = read_and_merge_configs() + write_yaml_file(merged_config, file_path=file_path) diff --git a/.github/validate.py b/.github/validate.py index c10c3f9..10b6459 100644 --- a/.github/validate.py +++ b/.github/validate.py @@ -1,7 +1,9 @@ import json import logging +import os from pathlib import Path +import yaml from jsonschema.validators import validator_for from referencing import Registry, Resource from referencing.exceptions import NoSuchResource @@ -49,8 +51,45 @@ def validate_against_schema(data): Validator = validator_for(schema) validator = Validator(schema, registry=registry) - try: - validator.validate(data) - except Exception as e: - logging.error(f"Validation error: {e}") - raise + validator.validate(data) + + +def read_and_merge_configs(base_dir="."): + """ + Reads all config.yml files in the directory structure and merges them. + """ + merged_config = {} + + for root, _, files in os.walk(base_dir): + if "config.yml" in files: + file_path = os.path.join(root, "config.yml") + + try: + with open(file_path, 'r') as stream: + config = yaml.safe_load(stream) + + # Store and remove global properties + global_properties = config.pop("properties", {}) + + # Merge global properties into network-specific properties + for _, network_config in config.get("networks", {}).items(): + local_properties = network_config.get("properties", {}) + network_config["properties"] = {**global_properties, **local_properties} + + # validate configuration + validate_against_schema(config) + + # get source id + source_id = config["source-id"] + # check if source id already exists + if source_id in merged_config: + raise Exception(f"Duplicate source_id: {source_id}") + + logging.info(f"Successfully validated source: {source_id}") + + merged_config[source_id] = config + + except Exception: + raise + + return merged_config diff --git a/.github/validate_registry.py b/.github/validate_registry.py index ef66ea7..fea8880 100644 --- a/.github/validate_registry.py +++ b/.github/validate_registry.py @@ -1,70 +1,9 @@ import logging -import os -import yaml - -from validate import validate_against_schema +from validate import read_and_merge_configs # Set up basic logging logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') - -def read_and_merge_configs(base_dir="."): - """ - Reads all config.yml files in the directory structure and merges them. - """ - merged_config = {} - - for root, _, files in os.walk(base_dir): - if "config.yml" in files: - file_path = os.path.join(root, "config.yml") - - try: - with open(file_path, 'r') as stream: - config = yaml.safe_load(stream) - - # Store and remove global properties - global_properties = config.pop("properties", {}) - - # Merge global properties into network-specific properties - for _, network_config in config.get("networks", {}).items(): - local_properties = network_config.get("properties", {}) - network_config["properties"] = {**global_properties, **local_properties} - - # validate configuration - validate_against_schema(config) - - # get source id - source_id = config["source-id"] - # check if source id already exists - if source_id in merged_config: - raise Exception(f"Duplicate source_id: {source_id}") - - logging.info(f"Successfully validated source: {source_id}") - - merged_config[source_id] = config - - except Exception as e: - logging.error(f"Error validating file {file_path}: {e}") - - return merged_config - - -def write_yaml_file(data, file_path="registry.yml"): - """ - Writes data to a YAML file. - """ - try: - with open(file_path, "w") as output_file: - yaml.dump(data, output_file, default_flow_style=False, indent=2) - logging.info(f"Successfully wrote to {file_path}") - except Exception as e: - logging.error(f"Error writing to file {file_path}: {e}") - - -def main(): - merged_config = read_and_merge_configs() - - if __name__ == "__main__": - main() + read_and_merge_configs() diff --git a/.github/workflows/build_registry.yml b/.github/workflows/build_registry.yml index bd29e3c..3b1600b 100644 --- a/.github/workflows/build_registry.yml +++ b/.github/workflows/build_registry.yml @@ -10,7 +10,7 @@ on: - '**.yml' jobs: - merge_yaml: + build_registry: runs-on: ubuntu-latest steps: @@ -22,7 +22,7 @@ jobs: working-directory: ${{ github.workspace }} - name: Merge Source configs and write Source-Registry - run: python .github/build_registry.py + run: python .github/build_registry.py .github/registry.yml working-directory: ${{ github.workspace }} - name: Commit and push changes diff --git a/.github/workflows/validate_registry.yml b/.github/workflows/validate_registry.yml index ae72cad..81718c1 100644 --- a/.github/workflows/validate_registry.yml +++ b/.github/workflows/validate_registry.yml @@ -1,4 +1,4 @@ -name: Merge Source configs and write Source-Registry +name: Merge Source configs and validate them on: push: @@ -6,7 +6,7 @@ on: - '**.yml' jobs: - merge_yaml: + validate_registry: runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index c1bafc8..3a779e5 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,7 @@ /workspace.xml # Editor-based HTTP Client requests /httpRequests/ -/.idea \ No newline at end of file +/.idea +# For local build the registry.yml will be built here +registry.yml +!.github/registry.yml \ No newline at end of file