diff --git a/.github/build_registry.py b/.github/build_registry.py new file mode 100644 index 0000000..7539aca --- /dev/null +++ b/.github/build_registry.py @@ -0,0 +1,71 @@ +import logging +import os + +import yaml + +from validate import validate_against_schema + +# Set up basic logging +logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') + + +def read_and_merge_configs(base_dir="."): + """ + Reads all config.yml files in the directory structure and merges them. + """ + merged_config = {} + + for root, _, files in os.walk(base_dir): + if "config.yml" in files: + file_path = os.path.join(root, "config.yml") + + try: + with open(file_path, 'r') as stream: + config = yaml.safe_load(stream) + + # Store and remove global properties + global_properties = config.pop("properties", {}) + + # Merge global properties into network-specific properties + for _, network_config in config.get("networks", {}).items(): + local_properties = network_config.get("properties", {}) + network_config["properties"] = {**global_properties, **local_properties} + + # validate configuration + validate_against_schema(config) + + # get source id + source_id = config["source-id"] + # check if source id already exists + if source_id in merged_config: + raise Exception(f"Duplicate source_id: {source_id}") + + logging.info(f"Successfully validated source: {source_id}") + + merged_config[source_id] = config + + except Exception as e: + logging.error(f"Error validating file {file_path}: {e}") + + return merged_config + + +def write_yaml_file(data, file_path=".github/registry.yml"): + """ + Writes data to a YAML file. + """ + try: + with open(file_path, "w") as output_file: + yaml.dump(data, output_file, default_flow_style=False, indent=2) + logging.info(f"Successfully wrote to {file_path}") + except Exception as e: + logging.error(f"Error writing to file {file_path}: {e}") + + +def main(): + merged_config = read_and_merge_configs() + write_yaml_file(merged_config) + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/.github/requirements.txt similarity index 100% rename from requirements.txt rename to .github/requirements.txt diff --git a/validate.py b/.github/validate.py similarity index 91% rename from validate.py rename to .github/validate.py index 86e33aa..c10c3f9 100644 --- a/validate.py +++ b/.github/validate.py @@ -49,4 +49,8 @@ def validate_against_schema(data): Validator = validator_for(schema) validator = Validator(schema, registry=registry) - validator.validate(data) + try: + validator.validate(data) + except Exception as e: + logging.error(f"Validation error: {e}") + raise diff --git a/main.py b/.github/validate_registry.py similarity index 96% rename from main.py rename to .github/validate_registry.py index 07be8ea..ef66ea7 100644 --- a/main.py +++ b/.github/validate_registry.py @@ -45,7 +45,7 @@ def read_and_merge_configs(base_dir="."): merged_config[source_id] = config except Exception as e: - raise + logging.error(f"Error validating file {file_path}: {e}") return merged_config @@ -64,7 +64,6 @@ def write_yaml_file(data, file_path="registry.yml"): def main(): merged_config = read_and_merge_configs() - write_yaml_file(merged_config) if __name__ == "__main__": diff --git a/.github/workflows/build_registry.yml b/.github/workflows/build_registry.yml new file mode 100644 index 0000000..bd29e3c --- /dev/null +++ b/.github/workflows/build_registry.yml @@ -0,0 +1,35 @@ +name: Merge Source configs and write Source-Registry + +on: + pull_request: + branches: + - main + types: + - closed + paths: + - '**.yml' + +jobs: + merge_yaml: + runs-on: ubuntu-latest + + steps: + - name: Check out the code + uses: actions/checkout@v2 + + - name: Install required Python packages + run: pip install -r .github/requirements.txt + working-directory: ${{ github.workspace }} + + - name: Merge Source configs and write Source-Registry + run: python .github/build_registry.py + working-directory: ${{ github.workspace }} + + - name: Commit and push changes + run: | + git config --local user.email "actions@github.com" + git config --local user.name "GitHub Actions" + git add .github/registry.yml + git commit -m "Update registry.yml" + git push + working-directory: ${{ github.workspace }} \ No newline at end of file diff --git a/.github/workflows/release_registry.yml b/.github/workflows/release_registry.yml deleted file mode 100644 index 25a6545..0000000 --- a/.github/workflows/release_registry.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Merge Source Configs and Create Release - -on: - push: - branches: - - main - paths: - - '**.yml' - pull_request: - types: - - closed - -jobs: - merge_yaml: - runs-on: ubuntu-latest - - steps: - - name: Check out the code - uses: actions/checkout@v2 - - - name: Get next version - uses: reecetech/version-increment@2023.9.3 - id: version - with: - scheme: semver - increment: patch - - - name: Merge Source Configs - run: python merge_configs.py --version ${{ steps.version.outputs.version }} - working-directory: ${{ github.workspace }} - - - name: Create a new Release - if: github.event.pull_request.merged == true || github.event.ref == 'refs/heads/main' - run: | - release_version=${{ steps.version.outputs.version }} - release_name="v$release_version" - release_body="An updated version of the KYVE Source-Registry." - release_url="https://api.github.com/repos/${GITHUB_REPOSITORY}/releases" - - # Create a release using the GitHub API - response=$(curl -X POST "$release_url" \ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -d '{ - "tag_name": "'"$release_version"'", - "name": "'"$release_name"'", - "body": "'"$release_body"'" - }') - - # Upload the merged YAML file as an asset - upload_url=$(echo "$response" | jq -r .upload_url) - upload_url=${upload_url/\{?name,label\}/} - curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Content-Type: application/octet-stream" \ - --data-binary "@registry.yml" \ - "$upload_url?name=registry.yml" \ No newline at end of file diff --git a/.github/workflows/validate_registry.yml b/.github/workflows/validate_registry.yml new file mode 100644 index 0000000..ae72cad --- /dev/null +++ b/.github/workflows/validate_registry.yml @@ -0,0 +1,22 @@ +name: Merge Source configs and write Source-Registry + +on: + push: + paths: + - '**.yml' + +jobs: + merge_yaml: + runs-on: ubuntu-latest + + steps: + - name: Check out the code + uses: actions/checkout@v2 + + - name: Install required Python packages + run: pip install -r .github/requirements.txt + working-directory: ${{ github.workspace }} + + - name: Merge Source configs and write Source-Registry + run: python .github/validate_registry.py + working-directory: ${{ github.workspace }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index f06c929..c1bafc8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,4 @@ /workspace.xml # Editor-based HTTP Client requests /httpRequests/ -/.idea -# ignoring built registry to avoid a push -registry.yml \ No newline at end of file +/.idea \ No newline at end of file