Testing of deployment of catalog on pull requests

.github/workflows/build_pull_request.yaml

@@ -5,26 +5,19 @@
 name: Deploy test static content to s3 bucket
 
 on:
-  pull_request_target:
+  pull_request:
     types:
       - opened
+      - reopened
       - synchronize
+      - closed
     branches:
       - 'main'
 
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
-# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
-concurrency:
-  group: "bucket_deploy"
-  cancel-in-progress: false
+concurrency: preview-${{ github.ref }}
 
 jobs:
-  deploy:
-    environment:
-      name: test_deploy
+  deploy-preview:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -42,15 +35,13 @@ jobs:
           SH_CLIENT_SECRET: ${{ secrets.SH_CLIENT_SECRET }}
         run: |
           cd generators/
-          python generate_indicators.py
+          python generate_indicators.py -ni
       # Upload build to S3
       - name: sync client s3
         uses: jakejarvis/[email protected]
-        with:
-          args: --delete --follow-symlinks
         env:
           SOURCE_DIR: 'build'
-          DEST_DIR: 'catalog/${GITHUB_REF##*/}'
+          DEST_DIR: 'catalog/${{ github.event.pull_request.head.ref }}'
           AWS_REGION: 'eu-central-1'
           AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}

.github/workflows/test_deploy_cleanup.yaml

@@ -0,0 +1,25 @@
+name: Delete catalog test deployment when pull request is closed / deleted
+
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - 'main'
+
+jobs:
+  delete_folder:
+    runs-on: ubuntu-latest
+    steps:
+      # Set up AWS credentials
+      - name: Set up AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-central-1
+
+      # Delete the folder from the S3 bucket
+      - name: Delete folder from S3
+        run: |
+          aws s3 rm s3://${{ secrets.AWS_S3_BUCKET }}/catalog/${{ github.event.pull_request.head.ref }} --recursive

generators/generate_indicators.py

@@ -3,7 +3,7 @@
 Indicator generator to harvest information from endpoints and generate catalog
 
 """
-
+import time
 import requests
 import json
 from pystac_client import Client
@@ -46,7 +46,17 @@
         as possible and generating a STAC catalog with the information''',
 )
 
-argparser.add_argument("-vd", action="store_true", help="if flag set validation will be run on generated catalogs")
+argparser.add_argument("-vd", action="store_true", help="validation flag, if set, validation will be run on generated catalogs")
+argparser.add_argument("-ni", action="store_true", help="no items flag, if set, items will not be saved")
+
+def recursive_save(stac_object, no_items=False):
+    stac_object.save_object()
+    for child in stac_object.get_children():
+        recursive_save(child, no_items)
+    if not no_items:
+        # try to save items if available
+        for item in stac_object.get_items():
+            item.save_object()
 
 def process_catalog_file(file_path, options):
     print("Processing catalog:", file_path)
@@ -62,9 +72,18 @@ def process_catalog_file(file_path, options):
             process_collection_file(config, "../collections/%s.yaml"%(collection), catalog)
 
         strategy = TemplateLayoutStrategy(item_template="${collection}/${year}")
-        catalog.normalize_hrefs(config["endpoint"], strategy=strategy)
-        catalog.save(dest_href="../build/%s"%config["id"])
+        catalog.normalize_hrefs("../build/%s"%config["id"], strategy=strategy)
 
+        print("Started creation of collection files")
+        start = time.time()
+        if options.ni:
+            recursive_save(catalog, options.ni)
+        else:
+            # For full catalog save with items this still seems to be faster
+            catalog.save(dest_href="../build/%s"%config["id"])
+        end = time.time()
+        print("Time consumed in saving: ", end - start)
+
         if options.vd:
             # try to validate catalog if flag was set
             print("Running validation of catalog %s"%file_path)