Fullfill automation (#333)

* feat: automation implemented until upload to GEE, pending update layer

* feat: update layer from automation

* feat: add envs in cron

* feat: fix automation

* feat: install python dependencies

* feat: implemented mask automation

* feat: update pip reqs

* feat: pyramidingPolycy: MODE

.github/workflows/cron_update_SPI24.yml

@@ -17,6 +17,15 @@ jobs:
       contents: "read"
       id-token: "write"
 
+    env:
+      RW_API_KEY: ${{ secrets.RW_API_KEY }}
+      BUCKET: ${{ vars.BUCKET }}
+      IMAGES_PREFIX_PATH: ${{ vars.IMAGES_PREFIX_PATH }}
+      GS_STAGING_PREFIX: ${{ vars.GS_STAGING_PREFIX }}
+      GEE_PROJECT_FOLDER: ${{ vars.GEE_PROJECT_FOLDER }}
+      MONTH_3_LAYER_ID: ${{ vars.MONTH_3_LAYER_ID }}
+      MONTH_24_LAYER_ID: ${{ vars.MONTH_24_LAYER_ID }}
+
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -28,7 +37,7 @@ jobs:
         run: |
           pip3 install --upgrade pip==22.0
           echo "Installing Python dependencies"
-          if [ -f ./python/requirements.txt ]; then pip install -r ./python/requirements.txt; fi
+          if [ -f ./automation/requirements.txt ]; then pip install -r ./automation/requirements.txt; fi
           pip install rasterio
           pip install earthengine-api
           pip install earthengine-api --upgrade
@@ -39,11 +48,11 @@ jobs:
       # Get file source name and put it on env
       - name: Run Python Script on File
         run: |
-          python python/getFilename24month.py
-      - name: Copy from Google storage to local
-        run: |
-          gcloud storage cp $WPS_24_MONTH_GS $WPS_24_MONTH_FILENAME.tif
-      # Get file source name and put it on env
-      - name: Project downloaded file
-        run: |
-          python python/project24month.py
+          echo "ENVS TO USE"
+          echo "BUCKET"
+          echo "IMAGES_PREFIX_PATH"
+          echo "GS_STAGING_PREFIX"
+          echo "GEE_PROJECT_FOLDER"
+          echo "RW_API_KEY"
+          echo "MONTH_24_LAYER_ID"
+          python automation/month_24_automation.py

.github/workflows/cron_update_SPI24_mask.yml

@@ -0,0 +1,62 @@
+name: Update 3-month SPI Anomaly (Long-term)
+
+on:
+  push:
+    branches:
+    - earthengine-actions
+  workflow_dispatch:
+    # Allows manual triggering of the action
+  schedule:
+    - cron: "0 12 * * *" # Runs daily at 12:00 UTC, adjust as needed
+
+jobs:
+  spi-24-month:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: "read"
+      id-token: "write"
+
+    env:
+      RW_API_KEY: ${{ secrets.RW_API_KEY }}
+      BUCKET: ${{ vars.BUCKET }}
+      IMAGES_PREFIX_PATH: ${{ vars.IMAGES_PREFIX_PATH }}
+      GS_STAGING_PREFIX: ${{ vars.GS_STAGING_PREFIX }}
+      GEE_PROJECT_FOLDER: ${{ vars.GEE_PROJECT_FOLDER }}
+      MONTH_3_LAYER_ID: ${{ vars.MONTH_3_LAYER_ID }}
+      MONTH_24_LAYER_ID: ${{ vars.MONTH_24_LAYER_ID }}
+      CARTO_MASK_TABLE_NAME: ${{ vars.CARTO_MASK_TABLE_NAME }}
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+      - uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+
+      - name: Upgrade pip & Install dependencies
+        run: |
+          pip3 install --upgrade pip==22.0
+          echo "Installing Python dependencies"
+          if [ -f ./automation/requirements.txt ]; then pip install -r ./automation/requirements.txt; fi
+          pip install rasterio
+          pip install earthengine-api
+          pip install earthengine-api --upgrade
+
+      - uses: "google-github-actions/auth@v2"
+        with:
+          credentials_json: "${{ secrets.GOOGLE_CREDENTIALS }}"
+
+      # Get file source name and put it on env
+      - name: Run Python Script on File
+        run: |
+          echo "ENVS TO USE"
+          echo "BUCKET"
+          echo "IMAGES_PREFIX_PATH"
+          echo "GS_STAGING_PREFIX"
+          echo "GEE_PROJECT_FOLDER"
+          echo "RW_API_KEY"
+          echo "MONTH_24_LAYER_ID"
+          echo "CARTO_MASK_TABLE_NAME"
+          python automation/mask_automation.py
+      

.github/workflows/cron_update_SPI3.yml

@@ -17,6 +17,15 @@ jobs:
       contents: "read"
       id-token: "write"
 
+    env:
+      RW_API_KEY: ${{ secrets.RW_API_KEY }}
+      BUCKET: ${{ vars.BUCKET }}
+      IMAGES_PREFIX_PATH: ${{ vars.IMAGES_PREFIX_PATH }}
+      GS_STAGING_PREFIX: ${{ vars.GS_STAGING_PREFIX }}
+      GEE_PROJECT_FOLDER: ${{ vars.GEE_PROJECT_FOLDER }}
+      MONTH_3_LAYER_ID: ${{ vars.MONTH_3_LAYER_ID }}
+      MONTH_24_LAYER_ID: ${{ vars.MONTH_24_LAYER_ID }}
+
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -28,7 +37,7 @@ jobs:
         run: |
           pip3 install --upgrade pip==22.0
           echo "Installing Python dependencies"
-          if [ -f ./python/requirements.txt ]; then pip install -r ./python/requirements.txt; fi
+          if [ -f ./automation/requirements.txt ]; then pip install -r ./automation/requirements.txt; fi
           pip install rasterio
           pip install earthengine-api
           pip install earthengine-api --upgrade
@@ -40,13 +49,12 @@ jobs:
       # Get file source name and put it on env
       - name: Run Python Script on File
         run: |
-          python python/getFilename3month.py
-
-      - name: Copy from Google storage to local
-        run: |
-          gcloud storage cp $WPS_3_MONTH_GS $WPS_3_MONTH_FILENAME.tif
-
-      # Get file source name and put it on env
-      - name: Project downloaded file
-        run: |
-          python python/project3month.py
+          echo "ENVS TO USE"
+          echo "BUCKET"
+          echo "IMAGES_PREFIX_PATH"
+          echo "GS_STAGING_PREFIX"
+          echo "GEE_PROJECT_FOLDER"
+          echo "RW_API_KEY"
+          echo "MONTH_3_LAYER_ID"
+          python automation/month_3_automation.py
+      

.gitignore

@@ -57,3 +57,6 @@ dev_docs/nginx-proxy/certs/*.crt
 
 automationvenv
 gcloud-key.json
+*.pyc
+__pycache__
+automation/.run-ee.sh

Makefile

@@ -0,0 +1,8 @@
+.PHONY: setup install
+
+setup: 
+	python3 -m venv automationvenv
+	source ./automationvenv/bin/activate
+
+install:
+	pip install -r automation/requirements.txt

automation/.run-ee.example.sh

@@ -0,0 +1,24 @@
+# obligatories for automation
+export GEE_SERVICE_ACCOUNT=
+export GOOGLE_APPLICATION_CREDENTIALS=
+export CLOUDSDK_CORE_PROJECT=
+export BUCKET="wps_pillar1a"
+export IMAGES_PREFIX_PATH="data_sources/Deltares/Data20"
+export GS_STAGING_PREFIX="wps-staging-delete-me"
+export GEE_PROJECT_FOLDER="projects/wpsi-208318/assets/wpsi"
+export RW_API_KEY=
+
+# depending on the script
+export MONTH_24_LAYER_ID="cdd0000b-34a9-4b3d-9640-8f574321223d"
+export MONTH_3_LAYER_ID="cdd0000b-34a9-4b3d-9640-8f57422f264d"
+
+## to inspect data
+# python automation/earthengine.py
+# python automation/automation.py listBlobs
+python automation/automation.py latestFiles
+# python automation/automation.py updateLayer cdd0000b-34a9-4b3d-9640-8f57422f264d projects/ee-lizsaccoccia/assets/spi3_112024
+# python automation/automation.py updateLayer cdd0000b-34a9-4b3d-9640-8f57422f264d helo
+
+## automations
+# python automation/month_24_automation.py
+# python automation/month_3_automation.py

automation/automation.py

@@ -0,0 +1,62 @@
+import sys
+
+def main():
+    """
+    Main function to handle command line arguments and execute the appropriate command.
+    This function expects three command line arguments:
+    1. command: The command to execute (e.g., 'updateLayer').
+    2. layerId: The ID of the layer to be updated.
+    3. assetId: The path of the asset id (e.g., projects/ee-lizsaccoccia/assets/spi3_112024).
+    The function performs the following steps:
+    1. Parses the command line arguments.
+    2. Validates that both layerId and assetId are provided.
+    3. Prints a success message if the arguments are valid.
+    4. Prints an error message and usage example if the arguments are missing or invalid.
+    5. Executes the 'updateLayer' command by calling the layer_set_asset_id function with the provided layerId and assetId.
+    Usage example:
+      python /home/carlos/wri/wri-wpsi/automation/rw-api.py updateLayer cdd0000b-34a9-4b3d-9640-8f57422f264d path/to/assetId
+    """
+    try:
+      command = sys.argv[1]
+      assert(command)
+    except:
+      print('command was not passed as argument')
+      print('example: python automation.py <command> ...args')
+      return
+
+    if command == 'updateLayer':
+      from src.services.resourcewatch import layer_set_asset_id
+      try:
+          layerId = sys.argv[2]
+          assetId = sys.argv[3]
+          assert(layerId and assetId) 
+          print(f'layer id: {layerId} and assetId {assetId} successfully loaded')
+      except:
+          print('layer id or assetId where not passed as arguments')
+          print(r'example: python automation.py updateLayer cdd0000b-34a9-4b3d-9640-8f57422f264d path/to/assetId')
+          return
+      layer_set_asset_id(layerId, assetId)
+
+    if command == 'listBlobs':
+      from src.services.cloudstorage import print_all_blobs
+      print_all_blobs()
+
+    if command == 'latestFiles':
+      from src.services.cloudstorage import latest_24_month, latest_3_month
+      print(latest_3_month(), latest_24_month())
+    if command == 'layerSql':
+      from src.services.resourcewatch import layer_set_sql
+      try:
+          layerId = sys.argv[2]
+          sql = sys.argv[3]
+          assert(layerId and sql)
+
+      except:
+          print('sql was not passed as argument')
+          print(r'example: python automation.py layerSql cdd0000b-34a9-4b3d-9640-8f57422f264d "SELECT * FROM wps_spi24_mask WHERE value > -1.5"')
+          return
+      layer_set_sql(layerId, sql)
+
+
+if __name__ == '__main__':
+    main()

automation/mask_automation.py

@@ -0,0 +1,36 @@
+import os
+import eeUtil as eu
+from src.services.mask import mask
+from src.services.cloudstorage import latest_24_month, download
+from src.services.project import project
+from src.services.cartosql import init, upload_to_carto, deleteRows
+
+# define constants
+BUCKET=os.environ["BUCKET"]
+GS_STAGING_PREFIX=os.environ["GS_STAGING_PREFIX"]
+MONTH_3_LAYER_ID=os.environ["MONTH_3_LAYER_ID"]
+GEE_PROJECT_FOLDER=os.environ["GEE_PROJECT_FOLDER"]
+CARTO_MASK_TABLE_NAME=os.environ["CARTO_MASK_TABLE_NAME"]
+
+# Login to gcloud and gee properly
+
+eu.init(bucket=BUCKET)
+
+# get the last 24 month image
+filename, blob_name = latest_24_month()
+print("****Mask automation****")
+print(filename, blob_name)
+
+# download the file from google storage
+download(blob_name, filename)
+
+# run the projection
+project(filename)
+
+# upload it to GEE and make it public
+masked_df = mask(filename)
+
+# carto
+init()
+deleteRows(CARTO_MASK_TABLE_NAME, "true")
+upload_to_carto(CARTO_MASK_TABLE_NAME, masked_df)

automation/month_24_automation.py

@@ -0,0 +1,34 @@
+import os
+import eeUtil as eu
+from src.services.resourcewatch import layer_set_asset_id
+from src.services.path import strip_extension
+from src.services.cloudstorage import latest_24_month, download
+from src.services.project import project
+
+# define constants
+BUCKET=os.environ["BUCKET"]
+GS_STAGING_PREFIX=os.environ["GS_STAGING_PREFIX"]
+MONTH_24_LAYER_ID=os.environ["MONTH_24_LAYER_ID"]
+GEE_PROJECT_FOLDER=os.environ["GEE_PROJECT_FOLDER"]
+
+# Login to gcloud and gee properly
+
+eu.init(bucket=BUCKET)
+
+# get the last 24 month image
+filename, blob_name = latest_24_month()
+print("****latest 24 month****")
+print(filename, blob_name)
+
+# download the file from google storage
+download(blob_name, filename)
+
+# run the projection
+project(filename)
+
+# upload it to GEE and make it public
+imageId = f"{GEE_PROJECT_FOLDER}/{strip_extension(filename)}"
+eu.upload(filename, imageId, gs_prefix=GS_STAGING_PREFIX, public=True, clean=False)
+
+# update the layerConfig.assetId in the given layerId from resourcewatch
+layer_set_asset_id(MONTH_24_LAYER_ID, imageId)

automation/month_3_automation.py

@@ -0,0 +1,34 @@
+import os
+import eeUtil as eu
+from src.services.resourcewatch import layer_set_asset_id
+from src.services.path import strip_extension
+from src.services.cloudstorage import latest_3_month, download
+from src.services.project import project
+
+# define constants
+BUCKET=os.environ["BUCKET"]
+GS_STAGING_PREFIX=os.environ["GS_STAGING_PREFIX"]
+MONTH_3_LAYER_ID=os.environ["MONTH_3_LAYER_ID"]
+GEE_PROJECT_FOLDER=os.environ["GEE_PROJECT_FOLDER"]
+
+# Login to gcloud and gee properly
+
+eu.init(bucket=BUCKET)
+
+# get the last 24 month image
+filename, blob_name = latest_3_month()
+print("****latest 3 month****")
+print(filename, blob_name)
+
+# download the file from google storage
+download(blob_name, filename)
+
+# run the projection
+project(filename)
+
+# upload it to GEE and make it public
+imageId = f"{GEE_PROJECT_FOLDER}/{strip_extension(filename)}"
+eu.upload(filename, imageId, gs_prefix=GS_STAGING_PREFIX, public=True, clean=False, ingest_params={"pyramidingPolicy": "MODE"})
+
+# update the layerConfig.assetId in the given layerId from resourcewatch
+layer_set_asset_id(MONTH_3_LAYER_ID, imageId)

python/readme.md → automation/readme.md

@@ -32,4 +32,10 @@ pip install -r python/requirements.txt
 ## Documentation
 
 - pipreqs to update requirements.txt [pipreqs](https://github.com/bndr/pipreqs)
-- pyenv [pyenv](https://github.com/pyenv/pyenv?tab=readme-ov-file#set-up-your-shell-environment-for-pyenv)
+- pyenv [pyenv](https://github.com/pyenv/pyenv?tab=readme-ov-file#set-up-your-shell-environment-for-pyenv)
+
+## Tests
+
+```bash
+python -m unittest discover -s automation
+```

automation/requirements.txt

@@ -0,0 +1,6 @@
+eeUtil==0.3.0
+eeUtil==0.3.0
+geopandas==1.0.1
+protobuf==5.28.3
+rasterio==1.4.2
+Requests==2.32.3