Merge pull request #138 from I-GUIDE/develop

Deploy to production
I-GUIDE · May 13, 2024 · a2188cd · a2188cd
2 parents c125252 + 10e60af
commit a2188cd
Show file tree

Hide file tree

Showing 116 changed files with 20,649 additions and 23,072 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -0,0 +1,77 @@
+name: Run I-GUIDE Builds and Config Saturations
+
+on:
+  workflow_dispatch:
+  push:
+    branches: ['*']
+
+env:
+  DOMAIN: iguide.cuahsi.io
+  TAG: latest
+  TESTING: false
+  OIDC_ISSUER: https://orcid.org
+  DATABASE_NAME: iguide_beta
+  DB_PROTOCOL: mongodb+srv
+  HYDROSHARE_META_READ_URL: https://www.hydroshare.org/hsapi2/resource/%s/json/
+  HYDROSHARE_FILE_READ_URL: https://www.hydroshare.org/hsapi/resource/%s/files/
+  VITE_APP_NAME: I-GUIDE
+  VITE_APP_URL: https://iguide-dev.cuahsi.io
+  VITE_APP_API_URL: https://iguide-dev.cuahsi.io/api
+  VITE_APP_LOGIN_URL: https://orcid.org/oauth/authorize
+  VITE_APP_GOOGLE_MAPS_API_KEY: ""
+  VITE_APP_SUPPORT_EMAIL: [email protected]
+  VITE_APP_CLIENT_ID: APP-4ZA8C8BYAH3QHNE9
+
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    steps:
+    - name: code checkout
+      uses: actions/checkout@v2
+
+    - name: Compile the root env file
+      env:
+        DB_HOST: ${{ secrets.DB_HOST }}
+        DB_USERNAME: ${{ secrets.DB_USERNAME }}
+        DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
+      run: |
+        variables=("OIDC_ISSUER" "DB_USERNAME" "DB_PASSWORD" "DB_HOST" "DATABASE_NAME" "DB_PROTOCOL" "TESTING" "VITE_APP_LOGIN_URL" "HYDROSHARE_META_READ_URL" "HYDROSHARE_FILE_READ_URL")
+
+        # Empty the .env file
+        > .env
+
+        # Loop through the variables and add them to the .env file
+        for var in "${variables[@]}"; do
+          echo "$var=${!var}" >> .env
+        done
+
+    - name: Compile the frontend env file
+      run: |
+        variables=("VITE_APP_NAME" "VITE_APP_API_URL" "VITE_APP_SUPPORT_EMAIL" "VITE_APP_URL" "VITE_APP_LOGIN_URL" "VITE_APP_CLIENT_ID" "VITE_APP_GOOGLE_MAPS_API_KEY")
+
+        # Empty the .env file
+        > frontend/.env
+
+        # Loop through the variables and add them to the .env file
+        for var in "${variables[@]}"; do
+          echo "$var=${!var}" >> frontend/.env
+        done
+
+    - name: Build and push docker images
+      env:
+        GOOGLE_PROJECT: ${{ secrets.GOOGLE_PROJECT }}
+      run: |
+        gcloud auth configure-docker us-central1-docker.pkg.dev
+        docker build -t us-central1-docker.pkg.dev/$GOOGLE_PROJECT/iguide/api:$TAG -f docker/api/Dockerfile .
+        docker build -t us-central1-docker.pkg.dev/$GOOGLE_PROJECT/iguide/frontend:$TAG -f docker/frontend/Dockerfile .
+        docker build -t us-central1-docker.pkg.dev/$GOOGLE_PROJECT/iguide/trigger:$TAG -f docker/triggers/Dockerfile .
+
+    - name: Deploy to GKE
+      env:
+        USE_GKE_GCLOUD_AUTH_PLUGIN: True
+        GOOGLE_PROJECT: ${{ secrets.GOOGLE_PROJECT }}
+      run: |
+        find ./kubernetes -type f | xargs -i sed -i "s/GOOGLE_PROJECT/$GOOGLE_PROJECT/g" {}
+        find ./kubernetes -type f | xargs -i sed -i "s/IGUIDE_TAG/$TAG/g" {}
+        find ./kubernetes -type f | xargs -i sed -i "s/IGUIDE_DOMAIN/$DOMAIN/g" {}
diff --git a/.github/workflows/dependabot.yml b/.github/workflows/dependabot.yml
@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "npm" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"
diff --git a/.github/workflows/deploy-dev.yaml b/.github/workflows/deploy-dev.yaml
@@ -0,0 +1,95 @@
+name: Deploy I-GUIDE to GKE Autopilot (Beta)
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+    - 'develop'
+
+env:
+  DOMAIN: iguide-dev.cuahsi.io
+  IP: iguide-dev
+  TESTING: false
+  OIDC_ISSUER: https://orcid.org
+  DATABASE_NAME: iguide_dev
+  DB_PROTOCOL: mongodb+srv
+  HYDROSHARE_META_READ_URL: https://www.hydroshare.org/hsapi2/resource/%s/json/
+  HYDROSHARE_FILE_READ_URL: https://www.hydroshare.org/hsapi/resource/%s/files/
+  VITE_APP_NAME: I-GUIDE
+  VITE_APP_URL: https://iguide-dev.cuahsi.io
+  VITE_APP_API_URL: https://iguide-dev.cuahsi.io/api
+  VITE_APP_LOGIN_URL: https://orcid.org/oauth/authorize
+  VITE_APP_SUPPORT_EMAIL: [email protected]
+  VITE_APP_CLIENT_ID: APP-4ZA8C8BYAH3QHNE9
+
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - name: code checkout
+      uses: actions/checkout@v2
+
+    - name: Install the gcloud cli
+      uses: google-github-actions/setup-gcloud@v0
+      with:
+        project_id: ${{ secrets.GOOGLE_PROJECT }}
+        service_account_key: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
+        install_components: 'gke-gcloud-auth-plugin'
+        export_default_credentials: true
+
+    - name: Compile the root env file
+      env:
+        DB_HOST: ${{ secrets.DB_HOST_BETA }}
+        DB_USERNAME: ${{ secrets.DB_USERNAME_BETA }}
+        DB_PASSWORD: ${{ secrets.DB_PASSWORD_BETA }}
+      run: |
+        variables=("OIDC_ISSUER" "DB_USERNAME" "DB_PASSWORD" "DB_HOST" "DATABASE_NAME" "DB_PROTOCOL" "TESTING" "VITE_APP_LOGIN_URL" "HYDROSHARE_META_READ_URL" "HYDROSHARE_FILE_READ_URL")
+
+        # Empty the .env file
+        > .env
+
+        # Loop through the variables and add them to the .env file
+        for var in "${variables[@]}"; do
+          echo "$var=${!var}" >> .env
+        done
+
+    - name: Compile the frontend env file
+      env: 
+        VITE_APP_GOOGLE_MAPS_API_KEY: ${{ secrets.VITE_APP_GOOGLE_MAPS_API_KEY }}
+
+      run: |
+        variables=("VITE_APP_NAME" "VITE_APP_API_URL" "VITE_APP_SUPPORT_EMAIL" "VITE_APP_URL" "VITE_APP_LOGIN_URL" "VITE_APP_CLIENT_ID" "VITE_APP_GOOGLE_MAPS_API_KEY")
+
+        # Empty the .env file
+        > frontend/.env
+
+        # Loop through the variables and add them to the .env file
+        for var in "${variables[@]}"; do
+          echo "$var=${!var}" >> frontend/.env
+        done
+
+    - name: Build and push docker images
+      env:
+        GOOGLE_PROJECT: ${{ secrets.GOOGLE_PROJECT }}
+      run: |
+        gcloud auth configure-docker us-central1-docker.pkg.dev
+        docker build -t us-central1-docker.pkg.dev/$GOOGLE_PROJECT/iguide/api:$GITHUB_SHA -f docker/api/Dockerfile .
+        docker push us-central1-docker.pkg.dev/$GOOGLE_PROJECT/iguide/api:$GITHUB_SHA
+        docker build -t us-central1-docker.pkg.dev/$GOOGLE_PROJECT/iguide/frontend:$GITHUB_SHA -f docker/frontend/Dockerfile .
+        docker push us-central1-docker.pkg.dev/$GOOGLE_PROJECT/iguide/frontend:$GITHUB_SHA
+        docker build -t us-central1-docker.pkg.dev/$GOOGLE_PROJECT/iguide/trigger:$GITHUB_SHA -f docker/triggers/Dockerfile .
+        docker push us-central1-docker.pkg.dev/$GOOGLE_PROJECT/iguide/trigger:$GITHUB_SHA
+
+    - name: Deploy to GKE
+      env:
+        USE_GKE_GCLOUD_AUTH_PLUGIN: True
+        GOOGLE_PROJECT: ${{ secrets.GOOGLE_PROJECT }}
+      run: |
+        gcloud container clusters get-credentials iguide-dev --region us-central1
+        find ./kubernetes -type f | xargs -i sed -i "s/GOOGLE_PROJECT/$GOOGLE_PROJECT/g" {}
+        find ./kubernetes -type f | xargs -i sed -i "s/IGUIDE_TAG/$GITHUB_SHA/g" {}
+        find ./kubernetes -type f | xargs -i sed -i "s/IGUIDE_DOMAIN/$DOMAIN/g" {}
+        kubectl apply -f kubernetes/
+        # Refresh pods
+        kubectl delete pods --all
diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
@@ -6,6 +6,25 @@ on:
     branches:
     - 'productionalization'
 
+env:
+  DOMAIN: iguide.cuahsi.io
+  TAG: latest
+  TESTING: false
+  IP: iguide
+  OIDC_ISSUER: https://orcid.org
+  DATABASE_NAME: iguide_beta
+  DB_PROTOCOL: mongodb+srv
+  HYDROSHARE_META_READ_URL: https://www.hydroshare.org/hsapi2/resource/%s/json/
+  HYDROSHARE_FILE_READ_URL: https://www.hydroshare.org/hsapi/resource/%s/files/
+  VITE_APP_NAME: I-GUIDE
+  VITE_APP_URL: https://iguide-dev.cuahsi.io
+  VITE_APP_API_URL: https://iguide-dev.cuahsi.io/api
+  VITE_APP_LOGIN_URL: https://orcid.org/oauth/authorize
+  VITE_APP_GOOGLE_MAPS_API_KEY: ""
+  VITE_APP_SUPPORT_EMAIL: [email protected]
+  VITE_APP_CLIENT_ID: APP-4ZA8C8BYAH3QHNE9
+
+
 jobs:
   deploy:
     runs-on: ubuntu-latest
@@ -23,18 +42,11 @@ jobs:
 
     - name: Compile the root env file
       env:
-        OIDC_ISSUER: ${{ secrets.OIDC_ISSUER }}
+        DB_HOST: ${{ secrets.DB_HOST }}
         DB_USERNAME: ${{ secrets.DB_USERNAME }}
         DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
-        DB_HOST: ${{ secrets.DB_HOST }}
-        DATABASE_NAME: ${{ secrets.DATABASE_NAME }}
-        DB_PROTOCOL: ${{ secrets.DB_PROTOCOL }}
-        TESTING: ${{ secrets.TESTING }}
-        VUE_APP_LOGIN_URL: ${{ secrets.VUE_APP_LOGIN_URL }}
-        HYDROSHARE_META_READ_URL: ${{ secrets.HYDROSHARE_META_READ_URL}}
-        HYDROSHARE_FILE_READ_URL: ${{ secrets.HYDROSHARE_FILE_READ_URL}}
       run: |
-        variables=("OIDC_ISSUER" "DB_USERNAME" "DB_PASSWORD" "DB_HOST" "DATABASE_NAME" "DB_PROTOCOL" "TESTING" "VUE_APP_LOGIN_URL" "HYDROSHARE_META_READ_URL" "HYDROSHARE_FILE_READ_URL")
+        variables=("OIDC_ISSUER" "DB_USERNAME" "DB_PASSWORD" "DB_HOST" "DATABASE_NAME" "DB_PROTOCOL" "TESTING" "VITE_APP_LOGIN_URL" "HYDROSHARE_META_READ_URL" "HYDROSHARE_FILE_READ_URL")
 
         # Empty the .env file
         > .env
@@ -45,16 +57,8 @@ jobs:
         done
 
     - name: Compile the frontend env file
-      env:
-        VUE_APP_NAME: ${{ secrets.VUE_APP_NAME }}
-        VUE_APP_API_URL: ${{ secrets.VUE_APP_API_URL }}
-        VUE_APP_SUPPORT_EMAIL: ${{ secrets.VUE_APP_SUPPORT_EMAIL }}
-        VUE_APP_URL: ${{ secrets.VUE_APP_URL }}
-        VUE_APP_LOGIN_URL: ${{ secrets.VUE_APP_LOGIN_URL }}
-        VUE_APP_CLIENT_ID: ${{ secrets.VUE_APP_CLIENT_ID }}
-        VUE_APP_GOOGLE_MAPS_API_KEY: ${{ secrets.VUE_APP_GOOGLE_MAPS_API_KEY }}
       run: |
-        variables=("VUE_APP_NAME" "VUE_APP_API_URL" "VUE_APP_SUPPORT_EMAIL" "VUE_APP_URL" "VUE_APP_LOGIN_URL" "VUE_APP_CLIENT_ID" "VUE_APP_GOOGLE_MAPS_API_KEY")
+        variables=("VITE_APP_NAME" "VITE_APP_API_URL" "VITE_APP_SUPPORT_EMAIL" "VITE_APP_URL" "VITE_APP_LOGIN_URL" "VITE_APP_CLIENT_ID" "VITE_APP_GOOGLE_MAPS_API_KEY")
 
         # Empty the .env file
         > frontend/.env
@@ -69,12 +73,12 @@ jobs:
         GOOGLE_PROJECT: ${{ secrets.GOOGLE_PROJECT }}
       run: |
         gcloud auth configure-docker us-central1-docker.pkg.dev
-        docker build -t us-central1-docker.pkg.dev/$GOOGLE_PROJECT/iguide/api:latest -f docker/api/Dockerfile .
-        docker push us-central1-docker.pkg.dev/$GOOGLE_PROJECT/iguide/api:latest
-        docker build -t us-central1-docker.pkg.dev/$GOOGLE_PROJECT/iguide/frontend:latest -f docker/frontend/Dockerfile .
-        docker push us-central1-docker.pkg.dev/$GOOGLE_PROJECT/iguide/frontend:latest
-        docker build -t us-central1-docker.pkg.dev/$GOOGLE_PROJECT/iguide/trigger:latest -f docker/triggers/Dockerfile .
-        docker push us-central1-docker.pkg.dev/$GOOGLE_PROJECT/iguide/trigger:latest
+        docker build -t us-central1-docker.pkg.dev/$GOOGLE_PROJECT/iguide/api:$TAG -f docker/api/Dockerfile .
+        docker push us-central1-docker.pkg.dev/$GOOGLE_PROJECT/iguide/api:$TAG
+        docker build -t us-central1-docker.pkg.dev/$GOOGLE_PROJECT/iguide/frontend:$TAG -f docker/frontend/Dockerfile .
+        docker push us-central1-docker.pkg.dev/$GOOGLE_PROJECT/iguide/frontend:$TAG
+        docker build -t us-central1-docker.pkg.dev/$GOOGLE_PROJECT/iguide/trigger:$TAG -f docker/triggers/Dockerfile .
+        docker push us-central1-docker.pkg.dev/$GOOGLE_PROJECT/iguide/trigger:$TAG
 
     - name: Deploy to GKE
       env:
@@ -83,6 +87,9 @@ jobs:
       run: |
         gcloud container clusters get-credentials iguide --region us-central1
         find ./kubernetes -type f | xargs -i sed -i "s/GOOGLE_PROJECT/$GOOGLE_PROJECT/g" {}
+        find ./kubernetes -type f | xargs -i sed -i "s/IGUIDE_TAG/$TAG/g" {}
+        find ./kubernetes -type f | xargs -i sed -i "s/IGUIDE_DOMAIN/$DOMAIN/g" {}
+        find ./kubernetes -type f | xargs -i sed -i "s/IGUIDE_IP/$IP/g" {}
         kubectl apply -f kubernetes/
         # Refresh pods
         kubectl delete pods --all
diff --git a/Makefile b/Makefile
@@ -18,10 +18,6 @@ down:
 build:
 	docker-compose build
 
-.PHONY: test
-test:
-	docker-compose exec api pytest tests
-
 .PHONY: format
 format:
 	docker-compose run api $(isort)

diff --git a/api/adapters/__init__.py b/api/adapters/__init__.py
@@ -0,0 +1,3 @@
+# import all adapters here to get them registered
+from api.adapters import hydroshare
+from api.adapters import s3
diff --git a/api/adapters/hydroshare.py b/api/adapters/hydroshare.py
@@ -1,14 +1,14 @@
 import requests
-from starlette import status
 from datetime import datetime
 from typing import List, Optional, Union
 from pydantic import BaseModel, EmailStr, HttpUrl
+
 from api.adapters.base import AbstractRepositoryMetadataAdapter, AbstractRepositoryRequestHandler
-from api.adapters.utils import RepositoryType
+from api.adapters.utils import RepositoryType, register_adapter
 from api.exceptions import RepositoryException
 from api.models import schema
 from api.models.catalog import DatasetMetadataDOC
-from api.models.user import Submission
+from api.models.user import Submission, SubmissionType
 
 
 class Creator(BaseModel):
@@ -127,6 +127,7 @@ def to_dataset_media_object(self):
         media_object.encodingFormat = self.content_type
         media_object.contentSize = f"{self.size/1000.00} KB"
         media_object.name = self.file_name
+        media_object.sha256 = self.checksum
         return media_object
 
 
@@ -162,14 +163,13 @@ def to_dataset_license(self):
 
 
 class _HydroshareRequestHandler(AbstractRepositoryRequestHandler):
-
     def get_metadata(self, record_id: str):
         hs_meta_url = self.settings.hydroshare_meta_read_url % record_id
         hs_file_url = self.settings.hydroshare_file_read_url % record_id
 
         def make_request(url, file_list=False) -> Union[dict, List[dict]]:
             response = requests.get(url)
-            if response.status_code != status.HTTP_200_OK:
+            if response.status_code != 200:
                 raise RepositoryException(status_code=response.status_code, detail=response.text)
             if not file_list:
                 return response.json()
@@ -179,7 +179,7 @@ def make_request(url, file_list=False) -> Union[dict, List[dict]]:
             # check if there are more results to fetch - by default, 100 files are returned from HydroShare
             while response.json()["next"]:
                 response = requests.get(response.json()["next"])
-                if response.status_code != status.HTTP_200_OK:
+                if response.status_code != 200:
                     raise RepositoryException(status_code=response.status_code, detail=response.text)
                 content_files.extend(response.json()["results"])
             return content_files
@@ -213,6 +213,9 @@ def update_submission(submission: Submission, repo_record_id: str) -> Submission
         return submission
 
 
+register_adapter(RepositoryType.HYDROSHARE, HydroshareMetadataAdapter)
+
+
 class _HydroshareResourceMetadata(BaseModel):
     title: str
     abstract: str