Merge pull request #565 from pangeo-data/staging

staging -> prod (#555, #556, #559, #561, #562, #564)
pangeo-data · Mar 16, 2020 · 88e5e4a · 88e5e4a
2 parents 9306274 + e55e623
commit 88e5e4a
Show file tree

Hide file tree

Showing 17 changed files with 70 additions and 34 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -208,9 +208,10 @@ jobs:
           name: Install helm
           when: always
           command: |
-            curl https://get.helm.sh/helm-v3.0.3-linux-amd64.tar.gz | \
+            curl https://get.helm.sh/helm-v3.1.2-linux-amd64.tar.gz | \
               tar -xzf -
             sudo mv linux-amd64/helm /usr/local/bin
+            helm version
             helm repo add pangeo https://pangeo-data.github.io/helm-chart/
             helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/
             helm repo add dask-gateway https://dask.org/dask-gateway-helm-repo/
@@ -220,31 +221,48 @@ jobs:
           name: Deploy dev.pangeo.io
           when: always
           command: |
-            hubploy deploy dev pangeo-deploy ${CIRCLE_BRANCH}
+            hubploy deploy dev pangeo-deploy ${CIRCLE_BRANCH} --cleanup-on-fail
 
       - run:
           name: Deploy ocean.pangeo.io
           when: always
           command: |
-            hubploy deploy ocean pangeo-deploy ${CIRCLE_BRANCH}
+            hubploy deploy ocean pangeo-deploy ${CIRCLE_BRANCH} --cleanup-on-fail
 
       - run:
           name: Deploy hydro.pangeo.io
           when: always
           command: |
-            hubploy deploy hydro pangeo-deploy ${CIRCLE_BRANCH}
+            hubploy deploy hydro pangeo-deploy ${CIRCLE_BRANCH} --cleanup-on-fail
 
       # - run:
       #     name: Deploy nasa.pangeo.io
       #     when: always
       #     command: |
       #       hubploy deploy nasa pangeo-deploy ${CIRCLE_BRANCH}
 
+      # NOTE: should move the dynamic IP into hubploy where credentials and awscli version already dealt with
+      # sleep 60 for now, but better to poll for readiness https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html
+      - run:
+          name: Add Runner IP to EKS Kubernetes API Whitelist
+          when: always
+          command: |
+            AWS_SHARED_CREDENTIALS_FILE=./deployments/icesat2/secrets/aws-config.txt
+            RUNNERIP=`curl --silent https://checkip.amazonaws.com`
+            aws --version
+            aws eks update-cluster-config --region us-west-2 --name pangeo --resources-vpc-config publicAccessCidrs=${RUNNERIP}/32 > /dev/null
+            sleep 90
       - run:
           name: Deploy aws-uswest2.pangeo.io
           when: always
           command: |
-            hubploy deploy icesat2 pangeo-deploy ${CIRCLE_BRANCH}
+            hubploy deploy icesat2 pangeo-deploy ${CIRCLE_BRANCH} --cleanup-on-fail
+      - run:
+          name: Revert to Original EKS IP Whitelist
+          when: always
+          command: |
+            AWS_SHARED_CREDENTIALS_FILE=./deployments/icesat2/secrets/aws-config.txt
+            aws eks update-cluster-config --region us-west-2 --name pangeo --resources-vpc-config publicAccessCidrs=${AWS_IP_WHITELIST} > /dev/null
 
       - run:
           name: Deploy ooi.pangeo.io
@@ -256,7 +274,7 @@ jobs:
             BODY="Deployment to ooi-${CIRCLE_BRANCH} failed on `date -u +'%D at %T UTC.'`"
             errormsg() { ./.circleci/send-email "${EMAIL}" "${SUBJECT}" "${BODY}"; }
             trap errormsg ERR
-            hubploy deploy ooi pangeo-deploy ${CIRCLE_BRANCH} --timeout 1200s
+            hubploy deploy ooi pangeo-deploy ${CIRCLE_BRANCH} --timeout 1200s --cleanup-on-fail
 
 workflows:
   version: 2

diff --git a/deployments/dev/config/prod.yaml b/deployments/dev/config/prod.yaml
@@ -4,7 +4,7 @@ pangeo:
       https:
         enabled: true
         hosts:
-        - hub.pangeo.io
+          - hub.pangeo.io
         letsencrypt:
           contactEmail: [email protected]
       service:

diff --git a/deployments/dev/config/staging.yaml b/deployments/dev/config/staging.yaml
@@ -3,7 +3,7 @@ pangeo:
     proxy:
       https:
         hosts:
-        - staging.hub.pangeo.io
+          - staging.hub.pangeo.io
         letsencrypt:
           contactEmail: [email protected]
       service:

diff --git a/deployments/hydro/config/prod.yaml b/deployments/hydro/config/prod.yaml
@@ -3,7 +3,7 @@ pangeo:
     proxy:
       https:
         hosts:
-        - hydro.pangeo.io
+          - hydro.pangeo.io
         letsencrypt:
           contactEmail: [email protected]
       service:

diff --git a/deployments/hydro/config/staging.yaml b/deployments/hydro/config/staging.yaml
@@ -3,7 +3,7 @@ pangeo:
     proxy:
       https:
         hosts:
-        - staging.hydro.pangeo.io
+          - staging.hydro.pangeo.io
         letsencrypt:
           contactEmail: [email protected]
       service:

diff --git a/deployments/icesat2/config/common.yaml b/deployments/icesat2/config/common.yaml
@@ -88,10 +88,10 @@ pangeo:
           c.KubeSpawner.profile_list = [
             {
                 'display_name': 'Default environment (4 vCPU, 16 Gb RAM)',
-                'default': 'true',
             },
             {
                 'display_name': 'IceSat-2 Hackweek 2020 (4 vCPU, 16 Gb RAM)',
+                'default': 'true',
                 'kubespawner_override': {
                     'image': 'uwhackweeks/icesat2:latest',
                 }
@@ -108,6 +108,12 @@ pangeo:
                     'image': 'uwhackweeks/icesat2:2019',
                 }
             },
+            {
+                'display_name': 'Latest ESIP Image (4 vCPU, 16 Gb RAM)',
+                'kubespawner_override': {
+                    'image': 'pangeo/pangeo-esip:latest',
+                }
+            },
             {
                 'display_name': 'Latest Pangeo-notebook Image (4 vCPU, 16 Gb RAM)',
                 'kubespawner_override': {
@@ -121,28 +127,39 @@ pangeo:
                 }
             },
             {
-                'display_name': 'AmesStereoPipeline *experimental* (4 vCPU, 16 Gb RAM)',
+                'display_name': 'AmesStereoPipeline *experimental*',
                 'kubespawner_override': {
                     'image': 'pangeo/pangeo-notebook:latest',
                 }
             },
             {
-                'display_name': 'ESIP Environment (4 vCPU, 16 Gb RAM)',
+                'display_name': 'Pangeo ML Env *experimental*',
                 'kubespawner_override': {
-                    'image': 'pangeo/pangeo-esip:latest',
+                    'mem_limit': '60G',
+                    'mem_guarantee': '25G',
+                    'image': 'pangeo/pangeo-ml:latest',
+                    'environment': {'NVIDIA_DRIVER_CAPABILITIES': 'compute,utility'},
+                    'tolerations': [{'key': 'nvidia.com/gpu','operator': 'Equal','value': 'present','effect': 'NoSchedule'}],
+                    'extra_resource_limits': {"nvidia.com/gpu": "1"}
+                }
+            },
+            {
+                'display_name': 'PangeoDev Pangeo-Image *experimental*',
+                'kubespawner_override': {
+                    'image': 'pangeodev/pangeo-notebook:latest',
                 }
             },
             {
-                'display_name': 'Pangeo ML Env (experimental)',
+                'display_name': 'PangeoDev ML-Image *experimental*',
                 'kubespawner_override': {
                     'mem_limit': '60G',
                     'mem_guarantee': '25G',
-                    'image': 'pangeo/pangeo-ml:latest',
+                    'image': 'pangeodev/ml-notebook:latest',
                     'environment': {'NVIDIA_DRIVER_CAPABILITIES': 'compute,utility'},
                     'tolerations': [{'key': 'nvidia.com/gpu','operator': 'Equal','value': 'present','effect': 'NoSchedule'}],
                     'extra_resource_limits': {"nvidia.com/gpu": "1"}
                 }
-            },
+            }
             ]
         cloneRepo: |
           # Hack solution for branding

diff --git a/deployments/icesat2/config/prod.yaml b/deployments/icesat2/config/prod.yaml
@@ -4,7 +4,7 @@ pangeo:
     proxy:
       https:
         hosts:
-        - aws-uswest2.pangeo.io
+          - aws-uswest2.pangeo.io
         letsencrypt:
           contactEmail: [email protected]
       service:

diff --git a/deployments/icesat2/config/staging.yaml b/deployments/icesat2/config/staging.yaml
@@ -4,7 +4,7 @@ pangeo:
     proxy:
       https:
         hosts:
-        - staging.aws-uswest2.pangeo.io
+          - staging.aws-uswest2.pangeo.io
         letsencrypt:
           contactEmail: [email protected]
       service:

diff --git a/deployments/nasa/config/prod.yaml b/deployments/nasa/config/prod.yaml
@@ -4,7 +4,7 @@ pangeo:
     proxy:
       https:
         hosts:
-        - nasa.pangeo.io
+          - nasa.pangeo.io
         letsencrypt:
           contactEmail: [email protected]
       service:

diff --git a/deployments/nasa/config/staging.yaml b/deployments/nasa/config/staging.yaml
@@ -4,7 +4,7 @@ pangeo:
     proxy:
       https:
         hosts:
-        - staging.nasa.pangeo.io
+          - staging.nasa.pangeo.io
         letsencrypt:
           contactEmail: [email protected]
       service:

diff --git a/deployments/ocean/config/prod.yaml b/deployments/ocean/config/prod.yaml
@@ -3,7 +3,7 @@ pangeo:
     proxy:
       https:
         hosts:
-        - ocean.pangeo.io
+          - ocean.pangeo.io
         letsencrypt:
           contactEmail: [email protected]
       service:

diff --git a/deployments/ocean/config/staging.yaml b/deployments/ocean/config/staging.yaml
@@ -3,7 +3,7 @@ pangeo:
     proxy:
       https:
         hosts:
-        - staging.ocean.pangeo.io
+          - staging.ocean.pangeo.io
         letsencrypt:
           contactEmail: [email protected]
       service:

diff --git a/deployments/ooi/config/prod.yaml b/deployments/ooi/config/prod.yaml
@@ -15,7 +15,7 @@ pangeo:
       https:
         enabled: true
         hosts:
-        - ooi.pangeo.io
+          - ooi.pangeo.io
         letsencrypt:
           contactEmail: [email protected]
       service:

diff --git a/deployments/ooi/config/staging.yaml b/deployments/ooi/config/staging.yaml
@@ -7,7 +7,7 @@ pangeo:
       https:
         enabled: true
         hosts:
-        - staging.ooi.pangeo.io
+          - staging.ooi.pangeo.io
         letsencrypt:
           contactEmail: [email protected]
       service:

diff --git a/pangeo-deploy/Chart.yaml b/pangeo-deploy/Chart.yaml
@@ -1,5 +1,12 @@
-apiVersion: v1
+apiVersion: v2
 appVersion: "1.0"
 description: A Helm chart for Kubernetes
 name: pangeo-deploy
 version: 0.1.0
+dependencies:
+  - name: pangeo
+    version: "v0.2.2-n049.h490c47a"
+    repository: https://pangeo-data.github.io/helm-chart/
+    import-values:
+      - child: rbac
+        parent: rbac
diff --git a/pangeo-deploy/requirements.yaml b/pangeo-deploy/requirements.yaml
diff --git a/requirements.txt b/requirements.txt
@@ -1,4 +1,5 @@
-git+https://github.com/yuvipanda/hubploy@a02fb01bbc4a698b2cc85f9677e9881dac8d438a
+git+https://github.com/yuvipanda/hubploy@d619b2da6cae00c1b14b05b176fb66a5413eb01f
 jupyter-repo2docker>=0.10.0
 pygithub
 azure-cli
+awscli>=1.18