Merge pull request #51 from NASA-IMPACT/bump-pangeo-2024.04.05 #155
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: docker-custom-images-gha | |
| on: | |
| push: | |
| branches: | |
| - main | |
| jobs: | |
| build-dynamic-matrix: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 3 | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: set-matrix | |
| id: set-matrix | |
| run: | | |
| sudo apt-get install -y jo | |
| CUSTOMDIRS=() | |
| DOCKERFILES=$(find ./docker-images/* -name "Dockerfile" ! -path '*/base/*' ! -path '*/examples/*' -prune) | |
| while read DOCKERFILE_PATH; do | |
| echo $DOCKERFILE_PATH | |
| PARENT_DIR=$(echo $DOCKERFILE_PATH | xargs -I{} dirname {}) | |
| DIRNAME=$(echo $PARENT_DIR | cut -d'/' -f4) | |
| CUSTOMDIRS+=("$DIRNAME") | |
| done < <(echo "$DOCKERFILES") | |
| IMAGE_MATRIX=$(jo -a "${CUSTOMDIRS[@]}") | |
| echo "IMAGE_MATRIX=${IMAGE_MATRIX}" >>$GITHUB_OUTPUT | |
| outputs: | |
| IMAGE_MATRIX: ${{ steps.set-matrix.outputs.IMAGE_MATRIX }} | |
| build-custom-images-gha: | |
| needs: build-dynamic-matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| IMAGE: ${{ fromJson(needs.build-dynamic-matrix.outputs.IMAGE_MATRIX) }} | |
| env: | |
| REGISTRY_BASE_URL: public.ecr.aws/i8x6m1u9 | |
| name: ${{ matrix.IMAGE }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 3 | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| aws-access-key-id: ${{ secrets.DEPLOY_USER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.DEPLOY_USER_AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-1 | |
| - name: create ecr repository if not | |
| uses: int128/[email protected] | |
| with: | |
| repository: ${{ matrix.IMAGE }} | |
| public: true | |
| - name: set the tags for docker | |
| id: tags | |
| run: | | |
| DATESTR=$(date +%F) | |
| datetag=${{ env.REGISTRY_BASE_URL }}/${{ matrix.IMAGE }}:$DATESTR | |
| echo "datetag=${datetag}" >>$GITHUB_OUTPUT | |
| if [ -f ./docker-images/custom/${{ matrix.IMAGE }}/environment.yml ]; then | |
| ENVIRON_DIGEST=$(sha256sum ./docker-images/custom/${{ matrix.IMAGE }}/environment.yml | cut -d' ' -f1) | |
| elif [ -f ./docker-images/custom/${{ matrix.IMAGE }}/standalone-environment.yml ]; then | |
| ENVIRON_DIGEST=$(sha256sum ./docker-images/custom/${{ matrix.IMAGE }}/standalone-environment.yml | cut -d' ' -f1) | |
| fi | |
| digesttag=${{ env.REGISTRY_BASE_URL }}/${{ matrix.IMAGE }}:$ENVIRON_DIGEST | |
| echo "digesttag=${digesttag}" >>$GITHUB_OUTPUT | |
| VERSION=$(cat ./docker-images/custom/${{ matrix.IMAGE }}/Dockerfile | grep VERSION | cut -d'=' -f 2) | |
| versiontag=${{ env.REGISTRY_BASE_URL }}/${{ matrix.IMAGE }}:$VERSION | |
| echo "versiontag=${versiontag}" >>$GITHUB_OUTPUT | |
| latesttag=${{ env.REGISTRY_BASE_URL }}/${{ matrix.IMAGE }}:latest | |
| echo "latesttag=${latesttag}" >>$GITHUB_OUTPUT | |
| - name: echo out ECR image URL and SHA tag | |
| run: | | |
| echo "#########################################################################" | |
| echo "YOUR NEW CUSTOM IMAGE URL + SHATAG IS:" | |
| echo "${{ steps.tags.outputs.digesttag }}" | |
| echo "#########################################################################" | |
| - name: check for changes | |
| id: changes | |
| run: | | |
| if ! git diff --quiet HEAD~ -- ./docker-images/custom/${{ matrix.IMAGE }}; then | |
| echo "./docker-images/custom/${{ matrix.IMAGE }} CHANGED, building!" | |
| echo "build=true" >>$GITHUB_OUTPUT | |
| else | |
| echo "./docker-images/custom/${{ matrix.IMAGE }} NOT CHANGED, skipping!" | |
| echo "build=false" >>$GITHUB_OUTPUT | |
| fi | |
| - name: set up docker buildx | |
| uses: docker/setup-buildx-action@v2 | |
| if: ${{ steps.changes.outputs.build == 'true' }} | |
| - name: docker login for ecr | |
| run: | |
| aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws | |
| if: ${{ steps.changes.outputs.build == 'true' }} | |
| - name: docker build image | |
| id: docker_build | |
| uses: docker/build-push-action@v3 | |
| with: | |
| push: true | |
| context: ./docker-images/custom/${{ matrix.IMAGE }} | |
| tags: | | |
| ${{ steps.tags.outputs.datetag }} | |
| ${{ steps.tags.outputs.versiontag }} | |
| ${{ steps.tags.outputs.digesttag }} | |
| ${{ steps.tags.outputs.latesttag }} | |
| build-args: | | |
| CONDA_ENV_NAME_OVERRIDE=${{ matrix.IMAGE }} | |
| if: ${{ steps.changes.outputs.build == 'true' }} | |
| template-and-cp-s3: | |
| runs-on: ubuntu-latest | |
| needs: build-custom-images-gha | |
| env: | |
| REGISTRY_BASE_URL: public.ecr.aws/i8x6m1u9 | |
| S3_BUCKET_NAME: veda-jh-environments-west1-sandbox-profile-list | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 3 | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: mikefarah/yq@master | |
| - name: configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| aws-access-key-id: ${{ secrets.DEPLOY_USER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.DEPLOY_USER_AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-1 | |
| - name: find all dockerfiles && create profileList yaml | |
| run: | | |
| ##################### | |
| # get a list of all dockerfiles and exclude the custom images we don't want to use | |
| DOCKERFILES=$(find ./docker-images/* -name "Dockerfile" ! -path '*/base/*' ! -path '*/examples/*' -prune) | |
| ##################### | |
| # build a small $PROFILE_VALUES file with our custom images we want to use | |
| PROFILE_VALUES=profilelist.yaml | |
| PROFILE_VALUES_JSON=profilelist.json | |
| INDEX=0 | |
| printf "profileList:\n" >> $PROFILE_VALUES | |
| while read DOCKERFILE_PATH; do | |
| VERSION=$(cat $DOCKERFILE_PATH | grep VERSION | cut -d'=' -f 2) | |
| NAME_AND_DESC=$(echo $DOCKERFILE_PATH | cut -d'/' -f 4) | |
| printf " - display_name: $NAME_AND_DESC\n" >> $PROFILE_VALUES | |
| printf " description: $NAME_AND_DESC\n" >> $PROFILE_VALUES | |
| printf " kubespawner_override:\n" >> $PROFILE_VALUES | |
| printf " image: $REGISTRY_BASE_URL/$NAME_AND_DESC:$VERSION\n" >> $PROFILE_VALUES | |
| if [ $INDEX -eq 0 ]; then | |
| printf " default: true\n" >> $PROFILE_VALUES | |
| fi | |
| let INDEX=${INDEX}+1 | |
| done < <(echo "$DOCKERFILES") | |
| # QC GH Actions CI/CD | |
| cat $PROFILE_VALUES | |
| # validate yaml manifest | |
| #yamllint $PROFILE_VALUES | |
| # convert to JSON so easier to deal with on Kubespawner image | |
| yq -o=json '.' $PROFILE_VALUES > $PROFILE_VALUES_JSON | |
| aws s3 cp $PROFILE_VALUES_JSON s3://${{ env.S3_BUCKET_NAME }} | |
| rm $PROFILE_VALUES $PROFILE_VALUES_JSON | |
| # - name: commit ./gitops-deploy changes back | |
| # uses: EndBug/add-and-commit@v8 | |
| # with: | |
| # default_author: github_actions |