From 1c206d78bfe4e8bff73aa7871cd07d97e07e79b3 Mon Sep 17 00:00:00 2001 From: Lucas Koontz Date: Mon, 22 Apr 2024 14:26:59 -0700 Subject: [PATCH] fix: add new ci --- .github/actions/build-push-ecr/action.yml | 50 --------------- .github/actions/setup-env/action.yml | 24 -------- .../workflows/dev-build-deploy-on-main.yaml | 59 +++++++++++++----- .../prod-build-deploy-on-release.yaml | 61 ++++++++++++++----- .gitignore | 12 +++- deployment/hashnode-starter-kit/values.yaml | 2 +- 6 files changed, 100 insertions(+), 108 deletions(-) delete mode 100644 .github/actions/build-push-ecr/action.yml delete mode 100644 .github/actions/setup-env/action.yml diff --git a/.github/actions/build-push-ecr/action.yml b/.github/actions/build-push-ecr/action.yml deleted file mode 100644 index dd300f26c..000000000 --- a/.github/actions/build-push-ecr/action.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Builds a docker image, then tags it with the github sha and pushes it to our Amazon ECR registry - -inputs: - module-name: - description: "Name of the module to build. Used as the default image name and src dir unless 'image-name' or 'src-path' are used." - required: true - build-for-environment: - description: "The backend environment we are building for (API calls are pointed to). This should be one of (development, staging, production)." - required: true - extra-build-args: - description: "Extra args passed to 'docker build'." - required: false - src-path: - description: "What folder to be (generally to find the Dockerfile in) default is root of repo" - required: false - - -runs: - using: 'composite' - steps: - - uses: ./.github/actions/setup-env - # https://github.com/aws-actions/amazon-ecr-login - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - shell: bash - run: | - # Env var parsing - INPUT_SRC_PATH=${{ inputs.src-path }} - IMAGE_NAME=${{ inputs.module-name }} - SRC_PATH=${INPUT_SRC_PATH:-"./"} - REPO_IMAGE=${{ steps.login-ecr.outputs.registry }}/$IMAGE_NAME - DOCKER_BUILDKIT=1 - ENVIRONMENT=${{ inputs.build-for-environment }} - BRANCH_NAME=${{env.ENV_NAME}} - IMAGE_TAG=$ENVIRONMENT-${{ env.SLUG }} - - # Create repo if needed - aws ecr create-repository --repository-name $IMAGE_NAME || true # Just let this fail if the repo already exists - - # Grab our cached images, if present - cd $SRC_PATH - CACHE_FROM_ARGS="--cache-to type=local,dest=/mnt/shared/layercache,mode=max --cache-from type=local,src=/mnt/shared/layercache" - BUILD_ARGS="--build-arg BUILD_FOR_ENVIRONMENT=$ENVIRONMENT --build-arg IMAGE_TAG=$IMAGE_TAG" - - # Finally, build our runner container - echo "Running: \n docker buildx build ${{ inputs.extra-build-args }} $BUILD_ARGS $CACHE_FROM_ARGS -t $REPO_IMAGE:$IMAGE_TAG -t $REPO_IMAGE:latest --push ." - docker buildx build ${{ inputs.extra-build-args }} $BUILD_ARGS $CACHE_FROM_ARGS -t $REPO_IMAGE:$IMAGE_TAG -t $REPO_IMAGE:latest --push . \ No newline at end of file diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml deleted file mode 100644 index d0a2622bc..000000000 --- a/.github/actions/setup-env/action.yml +++ /dev/null @@ -1,24 +0,0 @@ -# Set up all of the CI env vars required - -runs: - using: 'composite' - steps: - # Get clean environment variables via https://github.com/marketplace/actions/github-environment-variables-action - - uses: FranzDiebold/github-env-vars-action@v2 - - shell: bash - run: | - echo "REF_SLUG=${CI_REF_NAME_SLUG:-$CI_HEAD_REF_SLUG}" >> $GITHUB_ENV # Use whichever env ref is supplied (push or merge). - # Figure out the namespace and environment name, which should be up to 63 chars in length for Kubernetes - - uses: web3j/substr-action@v1.2 - id: substring-env-name - with: - value: '${{ env.CI_REPOSITORY_NAME }}-${{ env.REF_SLUG }}' - start: '0' - length: '63' - - id: set-envs - shell: bash - run: | - echo "SLUG=$CI_SHA" >> $GITHUB_ENV - # This will strip tailing dash characters, necessary because we can't have dash suffixed strings in Kubernetes - export STRIPPED_STRING=`echo "${{ steps.substring-env-name.outputs.result }}" | sed 's/-*$//g'` - echo "ENV_NAME=$STRIPPED_STRING" >> $GITHUB_ENV diff --git a/.github/workflows/dev-build-deploy-on-main.yaml b/.github/workflows/dev-build-deploy-on-main.yaml index 5e9dc32c8..121972349 100644 --- a/.github/workflows/dev-build-deploy-on-main.yaml +++ b/.github/workflows/dev-build-deploy-on-main.yaml @@ -3,40 +3,67 @@ name: Dev - Build and Deploy on main on: push: branches: - - master - main defaults: run: shell: bash -jobs: - build-hashnode-starter-kit: - runs-on: [self-hosted, dev] +jobs: + build: + runs-on: mdb-dev + env: + AWS_REGION: us-east-2 steps: - - uses: actions/checkout@v2 + - name: Generate token + # Use the Github App private key to request an installation + # token with read-only access to the organization's private + # repositories. This token is then used in the checkout step + # (but not in subsequent steps that create releases and upload + # assets, those still use the default token that has write + # access to the current repository). + id: generate_token + uses: tibdex/github-app-token@v1 + with: + app_id: ${{ secrets.ACTION_APP_ID }} + private_key: ${{ secrets.ACTION_APP_KEY }} + - name: Checkout tag + uses: actions/checkout@v4 with: - submodules: 'true' - - uses: ./.github/actions/setup-env - - uses: ./.github/actions/build-push-ecr + submodules: recursive + token: ${{ steps.generate_token.outputs.token }} + - name: Pull MindsDB Github Actions + uses: actions/checkout@v4 with: - module-name: ${{ env.CI_REPOSITORY_NAME_SLUG }} - build-for-environment: dev - # extra-build-args: "-f docker/mindsdb.Dockerfile" + repository: mindsdb/github-actions + path: github-actions + ssh-key: ${{ secrets.GH_ACTIONS_PULL_SSH }} + - uses: ./github-actions/setup-env + - uses: ./github-actions/build-push-ecr + with: + module-name: ${{ env.CI_REPOSITORY_SLUG }} + build-for-environment: development + # extra-build-args: -f docker/Dockerfile deploy: - runs-on: [self-hosted, dev] - needs: [build-hashnode-starter-kit] + runs-on: mdb-dev + needs: build environment: name: dev url: https://hashnode.dev.mindsdb.com/blog steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/setup-env + - uses: actions/checkout@v4 + - name: Pull MindsDB Github Actions + uses: actions/checkout@v4 + with: + repository: mindsdb/github-actions + path: github-actions + ssh-key: ${{ secrets.GH_ACTIONS_PULL_SSH }} + - uses: ./github-actions/setup-env - uses: DevOps-Nirvana/aws-helm-multi-deploy-nodocker@v2 with: environment-slug: dev k8s-namespace: dev - image-tag: dev-${{ env.SLUG }} + image-tag: development-${{ env.SLUG }} timeout: 600s diff --git a/.github/workflows/prod-build-deploy-on-release.yaml b/.github/workflows/prod-build-deploy-on-release.yaml index 0f06cf914..44054cf40 100644 --- a/.github/workflows/prod-build-deploy-on-release.yaml +++ b/.github/workflows/prod-build-deploy-on-release.yaml @@ -3,38 +3,69 @@ name: Prod - Build and Deploy on release on: release: types: [published] + workflow_dispatch: defaults: run: shell: bash + jobs: - build-hashnode-starter-kit: - runs-on: [self-hosted, dev] + build: + runs-on: mdb-dev + env: + AWS_REGION: us-east-2 steps: - - uses: actions/checkout@v2 + - name: Generate token + # Use the Github App private key to request an installation + # token with read-only access to the organization's private + # repositories. This token is then used in the checkout step + # (but not in subsequent steps that create releases and upload + # assets, those still use the default token that has write + # access to the current repository). + id: generate_token + uses: tibdex/github-app-token@v1 + with: + app_id: ${{ secrets.ACTION_APP_ID }} + private_key: ${{ secrets.ACTION_APP_KEY }} + - name: Checkout tag + uses: actions/checkout@v4 with: - submodules: 'true' - - uses: ./.github/actions/setup-env - - uses: ./.github/actions/build-push-ecr + submodules: recursive + token: ${{ steps.generate_token.outputs.token }} + - name: Pull MindsDB Github Actions + uses: actions/checkout@v4 with: - module-name: ${{ env.CI_REPOSITORY_NAME_SLUG }} - build-for-environment: prod - # extra-build-args: "-f docker/mindsdb.Dockerfile" + repository: mindsdb/github-actions + path: github-actions + ssh-key: ${{ secrets.GH_ACTIONS_PULL_SSH }} + - uses: ./github-actions/setup-env + - uses: ./github-actions/build-push-ecr + with: + module-name: ${{ env.CI_REPOSITORY_SLUG }} + build-for-environment: production + extra-build-args: -f docker/Dockerfile + image-ref: ${{ env.CI_REF_SLUG }} - deploy: - runs-on: [self-hosted, prod] - needs: [build-hashnode-starter-kit] + deploy-prod: + runs-on: mdb-prod + needs: [build] environment: name: prod url: https://mindsdb.com/blog steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/setup-env + - uses: actions/checkout@v4 + - name: Pull MindsDB Github Actions + uses: actions/checkout@v4 + with: + repository: mindsdb/github-actions + path: github-actions + ssh-key: ${{ secrets.GH_ACTIONS_PULL_SSH }} + - uses: ./github-actions/setup-env - uses: DevOps-Nirvana/aws-helm-multi-deploy-nodocker@v2 with: environment-slug: prod k8s-namespace: prod - image-tag: prod-${{ env.SLUG }} + image-tag: production-${{ env.CI_REF_SLUG }} timeout: 600s diff --git a/.gitignore b/.gitignore index 8a632414f..a1e5612c2 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,7 @@ /out/ # production -/build +build/ # misc .DS_Store @@ -41,6 +41,14 @@ next-env.d.ts .vscode/* !.vscode/extensions.json -# Kubernetes / DevOps Nirvana +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ + +# Kubernetes / Gitlab CI Automation / Helm requirements.lock +deployment/*/deployment/*/charts deployment/*/charts diff --git a/deployment/hashnode-starter-kit/values.yaml b/deployment/hashnode-starter-kit/values.yaml index 8d8d0679e..a43c14901 100644 --- a/deployment/hashnode-starter-kit/values.yaml +++ b/deployment/hashnode-starter-kit/values.yaml @@ -8,7 +8,7 @@ deployment: # Where the image comes from. This should be the ECR Repo / Gitlab Group / Repo Name image: - repository: 454861456664.dkr.ecr.us-east-2.amazonaws.com/hashnode-starter-kit + repository: 454861456664.dkr.ecr.us-east-2.amazonaws.com/mindsdb-hashnode-starter-kit # Uncomment these two temporarily (and disable probes below) to debug manually in Kubernetes # command: ["/bin/sh"] # args: ["-c", "while true; do echo hello; sleep 10;done"]