Skip to content
Build

Build #341

Sign in to view logs

Build

Build #341

Workflow file for this run

.github/workflows/build.yml at 4145173
# Build Pipeline for TIG
name: Build
# Controls when the workflow will run
on:
# Triggers the workflow on push events
push:
branches: [ develop, release/**, main, feature/**, issue/**, issues/**, dependabot/** ]
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}
jobs:
# First job in the workflow installs and verifies the software
build:
name: Build, Test, Verify, Publish
# The type of runner that the job will run on
runs-on: ubuntu-latest
steps:
#########################################################################
# Environment Setup
#########################################################################
# NOTE: This step is platform-specific
# Checks out this repository and sets up the build/test environment with
# gradle
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Install Poetry
uses: abatilo/actions-poetry@v3
with:
poetry-version: 1.3.2
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
#########################################################################
# Versioning (featuring weird gradle output work-arounds)
#########################################################################
# NOTE: This step is platform-specific
# Retrieve version information for use in the other versioning steps
- name: Get version
id: get-version
run: |
echo "the_service=${{ github.event.repository.name }}" >> $GITHUB_ENV
echo "the_env=$(printenv)" >> $GITHUB_ENV
echo "${{ github.event.repository.name }}"
echo "pyproject_name=$(poetry version | awk '{print $1}')" >> $GITHUB_ENV
poetry version > .temp_version.out
cat .temp_version.out
the_version=$(cat .temp_version.out |grep -v Downloading |grep -v '%' |sed -e 's/podaac-tig *//')
rm .temp_version.out
echo "old_version=$the_version" >> $GITHUB_ENV
echo "the_version=$the_version" >> $GITHUB_ENV
echo "Initial Version: $the_version"
# Pre-Alpha Logic - Use the project version number and add the short hash
# to it
- name: Bump pre-alpha version
# If triggered by push to a feature branch
if: |
startsWith(github.ref, 'refs/heads/issue') ||
startsWith(github.ref, 'refs/heads/dependabot/') ||
startsWith(github.ref, 'refs/heads/feature/')
# At pre-alpha, append git-commit to version, set it into gradle
# property, read the version out and set to build_service_version
run: |
the_version=$(echo "${{ env.the_version }}" | sed -e "s/-alpha.*//g")
the_version=$(echo "$the_version" | sed -e "s/-rc.*//g")
new_version="${the_version}+$(git rev-parse --short HEAD)"
echo "the_version=${new_version}" >> $GITHUB_ENV
echo "software_version=${new_version}" >> $GITHUB_ENV
echo "new_version=${new_version}" >> $GITHUB_ENV
echo "Github REF: ${{ github.ref }}"
# Alpha Logic - Use the project version number and add -alpha.1 or bump
# alpha number
- name: Bump alpha version
env:
VERSION: ${{ env.the_version }}
# If triggered by push to the develop branch
if: ${{ github.ref == 'refs/heads/develop' }}
run: |
if [[ ${VERSION} == *"-alpha"* ]]; then
alpha_number=$(echo "${VERSION}" | sed -e "s/^.*-alpha.//g")
alpha_number=$(echo "$alpha_number" | sed -e "s/-rc.*//g")
alpha_number=$((alpha_number+1))
the_version=$(echo "$the_version" | sed -e "s/-alpha.*//g")
the_version=$(echo "$the_version" | sed -e "s/-rc.*//g")
the_version="${the_version}-alpha.$alpha_number"
echo "software_version=${the_version}" >> $GITHUB_ENV
echo "the_version=${the_version}" >> $GITHUB_ENV
else
the_version="${{ env.the_version }}-alpha.1"
echo "software_version=${the_version}" >> $GITHUB_ENV
echo "the_version=${the_version}" >> $GITHUB_ENV
fi
echo "new_version=${the_version}" >> $GITHUB_ENV
echo "venue=sit" >> $GITHUB_ENV
echo "TARGET_ENV_UPPERCASE=SIT" >> $GITHUB_ENV
# Release Candidate Logic - Remove -alpha* and add -rc.1, or bump the rc
# number
- name: Bump rc version
if: ${{ startsWith(github.ref, 'refs/heads/release/') }}
env:
VERSION: ${{ env.the_version }}
COMMIT_VERSION: ${{ github.ref }}
run: |
commit_version=$COMMIT_VERSION
commit_version=$(echo "${commit_version}" |sed -e "s/^.*\///g")
commit_version=$(echo "${commit_version}" |sed -e "s/-alpha.*//g")
commit_version=$(echo "${commit_version}" |sed -e "s/-rc.*//g")
echo "COMMIT VERSION: $commit_version"
file_version=${VERSION}
file_version=$(echo "${file_version}" |sed -e "s/-alpha.*//g")
file_version=$(echo "${file_version}" |sed -e "s/-rc.*//g")
echo "FILE VERSION: $file_version"
if [[ "$commit_version" != "$file_version" ]]; then
echo "Commit version and file version are different, using commit version"
VERSION=$commit_version
fi
if [[ ${VERSION} == *"-rc"* ]]; then
echo "Bumping up the release candidate number from ${VERSION}"
rc_number=$(echo "${VERSION}" | sed -e "s/^.*-rc.//g")
rc_number=$(echo "${rc_number}" | sed -e "s/-alpha.*//g")
rc_number=$((rc_number+1))
the_version=$(echo "$the_version" | sed -e "s/-rc.*//g")
the_version=$(echo "$the_version" | sed -e "s/-alpha.*//g")
VERSION="${the_version}-rc.${rc_number}"
else
echo "Initializing the first release candidate for ${VERSION}"
VERSION=$(echo "${VERSION}" |sed -e "s/-alpha.*//g")
VERSION="${VERSION}-rc.1"
fi
echo "software_version=${VERSION}" >> $GITHUB_ENV
echo "the_version=${VERSION}" >> $GITHUB_ENV
echo "new_version=${VERSION}" >> $GITHUB_ENV
echo "venue=uat" >> $GITHUB_ENV
echo "TARGET_ENV_UPPERCASE=UAT" >> $GITHUB_ENV
# Release Logic
- name: Release version
# If triggered by push to the main branch
if: ${{ startsWith(github.ref, 'refs/heads/main') }}
env:
VERSION: ${{ env.the_version }}
# Remove -rc.* from end of version string
run: |
software_version=$(echo "${VERSION}" | sed -e s/-rc.*//g)
software_version=$(echo "${software_version}" | sed -e s/-alpha.*//g)
echo "software_version=$software_version" >> $GITHUB_ENV
echo "new_version=$software_version" >> $GITHUB_ENV
echo "the_version=$software_version" >> $GITHUB_ENV
echo "venue=ops" >> $GITHUB_ENV
echo "TARGET_ENV_UPPERCASE=OPS" >> $GITHUB_ENV
#########################################################################
# Versioning Summary
#########################################################################
- name: Versioning Summary
run: |
echo "the_service: ${{ env.the_service }}"
echo "old version : ${{ env.old_version }}"
echo "new version : ${{ env.new_version }}"
echo "the_env: ${{ env.the_env }}"
echo "software_version: ${{ env.software_version }}"
echo "GITHUB REF: ${{ github.ref }}"
echo "VENUE: ${{ env.venue }}"
echo "Target Env Uppercase: ${{ env.TARGET_ENV_UPPERCASE }}"
# NOTE: This step is platform-specific
# Update the version number in the application package itself
- name: Update version number in the application package
run: |
poetry version ${{ env.the_version }}
#########################################################################
# Install
#########################################################################
# NOTE: This step is platform-specific
# These are gradle-specific steps for installing the application
- name: Install Software
run: |
pip install setuptools -U
pip install pylint
pip install pytest
poetry install
#########################################################################
# Snyk
#########################################################################
- name: Run Snyk as a blocking step
uses: snyk/actions/python@master
env:
SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
with:
command: test
args: >
--org=${{ secrets.SNYK_ORG_ID }}
--project-name=${{ github.repository }}
--severity-threshold=high
--fail-on=all
- name: Run Snyk on Python
uses: snyk/actions/python@master
env:
SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
with:
command: monitor
args: >
--org=${{ secrets.SNYK_ORG_ID }}
--project-name=${{ github.repository }}
#########################################################################
# Test
#########################################################################
# This is where tests go
- name: Run Poetry Tests
run: |
poetry run pylint podaac
poetry run flake8 podaac
poetry run pytest --junitxml=build/reports/pytest.xml --cov=podaac/ --cov-report=html -m "not aws and not integration" tests/
# poetry run pytest --junitxml=build/reports/pytest.xml --cov=podaac/ --cov-report=xml:build/reports/coverage.xml -m "not aws and not integration" tests/
## TODO: Find out where the test report goes
#########################################################################
# Build
#########################################################################
- name: Install Software
run: |
poetry build
#########################################################################
# Publish new version numbers
#########################################################################
- name: Quick check for changes
id: check_changes
if: |
github.ref == 'refs/heads/develop' ||
github.ref == 'refs/heads/main' ||
startsWith(github.ref, 'refs/heads/release')
run: |
if [ -n "$(git status --porcelain)" ]; then
echo "changes=true" >> $GITHUB_OUTPUT
else
echo "changes=false" >> $GITHUB_OUTPUT
fi
- name: Commit Version Bump
# If building develop, a release branch, or main then we commit the version bump back to the repo
if: steps.check_changes.outputs.changes == 'true'
run: |
git config user.name "${GITHUB_ACTOR}"
git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
git commit -am "/version ${{ env.the_version }}"
git push
- name: Push Tag
env:
VERSION: ${{ env.the_version }}
if: |
github.ref == 'refs/heads/develop' ||
github.ref == 'refs/heads/main' ||
startsWith(github.ref, 'refs/heads/release')
run: |
git config user.name "${GITHUB_ACTOR}"
git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
git tag -a "${VERSION}" -m "Version ${VERSION}"
git push origin "${VERSION}"
#########################################################################
# Publish release to releases
#########################################################################
- name: Create Zip release
run: |
ls -al
cd terraform
ls -al
zip -r ../tig-terraform-${{ env.the_version }}.zip *
- name: Upload Release Artifacts
if: |
github.ref == 'refs/heads/develop' ||
github.ref == 'refs/heads/main' ||
startsWith(github.ref, 'refs/heads/release') ||
github.event.head_commit.message == '/deploy sit' ||
github.event.head_commit.message == '/deploy uat' ||
github.event.head_commit.message == '/deploy sandbox'
uses: ncipollo/[email protected]
with:
tag: ${{ env.the_version }}
artifacts: "*.zip"
token: ${{ secrets.GITHUB_TOKEN }}
body: "Version ${{ env.the_version }}"
makeLatest: "${{ github.ref == 'refs/heads/main' }}"
prerelease: "${{ github.ref != 'refs/heads/main' }}"
#########################################################################
# Publish to pypi.org
#########################################################################
- name: Publish to test.pypi.org
id: pypi-test-publish
if: |
github.ref == 'refs/heads/develop' ||
startsWith(github.ref, 'refs/heads/release')
env:
POETRY_PYPI_TOKEN_TESTPYPI: ${{secrets.TEST_PYPI_API_TOKEN}}
run: |
poetry config repositories.testpypi https://test.pypi.org/legacy/
poetry publish -r testpypi
- name: Publish to pypi.org
if: ${{ github.ref == 'refs/heads/main' }}
id: pypi-publish
env:
POETRY_PYPI_TOKEN_PYPI: ${{secrets.PYPI_API_TOKEN}}
run: |
poetry publish --skip-existing
## Due to observed delays between upload and availability, wait for the package to become available
- name: Wait for package
if: |
steps.pypi-test-publish.conclusion == 'success' ||
steps.pypi-publish.conclusion == 'success'
run: |
pip install tenacity logging
python3 ${GITHUB_WORKSPACE}/.github/workflows/wait-for-pypi.py ${{env.pyproject_name}}[harmony]==${{ env.software_version }}
#########################################################################
# Build and Publish Docker Container
#########################################################################
# Setup docker to build and push images
## Build and publish to GHCR
- name: Log in to the Container registry
if: |
steps.pypi-test-publish.conclusion == 'success' ||
steps.pypi-publish.conclusion == 'success'||
github.event.head_commit.message == '/deploy sit' ||
github.event.head_commit.message == '/deploy uat' ||
github.event.head_commit.message == '/deploy sandbox'
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Deploy Env Override
if: |
github.event.head_commit.message == '/deploy sit' ||
github.event.head_commit.message == '/deploy uat' ||
github.event.head_commit.message == '/deploy sandbox'
run: |
message="${{ github.event.head_commit.message }}"
trimmed_message=${message:1} # Remove leading slash
override_env=$(echo "$trimmed_message" | grep -oE '[^[:space:]]+$')
override_env_upper=$(echo "$trimmed_message" | awk '{print toupper($NF)}')
echo "THE_ENV=${override_env}" >> $GITHUB_ENV
echo "TARGET_ENV_UPPERCASE=${override_env_upper}" >> $GITHUB_ENV
- name: Lower Case Target Env
run: |
original_env_value="${TARGET_ENV_UPPERCASE}"
lowercase_value=$(echo "${original_env_value}" | tr '[:upper:]' '[:lower:]')
echo "TARGET_ENV_LOWERCASE=${lowercase_value}" >> $GITHUB_ENV
- name: Extract metadata (tags, labels) for Docker
if: |
steps.pypi-test-publish.conclusion == 'success' ||
steps.pypi-publish.conclusion == 'success' ||
github.event.head_commit.message == '/deploy sit' ||
github.event.head_commit.message == '/deploy uat' ||
github.event.head_commit.message == '/deploy sandbox'
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
flavor: |
latest=${{ github.ref == 'refs/heads/main' }}
tags: |
type=semver,pattern={{raw}},value=${{ env.the_version }}
type=raw,value=${{ env.TARGET_ENV_LOWERCASE }}
- name: Show meta outputs
run: |
echo "Tags: ${{ steps.meta.outputs.tags }}"
echo "labels: ${{ steps.meta.outputs.labels }}"
- name: Build and push Docker image
if: |
github.ref == 'refs/heads/develop' ||
github.ref == 'refs/heads/main' ||
startsWith(github.ref, 'refs/heads/release')
uses: docker/build-push-action@v3
with:
context: .
file: ./docker/lambdaDockerfileArm
push: true
pull: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
platforms: linux/arm/v7
build-args: |
SOURCE=${{ env.pyproject_name }}==${{ env.the_version }}
## Local tig docker builds
- name: Get Local Tig Build
if: |
github.event.head_commit.message == '/deploy sit' ||
github.event.head_commit.message == '/deploy uat' ||
github.event.head_commit.message == '/deploy sandbox'
run: |
local_tig=$(find dist -type f -name "*.whl")
echo "local_tig=${local_tig}" >> $GITHUB_ENV
- name: Build Local TIG and push Docker image
if: |
github.event.head_commit.message == '/deploy sit' ||
github.event.head_commit.message == '/deploy uat' ||
github.event.head_commit.message == '/deploy sandbox'
uses: docker/build-push-action@v3
with:
context: .
file: ./docker/lambdaDockerfileArm
push: true
pull: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
platforms: linux/arm/v7
build-args: |
DIST_PATH="dist/"
SOURCE=${{ env.local_tig }}
# #########################################################################
# # Build and Publish Documentation
# #########################################################################
# - name: Build Docs
# run: |
# poetry run sphinx-build -b html ./docs docs/_build/
# - name: Publish Docs
# uses: JamesIves/[email protected]
# with:
# branch: gh-pages # The branch the action should deploy to.
# folder: docs/_build/ # The folder the action should deploy.
# target-folder: ${{ env.software_version }}
#########################################################################
# Deploy to AWS via Terraform
#########################################################################
- uses: hashicorp/setup-terraform@v3
with:
terraform_version: 1.5.3
- name: Deploy Terraform
if: |
github.ref == 'refs/heads/develop' ||
github.ref == 'refs/heads/main' ||
startsWith(github.ref, 'refs/heads/release') ||
github.event.head_commit.message == '/deploy sit' ||
github.event.head_commit.message == '/deploy uat'
working-directory: terraform_deploy/
env:
AWS_ACCESS_KEY_ID: ${{ secrets[format('AWS_ACCESS_KEY_ID_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}
AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}
AWS_ACCOUNT_ID: ${{ secrets[format('AWS_ACCOUNT_ID_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}
AWS_DEFAULT_REGION: us-west-2
TF_VAR_tig_docker_image: "ghcr.io/podaac/tig:${{ env.DOCKER_METADATA_OUTPUT_VERSION }}"
TF_VAR_EARTH_DATA_LOGIN_CLIENT_ID: ${{ secrets[format('EARTH_DATA_LOGIN_CLIENT_ID_{0}', env.TARGET_ENV_UPPERCASE)] }}
TF_VAR_EARTH_DATA_LOGIN_PASSWORD: ${{ secrets[format('EARTH_DATA_LOGIN_PASSWORD_{0}', env.TARGET_ENV_UPPERCASE)] }}
run: |
ls -al
ls -al bin/
which python3
python3 --version
python3 override.py https://github.com/podaac/tig/releases/download/${{ env.the_version }}/tig-terraform-${{ env.the_version }}.zip "ghcr.io/podaac/tig:${{ env.DOCKER_METADATA_OUTPUT_VERSION }}"
ls -al
echo "Show override contents"
cat override.tf.json
echo "End show override contents"
terraform --version
source bin/config.sh ${{ env.TARGET_ENV_LOWERCASE }}
terraform plan -var-file=tfvars/"${{ env.TARGET_ENV_LOWERCASE }}".tfvars -var="app_version=${{ env.the_version }}" -out="tfplan"
terraform apply -auto-approve tfplan > /dev/null