Add per-image build matrix configuration (SOFTWARE-5904) #905
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
--- | |
name: Build and push container images | |
on: | |
push: | |
pull_request: | |
workflow_dispatch: | |
repository_dispatch: | |
types: [dispatch-build] | |
jobs: | |
make-date-tag: | |
runs-on: ubuntu-latest | |
outputs: | |
dtag: ${{ steps.mkdatetag.outputs.dtag }} | |
steps: | |
- name: make date tag | |
id: mkdatetag | |
run: echo "dtag=$(date +%Y%m%d-%H%M)" >> $GITHUB_OUTPUT | |
build-image-list: | |
runs-on: ubuntu-latest | |
outputs: | |
matrix: ${{ steps.set_matrix.outputs.matrix }} | |
steps: | |
- uses: actions/checkout@v3 | |
with: | |
fetch-depth: 0 | |
- id: image-list | |
run: | | |
ORG_DIR=opensciencegrid | |
# Get the list of files changed based on the type of event | |
# kicking off the GHA: | |
# 1. For the main branch, diff the previous state of main vs | |
# the current commit | |
# 2. For other branches (i.e., on someone's fork), diff main | |
# vs the current commit | |
# 3. For PRs, diff the base ref vs the current commit | |
# 4. For everything else (e.g., dispatches), build all images | |
if [[ $GITHUB_EVENT_NAME == 'pull_request' ]] || | |
[[ $GITHUB_EVENT_NAME == 'push' ]]; then | |
if [[ $GITHUB_EVENT_NAME == 'pull_request' ]]; then | |
BASE=$(git merge-base origin/$GITHUB_BASE_REF HEAD) | |
elif [[ $GITHUB_REF == 'refs/heads/main' ]]; then | |
BASE=${{github.event.before}} | |
else | |
BASE=origin/main | |
fi | |
# List image root dirs where files have changed and the | |
# root dir exists. Example value: | |
# "opensciencegrid/vo-frontend opensciencegrid/ospool-cm" | |
images=$(git diff --name-only \ | |
"$BASE" \ | |
"$GITHUB_SHA" | | |
egrep "^$ORG_DIR/" | | |
cut -d/ -f -2 | | |
sort | | |
uniq | | |
xargs -I {} find . -type d \ | |
-wholename ./{} \ | |
-printf "%P\n") | |
else | |
# List all image root dirs. Example value: | |
# "opensciencegrid/vo-frontend opensciencegrid/ospool-cm" | |
images=$(find $ORG_DIR -mindepth 1 \ | |
-maxdepth 1 \ | |
-type d \ | |
-printf "$ORG_DIR/%P\n") | |
fi | |
image_json=$(echo -n "${images:-dummy}" | jq -Rcs '.|split("\n") | map(select(. != ""))') | |
echo "$image_json" > image_list.json | |
echo "images=$(echo $images | tr '\n' ' ')" >> $GITHUB_OUTPUT | |
- name: Display image list | |
run: cat image_list.json | |
- name: Set up Python | |
uses: actions/setup-python@v2 | |
with: | |
python-version: '3.x' | |
- name: Run build-job-matrix script | |
id: run-python-script | |
run: python scripts/build-job-matrix.py ${{ steps.image-list.outputs.images }} | |
- name: Read JSON and set as output | |
id: set_matrix | |
run: | | |
json_content=$(cat scripts/grouped_output.json) | |
echo "::set-output name=matrix::$(echo $json_content | jq -c '.include')" | |
- name: Verify scripts/output.json | |
run: | | |
echo "Content of scripts/grouped_output.json:" | |
cat scripts/grouped_output.json | |
build: | |
runs-on: ubuntu-latest | |
needs: build-image-list | |
if: needs.build-image-list.outputs.images != '["dummy"]' | |
# Prevent a single build failure from killing the workflow. | |
# This is safe since subsequent pushes should fail on cache load | |
continue-on-error: true | |
strategy: | |
fail-fast: false | |
matrix: | |
include: ${{ fromJson(needs.build-image-list.outputs.matrix) }} | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Print raw matrix configuration | |
run: | | |
echo "Raw matrix configuration: ${{ matrix }}" | |
- name: Set environment variables | |
id: set-env-vars | |
run: | | |
CONFIG="${{ matrix.config }}" | |
BASE_OS=$(echo $CONFIG | awk -F'-' '{print $1}') | |
OSG_SERIES=$(echo $CONFIG | awk -F'-' '{print $2}') | |
BASE_REPO=$(echo $CONFIG | awk -F'-' '{print $3}') | |
CONTEXT="opensciencegrid/${{ matrix.name }}" | |
echo "BASE_OS=${BASE_OS}" >> $GITHUB_ENV | |
echo "OSG_SERIES=${OSG_SERIES}" >> $GITHUB_ENV | |
echo "BASE_REPO=${BASE_REPO}" >> $GITHUB_ENV | |
echo "CONTEXT=${CONTEXT}" >> $GITHUB_ENV | |
- name: Validate Environment Variables | |
run: | | |
echo "Validating environment variables:" | |
if [ -z "$BASE_OS" ] || [ -z "$OSG_SERIES" ] || [ -z "$BASE_REPO" ] || [ -z "$CONTEXT" ]; then | |
echo "Error: One or more environment variables are not set." | |
exit 1 | |
else | |
echo "All required environment variables are set." | |
fi | |
- name: Build Image | |
continue-on-error: ${{ matrix.yum_repo == 'development' }} | |
uses: opensciencegrid/[email protected] | |
with: | |
osg_series: ${{ env.OSG_SERIES }} | |
osg_repo: ${{ env.BASE_REPO }} | |
context: ${{ env.CONTEXT }} | |
base_os: ${{ env.BASE_OS }} | |
push: | |
runs-on: ubuntu-latest | |
if: >- | |
github.ref == 'refs/heads/main' && | |
github.event_name != 'pull_request' && | |
github.repository_owner == 'opensciencegrid' && | |
needs.build-image-list.outputs.images != '["dummy"]' | |
needs: [make-date-tag, build-image-list, build] | |
strategy: | |
fail-fast: false | |
matrix: | |
include: ${{ fromJson(needs.build-image-list.outputs.matrix) }} | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Print raw matrix configuration | |
run: | | |
echo "Raw matrix configuration: ${{ matrix }}" | |
- name: Set environment variables | |
id: set-env-vars | |
run: | | |
CONFIG="${{ matrix.config }}" | |
BASE_OS=$(echo $CONFIG | awk -F'-' '{print $1}') | |
OSG_SERIES=$(echo $CONFIG | awk -F'-' '{print $2}') | |
BASE_REPO=$(echo $CONFIG | awk -F'-' '{print $3}') | |
CONTEXT="opensciencegrid/${{ matrix.name }}" | |
echo "BASE_OS=${BASE_OS}" >> $GITHUB_ENV | |
echo "OSG_SERIES=${OSG_SERIES}" >> $GITHUB_ENV | |
echo "BASE_REPO=${BASE_REPO}" >> $GITHUB_ENV | |
echo "CONTEXT=${CONTEXT}" >> $GITHUB_ENV | |
- name: Validate Environment Variables | |
run: | | |
echo "Validating environment variables:" | |
if [ -z "$BASE_OS" ] || [ -z "$OSG_SERIES" ] || [ -z "$BASE_REPO" ] || [ -z "$CONTEXT" ]; then | |
echo "Error: One or more environment variables are not set." | |
exit 1 | |
else | |
echo "All required environment variables are set." | |
fi | |
- name: Push to Harbor (${{ env.OSG_SERIES }}-development) | |
if: env.OSG_SERIES != '23' | |
uses: opensciencegrid/push-container-action@main | |
with: | |
repo: development | |
osg_series: ${{ env.OSG_SERIES }} | |
context: ${{ env.CONTEXT }} | |
base_os: ${{ env.BASE_OS }} | |
registry_url: hub.opensciencegrid.org | |
registry_user: ${{ secrets.OSG_HARBOR_ROBOT_USER }} | |
registry_pass: ${{ secrets.OSG_HARBOR_ROBOT_PASSWORD }} | |
- name: Push to Harbor (${{ env.OSG_SERIES }}-testing) | |
if: always() | |
uses: opensciencegrid/push-container-action@main | |
with: | |
repo: testing | |
osg_series: ${{ env.OSG_SERIES }} | |
context: ${{ env.CONTEXT }} | |
base_os: ${{ env.BASE_OS }} | |
registry_url: hub.opensciencegrid.org | |
registry_user: ${{ secrets.OSG_HARBOR_ROBOT_USER }} | |
registry_pass: ${{ secrets.OSG_HARBOR_ROBOT_PASSWORD }} | |
- name: Push to Harbor (${{ env.OSG_SERIES }}-release) | |
if: always() | |
uses: opensciencegrid/push-container-action@main | |
with: | |
repo: release | |
osg_series: ${{ env.OSG_SERIES }} | |
context: ${{ env.CONTEXT }} | |
base_os: ${{ env.BASE_OS }} | |
registry_url: hub.opensciencegrid.org | |
registry_user: ${{ secrets.OSG_HARBOR_ROBOT_USER }} | |
registry_pass: ${{ secrets.OSG_HARBOR_ROBOT_PASSWORD }} | |
- name: Push to Docker Hub (${{ env.OSG_SERIES }}-development) | |
if: always() | |
uses: opensciencegrid/push-container-action@main | |
with: | |
repo: development | |
osg_series: ${{ env.OSG_SERIES }} | |
context: ${{ env.CONTEXT }} | |
base_os: ${{ env.BASE_OS }} | |
registry_url: docker.io | |
registry_user: ${{ secrets.DOCKER_USERNAME }} | |
registry_pass: ${{ secrets.DOCKER_PASSWORD }} | |
- name: Push to Docker Hub (${{ env.OSG_SERIES }}-testing) | |
if: always() | |
uses: opensciencegrid/push-container-action@main | |
with: | |
repo: testing | |
osg_series: ${{ env.OSG_SERIES }} | |
context: ${{ env.CONTEXT }} | |
base_os: ${{ env.BASE_OS }} | |
registry_url: docker.io | |
registry_user: ${{ secrets.DOCKER_USERNAME }} | |
registry_pass: ${{ secrets.DOCKER_PASSWORD }} | |
- name: Push to Docker Hub (${{env.OSG_SERIES }}-release) | |
if: always() | |
uses: opensciencegrid/push-container-action@main | |
with: | |
repo: release | |
osg_series: ${{ env.OSG_SERIES }} | |
context: ${{ env.CONTEXT }} | |
base_os: ${{ env.BASE_OS }} | |
registry_url: docker.io | |
registry_user: ${{ secrets.DOCKER_USERNAME }} | |
registry_pass: ${{ secrets.DOCKER_PASSWORD }} |