From ed8b671fd05c72333039cf963e80ddf9ab6ec128 Mon Sep 17 00:00:00 2001 From: apbassett <43486400+apbassett@users.noreply.github.com> Date: Wed, 18 Oct 2023 15:42:21 -0400 Subject: [PATCH] Adds workflow summary, misc workflow changes --- .github/templates/version_summary.md | 6 + .../{build-test-package.yml => build.yml} | 65 +++++++ .github/workflows/create-branch-build.yml | 9 +- .github/workflows/create-pr-build.yml | 9 +- .github/workflows/create-release-build.yml | 3 +- .github/workflows/get-dependency-details.yml | 17 +- .github/workflows/release.yml | 65 ------- .github/workflows/workflow-summary.yml | 67 +++++++ bin/build.sh | 171 ------------------ 9 files changed, 165 insertions(+), 247 deletions(-) create mode 100644 .github/templates/version_summary.md rename .github/workflows/{build-test-package.yml => build.yml} (77%) delete mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/workflow-summary.yml delete mode 100644 bin/build.sh diff --git a/.github/templates/version_summary.md b/.github/templates/version_summary.md new file mode 100644 index 0000000..748d281 --- /dev/null +++ b/.github/templates/version_summary.md @@ -0,0 +1,6 @@ +## This workflow run was built with the following packages: +
+ +| Name | Version | Notes | +| :-- | :-- | :-- | +| amalgam | {amalgam-version} | {amalgam-notes} | \ No newline at end of file diff --git a/.github/workflows/build-test-package.yml b/.github/workflows/build.yml similarity index 77% rename from .github/workflows/build-test-package.yml rename to .github/workflows/build.yml index c9b5afb..133b538 100644 --- a/.github/workflows/build-test-package.yml +++ b/.github/workflows/build.yml @@ -9,6 +9,9 @@ on: payload: required: false type: string + build-type: + required: false + type: string jobs: @@ -120,6 +123,14 @@ jobs: sed -i 's/dependencies/version/g' version.json find . -type d -name lib -exec sh -c 'mv {}/* "$(dirname {})"' \; + - name: Set workflow run info + if: needs.get-dependency-details.outputs.build-date != '' + run: | + cd amalgam/lib + jq '.version |= . + {"amalgam_display_title": ${{ needs.get-dependency-details.outputs.build-title }}}' version.json > temp.json && mv temp.json version.json + jq '.version |= . + {"amalgam_display_date": ${{ needs.get-dependency-details.outputs.build-date }}}' version.json > temp.json && mv temp.json version.json + cat version.json + - name: Build wheels run: | python3 -m build --wheel --outdir wheelhouse/ . @@ -150,6 +161,14 @@ jobs: path: dist/amalgam_lang-*.whl if-no-files-found: error + workflow-summary: + needs: ["build"] + uses: "./.github/workflows/workflow-summary.yml" + secrets: inherit + with: + payload: "${{ inputs.payload }}" + build-type: "${{ inputs.build-type }}" + test-3-8: needs: ["build"] uses: "./.github/workflows/pytest.yml" @@ -177,3 +196,49 @@ jobs: secrets: inherit with: python-version: "3.11" + + create-release: + if: inputs.build-type == 'release' + needs: ['build', 'test-3-8', 'test-3-9', 'test-3-10', 'test-3-11'] + runs-on: ubuntu-latest + environment: + name: pypi + permissions: + contents: write + id-token: write + + steps: + + - uses: actions/checkout@v3 + + - name: Download Artifacts + uses: actions/download-artifact@v3 + with: + path: ./tmp + + - name: Clean up dir + run: | + mkdir -p dist + find ./tmp -type f -name '*.whl' -exec mv -t ./dist {} + + find ./tmp -type f -name '*.tar.gz' -exec mv -t ./dist {} + + ls ./dist + + - name: Create Release + uses: ncipollo/release-action@v1 + with: + tag: ${{ inputs.version }} + commit: ${{ github.sha }} + name: "amalgam-lang-py ${{ inputs.version }}" + artifactErrorsFailBuild: true + generateReleaseNotes: true + makeLatest: legacy + artifacts: "dist/*" + artifactContentType: application/gzip + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 \ No newline at end of file diff --git a/.github/workflows/create-branch-build.yml b/.github/workflows/create-branch-build.yml index 996ba27..ac1b979 100644 --- a/.github/workflows/create-branch-build.yml +++ b/.github/workflows/create-branch-build.yml @@ -58,18 +58,19 @@ jobs: BRANCH_ITERATION=${{ github.run_attempt }}.${{ github.run_number }} echo "version=$(echo ${{ steps.next-semvers.outputs.patch }}-alpha+BR.${{ github.ref_name }}.${BRANCH_ITERATION})" >> $GITHUB_OUTPUT - build-test-package: + build: needs: ['set-branch-version'] - uses: "./.github/workflows/build-test-package.yml" + uses: "./.github/workflows/build.yml" secrets: inherit with: version: ${{ needs.set-branch-version.outputs.version }} payload: ${{ inputs.payload }} + build-type: 'branch' # This job is here to have only one final step to add for "Status Checks" - # in GitHub, instead of adding every leaf test from 'build-test-package' + # in GitHub, instead of adding every leaf test from 'build' final-check: - needs: ['build-test-package'] + needs: ['build'] if: always() && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) runs-on: ubuntu-latest steps: diff --git a/.github/workflows/create-pr-build.yml b/.github/workflows/create-pr-build.yml index b84cebe..0eb088e 100644 --- a/.github/workflows/create-pr-build.yml +++ b/.github/workflows/create-pr-build.yml @@ -45,17 +45,18 @@ jobs: PR_ITERATION=${{ github.run_attempt }}.${{ github.run_number }} echo "version=$(echo ${{ steps.next-semvers.outputs.patch }}-alpha+PR.${PR_NUMBER}.${PR_ITERATION})" >> $GITHUB_OUTPUT - build-test-package: + build: needs: ['set-pr-version'] - uses: "./.github/workflows/build-test-package.yml" + uses: "./.github/workflows/build.yml" secrets: inherit with: version: ${{ needs.set-pr-version.outputs.version }} + build-type: 'PR' # This job is here to have only one final step to add for "Status Checks" - # in GitHub, instead of adding every leaf test from 'build-test-package' + # in GitHub, instead of adding every leaf test from 'build' final-check: - needs: ['build-test-package'] + needs: ['build'] if: always() && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) runs-on: ubuntu-latest steps: diff --git a/.github/workflows/create-release-build.yml b/.github/workflows/create-release-build.yml index f96afff..a547d34 100644 --- a/.github/workflows/create-release-build.yml +++ b/.github/workflows/create-release-build.yml @@ -110,7 +110,8 @@ jobs: release: needs: ['construct-release-tag'] - uses: "./.github/workflows/release.yml" + uses: "./.github/workflows/build.yml" secrets: inherit with: version: ${{ needs.construct-release-tag.outputs.release-tag }} + build-type: 'release' \ No newline at end of file diff --git a/.github/workflows/get-dependency-details.yml b/.github/workflows/get-dependency-details.yml index 33abd94..384f7f1 100644 --- a/.github/workflows/get-dependency-details.yml +++ b/.github/workflows/get-dependency-details.yml @@ -24,6 +24,12 @@ on: run-id: description: "Run id to be used in GitHub CLI command for dependency query" value: ${{ jobs.get-dependency-details.outputs.run-id }} + build-date: + description: "The datetime of the run (non-release)" + value: ${{ jobs.get-dependency-details.outputs.build-date }} + build-title: + description: "The display title of the run (non-release)" + value: ${{ jobs.get-dependency-details.outputs.build-title }} defaults: run: @@ -36,6 +42,8 @@ jobs: outputs: run-type: ${{ steps.dependency.outputs.run-type }} run-id: ${{ steps.dependency.outputs.run-id }} + build-date: ${{ steps.dependency.outputs.build-date }} + build-title: ${{ steps.dependency.outputs.build-title }} steps: - uses: actions/checkout@v3 @@ -48,10 +56,10 @@ jobs: ID="" if ! [[ -n "${{ inputs.payload }}" && $(echo '${{ inputs.payload }}' | jq 'has("${{ inputs.repo }}")') == true ]]; then if [ "${{ inputs.skip-version-json-check }}" = true ]; then - echo "No JSON payload given, and skip-version-json-check flag is set. Exiting." + echo "No JSON payload given with repo ${{ inputs.repo }}, and skip-version-json-check flag is set. Exiting." exit 0 fi - echo "No JSON payload given, using version.json" + echo "No JSON payload given with repo ${{ inputs.repo }}, using version.json" ID=$(jq -r '.dependencies."${{ inputs.repo }}"' version.json) echo "Searching for ${{ inputs.repo }} build id '$ID'..." @@ -76,6 +84,11 @@ jobs: elif gh run view --repo ${{ inputs.owner }}/${{ inputs.repo }} "$ID" > /dev/null 2>&1; then echo "Found non-release build" echo "run-type=$(echo "run")" >> $GITHUB_OUTPUT + # Get the version and build date + build_date=$(gh run view --repo ${{ inputs.owner }}/${{ inputs.repo }} $ID --json createdAt | jq '.createdAt') + build_title=$(gh run view --repo ${{ inputs.owner }}/${{ inputs.repo }} $ID --json displayTitle | jq '.displayTitle') + echo "build-date=$(echo "$build_date")" >> $GITHUB_OUTPUT + echo "build-title=$(echo "$build_title")" >> $GITHUB_OUTPUT else echo "Build not found" exit 1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 54c8224..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Reusable WF - Release - -on: - workflow_call: - inputs: - version: - required: true - type: string - -defaults: - run: - shell: bash - -jobs: - - build-test-package: - uses: "./.github/workflows/build-test-package.yml" - secrets: inherit - with: - version: ${{ inputs.version }} - - create-release: - needs: ['build-test-package'] - runs-on: ubuntu-latest - environment: - name: pypi - permissions: - contents: write - id-token: write - - steps: - - - uses: actions/checkout@v3 - - - name: Download Artifacts - uses: actions/download-artifact@v3 - with: - path: ./tmp - - - name: Clean up dir - run: | - mkdir -p dist - find ./tmp -type f -name '*.whl' -exec mv -t ./dist {} + - find ./tmp -type f -name '*.tar.gz' -exec mv -t ./dist {} + - ls ./dist - - - name: Create Release - uses: ncipollo/release-action@v1 - with: - tag: ${{ inputs.version }} - commit: ${{ github.sha }} - name: "amalgam-lang-py ${{ inputs.version }}" - artifactErrorsFailBuild: true - generateReleaseNotes: true - makeLatest: legacy - artifacts: "dist/*" - artifactContentType: application/gzip - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - - name: Publish distribution to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 \ No newline at end of file diff --git a/.github/workflows/workflow-summary.yml b/.github/workflows/workflow-summary.yml new file mode 100644 index 0000000..882bbdb --- /dev/null +++ b/.github/workflows/workflow-summary.yml @@ -0,0 +1,67 @@ +name: Reusable WF - Output Workflow Summary + +on: + workflow_call: + inputs: + payload: + type: string + required: true + build-type: + type: string + required: true + +defaults: + run: + shell: bash + +jobs: + + output-summary: + runs-on: ubuntu-latest + steps: + + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Download Artifacts + uses: actions/download-artifact@v3 + with: + path: ./tmp + + - name: Clean up dir + run: | + mkdir -p dist + mkdir -p amlg + find ./tmp -type f -name '*.whl' -exec mv -t ./amlg {} + + + - name: Generate workflow summary + env: + GH_TOKEN: ${{ secrets.HOWSOAI_WORKFLOW_AUTOMATION_CLASSIC_TOKEN }} + run: | + pip install --user amlg/*-py3-none-any.whl + + site_packages=$(python -m site --user-site) + + if [[ $(cat $site_packages/amalgam/lib/version.json | jq 'has(".version.amalgam_build_date")') == true ]]; then + # Get amalgam build title and date + build_date=$(jq -r '.version."amalgam_build_date"' $site_packages/amalgam/lib/version.json) + build_date=$(python -c "from datetime import datetime; print(datetime.strptime($build_date, '%Y-%m-%dT%H:%M:%SZ').strftime('%A, %B %d, %I:%M %p'))") + amlg_version=$(jq -r '.version."amalgam_display_title"' $site_packages/amalgam/lib/version.json) + amlg_version=$(echo "$amlg_version ($build_date)") + amlg_notes=$(echo "Prerelease version") + else + # Get amalgam release version + amlg_version=$(jq -r '.version.amalgam' $site_packages/amalgam/lib/version.json) + amlg_notes=$(echo "Release version specified in \`version.json\`") + fi + # amalgam notes + sed -i "s|{amalgam-version}|$amlg_version|g" ./.github/templates/version_summary.md + sed -i "s|{amalgam-notes}|$amlg_notes|g" ./.github/templates/version_summary.md + # Remove unecessary quotations from variable substitutions + sed -i "s|\"||g" ./.github/templates/version_summary.md + # Output filled template file to workflow summary + cat ./.github/templates/version_summary.md >> $GITHUB_STEP_SUMMARY \ No newline at end of file diff --git a/bin/build.sh b/bin/build.sh deleted file mode 100644 index 75ba4b2..0000000 --- a/bin/build.sh +++ /dev/null @@ -1,171 +0,0 @@ -#!/bin/bash -# -# Build functions for the repository - predominantly used by build pipelines, to keep specific build logic output of pipeline yml files -# allowing control of build functions, outside of the generic build templates. -# -# usage: ./bin/build.sh {params} -# -##### - -source config/build.properties -# set -x -set -eu # fail on error, and undefined var usage - -# Updates the appropriate version file (defined in build.properties) -update_version() { - new_version=$1 - echo "Updating version to $new_version" - sed -i "s/__version__ = \(.*\)/__version__ = \"$new_version\"/" ${version_file} - cat ${version_file} -} - -# Logic for getting version information from appropriate .py file -get_version() { - cat ${version_file} | grep '^__version__' | sed 's/.*\=\(.*\)/\1/' | tr -d '"' | tr -d '[:space:]' -} - -# Used by Azure Devops Pipelines to set appropriate git config, for checking in files/tags -setup_git() { - branchName=${1} - git config --global user.email "devops@howso.com" - git config --global user.name "azuredevops" - git checkout $branchName -} - -# Used by Azure Devops Pipelines to add the version update back into git -git_add_version() { - git add ${version_file} -} - -# Install dependencies -install_deps() { - local pyversion=${1} - echo $pyversion - # --force-reinstall fixes an occasional bad installation of pip by Azure Devops, when it adds the python libraries into a container - python -m pip install -U --force-reinstall pip - python --version - pip --version - # Need to avoid building source, for speed, and avoiding build dependency errors - so force wheels, except where required - # Note - when hashes included, behavior of --prefer-binary still used sdist, when wheels available - pip install --only-binary :all: --no-binary ${no_binary_list} -r requirements-${pyversion}.txt -} - -# Install additional test time dependencies -install_test_deps() { - local pyversion=${1} - echo $pyversion - # --force-reinstall fixes an occasional bad installation of pip by Azure Devops, when it adds the python libraries into a container - python -m pip install -U --force-reinstall pip - python --version - pip --version - pip install --prefer-binary -r requirements-dev-${pyversion}.txt -} - -# Run pytest -test() { - local loglevel=${1:-INFO} - # For now doctest removed, as they are not well structured for testing - HOWSO_CONFIG=${HOWSO_CONFIG:-./config/latest-st-howso.yml} pytest -s --log-cli-level=${loglevel} -o junit_family=xunit2 --cov-report term --cov=${source_root_dir} --junitxml=junit/test-results.xml --cov-report=xml -} - -# Generate requirements.txt -# Requires pip-tools - `pip install pip-tools` -gen_requirements() { - local pyversion=${1} - echo $pyversion - rm -fv requirements-${pyversion}.txt - rm -fv requirements-dev-${pyversion}.txt - # https://github.com/jazzband/pip-tools/issues/973 describes use of --allow-unsafe - CUSTOM_COMPILE_COMMAND="./bin/build.sh gen_requirements $pyversion" pip-compile --resolver=backtracking --upgrade requirements.in --no-emit-index-url --allow-unsafe --output-file requirements-${pyversion}.txt - # Adds appropriate reference to newly created requirements-3.x.in to the requirements.dev.in - echo -c "requirements-${pyversion}.txt" | cat - requirements-dev.in > "requirements-dev-${pyversion}.in" - CUSTOM_COMPILE_COMMAND="./bin/build.sh gen_requirements $pyversion" pip-compile --resolver=backtracking --upgrade requirements-dev-${pyversion}.in --no-emit-index-url --output-file requirements-dev-${pyversion}.txt - # Removes temporary file. - rm -fv "requirements-dev-${pyversion}.in" - echo "$(echo '# NOTE - this file is automatically generated during CICD builds, do not update manually' | cat - requirements-${pyversion}.txt)" > requirements-${pyversion}.txt - echo "$(echo '# NOTE - this file is automatically generated during CICD builds, do not update manually' | cat - requirements-dev-${pyversion}.txt)" > requirements-dev-${pyversion}.txt -} - -# Install python modules required for building -install_build_deps() { - pip install twine wheel -} - -# Create license file -gen_licenses() { - pip install pip-licenses - pip-licenses --with-authors --with-urls --with-license-file --with-description --format=plain-vertical > ./LICENSE-3RD-PARTY.txt - # Include any additional non-generated licenses - if [ -f LICENSE-3RD-PARTY-EXTRA.txt ]; then - cat LICENSE-3RD-PARTY-EXTRA.txt >> LICENSE-3RD-PARTY.txt - fi - -} - -# Run pylint analysis -run_pylint() { - # Note - the formatting here allows azure devops to process the build issues - # https://docs.microsoft.com/en-us/azure/devops/pipelines/scripts/logging-commands?view=azure-devops&tabs=bash#example-log-an-error - pylint ${source_root_dir} --exit-zero --msg-template='##vso[task.logissue linenumber={line};columnnumber={column};code={msg_id};sourcepath={path};type=warning;] {category} - {obj} {msg}' -} - -# Run flake8 linter -run_flake8() { - # Note - the formatting here allows azure devops to process the build issues - # https://docs.microsoft.com/en-us/azure/devops/pipelines/scripts/logging-commands?view=azure-devops&tabs=bash#example-log-an-error - flake8 --exit-zero --format='##vso[task.logissue linenumber=%(row)d;columnnumber=%(col)d;code=%(code)s;sourcepath=%(path)s;type=warning;]%(text)s' ${source_root_dir} -} - -# Build the package -build() { - # Builds a source distribution and a universal wheel - python setup.py sdist bdist_wheel -} - -# Upload to artifactory - Note, this is utility for testing - actual uploading is from an Azure Devops Artifactory task -## relies on a file at ~/.pypirc -## -#[distutils] -#index-servers = -# local -# pypi -#[local] -#repository: https://dpbuild.jfrog.io/artifactory/api/pypi/pypi-edge -#username: -#password: -#[pypi] -#username = __token__ -#password = - -_upload() { - local repo=${1:-local} - twine upload -r ${repo} dist/* -} - - -clean() { - rm -rf build dist *.egg-info - rm -rf .pytest_cache - rm -rf .coverage - rm -rf junit - rm -rf htmlcov - rm -rf .eggs -} - - -# Show usage, and print functions -help() { - echo "usage: ./bin/build.sh {params}" - echo " where one of :-" - IFS=$'\n' - for f in $(declare -F); do - echo " ${f:11}" - done -} - -# Takes the cli params, and runs them, defaulting to 'help()' -if [ ! ${1:-} ]; then - help -else - "$@" -fi