Feature / Azure platform storage #1153
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Packaging | |
| on: | |
| # Build packages for pull requests, to make sure there are no packaging issues | |
| pull_request: | |
| # Re-run packaging jobs in main, to make sure there are no issues from the merge | |
| push: | |
| branches: | |
| - main | |
| # Build packages when releases are published on GitHub | |
| # Use the release:publish event rather than looking at all tags | |
| # Only release events will trigger the publishing jobs | |
| release: | |
| types: | |
| - published | |
| # Allow manual triggering of the packaging jobs | |
| # This will not publish packages, but they will be available as workflow assets | |
| workflow_dispatch: | |
| env: | |
| JAVA_VERSION: 11 | |
| JAVA_DISTRIBUTION: zulu | |
| PYTHON_VERSION: "3.11" | |
| NODE_VERSION: "16.x" | |
| jobs: | |
| # Java publishing is managed through Gradle, which does both build and publish | |
| # There is no neat way to split the build / publish phases across jobs | |
| # So, include publish to Maven as a conditional step in the build job | |
| platform_packages: | |
| runs-on: ubuntu-latest | |
| # BUILD_sql_h2 is turned on by default in plugins.gradle, for easy setup using the H2 SQL driver | |
| # However to publish packages, we do not want to include any SQL drivers by default | |
| # Consumers of the platform must decide which SQL drivers to include as part of the installation | |
| # Later we may create quick-start / sandbox images, in which case a default SQL driver could be included | |
| env: | |
| BUILD_sql_h2: false | |
| steps: | |
| # fetch-depth = 0 is needed to get tags for version info | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Java | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: ${{ env.JAVA_DISTRIBUTION }} | |
| java-version: ${{ env.JAVA_VERSION }} | |
| # Turn on Gradle dependency caching | |
| cache: gradle | |
| - name: Build JARs | |
| run: ./gradlew jar javadocJar sourcesJar -PincludeJavadoc=true -PincludeSources=true | |
| - name: Build packages for distribution | |
| run: ./gradlew installDist | |
| - name: Assemble platform package | |
| run: | | |
| VERSION=`dev/version.sh` | |
| mkdir -p build/dist/tracdap-platform-${VERSION} | |
| for MODULE in build/modules/*/install/*; do | |
| cp -R $MODULE build/dist/tracdap-platform-${VERSION} | |
| done | |
| cd build/dist | |
| tar -czvf tracdap-platform-${VERSION}.tgz tracdap-platform-${VERSION}/* | |
| - name: Assemble sandbox package | |
| run: | | |
| VERSION=`dev/version.sh` | |
| mkdir -p build/dist/tracdap-sandbox-${VERSION} | |
| for MODULE in build/modules/*/install/*; do | |
| cp -R $MODULE/* build/dist/tracdap-sandbox-${VERSION} | |
| done | |
| cd build/dist | |
| zip -r tracdap-sandbox-${VERSION}.zip tracdap-sandbox-${VERSION} | |
| # The dist for each plugin includes all its dependency JARs | |
| # We filter out JARs that are already included as part of the TRAC platform | |
| # This avoids putting the same JAR on the classpath twice when a plugin is installed | |
| - name: Assemble plugins package | |
| run: | | |
| VERSION=`dev/version.sh` | |
| mkdir -p build/dist/tracdap-plugins-${VERSION} | |
| for PLUGIN in build/plugins/*/install/*; do | |
| PLUGIN_NAME=`basename ${PLUGIN}` | |
| mkdir build/dist/tracdap-plugins-${VERSION}/${PLUGIN_NAME} | |
| cp ${PLUGIN}/*.jar build/dist/tracdap-plugins-${VERSION}/${PLUGIN_NAME}/ | |
| for JAR in ${PLUGIN}/lib/*.jar; do | |
| JAR_NAME=`basename ${JAR}` | |
| if [ ! -f build/dist/tracdap-sandbox-${VERSION}/lib/${JAR_NAME} ]; then | |
| cp ${JAR} build/dist/tracdap-plugins-${VERSION}/${PLUGIN_NAME}/ | |
| fi | |
| done | |
| done | |
| cd build/dist | |
| tar -czvf tracdap-plugins-${VERSION}.tgz tracdap-plugins-${VERSION}/* | |
| - name: Save packages | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: platform_packages | |
| path: | | |
| build/dist/*.tgz | |
| build/dist/*.zip | |
| retention-days: 7 | |
| # GitHub Secrets are not available in pull request builds, to prevent forked repos stealing secrets. | |
| # Secrets are needed to build the Java artifacts, which use GPG signing | |
| # Solution is to disable the artifact build on PRs (it will still run on merge to main) | |
| # It is possible to work around this, but doing so would open up the secret-stealing vulnerability | |
| - name: Build Maven artifacts | |
| if: ${{ github.event_name != 'pull_request' }} | |
| run: | | |
| mkdir -p ~/.gnupg/ | |
| printf "$GPG_KEY_BASE64" | base64 --decode > ~/.gnupg/keyring.gpg | |
| ./gradlew publishToMavenLocal \ | |
| -PincludeJavadoc=true \ | |
| -PincludeSources=true \ | |
| -Psigning.secretKeyRingFile=$HOME/.gnupg/keyring.gpg \ | |
| -Psigning.keyId=$GPG_KEY_ID \ | |
| -Psigning.password=$GPG_KEY_PASSPHRASE | |
| env: | |
| GPG_KEY_BASE64: ${{ secrets.GPG_KEY_BASE64 }} | |
| GPG_KEY_ID: ${{ secrets.GPG_KEY_ID }} | |
| GPG_KEY_PASSPHRASE: ${{ secrets.GPG_KEY_PASSPHRASE }} | |
| - name: Publish to Maven Central | |
| if: ${{ github.event_name == 'release' && github.event.action == 'published' }} | |
| run: | | |
| mkdir -p ~/.gnupg/ | |
| printf "$GPG_KEY_BASE64" | base64 --decode > ~/.gnupg/keyring.gpg | |
| ./gradlew \ | |
| publishToSonatype closeAndReleaseSonatypeStagingRepository \ | |
| -PincludeJavadoc=true \ | |
| -PincludeSources=true \ | |
| -PsonatypeUsername=$MAVEN_USERNAME \ | |
| -PsonatypePassword=$MAVEN_PASSWORD \ | |
| -Psigning.secretKeyRingFile=$HOME/.gnupg/keyring.gpg \ | |
| -Psigning.keyId=$GPG_KEY_ID \ | |
| -Psigning.password=$GPG_KEY_PASSPHRASE | |
| env: | |
| MAVEN_USERNAME: ${{ secrets.MAVEN_USERNAME }} | |
| MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }} | |
| GPG_KEY_BASE64: ${{ secrets.GPG_KEY_BASE64 }} | |
| GPG_KEY_ID: ${{ secrets.GPG_KEY_ID }} | |
| GPG_KEY_PASSPHRASE: ${{ secrets.GPG_KEY_PASSPHRASE }} | |
| python_runtime_package: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: PIP Upgrade | |
| run: python -m pip install --upgrade pip | |
| # fetch-depth = 0 is needed to get tags for version info | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install build environment dependencies | |
| run: | | |
| pip install -r tracdap-runtime/python/requirements.txt | |
| - name: Build runtime package | |
| run: python tracdap-runtime/python/build_runtime.py --target codegen dist | |
| - name: Save artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: python_runtime_package | |
| path: tracdap-runtime/python/build/dist | |
| retention-days: 7 | |
| web_api_package: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| # fetch-depth = 0 is needed to get tags for version info | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install dependencies | |
| run: | | |
| cd tracdap-api/packages/web | |
| npm install | |
| - name: Set TRAC version | |
| run: | | |
| cd tracdap-api/packages/web | |
| npm run tracVersion:posix | |
| - name: Build API | |
| run: | | |
| cd tracdap-api/packages/web | |
| npm run buildApi | |
| - name: Create tarball | |
| run: | | |
| cd tracdap-api/packages/web | |
| mkdir dist | |
| cd dist | |
| npm pack .. | |
| - name: Save artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: web_api_package | |
| path: tracdap-api/packages/web/dist | |
| retention-days: 7 | |
| publish_to_github: | |
| if: ${{ github.event_name == 'release' && github.event.action == 'published' }} | |
| runs-on: ubuntu-latest | |
| needs: | |
| - platform_packages | |
| - python_runtime_package | |
| - web_api_package | |
| steps: | |
| # fetch-depth = 0 is needed to get tags for version info | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Get TRAC version | |
| id: tracdap-version | |
| run: | | |
| tracdap_version=`dev/version.sh` | |
| echo "tracdap_version = ${tracdap_version}" | |
| echo "tracdap_version=${tracdap_version}" >> $GITHUB_OUTPUT | |
| - name: Fetch platform and sandbox packages | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: platform_packages | |
| path: . | |
| - name: Fetch Python runtime package | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: python_runtime_package | |
| path: tracdap-api-packages-${{ steps.tracdap-version.outputs.tracdap_version }}/python_runtime_package | |
| - name: Fetch web API package | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: web_api_package | |
| path: tracdap-api-packages-${{ steps.tracdap-version.outputs.tracdap_version }}/web_api_package | |
| - name: Build API packages tarball | |
| run: tar -cvzf tracdap-api-packages-${{ steps.tracdap-version.outputs.tracdap_version }}.tgz tracdap-api-packages-${{ steps.tracdap-version.outputs.tracdap_version }}/ | |
| - name: Publish platform package | |
| uses: actions/upload-release-asset@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| upload_url: ${{ github.event.release.upload_url }} | |
| asset_path: tracdap-platform-${{ steps.tracdap-version.outputs.tracdap_version }}.tgz | |
| asset_name: tracdap-platform-${{ steps.tracdap-version.outputs.tracdap_version }}.tgz | |
| asset_content_type: application/gzip | |
| - name: Publish sandbox package | |
| uses: actions/upload-release-asset@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| upload_url: ${{ github.event.release.upload_url }} | |
| asset_path: tracdap-sandbox-${{ steps.tracdap-version.outputs.tracdap_version }}.zip | |
| asset_name: tracdap-sandbox-${{ steps.tracdap-version.outputs.tracdap_version }}.zip | |
| asset_content_type: application/zip | |
| - name: Publish plugins package | |
| uses: actions/upload-release-asset@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| upload_url: ${{ github.event.release.upload_url }} | |
| asset_path: tracdap-plugins-${{ steps.tracdap-version.outputs.tracdap_version }}.tgz | |
| asset_name: tracdap-plugins-${{ steps.tracdap-version.outputs.tracdap_version }}.tgz | |
| asset_content_type: application/gzip | |
| - name: Publish API packages | |
| uses: actions/upload-release-asset@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| upload_url: ${{ github.event.release.upload_url }} | |
| asset_path: tracdap-api-packages-${{ steps.tracdap-version.outputs.tracdap_version }}.tgz | |
| asset_name: tracdap-api-packages-${{ steps.tracdap-version.outputs.tracdap_version }}.tgz | |
| asset_content_type: application/gzip | |
| publish_to_pypi: | |
| if: ${{ github.event_name == 'release' && github.event.action == 'published' }} | |
| runs-on: ubuntu-latest | |
| needs: | |
| - python_runtime_package | |
| steps: | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install Twine | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install twine | |
| - name: Fetch Python runtime package | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: python_runtime_package | |
| path: tracdap_dist/ | |
| - name: Publish to PyPI | |
| env: | |
| TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} | |
| run: | | |
| ls tracdap_dist/* | |
| twine upload --username __token__ tracdap_dist/* | |
| publish_to_npm: | |
| if: ${{ github.event_name == 'release' && github.event.action == 'published' }} | |
| runs-on: ubuntu-latest | |
| needs: | |
| - web_api_package | |
| steps: | |
| # fetch-depth = 0 is needed to get tags for version info | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Get TRAC version | |
| id: tracdap-version | |
| run: | | |
| tracdap_version=`dev/version.sh` | |
| echo "tracdap_version = ${tracdap_version}" | |
| echo "::set-output name=tracdap_version::${tracdap_version}" | |
| tracdap_tag=`dev/version_tag.sh ${tracdap_version}` | |
| echo "tracdap_tag = ${tracdap_tag}" | |
| echo "tracdap_tag=${tracdap_tag}" >> $GITHUB_OUTPUT | |
| - name: Fetch web API package | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: web_api_package | |
| path: tracdap_dist/ | |
| # NPM publish wants a folder in NPM layout | |
| # So, extract the tarball and publish using the NPM package files | |
| # Also, add an entry in .npmrc to tell NPM to use an auth token from the environment | |
| - name: Publish to NPM | |
| env: | |
| NPM_TOKEN: ${{ secrets.NPM_TOKEN }} | |
| NPM_TAG: ${{ steps.tracdap-version.outputs.tracdap_tag }} | |
| run: | | |
| tar -xzvf tracdap_dist/* | |
| cd package | |
| echo //registry.npmjs.org/:_authToken=\$\{NPM_TOKEN\} >> .npmrc | |
| npm publish --access public --tag ${NPM_TAG} |