Merge pull request #639 from aerospike/2024-Pipeline-Improvement

Scaffolded new CI/CD pipeline

.github/actions/get-artifact-for-stage-tests/action.yml

@@ -0,0 +1,82 @@
+name: 'Get artifact for stage tests'
+description: 'Downloads artifact either from Github artifacts or JFrog to current working dir'
+inputs:
+  get_from_jfrog:
+    description: Boolean. If false, get artifacts from Github
+    required: true
+  jfrog_build_version:
+    description: If getting from JFrog, what build version to get the artifact from?
+    required: false
+  dist_type_to_get:
+    description: 'Type of distribution to get (possible values: sdist, wheel)'
+    required: true
+  binding_nodejs_version:
+    description: 'If getting wheel, specify Nodejs version of binding (e.g possible value: 18)'
+    required: false
+  wheel_os:
+    description: 'If getting wheel, what os is it built for? Must be inside the wheel name / cibw build identifier (possible values: macosx, manylinux)'
+    required: false
+  wheel_cpu_arch:
+    description: 'If getting wheel, what CPU arch is it built for? Must be the same in wheel name / cibw build identifier used to build wheel (e.g possible value: x86_64)'
+  # Secrets
+  JFROG_PLATFORM_URL:
+    required: false
+  JFROG_ACCESS_TOKEN:
+    required: false
+  # Variables
+  JFROG_REPO_NAME:
+    required: false
+
+runs:
+  using: 'composite'
+  steps:
+  - if: ${{ inputs.get_from_jfrog == 'false' && inputs.dist_type_to_get == 'sdist' }}
+    name: 'Cat 1: Get Github artifact name for source distribution'
+    run: echo "GITHUB_ARTIFACT_NAME=sdist.build" >> $GITHUB_ENV
+    shell: bash
+
+  # If getting a wheel from Github, construct artifact name containing that wheel
+  # The artifact name is the build identifier used in cibuildwheel to build that wheel
+
+  # We also need the Nodejs tag for searching a wheel in JFrog
+  - name: 'Cat 2: Get Nodejs tag for build identifier'
+    if: ${{ inputs.dist_type_to_get == 'wheel' }}
+    # example: 3.9 -> cp39
+    run: echo "NODEJS_TAG=cp$(echo ${{ inputs.binding_nodejs_version }} | tr -d '.')" >> $GITHUB_ENV
+    shell: bash
+
+  - if: ${{ inputs.get_from_jfrog == 'false' && inputs.dist_type_to_get == 'wheel' }}
+    run: echo "GITHUB_ARTIFACT_NAME=${{ env.NODEJS_TAG }}-${{ inputs.wheel_os }}_${{ inputs.wheel_cpu_arch }}.build" >> $GITHUB_ENV
+    shell: bash
+
+  - uses: actions/download-artifact@v4
+    if: ${{ inputs.get_from_jfrog == 'false' }}
+    with:
+      name: ${{ env.GITHUB_ARTIFACT_NAME }}
+
+  # Either way when we download from JFrog or Github,
+  # we need the file name pattern to install the artifact using pip later on
+
+  - name: 'Using JFrog: Get file name glob pattern for sdist'
+    if: ${{ inputs.dist_type_to_get == 'sdist' }}
+    run: echo "ARTIFACT_FILE_NAME_PATTERN=*.tar.gz" >> $GITHUB_ENV
+    shell: bash
+
+  - name: 'Using JFrog: Get file name glob pattern for wheel'
+    if: ${{ inputs.dist_type_to_get == 'wheel' }}
+    run: echo "ARTIFACT_FILE_NAME_PATTERN=*${{ env.NODEJS_TAG }}*${{ inputs.wheel_os }}*${{ inputs.wheel_cpu_arch }}.whl" >> $GITHUB_ENV
+    shell: bash
+
+  # End codepath that downloads artifacts from Github
+  # Begin codepath that downloads from JFrog
+
+  - uses: jfrog/setup-jfrog-cli@v4
+    if: ${{ inputs.get_from_jfrog == 'true' }}
+    env:
+      JF_URL: ${{ inputs.JFROG_PLATFORM_URL }}
+      JF_ACCESS_TOKEN: ${{ inputs.JFROG_ACCESS_TOKEN }}
+
+  - name: Download artifact from JFrog
+    if: ${{ inputs.get_from_jfrog == 'true' }}
+    run: jf rt dl --fail-no-op --flat --build nodejs-client/${{ inputs.jfrog_build_version }} "${{ inputs.JFROG_REPO_NAME }}/**/${{ env.ARTIFACT_FILE_NAME_PATTERN }}"
+    shell: bash

.github/actions/setup-docker-on-macos/action.yml

@@ -0,0 +1,17 @@
+name: 'Install Docker on macOS runner'
+description: 'Install Docker using colima'
+
+runs:
+  using: "composite"
+  steps:
+  - name: Install Docker Engine
+    run: brew install colima
+    shell: bash
+
+  - name: Install Docker client
+    run: brew install docker
+    shell: bash
+
+  - name: Start Docker Engine
+    run: colima start
+    shell: bash

.github/actions/update-version/action.yml

@@ -0,0 +1,13 @@
+name: 'Update version'
+description: 'Update version in repo without committing. Repo must already be checked out'
+inputs:
+  new_version:
+    description: Version string to set
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+  - name: Update VERSION metadata
+    run: echo ${{ inputs.new_version }} > VERSION
+    shell: bash

.github/workflows/build-artifacts.yml

@@ -0,0 +1,105 @@
+name: Build artifacts
+run-name: Build artifacts (run_tests=${{ inputs.run_tests }}, use-server-rc=${{ inputs.use-server-rc }}, server-tag=${{ inputs.server-tag }})
+
+# Builds manylinux wheels and source distribution
+# Optionally run tests on manylinux wheels
+# Then upload artifacts to Github
+
+on:
+  workflow_dispatch:
+    inputs:
+      # There may be a case where we want to test these build debug flags on all platforms that support them
+      unoptimized:
+        description: 'macOS or Linux: Apply -O0 flag?'
+        type: boolean
+        required: false
+        default: false
+      include-debug-info-for-macos:
+        description: 'macOS: Build wheels for debugging?'
+        type: boolean
+        required: false
+        default: false
+      run_tests:
+        description: "Run integration tests with the wheels after building them"
+        required: true
+        type: boolean
+        default: false
+      use-server-rc:
+        type: boolean
+        required: true
+        default: false
+        description: 'Test against server release candidate? (e.g to test new server features)'
+      server-tag:
+        type: string
+        required: true
+        default: 'latest'
+        description: 'Server docker image tag (e.g to test a client backport version)'
+
+  workflow_call:
+    inputs:
+      # The "dev" tests test the artifacts against a server
+      # The dev-to-stage and stage-to-master workflow only need to build the artifacts, not test them
+      run_tests:
+        required: false
+        type: boolean
+        default: false
+      # workflow_call hack
+      is_workflow_call:
+        type: boolean
+        default: true
+        required: false
+      # This input is only used in workflow_call events
+      sha-to-build-and-test:
+        description: A calling workflow may want to run this workflow on a different ref than the calling workflow's ref
+        type: string
+        # Make it required to make things simple
+        required: true
+      # A calling workflow doesn't actually set values to the inputs below
+      # But that workflow needs to have default values for these inputs
+      unoptimized:
+        type: boolean
+        required: false
+        default: false
+      include-debug-info-for-macos:
+        type: boolean
+        required: false
+        default: false
+      use-server-rc:
+        required: false
+        default: false
+        type: boolean
+      server-tag:
+        type: string
+        required: false
+        default: 'latest'
+    secrets:
+      DOCKER_HUB_BOT_USERNAME:
+        required: true
+      DOCKER_HUB_BOT_PW:
+        required: true
+      MAC_M1_SELF_HOSTED_RUNNER_PW:
+        required: true
+
+jobs:
+  build-bindings:
+    strategy:
+      matrix:
+        platform-tag: [
+          "manylinux_x86_64",
+          "manylinux_aarch64",
+          "macosx_x86_64",
+          "macosx_arm64",
+          "win_amd64"
+        ]
+      fail-fast: false
+    uses: ./.github/workflows/build-bindings.yml
+    with:
+      platform-tag: ${{ matrix.platform-tag }}
+      # Can't use env context here, so just copy from build-sdist env var
+      sha-to-build-and-test: ${{ inputs.is_workflow_call == true && inputs.sha-to-build-and-test || github.sha }}
+      unoptimized: ${{ inputs.unoptimized }}
+      include-debug-info-for-macos: ${{ inputs.include-debug-info-for-macos }}
+      run_tests: ${{ inputs.run_tests }}
+      use-server-rc: ${{ inputs.use-server-rc }}
+      server-tag: ${{ inputs.server-tag }}
+    secrets: inherit