diff --git a/.github/actions/get-artifact-for-stage-tests/action.yml b/.github/actions/get-artifact-for-stage-tests/action.yml new file mode 100644 index 00000000..8b91fa5e --- /dev/null +++ b/.github/actions/get-artifact-for-stage-tests/action.yml @@ -0,0 +1,82 @@ +name: 'Get artifact for stage tests' +description: 'Downloads artifact either from Github artifacts or JFrog to current working dir' +inputs: + get_from_jfrog: + description: Boolean. If false, get artifacts from Github + required: true + jfrog_build_version: + description: If getting from JFrog, what build version to get the artifact from? + required: false + dist_type_to_get: + description: 'Type of distribution to get (possible values: sdist, wheel)' + required: true + binding_nodejs_version: + description: 'If getting wheel, specify Nodejs version of binding (e.g possible value: 18)' + required: false + wheel_os: + description: 'If getting wheel, what os is it built for? Must be inside the wheel name / cibw build identifier (possible values: macosx, manylinux)' + required: false + wheel_cpu_arch: + description: 'If getting wheel, what CPU arch is it built for? Must be the same in wheel name / cibw build identifier used to build wheel (e.g possible value: x86_64)' + # Secrets + JFROG_PLATFORM_URL: + required: false + JFROG_ACCESS_TOKEN: + required: false + # Variables + JFROG_REPO_NAME: + required: false + +runs: + using: 'composite' + steps: + - if: ${{ inputs.get_from_jfrog == 'false' && inputs.dist_type_to_get == 'sdist' }} + name: 'Cat 1: Get Github artifact name for source distribution' + run: echo "GITHUB_ARTIFACT_NAME=sdist.build" >> $GITHUB_ENV + shell: bash + + # If getting a wheel from Github, construct artifact name containing that wheel + # The artifact name is the build identifier used in cibuildwheel to build that wheel + + # We also need the Nodejs tag for searching a wheel in JFrog + - name: 'Cat 2: Get Nodejs tag for build identifier' + if: ${{ inputs.dist_type_to_get == 'wheel' }} + # example: 3.9 -> cp39 + run: echo "NODEJS_TAG=cp$(echo ${{ inputs.binding_nodejs_version }} | tr -d '.')" >> $GITHUB_ENV + shell: bash + + - if: ${{ inputs.get_from_jfrog == 'false' && inputs.dist_type_to_get == 'wheel' }} + run: echo "GITHUB_ARTIFACT_NAME=${{ env.NODEJS_TAG }}-${{ inputs.wheel_os }}_${{ inputs.wheel_cpu_arch }}.build" >> $GITHUB_ENV + shell: bash + + - uses: actions/download-artifact@v4 + if: ${{ inputs.get_from_jfrog == 'false' }} + with: + name: ${{ env.GITHUB_ARTIFACT_NAME }} + + # Either way when we download from JFrog or Github, + # we need the file name pattern to install the artifact using pip later on + + - name: 'Using JFrog: Get file name glob pattern for sdist' + if: ${{ inputs.dist_type_to_get == 'sdist' }} + run: echo "ARTIFACT_FILE_NAME_PATTERN=*.tar.gz" >> $GITHUB_ENV + shell: bash + + - name: 'Using JFrog: Get file name glob pattern for wheel' + if: ${{ inputs.dist_type_to_get == 'wheel' }} + run: echo "ARTIFACT_FILE_NAME_PATTERN=*${{ env.NODEJS_TAG }}*${{ inputs.wheel_os }}*${{ inputs.wheel_cpu_arch }}.whl" >> $GITHUB_ENV + shell: bash + + # End codepath that downloads artifacts from Github + # Begin codepath that downloads from JFrog + + - uses: jfrog/setup-jfrog-cli@v4 + if: ${{ inputs.get_from_jfrog == 'true' }} + env: + JF_URL: ${{ inputs.JFROG_PLATFORM_URL }} + JF_ACCESS_TOKEN: ${{ inputs.JFROG_ACCESS_TOKEN }} + + - name: Download artifact from JFrog + if: ${{ inputs.get_from_jfrog == 'true' }} + run: jf rt dl --fail-no-op --flat --build nodejs-client/${{ inputs.jfrog_build_version }} "${{ inputs.JFROG_REPO_NAME }}/**/${{ env.ARTIFACT_FILE_NAME_PATTERN }}" + shell: bash diff --git a/.github/actions/setup-docker-on-macos/action.yml b/.github/actions/setup-docker-on-macos/action.yml new file mode 100644 index 00000000..aa8e1da0 --- /dev/null +++ b/.github/actions/setup-docker-on-macos/action.yml @@ -0,0 +1,17 @@ +name: 'Install Docker on macOS runner' +description: 'Install Docker using colima' + +runs: + using: "composite" + steps: + - name: Install Docker Engine + run: brew install colima + shell: bash + + - name: Install Docker client + run: brew install docker + shell: bash + + - name: Start Docker Engine + run: colima start + shell: bash diff --git a/.github/actions/update-version/action.yml b/.github/actions/update-version/action.yml new file mode 100644 index 00000000..21a30634 --- /dev/null +++ b/.github/actions/update-version/action.yml @@ -0,0 +1,13 @@ +name: 'Update version' +description: 'Update version in repo without committing. Repo must already be checked out' +inputs: + new_version: + description: Version string to set + required: true + +runs: + using: "composite" + steps: + - name: Update VERSION metadata + run: echo ${{ inputs.new_version }} > VERSION + shell: bash diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml new file mode 100644 index 00000000..8a90cae0 --- /dev/null +++ b/.github/workflows/build-artifacts.yml @@ -0,0 +1,105 @@ +name: Build artifacts +run-name: Build artifacts (run_tests=${{ inputs.run_tests }}, use-server-rc=${{ inputs.use-server-rc }}, server-tag=${{ inputs.server-tag }}) + +# Builds manylinux wheels and source distribution +# Optionally run tests on manylinux wheels +# Then upload artifacts to Github + +on: + workflow_dispatch: + inputs: + # There may be a case where we want to test these build debug flags on all platforms that support them + unoptimized: + description: 'macOS or Linux: Apply -O0 flag?' + type: boolean + required: false + default: false + include-debug-info-for-macos: + description: 'macOS: Build wheels for debugging?' + type: boolean + required: false + default: false + run_tests: + description: "Run integration tests with the wheels after building them" + required: true + type: boolean + default: false + use-server-rc: + type: boolean + required: true + default: false + description: 'Test against server release candidate? (e.g to test new server features)' + server-tag: + type: string + required: true + default: 'latest' + description: 'Server docker image tag (e.g to test a client backport version)' + + workflow_call: + inputs: + # The "dev" tests test the artifacts against a server + # The dev-to-stage and stage-to-master workflow only need to build the artifacts, not test them + run_tests: + required: false + type: boolean + default: false + # workflow_call hack + is_workflow_call: + type: boolean + default: true + required: false + # This input is only used in workflow_call events + sha-to-build-and-test: + description: A calling workflow may want to run this workflow on a different ref than the calling workflow's ref + type: string + # Make it required to make things simple + required: true + # A calling workflow doesn't actually set values to the inputs below + # But that workflow needs to have default values for these inputs + unoptimized: + type: boolean + required: false + default: false + include-debug-info-for-macos: + type: boolean + required: false + default: false + use-server-rc: + required: false + default: false + type: boolean + server-tag: + type: string + required: false + default: 'latest' + secrets: + DOCKER_HUB_BOT_USERNAME: + required: true + DOCKER_HUB_BOT_PW: + required: true + MAC_M1_SELF_HOSTED_RUNNER_PW: + required: true + +jobs: + build-bindings: + strategy: + matrix: + platform-tag: [ + "manylinux_x86_64", + "manylinux_aarch64", + "macosx_x86_64", + "macosx_arm64", + "win_amd64" + ] + fail-fast: false + uses: ./.github/workflows/build-bindings.yml + with: + platform-tag: ${{ matrix.platform-tag }} + # Can't use env context here, so just copy from build-sdist env var + sha-to-build-and-test: ${{ inputs.is_workflow_call == true && inputs.sha-to-build-and-test || github.sha }} + unoptimized: ${{ inputs.unoptimized }} + include-debug-info-for-macos: ${{ inputs.include-debug-info-for-macos }} + run_tests: ${{ inputs.run_tests }} + use-server-rc: ${{ inputs.use-server-rc }} + server-tag: ${{ inputs.server-tag }} + secrets: inherit diff --git a/.github/workflows/build-bindings.yml b/.github/workflows/build-bindings.yml index 337c7f3e..338e7352 100644 --- a/.github/workflows/build-bindings.yml +++ b/.github/workflows/build-bindings.yml @@ -1,599 +1,359 @@ -name: Build binding -run-name: Build bindings (run_tests=${{ inputs.run_tests }}, use-server-rc=${{ inputs.use-server-rc }}, server-tag=${{ inputs.server-tag }}) +name: 'Build wheels' +run-name: 'Build wheels (nodejs-tags=${{ inputs.nodejs-tags }}, platform-tag=${{ inputs.platform-tag }}, unoptimized=${{ inputs.unoptimized }}, include-debug-info-for-macos=${{ inputs.include-debug-info-for-macos }}, run_tests=${{ inputs.run_tests }}, use-server-rc=${{ inputs.use-server-rc }}, server-tag=${{ inputs.server-tag }})' -# Builds manylinux bindings and source distribution -# If running tests, publish results in commit status -# Publishes bindings and source distribution on Github Actions if no tests are run or tests pass +# Build wheels on all (or select) Python versions supported by the Python client for a specific platform on: - push: - branches: ["stage", "master"] - pull_request: - branches: ["stage", "master"] - types: [ - # Default triggers - opened, - synchronize, - reopened, - # Additional triggers - labeled, - unlabeled - ] + workflow_dispatch: inputs: - # If we only want to check that the builds pass on an arbitrary branch - run_tests: - description: "Run integration tests" + # These are the usual cases for building wheels: + # + # 1. One wheel for *one* supported Python version. This is for running specialized tests that only need one Python version + # like valgrind or a failing QE test. And usually, we only need one wheel for debugging purposes. + # 2. Wheels for *all* supported Python versions for *one* supported platform. This is useful for testing workflow changes for a + # single OS or CPU architecture (e.g testing that changes to debugging modes work on all Python versions) + # 3. Wheels for *all* supported Python versions and *all* supported platforms. This is for building wheels for different + # CI/CD stages (e.g dev, stage, or master). We can also test debugging modes for all platforms that support them + # + # We're able to combine case 1 and 2 into one workflow by creating an input that takes in a JSON list of strings (Python tags) + # to build wheels for. Actual list inputs aren't supported yet, so it's actually a JSON list encoded as a string. + # + # However, it's harder to combine this workflow (case 1 + 2) with case 3, because matrix outputs don't exist yet + # in Github Actions. So all jobs in the cibuildwheel job would have to pass for a self hosted job to run. + # We want each platform to be tested independently of each other, + # so there is a wrapper workflow that has a list of platforms to test and reuses this workflow for each platform. + # If one platform fails, it will not affect the building and testing of another platform (we disable fail fast mode) + nodejs-tags: + type: string + description: Valid JSON list of Python tags to build the client for + required: false + default: '["v108", "v115", "v127", "v131"]' + platform-tag: + description: Platform to build the client for. + type: choice required: true + options: + - manylinux_x86_64 + - manylinux_aarch64 + - macosx_x86_64 + - macosx_arm64 + - win_amd64 + # Makes debugging via gh cli easier. + default: manylinux_x86_64 + unoptimized: + description: 'macOS or Linux: Apply -O0 flag?' + # Windows supports a different flag to disable optimizations, but we haven't added support for it yet type: boolean - default: true + required: false + default: false + include-debug-info-for-macos: + description: 'macOS: Build wheels for debugging?' + type: boolean + required: false + default: false + run_tests: + description: 'Run Aerospike server and run tests using built wheels?' + type: boolean + required: false + default: false use-server-rc: type: boolean required: true default: false description: 'Test against server release candidate?' - # If we are creating a backport and want to test an arbitrary branch against an older server version server-tag: required: true default: 'latest' description: 'Server docker image tag' + workflow_call: inputs: - # The dev tests test the artifacts against a server release - # The stage tests and release workflow only need to build the artifacts - run_tests: - description: "Run integration tests" - required: true + # See workflow call hack in update-version.yml + is_workflow_call: type: boolean default: true - commit_sha: - type: string required: false - # Calling workflow doesn't actually use these 2 options - # But we need to set default values for workflow calls - use-server-rc: - required: false - default: true - type: boolean - server-tag: + nodejs-tags: type: string required: false - default: 'latest' - secrets: - DOCKER_HUB_BOT_USERNAME: - required: false - DOCKER_HUB_BOT_PW: + default: '["v108", "v115", "v127", "v131"]' + platform-tag: + type: string + required: true + # Only used in workflow_call event + sha-to-build-and-test: + type: string + required: true + unoptimized: + type: boolean required: false - MAC_M1_SELF_HOSTED_RUNNER_PW: + default: false + include-debug-info-for-macos: + type: boolean required: false - workflow_dispatch: - inputs: - # If we only want to check that the builds pass on an arbitrary branch + default: false run_tests: - description: "Run integration tests" - required: true type: boolean - default: true + required: false + default: false use-server-rc: + required: false type: boolean - required: true default: false description: 'Test against server release candidate?' - # If we are creating a backport and want to test an arbitrary branch against an older server version server-tag: - required: true + required: false + type: string default: 'latest' description: 'Server docker image tag' + secrets: + # Just make all the secrets required to make things simpler... + DOCKER_HUB_BOT_USERNAME: + required: true + DOCKER_HUB_BOT_PW: + required: true + MAC_M1_SELF_HOSTED_RUNNER_PW: + required: true + +env: + COMMIT_SHA_TO_BUILD_AND_TEST: ${{ inputs.is_workflow_call == true && inputs.sha-to-build-and-test || github.sha }} + # Note that environment variables in Github are all strings + # Github mac m1 and windows runners don't support Docker / nested virtualization + # so we need to use self-hosted runners to test wheels for these platforms + RUN_INTEGRATION_TESTS_IN_CIBW: ${{ inputs.run_tests && (startsWith(inputs.platform-tag, 'manylinux') || inputs.platform-tag == 'macosx_x86_64') }} jobs: - manylinux: + # Maps don't exist in Github Actions, so we have to store the map using a script and fetch it in a job + # This uses up more billing minutes (rounded up to 1 minute for each job run), + # but this should be ok based on the minutes usage data for the aerospike organization + get-runner-os: + outputs: + runner-os: ${{ steps.get-runner-os.outputs.runner_os }} runs-on: ubuntu-22.04 + steps: + - id: get-runner-os + # Single source of truth for which runner OS to use for each platform tag + run: | + declare -A hashmap + hashmap[manylinux_x86_64]="ubuntu-22.04" + hashmap[manylinux_aarch64]="aerospike_arm_runners_2" + hashmap[macosx_x86_64]="macos-12-large" + hashmap[macosx_arm64]="macos-14" + hashmap[win_amd64]="windows-2022" + echo runner_os=${hashmap[${{ inputs.platform-tag }}]} >> $GITHUB_OUTPUT + # Bash >= 4 supports hashmaps + shell: bash + + cibuildbinding: + needs: get-runner-os strategy: - fail-fast: false matrix: - # nodejs versions to build bindings on - nodejs: [ - ["v108", 18], - ["v115", 20], - ["v127", 22], - ["v131", 23] - ] - platform: [ - "x86_64", - #"aarch64" - ] - + nodejs-tag: ${{ fromJSON(inputs.nodejs-tags) }} + fail-fast: false + runs-on: ${{ needs.get-runner-os.outputs.runner-os }} + env: + BUILD_IDENTIFIER: "${{ matrix.nodejs-tag }}-${{ inputs.platform-tag }}" + MACOS_OPENSSL_VERSION: 3 steps: + - name: Create status check message + run: echo STATUS_CHECK_MESSAGE="cibuildbinding (${{ env.BUILD_IDENTIFIER }})" >> $GITHUB_ENV + shell: bash - name: Show job status for commit uses: myrotvorets/set-commit-status-action@v2.0.0 + if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} with: - sha: ${{ github.sha }} - context: "Build bindings (${{ matrix.nodejs[0] }}-manylinux_${{ matrix.platform }})" + sha: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} + context: ${{ env.STATUS_CHECK_MESSAGE }} - uses: actions/checkout@v4 with: submodules: recursive - ref: ${{ inputs.commit_sha }} - - - uses: actions/setup-node@v4 + ref: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} + # We need the last tag before the ref, so we can relabel the version if needed + fetch-depth: 0 + + - name: 'Windows: Install C client deps' + if: ${{ inputs.platform-tag == 'win_amd64' }} + run: nuget restore + working-directory: aerospike-client-c/vs + + - name: 'macOS x86: Setup Docker using colima for testing' + if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' && inputs.platform-tag == 'macosx_x86_64' }} + uses: ./.github/actions/setup-docker-on-macos + + - name: 'macOS x86: run Aerospike server in Docker container and connect via localhost' + if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' && inputs.platform-tag == 'macosx_x86_64' }} + uses: ./.github/actions/run-ee-server with: - node-version: ${{ matrix.nodejs[1] }} - architecture: 'x64' - - - name: print - run: uname -m - - - name: Set up QEMU for cross compiling arm64 - if: ${{ matrix.platform == 'aarch64' }} - uses: docker/setup-qemu-action@v3 + use-server-rc: ${{ inputs.use-server-rc }} + server-tag: ${{ inputs.server-tag }} + docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} + docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} + + # TODO: combine this composite action and the above into one + - name: "Linux: run Aerospike server in Docker container and configure config.conf to connect to the server container's Docker IP address" + if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' && startsWith(inputs.platform-tag, 'manylinux') }} + uses: ./.github/actions/run-ee-server-for-ext-container with: - platforms: all - - - if: ${{ contains(github.event.pull_request.labels.*.name, 'new-server-features') }} - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} - password: ${{ secrets.DOCKER_HUB_BOT_PW }} - - - - name: Run Aerospike server release candidate with latest tag - if: ${{ contains(github.event.pull_request.labels.*.name, 'new-server-features') }} - run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server-rc:latest - - - name: Run Aerospike server - if: ${{ !contains(github.event.pull_request.labels.*.name, 'new-server-features') }} - run: | - npm i @types/mocha @types/yargs @types/semver @types/chai; - docker run -d -v $(pwd)/.github/assets/aerospike.conf:/etc/mail/aerospike.conf --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server --config-file /etc/mail/aerospike.conf - -# - name: Set config.conf to use Docker IP address of Aerospike server -# # config.conf should be copied into the cibuildwheel Docker container -# run: | -# export SERVER_DOCKER_IP=$(docker container inspect -f '{{ .NetworkSettings.IPAddress }}' aerospike) -# # Install crudini -# pip install crudini -c ../.github/workflows/requirements.txt -# sed -i "s/127.0.0.1:3000//" config.conf -# crudini --set config.conf enterprise-edition hosts ${SERVER_DOCKER_IP}:3000 -# working-directory: test - - - name: Enable tests + use-server-rc: ${{ inputs.use-server-rc }} + server-tag: ${{ inputs.server-tag }} + docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} + docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} + + # FIND NO SERVER TESTS AND RUN THEM + # + #- name: If not running tests against server, only run basic import test + # if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'false' }} + # # Use double quotes otherwise Windows will throw this error in cibuildwheel + # # 'import + # # ^ + # # SyntaxError: EOL while scanning string literal + # run: echo "TEST_COMMAND=python -c \"import aerospike\"" >> $GITHUB_ENV + # shell: bash + + - name: Otherwise, enable integration tests + if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' }} run: echo "TEST_COMMAND=cd ts-test; npm install typescript --save-dev; npx tsc; cd ..; npm run test dist/ -- --h 127.0.0.1 --port 3000" >> $GITHUB_ENV + shell: bash -# - name: Disable tests (only run basic import test) -# if: ${{ !inputs.run_tests }} -# run: echo "TEST_COMMAND=node -e 'aerospike = require(\".\/lib\/aerospike\")'" >> $GITHUB_ENV - - - uses: uraimo/run-on-arch-action@v2 - name: Build client arm - if: ${{ matrix.platform == 'aarch64' }} - id: runcmd - with: - arch: aarch64 - distro: ubuntu-22.04 + - name: Set unoptimize flag + if: ${{ inputs.unoptimized && (startsWith(inputs.platform-tag, 'manylinux') || startsWith(inputs.platform-tag, 'macosx')) }} + run: echo "UNOPTIMIZED=1" >> $GITHUB_ENV - # Set an output parameter `uname` for use in subsequent steps - run: | - apt update - apt install -y g++ libssl-dev zlib1g-dev make build-essential libuv1-dev wget curl - ./scripts/build-c-client.sh - wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.3/install.sh | bash; - source ~/.bashrc; - nvm i 22 - npm install + - name: Set include dsym flag + if: ${{ inputs.include-debug-info-for-macos && startsWith(inputs.platform-tag, 'macosx') }} + run: echo "INCLUDE_DSYM=1" >> $GITHUB_ENV - - name: Build client x64 - if: ${{ matrix.platform != 'aarch64' }} + - name: Build Binding run: | - sudo apt update - sudo apt install g++ libssl-dev zlib1g-dev; - sudo apt-get install -y make; - sudo apt install build-essential; - ./scripts/build-c-client.sh - npm install - env: - CFLAGS: '-Werror' - - - name: Test client - run: | - ${{ env.TEST_COMMAND }} - - - name: Send binding to test jobs + ./scripts/build-c-client.sh; + npm install; + + # NEED TO MAKE A NODEJS ACTION WHICH REPLACES THIS!!! + #- name: Build wheel + # uses: pypa/cibuildwheel@v2.20.0 + # env: + # CIBW_ENVIRONMENT_PASS_LINUX: ${{ inputs.unoptimized && 'UNOPTIMIZED' || '' }} + # CIBW_ENVIRONMENT_MACOS: SSL_LIB_PATH="$(brew --prefix openssl@${{ env.MACOS_OPENSSL_VERSION }})/lib/" CPATH="$(brew --prefix openssl@${{ env.MACOS_OPENSSL_VERSION }})/include/" STATIC_SSL=1 + # CIBW_BUILD: ${{ env.BUILD_IDENTIFIER }} + # CIBW_BUILD_FRONTEND: build + # CIBW_BEFORE_ALL_LINUX: > + # yum install openssl-devel -y && + # yum install python-devel -y && + # yum install python-setuptools -y + # # delvewheel is not enabled by default but we do need to repair the wheel + # CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel" + # CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair --add-path ./aerospike-client-c/vs/x64/Release -w {dest_dir} {wheel}" + # CIBW_TEST_COMMAND: ${{ env.TEST_COMMAND }} +# + - name: Upload wheels to GitHub uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} with: - name: ${{ matrix.nodejs[0] }}-manylinux_${{ matrix.platform }}.build - path: ./lib/binding/node-*-linux-*/ + path: ./lib/binding/* + name: ${{ env.BUILD_IDENTIFIER }}.node - name: Set final commit status uses: myrotvorets/set-commit-status-action@v2.0.0 - if: always() + if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} with: - sha: ${{ github.sha }} + sha: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} status: ${{ job.status }} - context: "Build bindings (${{ matrix.nodejs[0] }}-manylinux_${{ matrix.platform }})" + context: ${{ env.STATUS_CHECK_MESSAGE }} - macOS-x86: + test-self-hosted: + needs: cibuildbinding + # There's a top-level env variable for this but we can't use it here, unfortunately + if: ${{ inputs.run_tests && (inputs.platform-tag == 'macosx_arm64' || inputs.platform-tag == 'win_amd64') }} strategy: fail-fast: false matrix: - nodejs: [ - ["v108", 18], - ["v115", 20], - ["v127", 22], - ['v131', 23] - ] - runs-on: macos-13 + nodejs-tag: ${{ fromJSON(inputs.nodejs-tags) }} + runs-on: ${{ inputs.platform-tag == 'macosx_arm64' && fromJSON('["self-hosted", "macOS", "ARM64"]') || fromJSON('["self-hosted", "Windows", "X64"]') }} + env: + BUILD_IDENTIFIER: "${{ matrix.nodejs-tag }}-${{ inputs.platform-tag }}" steps: - - name: Show job status for commit - uses: myrotvorets/set-commit-status-action@v2.0.0 - with: - sha: ${{ github.sha }} - context: "Build bindings (${{ matrix.nodejs[0] }}-macosx_x86_64)" - - - uses: actions/checkout@v4 - with: - submodules: recursive - ref: ${{ inputs.commit_sha }} - - - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.nodejs[1] }} - architecture: 'x64' - -# - name: Install Docker Engine -# run: brew install colima -# -# - name: Install Docker client -# run: brew install docker -# -# - name: Start Docker Engine -# run: colima start -# -# - if: ${{ contains(github.event.pull_request.labels.*.name, 'new-server-features') }} -# uses: docker/login-action@v3 -# with: -# username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} -# password: ${{ secrets.DOCKER_HUB_BOT_PW }} -# -# - name: Run Aerospike server release candidate with latest tag -# if: ${{ contains(github.event.pull_request.labels.*.name, 'new-server-features') }} -# run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server-rc:latest -# -# - name: Run Aerospike server -# if: ${{ !contains(github.event.pull_request.labels.*.name, 'new-server-features') }} -# run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server -# -# - name: Enable tests -# run: echo "TEST_COMMAND=npm test -- --h 127.0.0.1 --port 3000 --t 30000" >> $GITHUB_ENV - -# - name: Disable tests (only run basic import test) -# if: ${{ !inputs.run_tests }} -# run: echo "TEST_COMMAND=node -e 'aerospike = require(\".\/lib\/aerospike\")'" >> $GITHUB_ENV - - - name: Build client - run: | - ./scripts/build-c-client.sh - npm install - env: - CFLAGS: '-Werror' - -# - name: Test client -# run: | -# ${{ env.TEST_COMMAND }} - - - name: Save macOS binding - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.nodejs[0] }}-macosx_x86_64.build - path: ./lib/binding/*/ - - - name: Set final commit status - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: always() - with: - status: ${{ job.status }} - sha: ${{ github.sha }} - context: "Build bindings (${{ matrix.nodejs[0] }}-macosx_x86_64)" - -# macOS-m1: -# runs-on: [ -# self-hosted, -# macOS, -# ARM64 -# ] -# strategy: -# matrix: -# nodejs-version: [ -# ["v108", "18"], -# ["v115", "20"], -# ["v127", 22] -# ] -# fail-fast: false -# steps: -# - name: Show job status for commit -# uses: myrotvorets/set-commit-status-action@v2.0.0 -# with: -# sha: ${{ github.sha }} -# context: "Build bindings (${{ matrix.nodejs-version[1] }}-macosx_arm64)" -# -# - uses: actions/checkout@v4 -# with: -# submodules: recursive -# ref: ${{ inputs.commit_sha }} -# -# - name: Install NVM -# run: | -# curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash -# source ~/.zshrc -# nvm -v -# nvm install ${{ matrix.nodejs-version[1] }} -# -# -# - name: Setup symlink folders -# run: | -# sudo rm -rf /usr/local/opt/openssl; -# sudo rm -rf /usr/local/opt/libuv; -# sudo mkdir -p /usr/local/opt; -# sudo chown -R $(whoami) /usr/local/opt -# -# - name: Install brew packages -# run: | -# brew install openssl@3.2.1 -# brew install libuv@1.47.0 -# -# - name: Set environment for building -# run: | -# echo "export PATH="/usr/local/bin/:/usr/local/opt/openssl/bin:$PATH" -# export LDFLAGS="-L/usr/local/opt/openssl/lib" -# export CPPFLAGS="-I/usr/local/opt/openssl/include" -# export EXT_CFLAGS="-I/usr/local/opt/openssl/include"" >> ~/.zshrc; -# source ~/.zshrc; -# -# - name: Setup symlink folders -# run: | -# sudo ln -s /usr/local/Cellar/libuv/1.47.0/ /usr/local/opt/libuv; -# sudo ln -s /usr/local/Cellar/openssl@3/3.2.1/ /usr/local/opt/openssl; -# -# # Self-hosted runner only -# # Need to be able to save Docker Hub credentials to keychain -# - run: security unlock-keychain -p ${{ secrets.MAC_M1_SELF_HOSTED_RUNNER_PW }} -# if: ${{ inputs.run_tests && inputs.use-server-rc }} -# -# - if: ${{ inputs.run_tests && inputs.use-server-rc }} -# uses: docker/login-action@v3 -# with: -# username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} -# password: ${{ secrets.DOCKER_HUB_BOT_PW }} -# -# - name: Use server rc -# if: ${{ inputs.run_tests && inputs.use-server-rc }} -# run: echo IMAGE_NAME="${{ vars.SERVER_RC_REPO_LINK }}:${{ inputs.server-tag }}" >> $GITHUB_ENV -# -# - name: Use server release -# if: ${{ inputs.run_tests && !inputs.use-server-rc }} -# run: echo IMAGE_NAME="${{ vars.SERVER_REPO_LINK }}:${{ inputs.server-tag }}" >> $GITHUB_ENV -# -# - name: Run server -# if: ${{ inputs.run_tests }} -# run: docker run -d -p 3000:3000 --name aerospike ${{ env.IMAGE_NAME }} -# -# - name: Build client -# run: | -# ./scripts/build-c-client.sh -# npm install -# env: -# CFLAGS: '-Werror' -# -# - name: Enable tests -# if: ${{ inputs.run_tests }} -# run: echo "TEST_COMMAND=npm test -- --h 127.0.0.1 --port 3000" >> $GITHUB_ENV -# -# - name: Disable tests (only run basic import test) -# if: ${{ !inputs.run_tests }} -# run: echo "TEST_COMMAND=node -e 'aerospike = require(\".\/lib\/aerospike\")'" >> $GITHUB_ENV -# -# -# - name: Test client -# run: | -# ${{ env.TEST_COMMAND }} -# -# - name: Save macOS wheel -# uses: actions/upload-artifact@v4 -# with: -# name: ${{ matrix.nodejs-version[0] }}-macosx_arm64.build -# path: ./lib/binding/*/ -# -# - name: Stop server -# if: ${{ always() && inputs.run_tests }} -# run: | -# docker container stop aerospike -# docker container prune -f -# -# - name: Set final commit status -# uses: myrotvorets/set-commit-status-action@v2.0.0 -# if: always() -# with: -# sha: ${{ github.sha }} -# status: ${{ job.status }} -# context: "Build bindings (${{ matrix.nodejs-version[0] }}-macosx_arm64)" - - test-npm-install: - runs-on: ubuntu-22.04 - needs: [manylinux, macOS-x86] - steps: - - uses: actions/checkout@v2 - with: - submodules: recursive - - - uses: ./.github/workflows/combine-bindings/ - - - uses: actions/setup-node@v4 - with: - node-version: ${{ env.LOWEST_SUPPORTED_NODEJS_VERSION }} - architecture: 'x64' - - - name: Modify the package.json - run: | - sudo npm install -g json - - - - name: Modify the package.json - run: | - sudo npm install -g json - json -I -f package.json -e "this.scripts.install=\"npm-run-all removeExtraBinaries build\"" - - - name: Run tests - run: | - mkdir -p testDir - cd testDir - pwd - sudo npm install .. - - test-yarn-install: - runs-on: ubuntu-22.04 - needs: [manylinux, macOS-x86] - steps: - - uses: actions/checkout@v2 - with: - submodules: recursive - - - uses: ./.github/workflows/combine-bindings/ - - - uses: actions/setup-node@v4 - with: - node-version: ${{ env.LOWEST_SUPPORTED_NODEJS_VERSION }} - architecture: 'x64' - - - name: Modify the package.json - run: | - sudo npm install -g json - json -I -f package.json -e "this.scripts.install=\"npm-run-all removeExtraBinaries build\"" - - - name: Run tests - run: | - mkdir -p testDir - yarn link - cd testDir - npm install --global yarn - yarn add link:.. - - test-pnpm-install: - runs-on: ubuntu-22.04 - needs: [manylinux, macOS-x86] - steps: - - uses: actions/checkout@v2 - with: - submodules: recursive - - - uses: ./.github/workflows/combine-bindings/ - - - uses: actions/setup-node@v4 - with: - node-version: ${{ env.LOWEST_SUPPORTED_NODEJS_VERSION }} - architecture: 'x64' - - - name: Modify the package.json - run: | - sudo npm install -g json - json -I -f package.json -e "this.scripts.install=\"npm-run-all removeExtraBinaries build\"" - - - name: Run tests - run: | - mkdir -p testDir - cd testDir - npm install --global pnpm - SHELL=bash pnpm setup - source /home/runner/.bashrc - pnpm install .. - - test-bun-install: - runs-on: ubuntu-latest - needs: [manylinux, macOS-x86] - steps: - - uses: actions/checkout@v2 - with: - submodules: recursive + - name: Create status check message + run: echo STATUS_CHECK_MESSAGE="Test on self hosted (${{ env.BUILD_IDENTIFIER }})" >> $GITHUB_ENV + shell: bash + + - name: Show job status for commit + uses: myrotvorets/set-commit-status-action@v2.0.0 + if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} + with: + sha: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} + context: ${{ env.STATUS_CHECK_MESSAGE }} + + - uses: actions/checkout@v4 + with: + ref: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} + + # Need to be able to save Docker Hub credentials to keychain + - if: ${{ inputs.platform-tag == 'macosx_arm64' && inputs.use-server-rc }} + run: security unlock-keychain -p ${{ secrets.MAC_M1_SELF_HOSTED_RUNNER_PW }} + + - uses: ./.github/actions/run-ee-server + with: + use-server-rc: ${{ inputs.use-server-rc }} + server-tag: ${{ inputs.server-tag }} + docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} + docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} + + - name: Download wheel + uses: actions/download-artifact@v4 + with: + name: ${{ env.BUILD_IDENTIFIER }}.build + + - name: Convert Python tag to Python version + # Don't use sed because we want this command to work on both mac and windows + # The command used in GNU sed is different than in macOS sed + run: | + NODEJS_TAG=${{ matrix.nodejs-tag }} + NODEJS_VERSION="${NODEJS_TAG/cp/}" + echo NODEJS_VERSION="${NODEJS_VERSION/3/3.}" >> $GITHUB_ENV + shell: bash - - uses: ./.github/workflows/combine-bindings/ + - uses: actions/setup-node@v5 + with: + nodejs-version: ${{ env.NODEJS_VERSION }} - - uses: actions/setup-node@v4 - with: - node-version: ${{ env.LOWEST_SUPPORTED_NODEJS_VERSION }} - architecture: 'x64' - - - name: Modify the package.json - run: | - sudo npm install -g json - json -I -f package.json -e "this.scripts.install=\"npm-run-all removeExtraBinaries build\"" + - name: Install wheel + run: | + ./scripts/build-c-client.sh + npm install + shell: bash - - name: Run tests - run: | - npm install -g bun - bun link - mkdir -p testDir - cd testDir - bun link aerospike - -# test-typescript-install: -# runs-on: ubuntu-22.04 -# needs: [manylinux, macOS-x86] -# steps: -# - uses: actions/checkout@v2 -# with: -# submodules: recursive -# -# - uses: ./.github/workflows/combine-bindings/ -# -# - uses: actions/setup-node@v4 -# with: -# node-version: ${{ env.LOWEST_SUPPORTED_NODEJS_VERSION }} -# architecture: 'x64' -# -# - if: ${{ contains(github.event.pull_request.labels.*.name, 'new-server-features') }} -# uses: docker/login-action@v3 -# with: -# username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} -# password: ${{ secrets.DOCKER_HUB_BOT_PW }} -# -# -# - name: Run Aerospike server release candidate with latest tag -# if: ${{ contains(github.event.pull_request.labels.*.name, 'new-server-features') }} -# run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server-rc:latest -# -# - name: Run Aerospike server -# if: ${{ !contains(github.event.pull_request.labels.*.name, 'new-server-features') }} -# run: | -# cd ts-test; -# npm i --save-dev @types/mocha; -# npm i --save-dev @types/yargs; -# npm i --save-dev @types/semver; -# npm i --save-dev @types/chai; -# tsc; -# cd ..; -# npm install; -# docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server -# -# -# -# - name: Wait for database to be ready -# # Should be ready after 3 seconds -# run: sleep 15 -# -# - name: Modify the package.json -# run: | -# sudo npm install -g json -# json -I -f package.json -e "this.scripts.install=\"npm-run-all removeExtraBinaries build\"" -# -# - name: Run tests -# run: | -# mkdir my-aerospike-project -# cd my-aerospike-project -# npm init -y -# npm install typescript ts-node --save-dev -# npm install .. -# cp ../examples/typescript.ts index.ts -# npx tsc index.ts -# node index.js \ No newline at end of file + - name: Connect to Docker container on remote machine with Docker daemon + if: ${{ inputs.platform-tag == 'win_amd64' }} + # DOCKER_HOST contains the IP address of the remote machine + run: | + $env:DOCKER_HOST_IP = $env:DOCKER_HOST | foreach {$_.replace("tcp://","")} | foreach {$_.replace(":2375", "")} + crudini --set config.conf enterprise-edition hosts ${env:DOCKER_HOST_IP}:3000 + working-directory: test + + # FIGURE OUT WHAT SETUP NEEDS TO HAPPEN + - run: | + npm install typescript --save-dev; + npx tsc; + + working-directory: ts-test + shell: bash + + - run: npx tsc + shell: bash + + - run: npm run test dist/ -- --h 127.0.0.1 --port 3000 + shell: bash + + - name: Show job status for commit + if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} + uses: myrotvorets/set-commit-status-action@v2.0.0 + with: + sha: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} + status: ${{ job.status }} + context: ${{ env.STATUS_CHECK_MESSAGE }} diff --git a/.github/workflows/bump-stage-and-upload-to-jfrog.yml b/.github/workflows/bump-stage-and-upload-to-jfrog.yml new file mode 100644 index 00000000..922c94de --- /dev/null +++ b/.github/workflows/bump-stage-and-upload-to-jfrog.yml @@ -0,0 +1,67 @@ +on: + workflow_call: + inputs: + passed-dev-tag: + type: string + description: Dev tag to fast forward the stage branch to + required: true + secrets: + # Used to bump version in Github + CLIENT_BOT_PAT: + required: true + # Used to upload to JFrog + JFROG_PLATFORM_URL: + required: true + JFROG_ACCESS_TOKEN: + required: true + +jobs: + ff-stage-to-dev-tag: + name: Fast forward stage branch to the dev tag that passed stage testing + uses: ./.github/workflows/fast-forward-merge.yml + with: + ref_to_merge: ${{ inputs.passed-dev-tag }} + base_branch: ${{ vars.STAGE_BRANCH_NAME }} + secrets: inherit + + promote-dev-build-to-rc: + name: Bump (promote) the dev version to an RC version in the stage branch + needs: ff-stage-to-dev-tag + uses: ./.github/workflows/bump-version.yml + with: + change: 'promote-dev-build-to-rc' + ref: ${{ vars.STAGE_BRANCH_NAME }} + secrets: inherit + + rebuild-artifacts-with-rc-version: + needs: promote-dev-build-to-rc + uses: ./.github/workflows/build-artifacts.yml + with: + sha-to-build-and-test: ${{ needs.promote-dev-build-to-rc.outputs.bump_sha }} + secrets: inherit + + upload-rc-artifacts-to-jfrog: + needs: [ + rebuild-artifacts-with-rc-version, + # We need the new RC version to label the build in JFrog + promote-dev-build-to-rc + ] + name: Upload artifacts to JFrog + uses: ./.github/workflows/upload-to-jfrog.yml + with: + version: ${{ needs.promote-dev-build-to-rc.outputs.new_version }} + secrets: inherit + + # See reason for deleting artifacts in dev-workflow-p2.yml + delete-artifacts: + needs: upload-rc-artifacts-to-jfrog + uses: ./.github/workflows/delete-artifacts.yml + + ff-dev-to-stage: + name: Fast forward dev branch to stage branch to include the bump to RC commit + needs: promote-dev-build-to-rc + uses: ./.github/workflows/fast-forward-merge.yml + with: + ref_to_merge: origin/${{ vars.STAGE_BRANCH_NAME }} + base_branch: ${{ vars.DEV_BRANCH_NAME }} + secrets: inherit diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml new file mode 100644 index 00000000..cec01cb0 --- /dev/null +++ b/.github/workflows/bump-version.yml @@ -0,0 +1,84 @@ +# Takes in how to bump version as input +# Commits changes to bump version and outputs the new version and commit hash as output + +name: Bump version +on: + workflow_dispatch: + inputs: + change: + type: choice + description: Python script name to update the version + required: true + options: + - bump-dev-num + - promote-dev-build-to-rc + - promote-rc-build-to-release + workflow_call: + inputs: + change: + # Since workflow_call doesn't support 'options' input type, + # we take in a string instead that must be a valid Python script name (excluding the .py part) + # Example: bump-dev-num for bump-dev-num.py + type: string + description: Python script name to change version + required: true + ref: + required: false + description: Commit to bump off of + type: string + # See workflow call hack in update-version.yml + is_workflow_call: + type: boolean + default: true + required: false + secrets: + CLIENT_BOT_PAT: + required: true + outputs: + new_version: + value: ${{ jobs.get-new-version.outputs.new_version }} + bump_sha: + value: ${{ jobs.update-version-in-repo.outputs.bump_sha }} + +jobs: + get-current-version: + name: Get new version string + runs-on: ubuntu-22.04 + outputs: + current_version: ${{ steps.get-current-version.outputs.current_version }} + steps: + # Checkout the branch where we want to bump the new version + - uses: actions/checkout@v4 + with: + token: ${{ secrets.CLIENT_BOT_PAT }} + ref: ${{ inputs.ref }} + + - name: Get current version + id: get-current-version + run: echo current_version=$(cat VERSION) >> $GITHUB_OUTPUT + + get-new-version: + runs-on: ubuntu-22.04 + needs: get-current-version + outputs: + new_version: ${{ steps.get-new-version.outputs.new_version }} + steps: + # Checkout branch where workflow is being called from + - uses: actions/checkout@v4 + + - name: Install library that parses PEP 440 versions + # NEED TO CHANGE THIS TO WORK WITH NODEJS + run: pip install parver -c requirements.txt + working-directory: .github/workflows + + - name: Get new version + id: get-new-version + run: echo new_version=$(python3 .github/workflows/${{ inputs.change }}.py ${{ needs.get-current-version.outputs.current_version }}) >> $GITHUB_OUTPUT + + update-version-in-repo: + needs: get-new-version + uses: ./.github/workflows/update-version.yml + with: + new_version: ${{ needs.get-new-version.outputs.new_version }} + ref: ${{ inputs.is_workflow_call && inputs.ref || github.ref }} + secrets: inherit diff --git a/.github/workflows/delete-artifacts.yml b/.github/workflows/delete-artifacts.yml new file mode 100644 index 00000000..a5b6a738 --- /dev/null +++ b/.github/workflows/delete-artifacts.yml @@ -0,0 +1,11 @@ +on: + workflow_call: + +jobs: + delete-artifacts: + runs-on: ubuntu-22.04 + steps: + - name: Remove artifacts with dev version + uses: geekyeggo/delete-artifact@v5 + with: + name: '*.node' diff --git a/.github/workflows/dev-to-stage.yml b/.github/workflows/dev-to-stage.yml new file mode 100644 index 00000000..5e8d203a --- /dev/null +++ b/.github/workflows/dev-to-stage.yml @@ -0,0 +1,60 @@ +name: Dev to stage + +on: + # This workflow manipulates the stage and dev branches regardless of the branch this workflow is run from + workflow_dispatch: + +jobs: + # We want to skip the stage tests if the changes made between dev and stage wouldn't affect the results of the stage tests + compare-latest-dev-tag-and-stage: + outputs: + # This is always available, but the stage tests only use it if they're being run (i.e not skipped) + latest-dev-tag: ${{ steps.get-dev-tag.outputs.latest-dev-tag }} + run_stage_tests: ${{ steps.run_stage_tests.outputs.run_stage_tests }} + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + with: + # Get all tags + fetch-depth: 0 + + - name: Get latest dev tag + run: echo latest-dev-tag=$(git describe --tags --abbrev=0 origin/${{ vars.DEV_BRANCH_NAME }}) >> $GITHUB_OUTPUT + id: get-dev-tag + + - name: Get number of files that were changed between dev and stage (with some exceptions) + run: echo NUM_FILES_CHANGED=$(git diff origin/${{ vars.STAGE_BRANCH_NAME }}..origin/${{ vars.DEV_BRANCH_NAME }} --name-only | grep --invert-match --count -e "^doc/" -e "^aerospike-stubs/" -e VERSION) >> $GITHUB_ENV + # We want this step to fail if a command failed while using pipes + shell: bash + + - name: If any files were changed besides the exceptions, run the stage tests + run: echo run_stage_tests=${{ env.NUM_FILES_CHANGED != '0' }} >> $GITHUB_OUTPUT + id: run_stage_tests + + run-stage-tests-on-jfrog-artifacts: + needs: compare-latest-dev-tag-and-stage + # All job outputs are unicode strings + if: ${{ needs.compare-latest-dev-tag-and-stage.outputs.run_stage_tests == 'true' }} + uses: ./.github/workflows/stage-tests.yml + # Need to pass in JFrog secrets + secrets: inherit + with: + use_jfrog_builds: true + jfrog-build-version-to-test: ${{ needs.compare-latest-dev-tag-and-stage.outputs.latest-dev-tag }} + + # Stage tests have passed or skipped + # so it is safe to update the stage branch with the changes in dev, promote the version to an RC, and rebuild and upload the RC to JFrog + # We store the subsequent jobs after the stage tests in a separate reusable workflow... + # because if stage tests were skipped, all subsequent jobs will be skipped by default too (both direct and indirect descendents) + # This means we have to add a manual check for each subsequent job that checks if the stage tests were skipped in order to run them + # It's easier to just add this manual check once to a reusable workflow that wraps around all the subsequent jobs + bump-stage-and-upload-to-jfrog: + needs: [ + run-stage-tests-on-jfrog-artifacts, + compare-latest-dev-tag-and-stage + ] + if: ${{ !cancelled() && needs.compare-latest-dev-tag-and-stage.result == 'success' && (needs.run-stage-tests-on-jfrog-artifacts.result == 'success' || needs.run-stage-tests-on-jfrog-artifacts.result == 'skipped') }} + uses: ./.github/workflows/bump-stage-and-upload-to-jfrog.yml + with: + passed-dev-tag: ${{ needs.compare-latest-dev-tag-and-stage.outputs.latest-dev-tag }} + secrets: inherit diff --git a/.github/workflows/dev-workflow-p1.yml b/.github/workflows/dev-workflow-p1.yml new file mode 100644 index 00000000..7dbd6527 --- /dev/null +++ b/.github/workflows/dev-workflow-p1.yml @@ -0,0 +1,37 @@ +name: Dev workflow (part 1) + +# 1. When a PR review is requested, run tests on that PR +# 2. If all of the tests pass, allow the PR to be merged into `dev` +# 3. Whenever a PR is merged to `dev`, bump version number in `dev` + +on: + pull_request: + types: + - review_requested + branches: + - 'dev*' + paths-ignore: + - 'docs/**' + - 'aerospike-stubs/**' + + # So we can test changes to the test-server-rc workflow + workflow_dispatch: + inputs: + run_server_release_tests: + description: 'DEBUG: Run server release tests in build-wheels workflow?' + type: boolean + default: false + +jobs: + test-with-server-release: + uses: ./.github/workflows/build-artifacts.yml + with: + run_tests: ${{ github.event_name == 'pull_request' && true || inputs.run_server_release_tests }} + sha-to-build-and-test: ${{ github.sha }} + secrets: inherit + + test-with-server-rc: + needs: test-with-server-release + if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'new-server-features') }} + uses: ./.github/workflows/test-server-rc.yml + secrets: inherit diff --git a/.github/workflows/dev-workflow-p2.yml b/.github/workflows/dev-workflow-p2.yml new file mode 100644 index 00000000..7427ec97 --- /dev/null +++ b/.github/workflows/dev-workflow-p2.yml @@ -0,0 +1,43 @@ +name: Dev workflow (part 2) + +on: + pull_request_target: + branches: + - 'dev*' + types: + - closed + workflow_dispatch: + +jobs: + bump-dev-number: + if: ${{ github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true }} + uses: ./.github/workflows/bump-version.yml + with: + change: 'bump-dev-num' + secrets: inherit + + rebuild-artifacts-with-new-dev-num: + needs: bump-dev-number + name: Rebuild artifacts with new dev number + uses: ./.github/workflows/build-artifacts.yml + with: + # On pull_request_target, the bump version commit will be ignored + # So we must pass it manually to the workflow + sha-to-build-and-test: ${{ needs.bump-dev-number.outputs.bump_sha }} + secrets: inherit + + upload-to-jfrog: + name: Upload artifacts to JFrog + needs: [ + bump-dev-number, + rebuild-artifacts-with-new-dev-num + ] + uses: ./.github/workflows/upload-to-jfrog.yml + with: + version: ${{ needs.bump-dev-number.outputs.new_version }} + secrets: inherit + + # We don't want the artifacts in JFrog to also exist in Github + delete-artifacts: + needs: upload-to-jfrog + uses: ./.github/workflows/delete-artifacts.yml diff --git a/.github/workflows/fast-forward-merge.yml b/.github/workflows/fast-forward-merge.yml new file mode 100644 index 00000000..6b1237d2 --- /dev/null +++ b/.github/workflows/fast-forward-merge.yml @@ -0,0 +1,43 @@ +# As of this writing, there are no Github Actions in the marketplace +# that allow fast forwarding using a personal access token +# So we need to use this for now +name: 'Fast forward merge' +on: + workflow_dispatch: + inputs: + ref_to_merge: + type: string + required: true + base_branch: + type: string + required: true + workflow_call: + inputs: + # If another branch, it must be origin/ + ref_to_merge: + type: string + required: true + base_branch: + type: string + required: true + secrets: + CLIENT_BOT_PAT: + required: true + +jobs: + merge: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # Fetch the whole history to prevent unrelated history errors + fetch-depth: '0' + ref: ${{ inputs.base_branch }} + token: ${{ secrets.CLIENT_BOT_PAT }} + + - name: Fast forward + run: git merge --ff-only ${{ inputs.ref_to_merge }} + + - name: Upload changes to remote head branch + run: git push diff --git a/.github/workflows/stage-tests.yml b/.github/workflows/stage-tests.yml new file mode 100644 index 00000000..a7664a3e --- /dev/null +++ b/.github/workflows/stage-tests.yml @@ -0,0 +1,208 @@ +name: Stage tests + +# Downloads artifacts either from Github or JFrog +# and runs category 1 (source distribution) and category 2 (binary distribution) tests +# The purpose is to test that our artifacts work on the Linux distros / OS versions that the client supports +# and QE doesn't have enough disk space for more Linux distros, so we have some tests here in Github Actions + +on: + workflow_call: + inputs: + use_jfrog_builds: + description: If false, download artifacts from Github + required: true + type: boolean + default: true + jfrog-build-version-to-test: + description: Used only when use_jfrog_builds is true. + type: string + required: false + # These will be set if use_jfrog_builds is false (i.e when someone is building the artifacts from scratch and stage testing them in a calling workflow) + # If use_jfrog_builds is true, only the defaults will be used + use-server-rc: + type: boolean + required: false + default: false + description: 'Test against server release candidate?' + server-tag: + type: string + required: false + default: 'latest' + description: 'Server docker image tag' + test-macos-x86: + required: false + type: boolean + default: false + description: 'Test macOS x86 wheels (unstable)' + +jobs: + linux-distro-tests: + strategy: + matrix: + test-case: [ + # Docker image + tag, test category, Docker image platform (cpu arch), Python version + ["amazonlinux:2023", 2, "linux/amd64", "3.9"], + ["amazonlinux:2023", 2, "linux/arm64", "3.9"], + ["amazonlinux:2023", 2, "linux/amd64", "3.11"], + ["amazonlinux:2023", 2, "linux/arm64", "3.11"], + ["ubuntu:22.04", 2, "linux/amd64", "3.10"], + ["ubuntu:22.04", 2, "linux/arm64", "3.10"], + # Bookworm is Debian 12 + ["python:3.8-bookworm", 2, "linux/amd64", "3.8"], + ["python:3.8-bookworm", 2, "linux/arm64", "3.8"], + ["python:3.9-bookworm", 2, "linux/amd64", "3.9"], + ["python:3.9-bookworm", 2, "linux/arm64", "3.9"], + ["python:3.10-bookworm", 2, "linux/amd64", "3.10"], + ["python:3.10-bookworm", 2, "linux/arm64", "3.10"], + ["python:3.11-bookworm", 2, "linux/amd64", "3.11"], + ["python:3.11-bookworm", 2, "linux/arm64", "3.11"], + ["python:3.12-bookworm", 2, "linux/amd64", "3.12"], + ["python:3.12-bookworm", 2, "linux/arm64", "3.12"], + ["amazonlinux:2023", 1, "linux/amd64", "3.9"], + ["redhat/ubi9", 1, "linux/amd64", "3.9"], + ] + fail-fast: false + env: + LINUX_DISTRO_CONTAINER_NAME: linux-distro + runs-on: ${{ matrix.test-case[2] == 'linux/amd64' && 'ubuntu-22.04' || 'aerospike_arm_runners_2' }} + steps: + # TODO: the checkout code is also duplicated in the macOS stage tests + # But it's only a few lines of code so I didn't bother to create a composite action for it. + - name: Get tests and Github action scripts + uses: actions/checkout@v4 + with: + ref: ${{ inputs.use_jfrog_builds && inputs.jfrog-build-version-to-test || github.sha }} + sparse-checkout: | + test + .github + + # Map test case tuple entries to env vars to make code easier to read + - run: echo "DISTRO_DOCKER_IMAGE_AND_TAG=${{ matrix.test-case[0] }}" >> $GITHUB_ENV + - run: echo "TEST_CATEGORY=${{ matrix.test-case[1] }}" >> $GITHUB_ENV + - run: echo "DISTRO_DOCKER_IMAGE_PLATFORM=${{ matrix.test-case[2] }}" >> $GITHUB_ENV + - run: echo "PYTHON_VERSION=${{ matrix.test-case[3] }}" >> $GITHUB_ENV + + - uses: ./.github/actions/get-artifact-for-stage-tests + with: + get_from_jfrog: ${{ inputs.use_jfrog_builds }} + # This input is only used if above input is true + jfrog_build_version: ${{ inputs.jfrog-build-version-to-test }} + dist_type_to_get: ${{ env.TEST_CATEGORY == '2' && 'wheel' || 'sdist' }} + # wheel* inputs are used only if running category 2 tests + wheel_python_version: ${{ env.PYTHON_VERSION }} + wheel_os: manylinux + wheel_cpu_arch: ${{ env.DISTRO_DOCKER_IMAGE_PLATFORM == 'linux/amd64' && 'x86_64' || 'aarch64' }} + JFROG_PLATFORM_URL: ${{ secrets.JFROG_PLATFORM_URL }} + JFROG_ACCESS_TOKEN: ${{ secrets.JFROG_ACCESS_TOKEN }} + JFROG_REPO_NAME: ${{ vars.JFROG_REPO_NAME }} + + - uses: ./.github/actions/run-ee-server + with: + use-server-rc: ${{ inputs.use-server-rc }} + server-tag: ${{ inputs.server-tag }} + docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} + docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} + + - name: Run distro container + # Run distro container on host network to access the Aerospike server using localhost (without having to change config.conf) + run: docker run --detach --network host --platform ${{ env.DISTRO_DOCKER_IMAGE_PLATFORM }} --name ${{ env.LINUX_DISTRO_CONTAINER_NAME }} ${{ env.DISTRO_DOCKER_IMAGE_AND_TAG }} tail -f /dev/null + + - name: Copy repo (and artifact) to container + run: docker cp . ${{ env.LINUX_DISTRO_CONTAINER_NAME }}:/aerospike-client-python + + - name: 'Amazon Linux 2023: install python 3.11 if applicable' + if: ${{ env.DISTRO_DOCKER_IMAGE_AND_TAG == 'amazonlinux:2023' && env.PYTHON_VERSION == '3.11' }} + run: docker exec ${{ env.LINUX_DISTRO_CONTAINER_NAME }} yum install -y python3.11 + + - name: 'Ubuntu 22.04 Install python 3.10 that comes by default (step 1)' + if: ${{ env.DISTRO_DOCKER_IMAGE_AND_TAG == 'ubuntu:22.04' }} + run: docker exec ${{ env.LINUX_DISTRO_CONTAINER_NAME }} apt update + + - name: 'Ubuntu 22.04 Install python 3.10 that comes by default (step 2)' + if: ${{ env.DISTRO_DOCKER_IMAGE_AND_TAG == 'ubuntu:22.04' }} + run: docker exec ${{ env.LINUX_DISTRO_CONTAINER_NAME }} apt install python3 python3-pip -y + + - name: Make sure pip is installed + if: ${{ env.DISTRO_DOCKER_IMAGE_AND_TAG == 'amazonlinux:2023' || env.DISTRO_DOCKER_IMAGE_AND_TAG == 'redhat/ubi9' || startsWith(env.DISTRO_DOCKER_IMAGE_AND_TAG, 'python') }} + run: docker exec ${{ env.LINUX_DISTRO_CONTAINER_NAME }} python${{ env.PYTHON_VERSION }} -m ensurepip + + - name: 'Cat 1: Install build dependencies using yum' + if: ${{ env.TEST_CATEGORY == '1' }} + run: docker exec ${{ env.LINUX_DISTRO_CONTAINER_NAME }} yum install -y openssl-devel glibc-devel autoconf automake libtool zlib-devel openssl-devel python-devel + + - name: 'Cat 1: Install pip build frontend' + if: ${{ env.TEST_CATEGORY == '1' }} + run: docker exec --workdir /aerospike-client-python/ ${{ env.LINUX_DISTRO_CONTAINER_NAME }} python${{ env.PYTHON_VERSION }} -m pip install -r requirements.txt + + - name: Install sdist or wheel distribution + run: docker exec --workdir /aerospike-client-python/ ${{ env.LINUX_DISTRO_CONTAINER_NAME }} python${{ env.PYTHON_VERSION }} -m pip install ${{ env.ARTIFACT_FILE_NAME_PATTERN }} + + - name: Install pytest + run: docker exec --workdir /aerospike-client-python/test ${{ env.LINUX_DISTRO_CONTAINER_NAME }} python${{ env.PYTHON_VERSION }} -m pip install pytest -c requirements.txt + + - name: Run tests + run: docker exec --workdir /aerospike-client-python/test ${{ env.LINUX_DISTRO_CONTAINER_NAME }} python${{ env.PYTHON_VERSION }} -m pytest new_tests/ + + macOS: + if: ${{ inputs.test-macos-x86 }} + strategy: + matrix: + runner-os: [ + macos-13, + macos-14-large + ] + python-version: [ + "3.8", + "3.9", + "3.10", + "3.11", + "3.12", + ] + fail-fast: false + runs-on: ${{ matrix.runner-os }} + steps: + - name: Get tests and Github action scripts + uses: actions/checkout@v4 + with: + ref: ${{ inputs.use_jfrog_builds && inputs.jfrog-build-version-to-test || github.sha }} + sparse-checkout: | + test + .github + + - uses: ./.github/actions/get-artifact-for-stage-tests + with: + # See comments in linux stage tests for how this works + get_from_jfrog: ${{ inputs.use_jfrog_builds }} + jfrog_build_version: ${{ inputs.jfrog-build-version-to-test }} + dist_type_to_get: 'wheel' + wheel_python_version: ${{ matrix.python-version }} + wheel_os: macosx + wheel_cpu_arch: x86_64 + JFROG_PLATFORM_URL: ${{ secrets.JFROG_PLATFORM_URL }} + JFROG_ACCESS_TOKEN: ${{ secrets.JFROG_ACCESS_TOKEN }} + JFROG_REPO_NAME: ${{ vars.JFROG_REPO_NAME }} + + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - uses: ./.github/actions/setup-docker-on-macos + + - uses: ./.github/actions/run-ee-server + with: + use-server-rc: ${{ inputs.use-server-rc }} + server-tag: ${{ inputs.server-tag }} + docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} + docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} + + - name: Install wheel + run: python3 -m pip install *.whl + + - name: Install test dependencies + run: python3 -m pip install pytest -c requirements.txt + working-directory: test + + - name: Run tests + run: python3 -m pytest new_tests/ + id: test + working-directory: test diff --git a/.github/workflows/test-server-rc.yml b/.github/workflows/test-server-rc.yml new file mode 100644 index 00000000..7bdc2f7c --- /dev/null +++ b/.github/workflows/test-server-rc.yml @@ -0,0 +1,170 @@ +name: Test cibuildwheel artifacts against server RC +run-name: Test cibuildwheel artifacts against server RC + +on: + workflow_call: + secrets: + DOCKER_HUB_BOT_USERNAME: + required: true + DOCKER_HUB_BOT_PW: + required: true + MAC_M1_SELF_HOSTED_RUNNER_PW: + required: true + +jobs: + manylinux: + strategy: + fail-fast: false + matrix: + # Python versions to build wheels on + nodejs: [ + ["18", "v108"], + ["20", "v115"], + ["22", "v127"], + ["23", "v131"], + ] + platform: [ + ["x86_64", "ubuntu-22.04"], + ["aarch64", "aerospike_arm_runners_2"] + ] + runs-on: ${{ matrix.platform[1] }} + steps: + - uses: actions/checkout@v4 + + - run: docker run -d --name manylinux quay.io/pypa/manylinux2014_${{ matrix.platform[0] }} tail -f /dev/null + + - uses: ./.github/actions/run-ee-server-for-ext-container + with: + use-server-rc: true + server-tag: latest + docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} + docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} + + - uses: actions/download-artifact@v4 + with: + name: ${{ matrix.nodejs[1] }}-manylinux_${{ matrix.platform[0] }}.build + path: ./ + + - name: Copy repo and wheel into manylinux container + run: docker cp ../aerospike-client-nodejs manylinux:/ + + - name: Install wheel + run: docker exec --workdir /aerospike-client-nodejs manylinux nodejs${{ matrix.nodejs[0] }} -m pip install *.whl + + - name: Install test prerequisites + run: docker exec --workdir /aerospike-client-nodejs/test manylinux nodejs${{ matrix.nodejs[0] }} -m pip install -r requirements.txt + + - name: Run tests + id: test + run: docker exec --workdir /aerospike-client-nodejs/test manylinux nodejs${{ matrix.nodejs[0] }} -m pytest new_tests/ + + macOS-x86: + # Skip macOS x86 testing since it's unstable + if: ${{ false }} + strategy: + fail-fast: false + matrix: + nodejs: [ + ["18", "v108"], + ["20", "v115"], + ["22", "v127"], + ["23", "v131"], + ] + runs-on: macos-12-large + steps: + - uses: actions/checkout@v4 + + - uses: ./.github/actions/setup-docker-on-macos + + - uses: ./.github/actions/run-ee-server + with: + use-server-rc: true + server-tag: latest + docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} + docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} + + - uses: actions/download-artifact@v4 + with: + name: ${{ matrix.nodejs[1] }}-macosx_x86_64.build + path: ./ + + - uses: actions/setup-nodejs@v4 + with: + nodejs-version: ${{ matrix.nodejs[0] }} + + - run: python${{ matrix.python[0] }} -m pip install *.whl + + - run: python${{ matrix.python[0] }} -m pip install -r requirements.txt + working-directory: test + + - run: python${{ matrix.python[0] }} -m pytest new_tests/ + id: test + working-directory: test + + macOS-m1: + # TODO: this shares a lot of steps as in the build wheels workflow + runs-on: [ + self-hosted, + macOS, + ARM64 + ] + strategy: + matrix: + python-version: [ + ["18", "v108"], + ["20", "v115"], + ["22", "v127"], + ["23", "v131"], + ] + fail-fast: false + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + with: + name: ${{ matrix.nodejs-version[1] }}-macosx_arm64.build + path: ./ + + # Update dependencies if needed + - name: Add brew to path + run: echo PATH=$PATH:/opt/homebrew/bin/ >> $GITHUB_ENV + + # TODO: INSTALL NODEJS + - name: Install or upgrade Python + run: brew install python@${{ matrix.python-version[0] }} + + # Self-hosted runner only + # Need to be able to save Docker Hub credentials to keychain + - run: security unlock-keychain -p ${{ secrets.MAC_M1_SELF_HOSTED_RUNNER_PW }} + + - uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} + password: ${{ secrets.DOCKER_HUB_BOT_PW }} + + - name: Run server RC + run: docker run -d -p 3000:3000 --name aerospike -e DEFAULT_TTL=2592000 ${{ vars.SERVER_RC_REPO_LINK }} + + - name: Create config.conf + run: cp config.conf.template config.conf + working-directory: test + + - name: Install wheel + run: python${{ matrix.nodejs-version[0] }} -m pip install --force-reinstall --break-system-packages *.whl + + - run: python${{ matrix.nodejs-version[0] }} -m pip install --force-reinstall --break-system-packages -r requirements.txt + working-directory: test + + - uses: ./.github/actions/wait-for-as-server-to-start + with: + container-name: aerospike + + - run: python${{ matrix.nodejs-version[0] }} -m pytest new_tests/ + id: test + working-directory: test + + - name: Stop server + if: ${{ always() }} + run: | + docker container stop aerospike + docker container prune -f diff --git a/.github/workflows/update-version.yml b/.github/workflows/update-version.yml new file mode 100644 index 00000000..ce155ed1 --- /dev/null +++ b/.github/workflows/update-version.yml @@ -0,0 +1,64 @@ +# Takes in a new version as input +# Changes the version in the whole repo and outputs the commit hash as output + +name: Update version in repo +on: + workflow_dispatch: + inputs: + new_version: + type: string + description: Version string to set in the repo + required: true + workflow_call: + inputs: + new_version: + type: string + description: Version string to set in the repo + required: true + ref: + type: string + description: commit/branch to change version + required: false + # A hack to tell if workflow is triggered by workflow_call or not + # Calling workflows should not set this input + # If workflow is triggered by workflow_dispatch, this should be set to the default boolean value: false + # https://github.com/actions/runner/discussions/1884#discussioncomment-6377587 + is_workflow_call: + type: boolean + default: true + required: false + secrets: + CLIENT_BOT_PAT: + required: true + outputs: + bump_sha: + value: ${{ jobs.update-version-in-repo.outputs.bump_sha }} + +jobs: + update-version-in-repo: + name: Update version in repo + runs-on: ubuntu-22.04 + outputs: + bump_sha: ${{ steps.get-bump-commit-sha.outputs.bump_sha }} + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.CLIENT_BOT_PAT }} + ref: ${{ inputs.ref }} + + - name: Update version in repo + uses: ./.github/actions/update-version + with: + new_version: ${{ inputs.new_version }} + + - name: Commit new version + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: 'Auto-bump version to ${{ inputs.new_version }} [skip ci]' + commit_author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> + tagging_message: ${{ inputs.new_version }} + branch: ${{ inputs.is_workflow_call && inputs.ref || github.ref }} + + - name: Output bump commit hash for next jobs to use + id: get-bump-commit-sha + run: echo "bump_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT diff --git a/.github/workflows/upload-to-jfrog.yml b/.github/workflows/upload-to-jfrog.yml new file mode 100644 index 00000000..5de0ada8 --- /dev/null +++ b/.github/workflows/upload-to-jfrog.yml @@ -0,0 +1,52 @@ +name: Upload to JFrog + +on: + workflow_call: + inputs: + version: + type: string + required: false + jfrog-repo-name: + type: string + required: false + default: ${{ vars.JFROG_REPO_NAME }} + secrets: + JFROG_PLATFORM_URL: + required: true + JFROG_ACCESS_TOKEN: + required: true + +jobs: + upload-to-jfrog: + name: Upload artifacts to JFrog + runs-on: ubuntu-22.04 + steps: + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + pattern: '*.node' + merge-multiple: true + path: artifacts + + - name: Set up JFrog credentials + uses: jfrog/setup-jfrog-cli@v3 + env: + JF_URL: ${{ secrets.JFROG_PLATFORM_URL }} + JF_ACCESS_TOKEN: ${{ secrets.JFROG_ACCESS_TOKEN }} + + - name: Upload manylinux builds from arbitrary branches to JFrog generic repo + if: ${{ inputs.jfrog-repo-name == vars.JFROG_GENERIC_REPO_NAME }} + run: jf rt upload "*manylinux*" ${{ vars.JFROG_GENERIC_REPO_NAME }}/${{ github.ref_name }}/ + working-directory: artifacts + + - name: Upload passing builds to JFrog Nodejs repo + if: ${{ inputs.jfrog-repo-name == vars.JFROG_REPO_NAME }} + # Source path must be in quotes if it contains an asterisk + # https://github.com/jfrog/jfrog-cli/issues/1775#issuecomment-1348986551 + run: jf rt upload --build-name nodejs-client --build-number $NEW_VERSION "artifacts/*" ${{ vars.JFROG_REPO_NAME }}/aerospike/$NEW_VERSION/ + env: + NEW_VERSION: ${{ inputs.version }} + + - name: Publish build info + if: ${{ inputs.jfrog-repo-name == vars.JFROG_REPO_NAME }} + run: jf rt build-publish nodejs-client ${{ inputs.version }} diff --git a/.github_former/actions/run-ee-server/action.yml b/.github_former/actions/run-ee-server/action.yml new file mode 100644 index 00000000..f7d4379f --- /dev/null +++ b/.github_former/actions/run-ee-server/action.yml @@ -0,0 +1,62 @@ +name: 'Run EE Server' +description: 'Run EE server' +inputs: + # All inputs in composite actions are strings + use-server-rc: + required: true + default: false + server-tag: + required: true + default: 'latest' + # Github Composite Actions can't access secrets + # so we need to pass them in as inputs + docker-hub-username: + required: false + docker-hub-password: + required: false + +runs: + using: "composite" + steps: + - name: Create config folder to store configs in + run: mkdir configs + shell: bash + + - name: Use release server + if: ${{ inputs.use-server-rc == 'false' }} + run: echo "SERVER_IMAGE=aerospike/aerospike-server-enterprise" >> $GITHUB_ENV + shell: bash + + - name: Use release candidate server + if: ${{ inputs.use-server-rc == 'true' }} + run: echo "SERVER_IMAGE=aerospike/aerospike-server-enterprise-rc" >> $GITHUB_ENV + shell: bash + + - name: Log into Docker Hub to get server RC + if: ${{ inputs.use-server-rc == 'true' }} + run: docker login --username ${{ inputs.docker-hub-username }} --password ${{ inputs.docker-hub-password }} + shell: bash + + - name: Get default aerospike.conf from Docker server EE container + run: | + docker run -d --name aerospike -p 3000-3002:3000-3002 $SERVER_IMAGE:${{ inputs.server-tag }} + sleep 5 + docker cp aerospike:/etc/aerospike/aerospike.conf ./configs/aerospike.conf + docker container stop aerospike + docker container rm aerospike + shell: bash + + - name: Enable security features using aerospike.conf + # Security stanza + run: echo -e "security {\n\tenable-quotas true\n}\n" >> ./aerospike.conf + working-directory: ./configs + shell: bash + + - name: Run enterprise edition server + run: docker run -tid -v $(pwd)/configs:/opt/aerospike/etc -p 3000:3000 --name aerospike $SERVER_IMAGE:${{ inputs.server-tag }} asd --config-file /opt/aerospike/etc/aerospike.conf + shell: bash + + - name: Create user in database for tests + # Use default admin user to create another user for testing + run: docker exec aerospike asadm --user admin --password admin --enable -e "manage acl create user superuser password superuser roles read-write-udf sys-admin user-admin data-admin" + shell: bash \ No newline at end of file diff --git a/.github/assets/aerospike.conf b/.github_former/assets/aerospike.conf similarity index 100% rename from .github/assets/aerospike.conf rename to .github_former/assets/aerospike.conf diff --git a/.github_former/workflows/build-bindings.yml b/.github_former/workflows/build-bindings.yml new file mode 100644 index 00000000..337c7f3e --- /dev/null +++ b/.github_former/workflows/build-bindings.yml @@ -0,0 +1,599 @@ +name: Build binding +run-name: Build bindings (run_tests=${{ inputs.run_tests }}, use-server-rc=${{ inputs.use-server-rc }}, server-tag=${{ inputs.server-tag }}) + +# Builds manylinux bindings and source distribution +# If running tests, publish results in commit status +# Publishes bindings and source distribution on Github Actions if no tests are run or tests pass + +on: + push: + branches: ["stage", "master"] + pull_request: + branches: ["stage", "master"] + types: [ + # Default triggers + opened, + synchronize, + reopened, + # Additional triggers + labeled, + unlabeled + ] + inputs: + # If we only want to check that the builds pass on an arbitrary branch + run_tests: + description: "Run integration tests" + required: true + type: boolean + default: true + use-server-rc: + type: boolean + required: true + default: false + description: 'Test against server release candidate?' + # If we are creating a backport and want to test an arbitrary branch against an older server version + server-tag: + required: true + default: 'latest' + description: 'Server docker image tag' + workflow_call: + inputs: + # The dev tests test the artifacts against a server release + # The stage tests and release workflow only need to build the artifacts + run_tests: + description: "Run integration tests" + required: true + type: boolean + default: true + commit_sha: + type: string + required: false + # Calling workflow doesn't actually use these 2 options + # But we need to set default values for workflow calls + use-server-rc: + required: false + default: true + type: boolean + server-tag: + type: string + required: false + default: 'latest' + secrets: + DOCKER_HUB_BOT_USERNAME: + required: false + DOCKER_HUB_BOT_PW: + required: false + MAC_M1_SELF_HOSTED_RUNNER_PW: + required: false + workflow_dispatch: + inputs: + # If we only want to check that the builds pass on an arbitrary branch + run_tests: + description: "Run integration tests" + required: true + type: boolean + default: true + use-server-rc: + type: boolean + required: true + default: false + description: 'Test against server release candidate?' + # If we are creating a backport and want to test an arbitrary branch against an older server version + server-tag: + required: true + default: 'latest' + description: 'Server docker image tag' + +jobs: + manylinux: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + # nodejs versions to build bindings on + nodejs: [ + ["v108", 18], + ["v115", 20], + ["v127", 22], + ["v131", 23] + ] + platform: [ + "x86_64", + #"aarch64" + ] + + steps: + + - name: Show job status for commit + uses: myrotvorets/set-commit-status-action@v2.0.0 + with: + sha: ${{ github.sha }} + context: "Build bindings (${{ matrix.nodejs[0] }}-manylinux_${{ matrix.platform }})" + + - uses: actions/checkout@v4 + with: + submodules: recursive + ref: ${{ inputs.commit_sha }} + + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.nodejs[1] }} + architecture: 'x64' + + - name: print + run: uname -m + + - name: Set up QEMU for cross compiling arm64 + if: ${{ matrix.platform == 'aarch64' }} + uses: docker/setup-qemu-action@v3 + with: + platforms: all + + - if: ${{ contains(github.event.pull_request.labels.*.name, 'new-server-features') }} + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} + password: ${{ secrets.DOCKER_HUB_BOT_PW }} + + + - name: Run Aerospike server release candidate with latest tag + if: ${{ contains(github.event.pull_request.labels.*.name, 'new-server-features') }} + run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server-rc:latest + + - name: Run Aerospike server + if: ${{ !contains(github.event.pull_request.labels.*.name, 'new-server-features') }} + run: | + npm i @types/mocha @types/yargs @types/semver @types/chai; + docker run -d -v $(pwd)/.github/assets/aerospike.conf:/etc/mail/aerospike.conf --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server --config-file /etc/mail/aerospike.conf + +# - name: Set config.conf to use Docker IP address of Aerospike server +# # config.conf should be copied into the cibuildwheel Docker container +# run: | +# export SERVER_DOCKER_IP=$(docker container inspect -f '{{ .NetworkSettings.IPAddress }}' aerospike) +# # Install crudini +# pip install crudini -c ../.github/workflows/requirements.txt +# sed -i "s/127.0.0.1:3000//" config.conf +# crudini --set config.conf enterprise-edition hosts ${SERVER_DOCKER_IP}:3000 +# working-directory: test + + - name: Enable tests + run: echo "TEST_COMMAND=cd ts-test; + npm install typescript --save-dev; + npx tsc; + cd ..; + npm run test dist/ -- --h 127.0.0.1 --port 3000" >> $GITHUB_ENV + +# - name: Disable tests (only run basic import test) +# if: ${{ !inputs.run_tests }} +# run: echo "TEST_COMMAND=node -e 'aerospike = require(\".\/lib\/aerospike\")'" >> $GITHUB_ENV + + - uses: uraimo/run-on-arch-action@v2 + name: Build client arm + if: ${{ matrix.platform == 'aarch64' }} + id: runcmd + with: + arch: aarch64 + distro: ubuntu-22.04 + + # Set an output parameter `uname` for use in subsequent steps + run: | + apt update + apt install -y g++ libssl-dev zlib1g-dev make build-essential libuv1-dev wget curl + ./scripts/build-c-client.sh + wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.3/install.sh | bash; + source ~/.bashrc; + nvm i 22 + npm install + + - name: Build client x64 + if: ${{ matrix.platform != 'aarch64' }} + run: | + sudo apt update + sudo apt install g++ libssl-dev zlib1g-dev; + sudo apt-get install -y make; + sudo apt install build-essential; + ./scripts/build-c-client.sh + npm install + env: + CFLAGS: '-Werror' + + - name: Test client + run: | + ${{ env.TEST_COMMAND }} + + - name: Send binding to test jobs + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.nodejs[0] }}-manylinux_${{ matrix.platform }}.build + path: ./lib/binding/node-*-linux-*/ + + - name: Set final commit status + uses: myrotvorets/set-commit-status-action@v2.0.0 + if: always() + with: + sha: ${{ github.sha }} + status: ${{ job.status }} + context: "Build bindings (${{ matrix.nodejs[0] }}-manylinux_${{ matrix.platform }})" + + macOS-x86: + strategy: + fail-fast: false + matrix: + nodejs: [ + ["v108", 18], + ["v115", 20], + ["v127", 22], + ['v131', 23] + ] + runs-on: macos-13 + steps: + - name: Show job status for commit + uses: myrotvorets/set-commit-status-action@v2.0.0 + with: + sha: ${{ github.sha }} + context: "Build bindings (${{ matrix.nodejs[0] }}-macosx_x86_64)" + + - uses: actions/checkout@v4 + with: + submodules: recursive + ref: ${{ inputs.commit_sha }} + + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.nodejs[1] }} + architecture: 'x64' + +# - name: Install Docker Engine +# run: brew install colima +# +# - name: Install Docker client +# run: brew install docker +# +# - name: Start Docker Engine +# run: colima start +# +# - if: ${{ contains(github.event.pull_request.labels.*.name, 'new-server-features') }} +# uses: docker/login-action@v3 +# with: +# username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} +# password: ${{ secrets.DOCKER_HUB_BOT_PW }} +# +# - name: Run Aerospike server release candidate with latest tag +# if: ${{ contains(github.event.pull_request.labels.*.name, 'new-server-features') }} +# run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server-rc:latest +# +# - name: Run Aerospike server +# if: ${{ !contains(github.event.pull_request.labels.*.name, 'new-server-features') }} +# run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server +# +# - name: Enable tests +# run: echo "TEST_COMMAND=npm test -- --h 127.0.0.1 --port 3000 --t 30000" >> $GITHUB_ENV + +# - name: Disable tests (only run basic import test) +# if: ${{ !inputs.run_tests }} +# run: echo "TEST_COMMAND=node -e 'aerospike = require(\".\/lib\/aerospike\")'" >> $GITHUB_ENV + + - name: Build client + run: | + ./scripts/build-c-client.sh + npm install + env: + CFLAGS: '-Werror' + +# - name: Test client +# run: | +# ${{ env.TEST_COMMAND }} + + - name: Save macOS binding + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.nodejs[0] }}-macosx_x86_64.build + path: ./lib/binding/*/ + + - name: Set final commit status + uses: myrotvorets/set-commit-status-action@v2.0.0 + if: always() + with: + status: ${{ job.status }} + sha: ${{ github.sha }} + context: "Build bindings (${{ matrix.nodejs[0] }}-macosx_x86_64)" + +# macOS-m1: +# runs-on: [ +# self-hosted, +# macOS, +# ARM64 +# ] +# strategy: +# matrix: +# nodejs-version: [ +# ["v108", "18"], +# ["v115", "20"], +# ["v127", 22] +# ] +# fail-fast: false +# steps: +# - name: Show job status for commit +# uses: myrotvorets/set-commit-status-action@v2.0.0 +# with: +# sha: ${{ github.sha }} +# context: "Build bindings (${{ matrix.nodejs-version[1] }}-macosx_arm64)" +# +# - uses: actions/checkout@v4 +# with: +# submodules: recursive +# ref: ${{ inputs.commit_sha }} +# +# - name: Install NVM +# run: | +# curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash +# source ~/.zshrc +# nvm -v +# nvm install ${{ matrix.nodejs-version[1] }} +# +# +# - name: Setup symlink folders +# run: | +# sudo rm -rf /usr/local/opt/openssl; +# sudo rm -rf /usr/local/opt/libuv; +# sudo mkdir -p /usr/local/opt; +# sudo chown -R $(whoami) /usr/local/opt +# +# - name: Install brew packages +# run: | +# brew install openssl@3.2.1 +# brew install libuv@1.47.0 +# +# - name: Set environment for building +# run: | +# echo "export PATH="/usr/local/bin/:/usr/local/opt/openssl/bin:$PATH" +# export LDFLAGS="-L/usr/local/opt/openssl/lib" +# export CPPFLAGS="-I/usr/local/opt/openssl/include" +# export EXT_CFLAGS="-I/usr/local/opt/openssl/include"" >> ~/.zshrc; +# source ~/.zshrc; +# +# - name: Setup symlink folders +# run: | +# sudo ln -s /usr/local/Cellar/libuv/1.47.0/ /usr/local/opt/libuv; +# sudo ln -s /usr/local/Cellar/openssl@3/3.2.1/ /usr/local/opt/openssl; +# +# # Self-hosted runner only +# # Need to be able to save Docker Hub credentials to keychain +# - run: security unlock-keychain -p ${{ secrets.MAC_M1_SELF_HOSTED_RUNNER_PW }} +# if: ${{ inputs.run_tests && inputs.use-server-rc }} +# +# - if: ${{ inputs.run_tests && inputs.use-server-rc }} +# uses: docker/login-action@v3 +# with: +# username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} +# password: ${{ secrets.DOCKER_HUB_BOT_PW }} +# +# - name: Use server rc +# if: ${{ inputs.run_tests && inputs.use-server-rc }} +# run: echo IMAGE_NAME="${{ vars.SERVER_RC_REPO_LINK }}:${{ inputs.server-tag }}" >> $GITHUB_ENV +# +# - name: Use server release +# if: ${{ inputs.run_tests && !inputs.use-server-rc }} +# run: echo IMAGE_NAME="${{ vars.SERVER_REPO_LINK }}:${{ inputs.server-tag }}" >> $GITHUB_ENV +# +# - name: Run server +# if: ${{ inputs.run_tests }} +# run: docker run -d -p 3000:3000 --name aerospike ${{ env.IMAGE_NAME }} +# +# - name: Build client +# run: | +# ./scripts/build-c-client.sh +# npm install +# env: +# CFLAGS: '-Werror' +# +# - name: Enable tests +# if: ${{ inputs.run_tests }} +# run: echo "TEST_COMMAND=npm test -- --h 127.0.0.1 --port 3000" >> $GITHUB_ENV +# +# - name: Disable tests (only run basic import test) +# if: ${{ !inputs.run_tests }} +# run: echo "TEST_COMMAND=node -e 'aerospike = require(\".\/lib\/aerospike\")'" >> $GITHUB_ENV +# +# +# - name: Test client +# run: | +# ${{ env.TEST_COMMAND }} +# +# - name: Save macOS wheel +# uses: actions/upload-artifact@v4 +# with: +# name: ${{ matrix.nodejs-version[0] }}-macosx_arm64.build +# path: ./lib/binding/*/ +# +# - name: Stop server +# if: ${{ always() && inputs.run_tests }} +# run: | +# docker container stop aerospike +# docker container prune -f +# +# - name: Set final commit status +# uses: myrotvorets/set-commit-status-action@v2.0.0 +# if: always() +# with: +# sha: ${{ github.sha }} +# status: ${{ job.status }} +# context: "Build bindings (${{ matrix.nodejs-version[0] }}-macosx_arm64)" + + test-npm-install: + runs-on: ubuntu-22.04 + needs: [manylinux, macOS-x86] + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + + - uses: ./.github/workflows/combine-bindings/ + + - uses: actions/setup-node@v4 + with: + node-version: ${{ env.LOWEST_SUPPORTED_NODEJS_VERSION }} + architecture: 'x64' + + - name: Modify the package.json + run: | + sudo npm install -g json + + + - name: Modify the package.json + run: | + sudo npm install -g json + json -I -f package.json -e "this.scripts.install=\"npm-run-all removeExtraBinaries build\"" + + - name: Run tests + run: | + mkdir -p testDir + cd testDir + pwd + sudo npm install .. + + test-yarn-install: + runs-on: ubuntu-22.04 + needs: [manylinux, macOS-x86] + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + + - uses: ./.github/workflows/combine-bindings/ + + - uses: actions/setup-node@v4 + with: + node-version: ${{ env.LOWEST_SUPPORTED_NODEJS_VERSION }} + architecture: 'x64' + + - name: Modify the package.json + run: | + sudo npm install -g json + json -I -f package.json -e "this.scripts.install=\"npm-run-all removeExtraBinaries build\"" + + - name: Run tests + run: | + mkdir -p testDir + yarn link + cd testDir + npm install --global yarn + yarn add link:.. + + test-pnpm-install: + runs-on: ubuntu-22.04 + needs: [manylinux, macOS-x86] + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + + - uses: ./.github/workflows/combine-bindings/ + + - uses: actions/setup-node@v4 + with: + node-version: ${{ env.LOWEST_SUPPORTED_NODEJS_VERSION }} + architecture: 'x64' + + - name: Modify the package.json + run: | + sudo npm install -g json + json -I -f package.json -e "this.scripts.install=\"npm-run-all removeExtraBinaries build\"" + + - name: Run tests + run: | + mkdir -p testDir + cd testDir + npm install --global pnpm + SHELL=bash pnpm setup + source /home/runner/.bashrc + pnpm install .. + + test-bun-install: + runs-on: ubuntu-latest + needs: [manylinux, macOS-x86] + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + + - uses: ./.github/workflows/combine-bindings/ + + - uses: actions/setup-node@v4 + with: + node-version: ${{ env.LOWEST_SUPPORTED_NODEJS_VERSION }} + architecture: 'x64' + + - name: Modify the package.json + run: | + sudo npm install -g json + json -I -f package.json -e "this.scripts.install=\"npm-run-all removeExtraBinaries build\"" + + - name: Run tests + run: | + npm install -g bun + bun link + mkdir -p testDir + cd testDir + bun link aerospike + +# test-typescript-install: +# runs-on: ubuntu-22.04 +# needs: [manylinux, macOS-x86] +# steps: +# - uses: actions/checkout@v2 +# with: +# submodules: recursive +# +# - uses: ./.github/workflows/combine-bindings/ +# +# - uses: actions/setup-node@v4 +# with: +# node-version: ${{ env.LOWEST_SUPPORTED_NODEJS_VERSION }} +# architecture: 'x64' +# +# - if: ${{ contains(github.event.pull_request.labels.*.name, 'new-server-features') }} +# uses: docker/login-action@v3 +# with: +# username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} +# password: ${{ secrets.DOCKER_HUB_BOT_PW }} +# +# +# - name: Run Aerospike server release candidate with latest tag +# if: ${{ contains(github.event.pull_request.labels.*.name, 'new-server-features') }} +# run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server-rc:latest +# +# - name: Run Aerospike server +# if: ${{ !contains(github.event.pull_request.labels.*.name, 'new-server-features') }} +# run: | +# cd ts-test; +# npm i --save-dev @types/mocha; +# npm i --save-dev @types/yargs; +# npm i --save-dev @types/semver; +# npm i --save-dev @types/chai; +# tsc; +# cd ..; +# npm install; +# docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server +# +# +# +# - name: Wait for database to be ready +# # Should be ready after 3 seconds +# run: sleep 15 +# +# - name: Modify the package.json +# run: | +# sudo npm install -g json +# json -I -f package.json -e "this.scripts.install=\"npm-run-all removeExtraBinaries build\"" +# +# - name: Run tests +# run: | +# mkdir my-aerospike-project +# cd my-aerospike-project +# npm init -y +# npm install typescript ts-node --save-dev +# npm install .. +# cp ../examples/typescript.ts index.ts +# npx tsc index.ts +# node index.js \ No newline at end of file diff --git a/.github/workflows/codeql.yml b/.github_former/workflows/codeql.yml similarity index 100% rename from .github/workflows/codeql.yml rename to .github_former/workflows/codeql.yml diff --git a/.github/workflows/combine-bindings/action.yml b/.github_former/workflows/combine-bindings/action.yml similarity index 100% rename from .github/workflows/combine-bindings/action.yml rename to .github_former/workflows/combine-bindings/action.yml diff --git a/.github/workflows/tests.yml b/.github_former/workflows/tests.yml similarity index 100% rename from .github/workflows/tests.yml rename to .github_former/workflows/tests.yml diff --git a/.github/workflows/windows-build.yml b/.github_former/workflows/windows-build.yml similarity index 100% rename from .github/workflows/windows-build.yml rename to .github_former/workflows/windows-build.yml