Add GitHub Actions workflow to build repro-cassandra jar #11
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - 'repro-*' | |
| pull_request: | |
| paths-ignore: | |
| - 'docs/**' | |
| env: | |
| # An envar that signals to tests we are executing in the CI environment | |
| CONTINUOUS_INTEGRATION: true | |
| # allow overriding Maven command | |
| MAVEN: ./mvnw | |
| # maven.wagon.rto is in millis, defaults to 30m | |
| MAVEN_OPTS: "-Xmx512M -XX:+ExitOnOutOfMemoryError -Dmaven.wagon.rto=60000" | |
| MAVEN_INSTALL_OPTS: "-Xmx3G -XX:+ExitOnOutOfMemoryError -Dmaven.wagon.rto=60000" | |
| MAVEN_FAST_INSTALL: "-B --strict-checksums -V --quiet -T C1 -DskipTests -Dmaven.source.skip=true -Dair.check.skip-all -P gib -Dgib.referenceBranch=refs/remotes/origin/${{ github.event.pull_request.base.ref }}" | |
| MAVEN_TEST: "-B --strict-checksums -Dmaven.source.skip=true -Dair.check.skip-all --fail-at-end -P gib -Dgib.referenceBranch=refs/remotes/origin/${{ github.event.pull_request.base.ref }}" | |
| RETRY: .github/bin/retry | |
| # Testcontainers kills image pulls if they don't make progress for > 30s and retries for 2m before failing. This means | |
| # that if an image doesn't download all it's layers within ~2m then any other concurrent pull will be killed because | |
| # the Docker daemon only downloads 3 layers concurrently which prevents the other pull from making any progress. | |
| # This value should be greater than the time taken for the longest image pull. | |
| TESTCONTAINERS_PULL_PAUSE_TIMEOUT: 600 | |
| TEST_REPORT_RETENTION_DAYS: 5 | |
| # Cancel previous PR builds. | |
| concurrency: | |
| # Cancel all workflow runs except latest within a concurrency group. This is achieved by defining a concurrency group for the PR. | |
| # Non-PR builds have singleton concurrency groups. | |
| group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.number || github.sha }} | |
| cancel-in-progress: true | |
| jobs: | |
| maven-checks: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| java-version: | |
| - 17 | |
| timeout-minutes: 45 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 # checkout all commits, as the build result depends on `git describe` equivalent | |
| - uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'zulu' | |
| java-version: ${{ matrix.java-version }} | |
| cache: 'maven' | |
| - name: Configure Problem Matchers | |
| run: echo "::add-matcher::.github/problem-matcher.json" | |
| - name: Maven Checks | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| $RETRY $MAVEN clean install -B --strict-checksums -V -T C1 -DskipTests -P ci -pl '!:trino-server-rpm' | |
| - name: Test Server RPM | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| $RETRY bash -c "$MAVEN verify -B --strict-checksums -P ci -pl :trino-server-rpm || find core/trino-server-rpm/ -exec ls -ald {} +" | |
| - name: Clean Maven Output | |
| run: $MAVEN clean -pl '!:trino-server,!:trino-cli' | |
| - uses: docker/setup-qemu-action@v1 | |
| with: | |
| platforms: arm64 | |
| - name: Test Docker Image | |
| run: core/docker/build.sh | |
| error-prone-checks: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB | |
| - name: Fetch base ref to find merge-base for GIB | |
| run: .github/bin/git-fetch-base-ref.sh | |
| - uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'zulu' | |
| java-version: 17 | |
| cache: 'maven' | |
| - name: Configure Problem Matchers | |
| run: echo "::add-matcher::.github/problem-matcher.json" | |
| - name: Maven Install | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| $RETRY $MAVEN clean install ${MAVEN_FAST_INSTALL} -pl '!:trino-docs,!:trino-server,!:trino-server-rpm' | |
| - name: Error Prone Checks | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| # Run Error Prone on one module with a retry to ensure all runtime dependencies are fetched | |
| $RETRY $MAVEN ${MAVEN_TEST} -T C1 clean test-compile -P gib,errorprone-compiler -pl ':trino-spi' | |
| # The main Error Prone run | |
| $MAVEN ${MAVEN_TEST} -T C1 clean test-compile -P gib,errorprone-compiler \ | |
| -pl '!:trino-docs,!:trino-server,!:trino-server-rpm' | |
| web-ui-checks: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 # checkout all commits: it's not needed here, but it's needed almost always, so let's do this for completeness | |
| - name: Web UI Checks | |
| run: core/trino-main/bin/check_webui.sh | |
| test-jdbc-compatibility: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 # checkout tags so version in Manifest is set properly | |
| - name: Fetch base ref to find merge-base for GIB | |
| run: .github/bin/git-fetch-base-ref.sh | |
| - uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'zulu' | |
| java-version: 17 | |
| cache: 'maven' | |
| - name: Configure Problem Matchers | |
| run: echo "::add-matcher::.github/problem-matcher.json" | |
| - name: Maven Install | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| $RETRY $MAVEN clean install ${MAVEN_FAST_INSTALL} -pl '!:trino-docs,!:trino-server,!:trino-server-rpm' | |
| - name: Test old JDBC vs current server | |
| run: | | |
| if [ ! -f gib-impacted.log ] || grep -q testing/trino-test-jdbc-compatibility-old-driver gib-impacted.log; then | |
| testing/trino-test-jdbc-compatibility-old-driver/bin/run_tests.sh | |
| fi | |
| - name: Test current JDBC vs old server | |
| if: always() | |
| run: | | |
| if [ ! -f gib-impacted.log ] || grep -q testing/trino-test-jdbc-compatibility-old-server gib-impacted.log; then | |
| $MAVEN test ${MAVEN_TEST} -pl :trino-test-jdbc-compatibility-old-server | |
| fi | |
| - name: Upload test results | |
| uses: actions/upload-artifact@v2 | |
| # Upload all test reports only on failure, because the artifacts are large | |
| if: failure() | |
| with: | |
| name: result ${{ github.job }} | |
| path: | | |
| **/target/surefire-reports | |
| **/target/checkstyle-* | |
| - name: Upload test report | |
| uses: actions/upload-artifact@v2 | |
| # Always upload the test report for the annotate.yml workflow, | |
| # but only the single XML file to keep the artifact small | |
| if: always() | |
| with: | |
| # Name prefix is checked in the `Annotate checks` workflow | |
| name: test report ${{ github.job }} | |
| path: | | |
| **/surefire-reports/TEST-*.xml | |
| retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} | |
| hive-tests: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: | |
| - config-empty | |
| - config-hdp3 | |
| # TODO: config-cdh5 | |
| # TODO: config-apache-hive3 | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB | |
| - name: Fetch base ref to find merge-base for GIB | |
| run: .github/bin/git-fetch-base-ref.sh | |
| - uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'zulu' | |
| java-version: 17 | |
| cache: 'maven' | |
| - name: Configure Problem Matchers | |
| run: echo "::add-matcher::.github/problem-matcher.json" | |
| - name: Install Hive Module | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| $RETRY $MAVEN clean install ${MAVEN_FAST_INSTALL} -am -pl :trino-hive-hadoop2 | |
| - name: Run Hive Tests | |
| run: | | |
| source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh && | |
| plugin/trino-hive-hadoop2/bin/run_hive_tests.sh | |
| - name: Run Hive S3 Tests | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESSKEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRETKEY }} | |
| S3_BUCKET: "presto-ci-test" | |
| S3_BUCKET_ENDPOINT: "s3.us-east-2.amazonaws.com" | |
| run: | | |
| if [ "${AWS_ACCESS_KEY_ID}" != "" ]; then | |
| source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh && | |
| plugin/trino-hive-hadoop2/bin/run_hive_s3_tests.sh | |
| fi | |
| - name: Run Hive Glue Tests | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESSKEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRETKEY }} | |
| AWS_REGION: us-east-2 | |
| run: | | |
| if [ "${AWS_ACCESS_KEY_ID}" != "" ]; then | |
| $MAVEN test ${MAVEN_TEST} -pl :trino-hive -P test-hive-glue | |
| fi | |
| - name: Run Hive Azure ABFS Access Key Tests | |
| if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage | |
| env: | |
| ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }} | |
| ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }} | |
| ABFS_ACCESS_KEY: ${{ secrets.AZURE_ABFS_ACCESSKEY }} | |
| run: | | |
| if [ "${ABFS_CONTAINER}" != "" ]; then | |
| source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh && | |
| plugin/trino-hive-hadoop2/bin/run_hive_abfs_access_key_tests.sh | |
| fi | |
| - name: Run Hive Azure ABFS OAuth Tests | |
| if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage | |
| env: | |
| ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }} | |
| ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }} | |
| ABFS_OAUTH_ENDPOINT: ${{ secrets.AZURE_ABFS_OAUTH_ENDPOINT }} | |
| ABFS_OAUTH_CLIENTID: ${{ secrets.AZURE_ABFS_OAUTH_CLIENTID }} | |
| ABFS_OAUTH_SECRET: ${{ secrets.AZURE_ABFS_OAUTH_SECRET }} | |
| run: | | |
| if [ -n "$ABFS_CONTAINER" ]; then | |
| source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh && | |
| plugin/trino-hive-hadoop2/bin/run_hive_abfs_oauth_tests.sh | |
| fi | |
| - name: Run Hive Azure WASB Tests | |
| if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage | |
| env: | |
| WASB_CONTAINER: ${{ secrets.AZURE_WASB_CONTAINER }} | |
| WASB_ACCOUNT: ${{ secrets.AZURE_WASB_ACCOUNT }} | |
| WASB_ACCESS_KEY: ${{ secrets.AZURE_WASB_ACCESSKEY }} | |
| run: | | |
| if [ "${WASB_CONTAINER}" != "" ]; then | |
| source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh && | |
| plugin/trino-hive-hadoop2/bin/run_hive_wasb_tests.sh | |
| fi | |
| - name: Run Hive Azure ADL Tests | |
| if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage | |
| env: | |
| ADL_NAME: ${{ secrets.AZURE_ADL_NAME }} | |
| ADL_CLIENT_ID: ${{ secrets.AZURE_ADL_CLIENTID }} | |
| ADL_CREDENTIAL: ${{ secrets.AZURE_ADL_CREDENTIAL }} | |
| ADL_REFRESH_URL: ${{ secrets.AZURE_ADL_REFRESHURL }} | |
| run: | | |
| if [ "${ADL_NAME}" != "" ]; then | |
| source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh && | |
| plugin/trino-hive-hadoop2/bin/run_hive_adl_tests.sh | |
| fi | |
| - name: Run Hive Alluxio Tests | |
| run: | | |
| source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh && | |
| plugin/trino-hive-hadoop2/bin/run_hive_alluxio_tests.sh | |
| - name: Upload test results | |
| uses: actions/upload-artifact@v2 | |
| # Upload all test reports only on failure, because the artifacts are large | |
| if: failure() | |
| with: | |
| name: result ${{ github.job }} | |
| path: | | |
| **/target/surefire-reports | |
| **/target/checkstyle-* | |
| - name: Upload test report | |
| uses: actions/upload-artifact@v2 | |
| # Always upload the test report for the annotate.yml workflow, | |
| # but only the single XML file to keep the artifact small | |
| if: always() | |
| with: | |
| # Name prefix is checked in the `Annotate checks` workflow | |
| name: test report ${{ github.job }} (${{ matrix.config }}) | |
| path: | | |
| **/surefire-reports/TEST-*.xml | |
| retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} | |
| test-other-modules: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB | |
| - name: Fetch base ref to find merge-base for GIB | |
| run: .github/bin/git-fetch-base-ref.sh | |
| - uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'zulu' | |
| java-version: 17 | |
| cache: 'maven' | |
| - name: Configure Problem Matchers | |
| run: echo "::add-matcher::.github/problem-matcher.json" | |
| - name: Maven Install | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| $RETRY $MAVEN clean install ${MAVEN_FAST_INSTALL} -pl '!:trino-docs,!:trino-server,!:trino-server-rpm' | |
| - name: Maven Tests | |
| run: | | |
| $MAVEN test ${MAVEN_TEST} -pl ' | |
| !:trino-main, | |
| !:trino-tests, | |
| !:trino-faulttolerant-tests, | |
| !:trino-raptor-legacy, | |
| !:trino-accumulo, | |
| !:trino-cassandra, | |
| !:trino-clickhouse, | |
| !:trino-delta-lake, | |
| !:trino-hive, | |
| !:trino-elasticsearch, | |
| !:trino-mongodb, | |
| !:trino-kafka, | |
| !:trino-pinot, | |
| !:trino-redis, | |
| !:trino-mysql, | |
| !:trino-postgresql, | |
| !:trino-sqlserver, | |
| !:trino-mariadb, | |
| !:trino-oracle, | |
| !:trino-kudu, | |
| !:trino-druid, | |
| !:trino-iceberg, | |
| !:trino-phoenix5, | |
| !:trino-jdbc,!:trino-base-jdbc,!:trino-thrift,!:trino-memory, | |
| !:trino-docs,!:trino-server,!:trino-server-rpm, | |
| !:trino-test-jdbc-compatibility-old-server, | |
| !:trino-singlestore, | |
| !:trino-bigquery' | |
| - name: Upload test results | |
| uses: actions/upload-artifact@v2 | |
| # Upload all test reports only on failure, because the artifacts are large | |
| if: failure() | |
| with: | |
| name: result ${{ github.job }} | |
| path: | | |
| **/target/surefire-reports | |
| **/target/checkstyle-* | |
| - name: Upload test report | |
| uses: actions/upload-artifact@v2 | |
| # Always upload the test report for the annotate.yml workflow, | |
| # but only the single XML file to keep the artifact small | |
| if: always() | |
| with: | |
| # Name prefix is checked in the `Annotate checks` workflow | |
| name: test report ${{ github.job }} | |
| path: | | |
| **/surefire-reports/TEST-*.xml | |
| retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} | |
| build-test-matrix: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB | |
| - name: Fetch base ref to find merge-base for GIB | |
| run: .github/bin/git-fetch-base-ref.sh | |
| - uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'zulu' | |
| java-version: 17 | |
| - name: Maven validate | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| $RETRY $MAVEN validate ${MAVEN_FAST_INSTALL} -P disable-check-spi-dependencies -pl '!:trino-docs,!:trino-server,!:trino-server-rpm' | |
| - name: Set matrix | |
| id: set-matrix | |
| run: | | |
| # GIB doesn't run on master, so make sure the file always exist | |
| touch gib-impacted.log | |
| cat <<EOF > .github/test-matrix.yaml | |
| include: | |
| - { modules: core/trino-main } | |
| - { modules: testing/trino-tests } | |
| - { modules: testing/trino-faulttolerant-tests, profile: test-fault-tolerant-hive-1 } | |
| - { modules: testing/trino-faulttolerant-tests, profile: test-fault-tolerant-hive-2 } | |
| - { modules: testing/trino-faulttolerant-tests, profile: test-fault-tolerant-delta } | |
| - { modules: testing/trino-faulttolerant-tests, profile: test-fault-tolerant-iceberg } | |
| - { modules: plugin/trino-raptor-legacy } | |
| - { modules: plugin/trino-accumulo } | |
| - { modules: plugin/trino-cassandra } | |
| - { modules: plugin/trino-clickhouse } | |
| - { modules: plugin/trino-delta-lake } | |
| - { modules: plugin/trino-hive } | |
| - { modules: plugin/trino-hive, profile: test-parquet } | |
| - { modules: plugin/trino-elasticsearch } | |
| - { modules: plugin/trino-elasticsearch, profile: test-stats } | |
| - { modules: plugin/trino-mongodb } | |
| - { modules: plugin/trino-kafka } | |
| - { modules: plugin/trino-pinot } | |
| - { modules: plugin/trino-redis } | |
| - { modules: plugin/trino-mysql } | |
| - { modules: plugin/trino-postgresql } | |
| - { modules: plugin/trino-sqlserver } | |
| - { modules: plugin/trino-singlestore } | |
| - { modules: plugin/trino-mariadb } | |
| - { modules: plugin/trino-oracle } | |
| - { modules: plugin/trino-kudu } | |
| - { modules: plugin/trino-druid } | |
| - { modules: plugin/trino-iceberg } | |
| - { modules: plugin/trino-phoenix5 } | |
| - { modules: [ client/trino-jdbc, plugin/trino-base-jdbc, plugin/trino-thrift, plugin/trino-memory ] } | |
| - { modules: plugin/trino-bigquery } | |
| EOF | |
| ./.github/bin/build-matrix-from-impacted.py -v -i gib-impacted.log -m .github/test-matrix.yaml -o matrix.json | |
| echo "Matrix: $(jq '.' matrix.json)" | |
| echo "::set-output name=matrix::$(jq -c '.' matrix.json)" | |
| test: | |
| runs-on: ubuntu-latest | |
| needs: build-test-matrix | |
| if: needs.build-test-matrix.outputs.matrix != '{}' | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.build-test-matrix.outputs.matrix) }} | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB | |
| - name: Fetch base ref to find merge-base for GIB | |
| run: .github/bin/git-fetch-base-ref.sh | |
| - uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'zulu' | |
| java-version: 17 | |
| cache: 'maven' | |
| - name: Configure Problem Matchers | |
| run: echo "::add-matcher::.github/problem-matcher.json" | |
| - name: Cleanup node | |
| # This is required as a virtual environment update 20210219.1 left too little space for MemSQL to work | |
| if: matrix.modules == 'plugin/trino-singlestore' | |
| run: .github/bin/cleanup-node.sh | |
| - name: Maven Install | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| $RETRY $MAVEN clean install ${MAVEN_FAST_INSTALL} -am -pl "${{ matrix.modules }}" | |
| - name: Maven Tests | |
| if: matrix.modules != 'plugin/trino-singlestore' | |
| run: $MAVEN test ${MAVEN_TEST} -pl ${{ matrix.modules }} ${{ matrix.profile != '' && format('-P {0}', matrix.profile) || '' }} | |
| # Additional tests for selected modules | |
| - name: Cloud Delta Lake Tests | |
| env: | |
| ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }} | |
| ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }} | |
| ABFS_ACCESSKEY: ${{ secrets.AZURE_ABFS_ACCESSKEY }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESSKEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRETKEY }} | |
| AWS_REGION: us-east-2 | |
| # Run tests if any of the secrets is present. Do not skip tests when one secret renamed, or secret name has a typo. | |
| if: >- | |
| contains(matrix.modules, 'trino-delta-lake') && | |
| (env.ABFS_ACCOUNT != '' || env.ABFS_CONTAINER != '' || env.ABFS_ACCESSKEY != '' || env.AWS_ACCESS_KEY_ID != '' || env.AWS_SECRET_ACCESS_KEY != '') | |
| run: | | |
| $MAVEN test ${MAVEN_TEST} -P cloud-tests -pl :trino-delta-lake \ | |
| -Dhive.hadoop2.azure-abfs-container="${ABFS_CONTAINER}" \ | |
| -Dhive.hadoop2.azure-abfs-account="${ABFS_ACCOUNT}" \ | |
| -Dhive.hadoop2.azure-abfs-access-key="${ABFS_ACCESSKEY}" | |
| - name: Memsql Tests | |
| env: | |
| MEMSQL_LICENSE: ${{ secrets.MEMSQL_LICENSE }} | |
| if: matrix.modules == 'plugin/trino-singlestore' && env.MEMSQL_LICENSE != '' | |
| run: | | |
| $MAVEN test ${MAVEN_TEST} -pl :trino-singlestore -Dmemsql.license=${MEMSQL_LICENSE} | |
| - name: Cloud BigQuery Tests | |
| env: | |
| BIGQUERY_CREDENTIALS_KEY: ${{ secrets.BIGQUERY_CREDENTIALS_KEY }} | |
| if: matrix.modules == 'plugin/trino-bigquery' && env.BIGQUERY_CREDENTIALS_KEY != '' | |
| run: | | |
| $MAVEN test ${MAVEN_TEST} -pl :trino-bigquery -Pcloud-tests -Dbigquery.credentials-key="${BIGQUERY_CREDENTIALS_KEY}" | |
| - name: Cloud BigQuery Case Insensitive Mapping Tests | |
| env: | |
| BIGQUERY_CASE_INSENSITIVE_CREDENTIALS_KEY: ${{ secrets.BIGQUERY_CASE_INSENSITIVE_CREDENTIALS_KEY }} | |
| if: matrix.modules == 'plugin/trino-bigquery' && env.BIGQUERY_CASE_INSENSITIVE_CREDENTIALS_KEY != '' | |
| run: | | |
| $MAVEN test ${MAVEN_TEST} -pl :trino-bigquery -Pcloud-tests-case-insensitive-mapping -Dbigquery.credentials-key="${BIGQUERY_CASE_INSENSITIVE_CREDENTIALS_KEY}" | |
| - name: Iceberg Glue Catalog Tests | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESSKEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRETKEY }} | |
| AWS_REGION: us-east-2 | |
| S3_BUCKET: presto-ci-test | |
| if: contains(matrix.modules, 'plugin/trino-iceberg') && (env.AWS_ACCESS_KEY_ID != '' || env.AWS_SECRET_ACCESS_KEY != '') | |
| run: | | |
| $MAVEN test ${MAVEN_TEST} -pl :trino-iceberg -P test-glue-catalog -Ds3.bucket=${S3_BUCKET} | |
| - name: Sanitize artifact name | |
| if: always() | |
| run: | | |
| # Generate a valid artifact name and make it available to next steps as | |
| # an environment variable ARTIFACT_NAME | |
| # ", :, <, >, |, *, ?, \, / are not allowed in artifact names, replace it with an underscore | |
| name=$(echo -n "${{ matrix.modules }}" | sed -e 's/[":<>|\*\?\\\/]/_/g') | |
| echo "ARTIFACT_NAME=$name" >> $GITHUB_ENV | |
| - name: Upload test results | |
| uses: actions/upload-artifact@v2 | |
| # Upload all test reports only on failure, because the artifacts are large | |
| if: failure() | |
| with: | |
| name: result ${{ env.ARTIFACT_NAME }} | |
| path: | | |
| **/target/surefire-reports | |
| **/target/checkstyle-* | |
| - name: Upload test report | |
| uses: actions/upload-artifact@v2 | |
| # Always upload the test report for the annotate.yml workflow, | |
| # but only the single XML file to keep the artifact small | |
| if: always() | |
| with: | |
| # Name prefix is checked in the `Annotate checks` workflow | |
| name: test report ${{ github.job }} (${{ env.ARTIFACT_NAME }}) | |
| path: | | |
| **/surefire-reports/TEST-*.xml | |
| retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} | |
| build-pt: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| have_azure_secrets: ${{ steps.check-secrets.outputs.have_azure_secrets }} | |
| have_databricks_secrets: ${{ steps.check-databricks-secrets.outputs.have_databricks_secrets }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB | |
| - name: Fetch base ref to find merge-base for GIB | |
| run: .github/bin/git-fetch-base-ref.sh | |
| - uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'zulu' | |
| java-version: 17 | |
| cache: 'maven' | |
| - name: Check secrets | |
| run: | | |
| if [[ "${{ secrets.AZURE_ABFS_CONTAINER }}" != "" && \ | |
| "${{ secrets.AZURE_ABFS_ACCOUNT }}" != "" && \ | |
| "${{ secrets.AZURE_ABFS_ACCESSKEY }}" != "" | |
| ]]; \ | |
| then | |
| echo "Secrets to run product tests were configured in the repo" | |
| echo "::set-output name=have_azure_secrets::true" | |
| else | |
| echo "Secrets to run product tests were not configured in the repo" | |
| echo "::set-output name=have_azure_secrets::false" | |
| fi | |
| id: check-secrets | |
| - name: Check Delta Databricks secrets | |
| id: check-databricks-secrets | |
| run: | | |
| if [[ "${{ secrets.DATABRICKS_TOKEN }}" != "" ]]; \ | |
| then | |
| echo "Secrets to run Delta Databricks product tests were configured in the repo" | |
| echo "::set-output name=have_databricks_secrets::true" | |
| else | |
| echo "Secrets to run Delta Databricks product tests were not configured in the repo" | |
| echo "::set-output name=have_databricks_secrets::false" | |
| fi | |
| - name: Maven Install | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| # GIB needs to be explicitly disabled, because the gib profile enables it, but the trino-server module requires all of its dependencies to be built | |
| $RETRY $MAVEN clean install ${MAVEN_FAST_INSTALL} -Dgib.disable -pl '!:trino-docs,!:trino-server-rpm' | |
| - name: Map impacted plugins to features | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| $MAVEN validate ${MAVEN_FAST_INSTALL} -pl '!:trino-docs,!:trino-server-rpm' | |
| # GIB doesn't run on master, so make sure the file always exist | |
| touch gib-impacted.log | |
| testing/trino-plugin-reader/target/trino-plugin-reader-*-executable.jar -i gib-impacted.log -p core/trino-server/target/trino-server-*-hardlinks/plugin > impacted-features.log | |
| echo "Impacted plugin features:" | |
| cat impacted-features.log | |
| - name: Product tests artifact | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: product tests and server tarball | |
| path: | | |
| core/trino-server/target/*.tar.gz | |
| impacted-features.log | |
| testing/trino-product-tests-launcher/target/*-executable.jar | |
| testing/trino-product-tests/target/*-executable.jar | |
| client/trino-cli/target/*-executable.jar | |
| retention-days: 1 | |
| pt: | |
| runs-on: ubuntu-latest | |
| # explicitly define the name to avoid adding the value of the `ignore exclusion if` matrix item | |
| name: pt (${{ matrix.config }}, ${{ matrix.suite }}, ${{ matrix.jdk }}) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: | |
| - default | |
| - hdp3 | |
| - cdh5 | |
| # TODO: config-apache-hive3 | |
| suite: | |
| - suite-1 | |
| - suite-2 | |
| - suite-3 | |
| # suite-4 does not exist | |
| - suite-5 | |
| - suite-azure | |
| - suite-delta-lake-databricks | |
| exclude: | |
| - config: cdh5 | |
| ignore exclusion if: >- | |
| ${{ github.event_name != 'pull_request' | |
| || contains(github.event.pull_request.labels.*.name, 'tests:all') | |
| || contains(github.event.pull_request.labels.*.name, 'tests:hive') | |
| }} | |
| - config: default | |
| ignore exclusion if: >- | |
| ${{ github.event_name != 'pull_request' | |
| || contains(github.event.pull_request.labels.*.name, 'tests:all') | |
| || contains(github.event.pull_request.labels.*.name, 'tests:hive') | |
| }} | |
| - suite: suite-azure | |
| config: default | |
| - suite: suite-azure | |
| config: cdh5 | |
| - suite: suite-azure | |
| ignore exclusion if: >- | |
| ${{ needs.build-pt.outputs.have_azure_secrets == 'true' }} | |
| - suite: suite-delta-lake-databricks | |
| config: cdh5 | |
| - suite: suite-delta-lake-databricks | |
| config: hdp3 | |
| - suite: suite-delta-lake-databricks | |
| ignore exclusion if: >- | |
| ${{ needs.build-pt.outputs.have_databricks_secrets == 'true' }} | |
| ignore exclusion if: | |
| # Do not use this property outside of the matrix configuration. | |
| # | |
| # This is added to all matrix entries so they may be conditionally | |
| # excluded by adding them to the excludes list with a GHA expression | |
| # for this property. | |
| # - If the expression evaluates to true, it will never match the a | |
| # actual value of the property, and will therefore not be excluded. | |
| # - If the expression evaluates to false, it will match the actual | |
| # value of the property, and the exclusion will apply normally. | |
| - false | |
| include: | |
| # this suite is not meant to be run with different configs | |
| - config: default | |
| suite: suite-6-non-generic | |
| # this suite is not meant to be run with different configs | |
| - config: default | |
| suite: suite-7-non-generic | |
| # this suite is not meant to be run with different configs | |
| - config: default | |
| suite: suite-8-non-generic | |
| # this suite is not meant to be run with different configs | |
| - config: default | |
| suite: suite-tpcds | |
| # this suite is not meant to be run with different configs | |
| - config: default | |
| suite: suite-oauth2 | |
| # this suite is not meant to be run with different configs | |
| - config: default | |
| suite: suite-ldap | |
| # this suite is not meant to be run with different configs | |
| - config: default | |
| suite: suite-compatibility | |
| # this suite is designed specifically for apache-hive3. TODO remove the suite once we can run all regular tests on apache-hive3. | |
| - config: apache-hive3 | |
| suite: suite-hms-only | |
| # suite-1 does not contain Kerberos/encrypted Hive environments | |
| - config: hdp3 | |
| suite: suite-1 | |
| # suite-2 contains Kerberos/encrypted Hive environments | |
| - config: hdp3 | |
| suite: suite-2 | |
| # this suite is not meant to be run with different configs | |
| - config: default | |
| suite: suite-all | |
| # this suite is not meant to be run with different configs | |
| - config: default | |
| suite: suite-delta-lake-oss | |
| # PT Launcher's timeout defaults to 2h, add some margin | |
| timeout-minutes: 130 | |
| needs: build-pt | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 # checkout all commits, as the build result depends on `git describe` equivalent | |
| - uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'zulu' | |
| java-version: 17 | |
| - uses: dorny/paths-filter@v2 | |
| id: filter | |
| with: | |
| filters: | | |
| product-tests: | |
| - 'testing/trino-product-tests*/**' | |
| - 'testing/trino-testing-services/**' | |
| # run all tests when there are any changes in the trino-server Maven module | |
| # because it uses Provisio to determine its dependencies | |
| - 'core/trino-server/**' | |
| - '.github/**' | |
| - name: Product tests artifact | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: product tests and server tarball | |
| - name: Fix artifact permissions | |
| run: | | |
| find . -type f -name \*-executable.jar -exec chmod 0777 {} \; | |
| - name: Enable impact analysis | |
| if: steps.filter.outputs.product-tests == 'false' && !contains(github.event.pull_request.labels.*.name, 'tests:all') && !contains(github.event.pull_request.labels.*.name, 'tests:all-product') | |
| run: echo "PTL_OPTS=--impacted-features impacted-features.log" >> $GITHUB_ENV | |
| - name: Product Tests | |
| env: | |
| ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }} | |
| ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }} | |
| ABFS_ACCESS_KEY: ${{ secrets.AZURE_ABFS_ACCESSKEY }} | |
| S3_BUCKET: trino-ci-test | |
| AWS_REGION: us-east-2 | |
| DATABRICKS_AWS_ACCESS_KEY_ID: ${{ secrets.DATABRICKS_AWS_ACCESS_KEY_ID }} | |
| DATABRICKS_AWS_SECRET_ACCESS_KEY: ${{ secrets.DATABRICKS_AWS_SECRET_ACCESS_KEY }} | |
| DATABRICKS_73_JDBC_URL: ${{ secrets.DATABRICKS_73_JDBC_URL }} | |
| DATABRICKS_91_JDBC_URL: ${{ secrets.DATABRICKS_91_JDBC_URL }} | |
| DATABRICKS_104_JDBC_URL: ${{ secrets.DATABRICKS_104_JDBC_URL }} | |
| DATABRICKS_LOGIN: token | |
| DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} | |
| run: | | |
| testing/bin/ptl suite run \ | |
| --suite ${{ matrix.suite }} \ | |
| --config config-${{ matrix.config }} \ | |
| $PTL_OPTS \ | |
| --bind=off --logs-dir logs/ --timeout 2h | |
| - name: Upload test logs and results | |
| uses: actions/upload-artifact@v2 | |
| # Upload all test reports only on failure, because the artifacts are large | |
| if: failure() | |
| with: | |
| name: result pt (${{ matrix.config }}, ${{ matrix.suite }}, ${{ matrix.jdk }}) | |
| path: | | |
| testing/trino-product-tests/target/* | |
| logs/* | |
| - name: Upload test report | |
| uses: actions/upload-artifact@v2 | |
| # Always upload the test report for the annotate.yml workflow, | |
| # but only the single XML file to keep the artifact small | |
| if: always() | |
| with: | |
| # Name prefix is checked in the `Annotate checks` workflow | |
| name: test report pt (${{ matrix.config }}, ${{ matrix.suite }}, ${{ matrix.jdk }}) | |
| path: testing/trino-product-tests/target/reports/**/testng-results.xml | |
| retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} |