integration.yml

name: Integration

on:

  # Integration tests always run on pull requests
  pull_request:

  # Re-run integration tests in main, to make sure there are no issues from the merge
  push:
    branches:
      - main

  # Allow manual triggering of integration tests
  workflow_dispatch:


# Use baseline language versions for integration builds
env:
  JAVA_VERSION: 17
  JAVA_DISTRIBUTION: zulu
  PYTHON_VERSION: "3.11"
  NODE_VERSION: 18


jobs:

  web-api:

    runs-on: ubuntu-latest
    timeout-minutes: 20

    steps:

      - name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Setup - Java
        uses: actions/setup-java@v3
        with:
          distribution: ${{ env.JAVA_DISTRIBUTION }}
          java-version: ${{ env.JAVA_VERSION }}

          # Turn on Gradle dependency caching
          cache: gradle

      - name: Setup -Node.js
        uses: actions/setup-node@v3
        with:
          node-version: ${{ env.NODE_VERSION }}

      - name: Platform - Build distribution files
        run: ./gradlew installDist

      - name: Platform - Install the sandbox
        run: |
          VERSION=`dev/version.sh`
          mkdir -p build/dist/tracdap-sandbox-${VERSION}
          for MODULE in build/modules/*/install/*; do
            cp -R $MODULE/* build/dist/tracdap-sandbox-${VERSION}
          done
          cd build/dist
          ln -s tracdap-sandbox-${VERSION}/ current

      # Currently not starting svc-orch, we'd need to build the runtime and deploy an execution venv
      # OUr JavaScript tests only do data and metadata so far, no spawning / checking jobs
      # We should aim to get there though!

      - name: Platform - Run setup tools
        run: |
          cd build/dist/current
          deploy_dir=`pwd`
          
          mkdir metadata
          mkdir -p data/storage1
          
          echo SECRET_KEY=xHjFeFlo$k= >> etc/env.sh
          sed -i "s#/path/to#${deploy_dir}#g" etc/trac-platform.yaml
          
          bin/secret-tool run --task init_secrets
          bin/secret-tool run --task create_root_auth_key EC 256
          bin/deploy-metadb run --task deploy_schema
          bin/deploy-metadb run --task add_tenant ACME_CORP "ACME Rockets always get the Coyote!"

      - name: Platform - Start
        run: |
          cd build/dist/current
          
          bin/tracdap-svc-meta start
          bin/tracdap-svc-data start
          bin/tracdap-gateway start

      - name: Web API - Install dependencies
        run: |
          cd tracdap-api/packages/web
          npm install

      - name: Web API - Set TRAC version
        run: |
          cd tracdap-api/packages/web
          npm run tracVersion:posix

      - name: Web API - Build package
        run: |
          cd tracdap-api/packages/web
          npm run buildApi

      - name: Examples - Install dependencies
        run: |
          cd examples/apps/javascript
          npm install

      - name: Examples - Run all
        run: |
          cd examples/apps/javascript
          npm run examples

      - name: Platform - Shut down
        if: always()
        run: |
          cd build/dist/current
          
          bin/tracdap-gateway stop
          bin/tracdap-svc-data stop
          bin/tracdap-svc-meta stop
  

  slow-tests:

    runs-on: ubuntu-latest
    timeout-minutes: 20

    steps:

      # fetch-depth = 0 is needed to get tags for version info
      - name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Set up Java
        uses: actions/setup-java@v3
        with:
          distribution: ${{ env.JAVA_DISTRIBUTION }}
          java-version: ${{ env.JAVA_VERSION }}
          cache: gradle

      - name: Build
        run: ./gradlew testClasses

      - name: Slow tests
        run: ./gradlew integration -DintegrationTags="slow"

      # If the tests fail, make the output available for download
      - name: Store failed test results
        uses: actions/upload-artifact@v3
        if: failure()
        with:
          name: junit-test-results
          path: build/modules/*/reports/**
          retention-days: 7


  end-to-end:

    runs-on: ubuntu-latest
    timeout-minutes: 20

    steps:

      # fetch-depth = 0 is needed to get tags for version info
      - name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Java - Setup
        uses: actions/setup-java@v3
        with:
          distribution: ${{ env.JAVA_DISTRIBUTION }}
          java-version: ${{ env.JAVA_VERSION }}

          # Turn on Gradle dependency caching
          cache: gradle

      - name: Java - Build platform
        run: ./gradlew testClasses

      - name: Python - setup
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Python - PIP
        run: python -m pip install --upgrade pip

      - name: Python - Install build dependencies
        run: |
          pip install -r tracdap-runtime/python/requirements.txt

      - name: Python - Build runtime package
        run: python tracdap-runtime/python/build_runtime.py --target codegen dist

      - name: Run end to end tests
        run: ./gradlew integration -DintegrationTags="int-e2e"

      # If the tests fail, make the output available for download
      - name: Store failed test results
        uses: actions/upload-artifact@v3
        if: failure()
        with:
          name: junit-test-results
          path: build/modules/*/reports/**
          retention-days: 7


  int-metadb:

    runs-on: ubuntu-latest
    timeout-minutes: 20

    container:
      image: ubuntu:latest

    strategy:

      # Try to finish all jobs - it can be helpful to see if some succeed and others fail
      fail-fast: false

      matrix:

        database:

          - { DB_NAME: MySQL,
              DB_IMAGE: 'mysql:8.0',
              DB_PORT: 3306,
              DB_OPTIONS: '--health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3',
              BUILD_sql_mysql: true,
              TRAC_CONFIG_FILE: '.github/config/int-metadb-mysql.yaml',
              TRAC_SECRET_KEY: wDeq3x-NjaLL7,
              MYSQL_DATABASE: trac,
              MYSQL_USER: trac_admin,
              MYSQL_PASSWORD: trac_admin,
              METADB_SECRET: trac_admin,
              MYSQL_ALLOW_EMPTY_PASSWORD: yes }

          - { DB_NAME: MariaDB,
              DB_IMAGE: 'mariadb:10.3',
              DB_PORT: 3306,
              DB_OPTIONS: '--health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3',
              BUILD_sql_mariadb: true,
              TRAC_CONFIG_FILE: '.github/config/int-metadb-mariadb.yaml',
              TRAC_SECRET_KEY: uYhnKwq8+esS,
              MYSQL_DATABASE: trac,
              MYSQL_USER: trac_admin,
              MYSQL_PASSWORD: trac_admin,
              METADB_SECRET: trac_admin,
              MYSQL_ALLOW_EMPTY_PASSWORD: yes }

          - { DB_NAME: PostgreSQL,
              DB_IMAGE: 'postgres:10-alpine',
              DB_PORT: 5432,
              DB_OPTIONS: '--health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5',
              BUILD_sql_postgresql: true,
              TRAC_CONFIG_FILE: '.github/config/int-metadb-postgresql.yaml',
              TRAC_SECRET_KEY: hjXks83bX=wxMr,
              POSTGRES_DB: trac,
              POSTGRES_USER: trac_admin,
              METADB_SECRET: trac_admin,
              POSTGRES_PASSWORD: trac_admin }

          - { DB_NAME: SQLServer,
              DB_IMAGE: 'mcr.microsoft.com/mssql/server:2019-latest',
              DB_PORT: 1433,
              DB_OPTIONS: '-e "NO_DB_OPTIONS=not_used"',  # docker run -e flag sets an env variable, passing '' causes errors
              BUILD_sql_sqlserver: true,
              TRAC_CONFIG_FILE: '.github/config/int-metadb-sqlserver.yaml',
              TRAC_SECRET_KEY: unHkj>weN2jSl,
              MSSQL_PID: Developer,
              ACCEPT_EULA: Y,
              SA_PASSWORD: "tR4c_aDm!n",
              METADB_SECRET: "tR4c_aDm!n" }

    env: ${{ matrix.database }}

    services:

      metadb:

        image: ${{ matrix.database.DB_IMAGE }}
        env: ${{ matrix.database }}
        ports:
          - ${{ matrix.database.DB_PORT }}:${{ matrix.database.DB_PORT }}
        options: ${{ matrix.database.DB_OPTIONS }}

    steps:

      # fetch-depth = 0 is needed to get tags for version info
      - name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Set up Java
        uses: actions/setup-java@v3
        with:
          distribution: ${{ env.JAVA_DISTRIBUTION }}
          java-version: ${{ env.JAVA_VERSION }}
          cache: gradle

      - name: Build
        run: ./gradlew trac-svc-meta:testClasses

      # Auth tool will also create the secrets file if it doesn't exist
      - name: Prepare secrets
        run: |
          ./gradlew secret-tool:run --args="--config ${{ env.TRAC_CONFIG_FILE }} --task init_secrets"
          ./gradlew secret-tool:run --args="--config ${{ env.TRAC_CONFIG_FILE }} --task create_root_auth_key EC 256"
          
          echo "${METADB_SECRET}" | ./gradlew secret-tool:run --args="--config ${{ env.TRAC_CONFIG_FILE }} --task add_secret metadb_secret"

      # The name and description of the test tenant are verified in one of the test cases so they need to match
      # MetadataReapApiTest listTenants()
      - name: Prepare database
        run: |
          ./gradlew deploy-metadb:run --args="\
          --config ${{ env.TRAC_CONFIG_FILE }} \
          --secret-key ${{ env.TRAC_SECRET_KEY }} \
          --task deploy_schema \
          --task add_tenant ACME_CORP 'Test tenant [ACME_CORP]'"

      - name: Integration tests
        run: ./gradlew trac-svc-meta:integration -DintegrationTags="int-metadb"

      # If the tests fail, make the output available for download
      - name: Store failed test results
        uses: actions/upload-artifact@v3
        if: failure()
        with:
          name: junit-test-results
          path: build/modules/*/reports/**
          retention-days: 7


  int-storage:

    strategy:

      # Try to finish all jobs - it can be helpful to see if some succeed and others fail
      fail-fast: false

      matrix:

        storage:

          - { SERVICE: storage,
              TARGET: aws,
              PROTOCOL: S3,
              ENABLE_IF: TRAC_AWS_BUCKET,
              BUILD_aws_storage: true,
              INSTALL_SCRIPT: ".github/scripts/int-storage-s3-install.sh",
              SETUP_SCRIPT: ".github/scripts/int-storage-s3-setup.sh",
              TRAC_CONFIG_FILE: '.github/config/int-storage-s3.yaml',
              TRAC_SECRET_KEY: short-lived-secret,
              PYTHON_TESTS: int_storage_aws*.py,
              JAVA_TESTS: int-storage }

          - { SERVICE: storage,
              TARGET: gcp,
              PROTOCOL: GCS,
              ENABLE_IF: TRAC_GCP_BUCKET,
              BUILD_gcp_storage: true,
              INSTALL_SCRIPT: ".github/scripts/int-storage-gcs-install.sh",
              SETUP_SCRIPT: ".github/scripts/int-storage-gcs-setup.sh",
              TRAC_CONFIG_FILE: '.github/config/int-storage-gcs.yaml',
              TRAC_SECRET_KEY: short-lived-secret,
              PYTHON_TESTS: int_storage_gcp*.py,
              JAVA_TESTS: int-storage }

          - { SERVICE: storage,
              TARGET: azure,
              PROTOCOL: BLOB,
              ENABLE_IF: TRAC_AZURE_CONTAINER,
              BUILD_azure_storage: true,
              INSTALL_SCRIPT: ".github/scripts/int-storage-azure-install.sh",
              SETUP_SCRIPT: ".github/scripts/int-storage-azure-setup.sh",
              TRAC_CONFIG_FILE: '.github/config/int-storage-azure.yaml',
              TRAC_SECRET_KEY: short-lived-secret,
              PYTHON_TESTS: int_storage_azure*.py,
              JAVA_TESTS: int-storage }

    uses: ./.github/workflows/integration-cloud.yaml

    with:
      service: ${{ matrix.storage.SERVICE }}
      target: ${{ matrix.storage.TARGET }}
      protocol: ${{ matrix.storage.PROTOCOL }}
      matrix: ${{ toJson( matrix.storage ) }}
      enabled: ${{ vars[ matrix.storage.ENABLE_IF ] != '' }}
      has_java: ${{ matrix.storage.JAVA_TESTS != '' }}
      has_python: ${{ matrix.storage.PYTHON_TESTS != '' }}

    secrets: inherit

    # Permissions required to use IdP to connect to cloud platforms
    permissions:
      id-token: write   # This is required for requesting the JWT
      contents: read    # This is required for actions/checkout