connectors_up_to_date.yml

name: Connectors up-to-date
concurrency:
  group: ${{ github.workflow }}
  cancel-in-progress: false

on:
  schedule:
    # Runs everyday Saturday at 12:00 UTC
    - cron: "0 12 * * 6"
  workflow_dispatch:
    inputs:
      connectors-options:
        description: "Options to pass to the 'airbyte-ci connectors' command group."
        default: "--language=python --language=low-code --language=manifest-only"
jobs:
  generate_matrix:
    name: Generate matrix
    runs-on: ubuntu-latest
    outputs:
      generated_matrix: ${{ steps.generate_matrix.outputs.generated_matrix }}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v4
      - name: Run airbyte-ci connectors list [SCHEDULED TRIGGER]
        if: github.event_name == 'schedule'
        id: airbyte-ci-connectors-list-scheduled
        uses: ./.github/actions/run-airbyte-ci
        with:
          context: "master"
          subcommand: 'connectors --language=python --language=low-code --language=manifest-only  --metadata-query="\"-rc.\" not in data.dockerImageTag and \"source-declarative-manifest\" not in data.dockerRepository" list --output=selected_connectors.json'
      - name: Run airbyte-ci connectors list [MANUAL TRIGGER]
        if: github.event_name == 'workflow_dispatch'
        id: airbyte-ci-connectors-list-workflow-dispatch
        uses: ./.github/actions/run-airbyte-ci
        with:
          context: "master"
          subcommand: 'connectors ${{ github.event.inputs.connectors-options }} --metadata-query="\"-rc.\" not in data.dockerImageTag and \"source-declarative-manifest\" not in data.dockerRepository" list --output=selected_connectors.json'
      # We generate a dynamic matrix from the list of selected connectors.
      # A matrix is required in this situation because the number of connectors is large and running them all in a single job would exceed the maximum job time limit of 6 hours.
      - name: Generate matrix - 100 connectors per job
        id: generate_matrix
        run: |
          matrix=$(jq -c -r '{include: [.[] | "--name=" + .] | to_entries | group_by(.key / 100 | floor) | map(map(.value) | {"connector_names": join(" ")})}' selected_connectors.json)
          echo "::set-output name=generated_matrix::$matrix"

  run_connectors_up_to_date:
    needs: generate_matrix
    name: Connectors up-to-date
    runs-on: connector-up-to-date-medium
    continue-on-error: true
    strategy:
      matrix: ${{fromJson(needs.generate_matrix.outputs.generated_matrix)}}
    permissions:
      pull-requests: write
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v4
      - name: Run airbyte-ci connectors up-to-date [WORKFLOW]
        id: airbyte-ci-connectors-up-to-date-workflow-dispatch
        uses: ./.github/actions/run-airbyte-ci
        with:
          context: "master"
          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_3 }}
          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
          gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
          s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          subcommand: "connectors --concurrency=10 ${{ matrix.connector_names}} up-to-date  --create-prs --auto-merge"