Mitlify POC #3514
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: ci-slow | |
| on: | |
| workflow_dispatch: | |
| workflow_call: | |
| push: | |
| branches: [main] | |
| paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] | |
| pull_request: | |
| types: [opened, synchronize, ready_for_review] | |
| paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] | |
| concurrency: | |
| # New commit on branch cancels running workflows of the same branch | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| run-slow-ci-label-is-set: | |
| runs-on: ubuntu-latest | |
| if: github.event.pull_request.draft == false | |
| steps: | |
| # using this instead of contains(github.event.pull_request.labels.*.name, 'run-slow-ci') | |
| # to make it dynamic, otherwise github context is fixed at the moment of trigger event. | |
| # With dynamic approach dev can set label and rerun this flow to make it running. | |
| - name: Get PR labels | |
| id: pr-labels | |
| uses: actions/github-script@v5 | |
| with: | |
| script: | | |
| const prNumber = ${{ github.event.pull_request.number }}; | |
| const { data: labels } = await github.rest.issues.listLabelsOnIssue({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: prNumber, | |
| }); | |
| const labelNames = labels.map(label => label.name); | |
| core.setOutput('all-labels', labelNames.join(',')); | |
| - name: Slow CI label not set | |
| if: ${{ !contains(steps.pr-labels.outputs.all-labels, 'run-slow-ci') }} | |
| run: | | |
| echo "Please add the 'run-slow-ci' label to this PR before merging." | |
| exit 1 | |
| docstring-check: | |
| if: github.event.pull_request.draft == false | |
| needs: run-slow-ci-label-is-set | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: Set up Python | |
| uses: actions/[email protected] | |
| with: | |
| python-version: '3.10' | |
| - name: Install current package as editable | |
| run: | | |
| curl -LsSf https://astral.sh/uv/install.sh | sh | |
| source $HOME/.cargo/env | |
| uv pip install --system darglint | |
| - name: Check docstrings | |
| run: bash scripts/docstring.sh | |
| mysql-db-migration-testing-full: | |
| if: github.event.pull_request.draft == false | |
| needs: run-slow-ci-label-is-set | |
| env: | |
| ZENML_ANALYTICS_OPT_IN: false | |
| ZENML_DEBUG: true | |
| runs-on: arc-runner-set | |
| steps: | |
| - uses: actions/[email protected] | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python 3.9 | |
| uses: actions/[email protected] | |
| with: | |
| python-version: '3.9' | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| if: github.event.pull_request.head.repo.fork == false | |
| - name: Test migrations across versions | |
| run: bash scripts/test-migrations.sh mysql full | |
| mysql-db-migration-testing-random: | |
| if: github.event.pull_request.draft == false | |
| needs: run-slow-ci-label-is-set | |
| env: | |
| ZENML_ANALYTICS_OPT_IN: false | |
| ZENML_DEBUG: true | |
| runs-on: arc-runner-set | |
| steps: | |
| - uses: actions/[email protected] | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python 3.9 | |
| uses: actions/[email protected] | |
| with: | |
| python-version: '3.9' | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| if: github.event.pull_request.head.repo.fork == false | |
| - name: Test migrations across versions | |
| run: bash scripts/test-migrations.sh mysql random | |
| sqlite-db-migration-testing-full: | |
| needs: run-slow-ci-label-is-set | |
| runs-on: arc-runner-set | |
| env: | |
| ZENML_ANALYTICS_OPT_IN: false | |
| ZENML_DEBUG: true | |
| if: github.event.pull_request.draft == false | |
| steps: | |
| - name: Checkout code | |
| uses: actions/[email protected] | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python 3.9 | |
| uses: actions/[email protected] | |
| with: | |
| python-version: '3.9' | |
| - name: Test migrations across versions | |
| run: bash scripts/test-migrations.sh sqlite full | |
| mariadb-db-migration-testing: | |
| if: github.event.pull_request.draft == false | |
| needs: run-slow-ci-label-is-set | |
| env: | |
| ZENML_ANALYTICS_OPT_IN: false | |
| ZENML_DEBUG: true | |
| runs-on: arc-runner-set | |
| steps: | |
| - uses: actions/[email protected] | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python 3.9 | |
| uses: actions/[email protected] | |
| with: | |
| python-version: '3.9' | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| if: github.event.pull_request.head.repo.fork == false | |
| - name: Test migrations across versions | |
| run: bash scripts/test-migrations.sh mariadb full | |
| small-checks: | |
| if: github.event.pull_request.draft == false | |
| needs: run-slow-ci-label-is-set | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: Set up Python | |
| uses: actions/[email protected] | |
| with: | |
| python-version: '3.8' | |
| - name: Install uv | |
| run: | | |
| curl -LsSf https://astral.sh/uv/install.sh | sh | |
| source $HOME/.cargo/env | |
| - name: Create virtual environment | |
| run: | | |
| uv venv | |
| - name: Check mlstacks compatibility | |
| run: | | |
| source .venv/bin/activate | |
| uv pip install -e . | |
| uv pip install mlstacks | |
| - name: Check for broken dependencies | |
| run: | | |
| source .venv/bin/activate | |
| uv pip check | |
| - name: Markdown link check | |
| uses: gaurav-nelson/[email protected] | |
| with: | |
| use-quiet-mode: 'yes' | |
| use-verbose-mode: 'no' | |
| folder-path: ./examples, ./docs/book, ./src | |
| file-path: ./README.md, ./LICENSE, ./RELEASE_NOTES.md, CODE-OF-CONDUCT.md, | |
| CONTRIBUTING.md, CLA.md, RELEASE_NOTES.md, ROADMAP.md | |
| config-file: .github/workflows/markdown_check_config.json | |
| continue-on-error: true | |
| - name: Security check | |
| run: | | |
| source .venv/bin/activate | |
| uv pip install bandit | |
| bash scripts/check-security.sh | |
| - name: Check for alembic branch divergence | |
| env: | |
| ZENML_DEBUG: 0 | |
| run: | | |
| source .venv/bin/activate | |
| uv pip install alembic | |
| bash scripts/check-alembic-branches.sh | |
| - name: Install latest dashboard (test gitignore) | |
| run: bash scripts/install-dashboard.sh | |
| - name: Test for relative links in docs | |
| run: python scripts/ensure_absolute_links.py docs/book | |
| custom-ubuntu-unit-test: | |
| if: github.event.pull_request.draft == false | |
| needs: run-slow-ci-label-is-set | |
| strategy: | |
| matrix: | |
| os: [arc-runner-set] | |
| python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] | |
| fail-fast: false | |
| uses: ./.github/workflows/unit-test.yml | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| os: ${{ matrix.os }} | |
| secrets: inherit | |
| windows-unit-test: | |
| if: github.event.pull_request.draft == false | |
| needs: run-slow-ci-label-is-set | |
| strategy: | |
| matrix: | |
| os: [windows-latest] | |
| python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] | |
| fail-fast: false | |
| uses: ./.github/workflows/unit-test.yml | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| os: ${{ matrix.os }} | |
| secrets: inherit | |
| macos-unit-test: | |
| if: github.event.pull_request.draft == false | |
| needs: run-slow-ci-label-is-set | |
| strategy: | |
| matrix: | |
| os: [macos-latest] | |
| python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] | |
| # Python 3.8 and 3.9 is on macos-13 but not macos-latest (macos-14-arm64) | |
| # https://github.com/actions/setup-python/issues/696#issuecomment-1637587760 | |
| exclude: | |
| - {python-version: '3.8', os: macos-latest} | |
| - {python-version: '3.9', os: macos-latest} | |
| include: | |
| - {python-version: '3.8', os: macos-13} | |
| - {python-version: '3.9', os: macos-13} | |
| fail-fast: false | |
| uses: ./.github/workflows/unit-test.yml | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| os: ${{ matrix.os }} | |
| secrets: inherit | |
| windows-integration-test: | |
| if: github.event.pull_request.draft == false | |
| needs: run-slow-ci-label-is-set | |
| strategy: | |
| matrix: | |
| os: [windows-latest] | |
| python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] | |
| test_environment: [default] | |
| fail-fast: false | |
| uses: ./.github/workflows/integration-test-slow.yml | |
| with: | |
| os: ${{ matrix.os }} | |
| python-version: ${{ matrix.python-version }} | |
| test_environment: ${{ matrix.test_environment }} | |
| secrets: inherit | |
| macos-integration-test: | |
| if: github.event.pull_request.draft == false | |
| needs: run-slow-ci-label-is-set | |
| strategy: | |
| matrix: | |
| os: [macos-13] | |
| python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] | |
| # Python 3.8 and 3.9 is on macos-13 but not macos-latest (macos-14-arm64) | |
| # https://github.com/actions/setup-python/issues/696#issuecomment-1637587760 | |
| exclude: | |
| - {python-version: '3.8', os: macos-latest} | |
| - {python-version: '3.9', os: macos-latest} | |
| include: | |
| - {python-version: '3.8', os: macos-13} | |
| - {python-version: '3.9', os: macos-13} | |
| test_environment: [default] | |
| fail-fast: false | |
| uses: ./.github/workflows/integration-test-slow.yml | |
| with: | |
| os: ${{ matrix.os }} | |
| python-version: ${{ matrix.python-version }} | |
| test_environment: ${{ matrix.test_environment }} | |
| secrets: inherit | |
| custom-ubuntu-integration-test: | |
| if: github.event.pull_request.draft == false | |
| needs: run-slow-ci-label-is-set | |
| strategy: | |
| matrix: | |
| os: [arc-runner-set] | |
| python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] | |
| test_environment: | |
| - default | |
| - docker-server-docker-orchestrator-mysql | |
| - docker-server-docker-orchestrator-mariadb | |
| exclude: | |
| # docker is time-consuming to run, so we only run it on 3.8 | |
| - test_environment: docker-server-docker-orchestrator-mysql | |
| python-version: '3.9' | |
| - test_environment: docker-server-docker-orchestrator-mysql | |
| python-version: '3.10' | |
| - test_environment: docker-server-docker-orchestrator-mysql | |
| python-version: '3.11' | |
| - test_environment: docker-server-docker-orchestrator-mariadb | |
| python-version: '3.9' | |
| - test_environment: docker-server-docker-orchestrator-mariadb | |
| python-version: '3.10' | |
| - test_environment: docker-server-docker-orchestrator-mariadb | |
| python-version: '3.11' | |
| fail-fast: false | |
| uses: ./.github/workflows/integration-test-slow.yml | |
| with: | |
| os: ${{ matrix.os }} | |
| python-version: ${{ matrix.python-version }} | |
| test_environment: ${{ matrix.test_environment }} | |
| secrets: inherit |