microk8s-tests.yml

name: Test Kubeflow

# The small `edge` bundle has been deprecated, `lite` bundle takes 40mins to run which is too slow for gh action.
# Disable this one for now, please check `nw-deploy-kubeflow` on Jenkins.
on: workflow_dispatch

env:
  DOCKER_USERNAME: jujuqabot
  JUJU_BUILD_NUMBER: 888

permissions:
  contents: read

jobs:
  build:
    name: Test Kubeflow
    runs-on: ubuntu-latest
    if: github.event.pull_request.draft == false
    strategy:
      fail-fast: false
      matrix:
        microk8s: [1.21/stable]

    steps:
    - name: Checking out repo
      uses: actions/checkout@v3

    - name: Set up Go
      uses: actions/setup-go@v3
      with:
        go-version-file: 'go.mod'
        cache: true

    - name: setup env
      shell: bash
      run: |
        echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
        echo "$(go env GOPATH)/bin" >> $GITHUB_PATH

    - uses: balchua/microk8s-actions@v0.3.1
      with:
        channel: '${{ matrix.microk8s }}'
        # enable now to give microk8s more time to settle down.
        addons: '["dns", "storage", "dashboard", "ingress", "metallb:10.64.140.43-10.64.140.49"]'

    - name: Install Dependencies
      run: |
        set -eux
        for snap in charm juju-helpers juju-wait; do
            sudo snap install $snap --classic
        done

        sudo apt update
        sudo apt install -y libssl-dev python3-setuptools
        sudo usermod -a -G microk8s $USER

    - name: Build juju and operator image
      run: |
        sg microk8s <<EOF
          set -eux
          JUJU_BUILD_NUMBER=$JUJU_BUILD_NUMBER DOCKER_USERNAME=$DOCKER_USERNAME make microk8s-operator-update
          microk8s.ctr images list | grep juju
          juju version --all

          # clean up disk space because the runner only has 14G which is not enough.
          go clean -cache -modcache -i -r
          docker system prune --all --force
        EOF

    - name: Deploy Kubeflow
      run: |
        sg microk8s <<EOF
          set -eux

          microk8s kubectl wait --for=condition=available -nkube-system deployment/coredns deployment/hostpath-provisioner --timeout=10m
          juju bootstrap microk8s --debug uk8s --config=caas-image-repo=$DOCKER_USERNAME --config test-mode=true --model-default test-mode=true

          juju add-model kubeflow microk8s --config logging-config="<root>=DEBUG;unit=DEBUG"
          juju deploy kubeflow-lite --trust --revision 60
          sleep 10
          microk8s kubectl patch role -n kubeflow istio-ingressgateway-operator -p '{"apiVersion":"rbac.authorization.k8s.io/v1","kind":"Role","metadata":{"name":"istio-ingressgateway-operator"},"rules":[{"apiGroups":["*"],"resources":["*"],"verbs":["*"]}]}'
          sleep 6m
          juju --debug wait -wv -m kubeflow -t 1200

          microk8s kubectl get pods -l 'juju-operator' -A -o custom-columns='Name:metadata.name,Image:spec.containers[0].image'
        EOF

    - name: Test kubeflow
      # TODO: disable test for now because some files required by kubeflow tests are not accessible now.
      # URL fetch failure on https://people.canonical.com/~knkski/train-images-idx3-ubyte.gz: 404 -- Not Found
      if: ${{ false }}
      run: |
        sg microk8s <<EOF
          set -eux
          cd bundle-kubeflow
          tox -e tests -- -m edge
        EOF

    - name: Juju status
      run: juju status --relations --color --storage
      if: ${{ failure() }}

    - name: Juju status (YAML)
      run: juju status --relations --color --storage --format=yaml
      if: ${{ failure() }}

    - name: MicroK8s status
      run: sudo microk8s status
      if: ${{ failure() }}

    - name: Get MicroK8s pods
      run: |
        sudo microk8s kubectl get pods -A -o wide
      if: ${{ failure() }}

    - name: Describe MicroK8s pods
      run: sudo microk8s kubectl describe pods -nkubeflow
      if: ${{ failure() }}

    - name: Generate debug log
      run: juju debug-log --replay --no-tail > juju-debug.log
      if: ${{ failure() }}

    - name: Upload debug log
      uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v2
      with:
        name: juju-debug-actions
        path: juju-debug.log
      if: ${{ failure() }}

    - name: Get pipeline logs
      run: |
        set -eux
        pods=$(sudo microk8s kubectl get -nkubeflow pods -l workflows.argoproj.io/completed="true" -o custom-columns=:metadata.name --no-headers)
        for pod in $pods; do
          containers=$(sudo microk8s kubectl get -nkubeflow pods -o jsonpath="{.spec.containers[*].name}" $pod)
          for container in $containers; do
            sudo microk8s kubectl logs -nkubeflow --timestamps $pod -c $container
            printf '\n'
          done
          printf '\n\n'
        done
      if: ${{ failure() }}

    - name: Generate inspect tarball
      run: >
        sudo microk8s inspect |
        grep -Po "Report tarball is at \K.+" |
        sudo xargs -I {} mv {} inspection-report-${{ strategy.job-index }}.tar.gz
      if: ${{ failure() }}

    - name: Upload inspect tarball
      uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v2
      with:
        name: inspection-report-actions
        path: ./inspection-report-${{ strategy.job-index }}.tar.gz
      if: ${{ failure() }}

    - name: Generate kubectl describe
      run: |
        set -eux
        mkdir describe
        for resource in $(kubectl api-resources -o name | sort); do
            kubectl describe $resource -A > describe/"$resource".describe || true
        done
      if: ${{ failure() }}

    - name: Upload kubectl describe
      uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v2
      with:
        name: kubectl-describe-actions
        path: describe/*.describe
      if: ${{ failure() }}

    - name: Generate kubeflow pod logs
      run: |
        set -eux
        mkdir stdout
        for pod in $(kubectl get pods -nkubeflow -o custom-columns=:metadata.name --no-headers); do
            for container in $(kubectl get pods -nkubeflow -o jsonpath="{.spec.containers[*].name}" $pod); do
              kubectl logs -nkubeflow --timestamps $pod -c $container > stdout/$pod-$container.log
            done
        done
      if: ${{ failure() }}

    - name: Upload kubeflow pod logs
      uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v2
      with:
        name: kubectl-stdout-actions
        path: stdout/*.log
      if: ${{ failure() }}