Jenkinsfile

pipeline {
  agent any
  environment {
    DEEPOPS_FULL_INSTALL = ''
    DEEPOPS_VAGRANT_OS = 'ubuntu'
    DEEPOPS_OS_VERSION = '20.04'
  }
  stages {
    stage('Stop Any Old Builds') {
      steps {
        milestone label: '', ordinal:  Integer.parseInt(env.BUILD_ID) - 1
        milestone label: '', ordinal:  Integer.parseInt(env.BUILD_ID)
      }
    }
    stage('Cluster Up') {
      steps {
        // TODO: ideally lock should work with declared stages
        lock(resource: null, label: 'gpu', quantity: 1, variable: 'GPUDATA') {
          echo "Reset repo and unmunge files"
          sh '''
            git reset --hard
            rm -rf config
          '''

          echo "Munge files for testing"
          sh '''
            bash -x ./workloads/jenkins/scripts/munge-files.sh
          '''

          echo "Tear down any Vagrant that was not cleaned up"
	    sh '''
	    pwd
	    cd virtual && ./vagrant_shutdown.sh || true
          '''

          echo "Vagrant Up"
          sh '''
            bash -x ./workloads/jenkins/scripts/vagrant-startup.sh
          '''

          echo "Cluster Up"
          sh '''
            bash -x ./workloads/jenkins/scripts/test-cluster-up.sh
          '''

          echo "Get K8S Cluster Status"
          sh '''
            bash -x ./workloads/jenkins/scripts/get-k8s-debug.sh
          '''

          echo "Verify we can run a GPU job"
          sh '''
            timeout 500 bash -x ./workloads/jenkins/scripts/run-gpu-job.sh
          '''

          echo "Verify ingress config"
          sh '''
             bash -x ./workloads/jenkins/scripts/verify-ingress-config.sh
          '''

          echo "Verify registry mirror"
          sh '''
             bash -x ./workloads/jenkins/scripts/test-deepops-registry-mirror.sh
          '''

          echo "Verify in-cluster k8s docker registry"
          sh '''
             bash -x ./workloads/jenkins/scripts/test-local-registry.sh
          '''

          echo "Verify rsyslog forwarding is working for the k8s cluster"
          sh '''
             bash -x ./workloads/jenkins/scripts/test-rsyslog-k8s.sh
          '''

          echo "Test Monitoring installation"
          sh '''
            timeout 800 bash -x ./workloads/jenkins/scripts/test-monitoring.sh
          '''

          echo "Test Dashboard installation"
          sh '''
            timeout 180 bash -x ./workloads/jenkins/scripts/test-dashboard.sh
          '''

          echo "Start new virtual environment pre-Slurm checks"
          sh '''
            bash -x ./workloads/jenkins/scripts/vagrant-startup.sh
          '''

          echo "Set up Slurm"
          sh '''
            bash -x ./workloads/jenkins/scripts/test-setup-slurm.sh
          '''

          echo "Get Slurm Cluster Status"
          sh '''
            bash -x ./workloads/jenkins/scripts/get-slurm-debug.sh
          '''

          echo "Test Slurm"
          sh '''
            timeout 60 bash -x ./workloads/jenkins/scripts/test-slurm-job.sh
          '''

          echo "Test NFS"
          sh '''
            timeout 60 bash -x ./workloads/jenkins/scripts/test-slurm-nfs-mount.sh
          '''

          echo "Test MPI"
          sh '''
            timeout 60 bash -x ./workloads/jenkins/scripts/test-mpi-job.sh
          '''

          echo "Test Enroot"
          sh '''
            timeout 120 bash -x ./workloads/jenkins/scripts/test-slurm-enroot-job.sh
          '''

          echo "Verify rsyslog forwarding is working for the slurm cluster"
          sh '''
             bash -x ./workloads/jenkins/scripts/test-rsyslog-slurm.sh
          '''

          echo "Test GPU job"
          sh '''
            timeout 60 bash -x ./workloads/jenkins/scripts/test-slurm-gpu.sh
          '''

          echo "Test DCGM metrics"
          sh '''
             timeout 600 bash -x ./workloads/jenkins/scripts/test-dcgm-metrics.sh slurm-node
          '''
        }
      }
    }
  }
  post {
    always {
      sh '''
        pwd
        cd virtual && ./vagrant_shutdown.sh
      '''
    }
  }
}