From 1d623821f562579bd9362fc8288c1e7713dcec05 Mon Sep 17 00:00:00 2001
From: Charles Kawczynski <kawczynski.charles@gmail.com>
Date: Sat, 24 Feb 2024 10:44:08 -0500
Subject: [PATCH] Update buildkite config

---
 .buildkite/pipeline.yml | 66 +++++++++++------------------------------
 1 file changed, 17 insertions(+), 49 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 4fbe2be2d..bc7f0e5fd 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,6 +1,15 @@
+agents:
+  queue: new-central
+  modules: climacommon/2024_02_20
+  partition: expansion
+
 env:
-  JULIA_VERSION: "1.10.0"
-  CUDA_VERSION: "12.2"
+  JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite"
+  OPENBLAS_NUM_THREADS: 1
+  JULIA_NVTX_CALLBACKS: gc
+  JULIA_CPU_TARGET: 'broadwell;skylake;icelake;cascadelake;epyc'
+  OMPI_MCA_opal_warn_on_missing_libcuda: 0
+  JULIA_MAX_NUM_PRECOMPILE_FILES: 100
 
 steps:
   - label: "init cpu environments :computer:"
@@ -13,14 +22,6 @@ steps:
       - "julia --project=examples -e 'using Pkg; Pkg.precompile()'"
       - "julia --project=examples -e 'using Pkg; Pkg.status()'"
 
-    agents:
-      config: cpu
-      queue: central
-      slurm_ntasks: 1
-    #   slurm_cpus_per_task: 8
-    # env:
-    #   JULIA_NUM_PRECOMPILE_TASKS: 8
-
   - label: "init gpu environments :computer:"
     key: "init_gpu_env"
     command:
@@ -31,13 +32,8 @@ steps:
       - "julia --project=gpuenv -e 'using Pkg; Pkg.status()'"
 
     agents:
-      config: gpu
-      queue: central
-      slurm_ntasks: 1
       slurm_gres: "gpu:1"
-    #   slurm_cpus_per_task: 8
-    # env:
-    #   JULIA_NUM_PRECOMPILE_TASKS: 8
+      partition: gpu
 
   - wait
 
@@ -47,26 +43,19 @@ steps:
       - label: "CPU unit tests"
         command: "julia --color=yes --project=examples test/runtests.jl"
         agents:
-          config: cpu
-          queue: central
-          slurm_ntasks: 1
           slurm_mem_per_cpu: 6G
 
       - label: "Threaded CPU unit tests"
         command: "julia --threads 8 --color=yes --project=examples test/runtests.jl"
         agents:
-          config: cpu
-          queue: central
           slurm_cpus_per_task: 8
           slurm_mem_per_cpu: 6G
 
       - label: "GPU unit tests"
         command: "julia --color=yes --project=gpuenv test/runtests_gpu.jl"
         agents:
-          config: gpu
-          queue: central
-          slurm_ntasks: 1
           slurm_gres: "gpu:1"
+          partition: gpu
 
   - group: "Flame graphs"
     steps:
@@ -75,68 +64,47 @@ steps:
         command: "julia --color=yes --project=examples perf/flame.jl gray_atm.jl"
         artifact_paths: "flame_graphs/gray_atm/*"
         agents:
-          config: cpu
-          queue: central
-          slurm_ntasks: 1
           slurm_mem_per_cpu: 6G
 
       - label: "Flame Graph: Clear Sky"
         command: "julia --color=yes --project=examples perf/flame.jl clear_sky.jl"
         artifact_paths: "flame_graphs/clear_sky/*"
         agents:
-          config: cpu
-          queue: central
-          slurm_ntasks: 1
           slurm_mem_per_cpu: 6G
 
       - label: "Flame Graph: All Sky"
         command: "julia --color=yes --project=examples perf/flame.jl all_sky.jl"
         artifact_paths: "flame_graphs/all_sky/*"
         agents:
-          config: cpu
-          queue: central
-          slurm_ntasks: 1
           slurm_mem_per_cpu: 6G
 
   - group: "Benchmarks"
     steps:
-  
+
       - label: "Benchmarks"
         command: "julia --color=yes --project=examples perf/benchmark.jl"
         agents:
-          config: cpu
-          queue: central
-          slurm_ntasks: 1
           slurm_mem_per_cpu: 6G
-  
+
       - label: "Benchmarks (Float32)"
         command: "julia --color=yes --project=examples perf/benchmark.jl Float32"
         agents:
-          config: cpu
-          queue: central
-          slurm_ntasks: 1
           slurm_mem_per_cpu: 6G
 
       - label: "Threaded benchmarks"
         command: "julia --threads=8 --color=yes --project=examples perf/benchmark.jl"
         agents:
-          config: cpu
-          queue: central
           slurm_cpus_per_task: 8
           slurm_mem_per_cpu: 6G
 
       - label: "GPU benchmarks"
         command: "julia --color=yes --project=gpuenv perf/benchmark.jl"
         agents:
-          config: gpu
-          queue: central
-          slurm_ntasks: 1
           slurm_gres: "gpu:1"
+          partition: gpu
 
       - label: "GPU All-sky DYAMOND benchmark"
         command: "julia --color=yes --project=gpuenv test/all_sky_dyamond_gpu_benchmark.jl"
         agents:
-          config: gpu
-          queue: central
-          slurm_ntasks: 1
           slurm_gres: "gpu:1"
+          partition: gpu