[CI] Avoid concurrent LLVM builds

scripts/buildkite/build_llvm.sh

@@ -50,6 +50,16 @@ LLVM_PROJECT_DIR=${LLVM_TAR_DIR}/llvm-project-${LLVM_VERSION}
 LLVM_INSTALL_DIR=${LLVMROOT}/${LLVM_VERSION}
 LLVM_INSTALL_DIR=$(add_device_extensions ${LLVM_INSTALL_DIR} ${GPU})
 
+# If there's another process building it, wait.
+# Otherwise, make the dir quickly so others don't attempt at the same time
+if [ -d ${LLVM_INSTALL_DIR} ]; then
+  wait_for_file ${LLVM_INSTALL_DIR} bin/mlir-opt
+  echo "LLVM built by another process, results in ${LLVM_INSTALL_DIR}"
+  exit 0
+else
+  mkdir -p ${LLVM_INSTALL_DIR}
+fi
+
 # Environment setup
 echo "--- ENVIRONMENT"
 if [ ! "${COMPILER}" ]; then

scripts/ci/common.sh

@@ -25,28 +25,28 @@ git_commit() {
 
 # Check if a program is in the PATH
 check_program() {
-  PROG=$1
-  if ! which $PROG > /dev/null; then
-    echo "ERROR: '$PROG' not found!"
+  local PROG=${1}
+  if ! which ${PROG} > /dev/null; then
+    echo "ERROR: '${PROG}' not found!"
     exit 1
   fi
 }
 
 # Echoes and runs a program
 echo_run() {
-  PROGRAM=$*
+  local PROGRAM=$*
   echo "${PROGRAM}"
   ${PROGRAM}
 }
 
 # Get the LLVM version for this build
 llvm_version() {
-  LLVM_VERSION_FILE=$(git_root)/build_tools/llvm_version.txt
+  local LLVM_VERSION_FILE=$(git_root)/build_tools/llvm_version.txt
   if [ ! -f "${LLVM_VERSION_FILE}" ]; then
     echo "ERROR: cannot find ${LLVM_VERSION_FILE} for ${PWD}!"
     exit 1
   fi
-  LLVM_VERSION=$(cat "${LLVM_VERSION_FILE}")
+  local LLVM_VERSION=$(cat "${LLVM_VERSION_FILE}")
   if [ ! "${LLVM_VERSION}" ]; then
     echo "ERROR: cannot find LLVM version in ${LLVM_VERSION_FILE}!"
     exit 1
@@ -69,3 +69,19 @@ add_device_extensions() {
 
   echo ${BASE}
 }
+
+# Wait for a file to appear on an existing directory
+waif_for_file() {
+  local DIR=${1}
+  local FILE=$(realpath ${1}/${2})
+
+  if [ ! -d ${DIR} ]; then
+    echo "ERROR: Directory ${DIR} not found"
+  fi
+  echo -n "Waiting for ${FILE}..."
+  while [ ! -f ${FILE} ]; do
+    sleep 30
+    echo -n "."
+  done
+  echo " Found"
+}