From 3363c14dca8f6cc1f0e7f132d8c25cb58f18d8d5 Mon Sep 17 00:00:00 2001
From: Mike Sul <mike.sul@foundries.io>
Date: Thu, 31 Oct 2024 15:18:54 +0100
Subject: [PATCH] lmp: Push ostree repo with retries

Run `fiopush` and `fiocheck` commands with up to 3 attempts.

Signed-off-by: Mike Sul <mike.sul@foundries.io>
---
 lmp/bb-config.sh      | 12 +++++++-----
 lmp/fiocheck.sh       |  7 +++++++
 lmp/fiopush.sh        |  6 ++++++
 lmp/run-with-retry.sh | 34 ++++++++++++++++++++++++++++++++++
 4 files changed, 54 insertions(+), 5 deletions(-)
 create mode 100755 lmp/fiocheck.sh
 create mode 100755 lmp/fiopush.sh
 create mode 100755 lmp/run-with-retry.sh

diff --git a/lmp/bb-config.sh b/lmp/bb-config.sh
index 7c157fa4..66fec8d5 100755
--- a/lmp/bb-config.sh
+++ b/lmp/bb-config.sh
@@ -18,8 +18,8 @@ DOCKER_MAX_CONCURRENT_DOWNLOADS="${DOCKER_MAX_CONCURRENT_DOWNLOADS-3}"
 DOCKER_MAX_DOWNLOAD_ATTEMPTS="${DOCKER_MAX_DOWNLOAD_ATTEMPTS-5}"
 MFGTOOL_FLASH_IMAGE="${MFGTOOL_FLASH_IMAGE-lmp-factory-image}"
 USE_FIOTOOLS="${USE_FIOTOOLS-1}"
-FIO_CHECK_CMD="${FIO_CHECK_CMD-/usr/bin/fiocheck}"
-FIO_PUSH_CMD="${FIO_PUSH_CMD-/usr/bin/fiopush}"
+FIO_PUSH_BIN="${FIO_PUSH_BIN-/usr/bin/fiopush}"
+FIO_CHECK_BIN="${FIO_CHECK_BIN-/usr/bin/fiocheck}"
 OSTREE_API_VERSION="${OSTREE_API_VERSION-v2}"
 DEV_MODE="${DEV_MODE-0}"
 BUILD_SDK="${BUILD_SDK-0}"
@@ -99,9 +99,11 @@ GARAGE_PUSH_RETRIES = "${GARAGE_PUSH_RETRIES-5}"
 GARAGE_PUSH_RETRIES_SLEEP = "${GARAGE_PUSH_RETRIES_SLEEP-10}"
 DOCKER_COMPOSE_APP = "${DOCKER_COMPOSE_APP}"
 USE_FIOTOOLS = "${USE_FIOTOOLS}"
-FIO_CHECK_CMD = "${FIO_CHECK_CMD}"
-FIO_PUSH_CMD = "${FIO_PUSH_CMD}"
-FIO_CHECK_PUSH = "${FIO_PUSH_CMD}"
+FIO_CHECK_BIN = "${FIO_CHECK_BIN}"
+FIO_PUSH_BIN = "${FIO_PUSH_BIN}"
+FIO_CHECK_CMD = "${HERE}/fiocheck.sh"
+FIO_PUSH_CMD = "${HERE}/fiopush.sh"
+FIO_CHECK_PUSH = "${HERE}/fiopush.sh"
 OSTREE_API_VERSION = "${OSTREE_API_VERSION}"
 
 # Apps preloading params
diff --git a/lmp/fiocheck.sh b/lmp/fiocheck.sh
new file mode 100755
index 00000000..f50bb7a5
--- /dev/null
+++ b/lmp/fiocheck.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+HERE=$(dirname $(readlink -f $0))
+CHECK_BIN="${FIO_CHECK_BIN-/usr/bin/fiocheck}"
+
+"${HERE}/run-with-retry.sh" "${CHECK_BIN}" $@
+
diff --git a/lmp/fiopush.sh b/lmp/fiopush.sh
new file mode 100755
index 00000000..11e68678
--- /dev/null
+++ b/lmp/fiopush.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+HERE=$(dirname $(readlink -f $0))
+PUSH_BIN="${FIO_PUSH_BIN-/usr/bin/fiopush}"
+
+"${HERE}/run-with-retry.sh" "${PUSH_BIN}" $@
diff --git a/lmp/run-with-retry.sh b/lmp/run-with-retry.sh
new file mode 100755
index 00000000..7a0add6f
--- /dev/null
+++ b/lmp/run-with-retry.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+num_attempts=3
+
+# Check if at least one argument is provided, the first argument is treated
+# as the command to execute.
+if [[ $# -lt 1 ]]; then
+  echo "Usage: $0 <command> [args...]"
+  exit 1
+fi
+
+command=$1
+shift  # Shift out the command
+
+# Track the attempt count
+attempt=1
+
+# Try to execute the command with retries
+while [[ $attempt -le $num_attempts ]]; do
+  echo "Attempt $attempt/$num_attempts: Running '$command $@'"
+  $command "$@"
+
+  # Check if the command succeeded
+  if [[ $? -eq 0 ]]; then
+    echo "Command succeeded on attempt #$attempt"
+    exit 0
+  fi
+
+  echo "Command failed on attempt #$attempt. Retrying..."
+  attempt=$((attempt + 1))
+done
+
+echo "Command failed after $num_attempts attempts."
+exit 1