[integration tests] handle long timeouts in storage checks

.github/integration/tests/sda/22_error_test.sh

@@ -74,7 +74,7 @@ curl -k -u guest:guest "$URI/api/exchanges/sda/sda/publish" \
 until [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT event FROM sda.file_event_log WHERE correlation_id = '$CORRID' ORDER BY ID DESC LIMIT 1;")" = "error" ]; do
     echo "waiting for file error to be logged by ingest"
     RETRY_TIMES=$((RETRY_TIMES + 1))
-    if [ "$RETRY_TIMES" -eq 30 ]; then
+    if [ "$RETRY_TIMES" -eq 61 ]; then
         echo "::error::Time out while waiting for ingest to set file error"
         exit 1
     fi
@@ -123,7 +123,7 @@ until [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT event FROM sda.fil
     echo "waiting for file error to be logged by verify"
     date
     RETRY_TIMES=$((RETRY_TIMES + 1))
-    if [ "$RETRY_TIMES" -eq 30 ]; then
+    if [ "$RETRY_TIMES" -eq 61 ]; then
         echo "::error::Time out while waiting for verify to set file error"
         exit 1
     fi

.github/integration/tests/sda/92_handle_file_errors.sh

@@ -99,11 +99,15 @@ curl -s -u guest:guest 'http://rabbitmq:15672/api/exchanges/sda/sda/publish' \
 
 sleep 10
 
-
-if [ $((stream_size++)) -eq "$(curl -s -u guest:guest http://rabbitmq:15672/api/queues/sda/error_stream | jq '.messages_ready')" ]; then
-    echo "missing file not moved to error"
-    exit 1
-fi
+RETRY_TIMES=0
+until [ $((stream_size++)) -eq "$(curl -s -u guest:guest http://rabbitmq:15672/api/queues/sda/error_stream | jq '.messages_ready')" ]; do
+    RETRY_TIMES=$((RETRY_TIMES + 1))
+    if [ "$RETRY_TIMES" -eq 61 ]; then
+        echo "missing file not moved to error"
+        exit 1
+    fi
+    sleep 2
+done
 
 CORRID=$(
     curl -s -X POST \