[release-18.0] Add a retry to remove the vttablet directory during up…

…grade/downgrade backup tests (vitessio#14753) (vitessio#14758)

Co-authored-by: vitess-bot[bot] <108069721+vitess-bot[bot]@users.noreply.github.com>

.github/workflows/upgrade_downgrade_test_backups_manual.yml

@@ -82,6 +82,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/upgrade_downgrade_test_backups_manual.yml'
+            - 'examples/**'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml

@@ -85,6 +85,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml'
+            - 'examples/**'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

examples/backups/stop_tablets.sh

@@ -30,7 +30,25 @@ for tablet in 100 200 300; do
       CELL=zone1 TABLET_UID=$uid ../common/scripts/mysqlctl-down.sh
       echo "Removing tablet directory zone1-$uid"
       vtctldclient DeleteTablets --allow-primary zone1-$uid
-      rm -Rf $VTDATAROOT/vt_0000000$uid
+
+      for ((i=0; i<30; i++)); do
+          # Redirect stderr to a temporary file
+          temp_file=$(mktemp)
+          rm -Rf $VTDATAROOT/vt_0000000$uid 2>"$temp_file"
+
+          if grep -q 'Directory not empty' "$temp_file"; then
+              echo "Directory not empty, retrying..."
+          elif [ ! -s "$temp_file" ]; then
+              echo "Deletion succeeded."
+              rm -f "$temp_file"
+              break
+          else
+              echo "An error occurred."
+              cat "$temp_file"
+          fi
+          rm -f "$temp_file"
+          sleep 1
+      done
     done
   fi
 done