Beat itest [1/3]: remove the usage of standby nodes

Makefile

@@ -208,7 +208,7 @@ endif
 itest-only: db-instance
 	@$(call print, "Running integration tests with ${backend} backend.")
 	rm -rf itest/*.log itest/.logs-*; date
-	EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_part.sh 0 1 $(TEST_FLAGS) $(ITEST_FLAGS) -test.v
+	EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_part.sh 0 1 $(SHUFFLE_SEED) $(TEST_FLAGS) $(ITEST_FLAGS) -test.v
 	$(COLLECT_ITEST_COVERAGE)
 
 #? itest: Build and run integration tests
@@ -221,7 +221,7 @@ itest-race: build-itest-race itest-only
 itest-parallel: build-itest db-instance
 	@$(call print, "Running tests")
 	rm -rf itest/*.log itest/.logs-*; date
-	EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_parallel.sh $(ITEST_PARALLELISM) $(NUM_ITEST_TRANCHES) $(TEST_FLAGS) $(ITEST_FLAGS)
+	EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_parallel.sh $(ITEST_PARALLELISM) $(NUM_ITEST_TRANCHES) $(SHUFFLE_SEED) $(TEST_FLAGS) $(ITEST_FLAGS)
 	$(COLLECT_ITEST_COVERAGE)
 
 #? itest-clean: Kill all running itest processes

itest/list_on_test.go

@@ -2,7 +2,9 @@
 
 package itest
 
-import "github.com/lightningnetwork/lnd/lntest"
+import (
+	"github.com/lightningnetwork/lnd/lntest"
+)
 
 var allTestCases = []*lntest.TestCase{
 	{

itest/lnd_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/lightningnetwork/lnd/lntest/port"
 	"github.com/lightningnetwork/lnd/lntest/wait"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/exp/rand"
 	"google.golang.org/grpc/grpclog"
 )
 
@@ -61,6 +62,13 @@ var (
 			"0-based index specified by the -runtranche flag",
 	)
 
+	// shuffleSeedFlag is the source of randomness used to shuffle the test
+	// cases. If not specified, the test cases won't be shuffled.
+	shuffleSeedFlag = flag.Uint64(
+		"shuffleseed", 0, "if set, shuffles the test cases using this "+
+			"as the source of randomness",
+	)
+
 	// testCasesRunTranche is the 0-based index of the split test cases
 	// tranche to run in the current invocation.
 	testCasesRunTranche = flag.Uint(
@@ -160,6 +168,32 @@ func TestLightningNetworkDaemon(t *testing.T) {
 		harnessTest.CurrentHeight()-height)
 }
 
+// maybeShuffleTestCases shuffles the test cases if the flag `shuffleseed` is
+// set and not 0. In parallel tests we want to shuffle the test cases so they
+// are executed in a random order. This is done to even out the blocks mined in
+// each test tranche so they can run faster.
+//
+// NOTE: Because the parallel tests are initialized with the same seed (job
+// ID), they will always have the same order.
+func maybeShuffleTestCases() {
+	// Exit if not set.
+	if shuffleSeedFlag == nil {
+		return
+	}
+
+	// Exit if set to 0.
+	if *shuffleSeedFlag == 0 {
+		return
+	}
+
+	// Init the seed and shuffle the test cases.
+	rand.Seed(*shuffleSeedFlag)
+	rand.Shuffle(len(allTestCases), func(i, j int) {
+		allTestCases[i], allTestCases[j] =
+			allTestCases[j], allTestCases[i]
+	})
+}
+
 // getTestCaseSplitTranche returns the sub slice of the test cases that should
 // be run as the current split tranche as well as the index and slice offset of
 // the tranche.
@@ -182,6 +216,9 @@ func getTestCaseSplitTranche() ([]*lntest.TestCase, uint, uint) {
 		runTranche = 0
 	}
 
+	// Shuffle the test cases if the `shuffleseed` flag is set.
+	maybeShuffleTestCases()
+
 	numCases := uint(len(allTestCases))
 	testsPerTranche := numCases / numTranches
 	trancheOffset := runTranche * testsPerTranche

make/testing_flags.mk

@@ -10,6 +10,7 @@ COVER_PKG = $$(go list -deps -tags="$(DEV_TAGS)" ./... | grep '$(PKG)' | grep -v
 NUM_ITEST_TRANCHES = 4
 ITEST_PARALLELISM = $(NUM_ITEST_TRANCHES)
 POSTGRES_START_DELAY = 5
+SHUFFLE_SEED = 0
 
 # If rpc option is set also add all extra RPC tags to DEV_TAGS
 ifneq ($(with-rpc),)
@@ -27,6 +28,11 @@ ifneq ($(parallel),)
 ITEST_PARALLELISM = $(parallel)
 endif
 
+# Set the seed for shuffling the test cases.
+ifneq ($(shuffleseed),)
+SHUFFLE_SEED = $(shuffleseed)
+endif
+
 # Windows needs to append a .exe suffix to all executable files, otherwise it
 # won't run them.
 ifneq ($(windows),)

scripts/itest_parallel.sh

@@ -3,17 +3,18 @@
 # Get all the variables.
 PROCESSES=$1
 TRANCHES=$2
+SHUFFLE_SEED=$3
 
-# Here we also shift 2 times and get the rest of our flags to pass on in $@.
-shift 2
+# Here we also shift 3 times and get the rest of our flags to pass on in $@.
+shift 3
 
 # Create a variable to hold the final exit code.
 exit_code=0
 
 # Run commands using xargs in parallel and capture their PIDs
 pids=()
 for ((i=0; i<PROCESSES; i++)); do 
-    scripts/itest_part.sh $i $TRANCHES $@ &
+    scripts/itest_part.sh $i $TRANCHES $SHUFFLE_SEED $@ &
     pids+=($!)
 done
 

scripts/itest_part.sh

@@ -5,11 +5,11 @@ WORKDIR=$(pwd)/itest
 
 TRANCHE=$1
 NUM_TRANCHES=$2
+SHUFFLE_SEED_PARAM=$3
 
-# Shift the passed parameters by two, giving us all remaining testing flags in
+# Shift the passed parameters by three, giving us all remaining testing flags in
 # the $@ special variable.
-shift
-shift
+shift 3
 
 # Windows insists on having the .exe suffix for an executable, we need to add
 # that here if necessary.
@@ -18,9 +18,9 @@ LND_EXEC="$WORKDIR"/lnd-itest"$EXEC_SUFFIX"
 BTCD_EXEC="$WORKDIR"/btcd-itest"$EXEC_SUFFIX"
 export GOCOVERDIR="$WORKDIR/cover"
 mkdir -p "$GOCOVERDIR"
-echo $EXEC "$@" -logoutput -logdir=.logs-tranche$TRANCHE -lndexec=$LND_EXEC -btcdexec=$BTCD_EXEC -splittranches=$NUM_TRANCHES -runtranche=$TRANCHE
+echo $EXEC "$@" -logoutput -logdir=.logs-tranche$TRANCHE -lndexec=$LND_EXEC -btcdexec=$BTCD_EXEC -splittranches=$NUM_TRANCHES -runtranche=$TRANCHE -shuffleseed=$SHUFFLE_SEED_PARAM
 
 # Exit code 255 causes the parallel jobs to abort, so if one part fails the
 # other is aborted too.
 cd "$WORKDIR" || exit 255
-$EXEC "$@" -logoutput -logdir=.logs-tranche$TRANCHE -lndexec=$LND_EXEC -btcdexec=$BTCD_EXEC -splittranches=$NUM_TRANCHES -runtranche=$TRANCHE || exit 255
+$EXEC "$@" -logoutput -logdir=.logs-tranche$TRANCHE -lndexec=$LND_EXEC -btcdexec=$BTCD_EXEC -splittranches=$NUM_TRANCHES -runtranche=$TRANCHE -shuffleseed=$SHUFFLE_SEED_PARAM || exit 255