Improve setupDelegation reliability and logging (seems to fix CI) (#…

…4602)

- [x] Log failures to delegate - I don't think we'll see the problem
with this, but let's rule it out, and ignoring the result is regardless
bad practice.
- [x] Reduce the number of `rewardWallet` wallets we prepare from 30 to
25 (to reduce likeliehood of the below change causing problems)
- [x] Stop deleting wallets as quickly as possible in `setupDelegation`.
This appears to fix the problem of dropped txs without causing new
problems from strained resources.

### Issue Number

ADP-3362

lib/integration/framework/Test/Integration/Framework/Logging.hs

@@ -89,6 +89,7 @@ data TestsLog
     | MsgCluster ClusterLog
     | MsgPoolGarbageCollectionEvent PoolGarbageCollectionEvent
     | MsgServerError SomeException
+    | MsgRewardWalletDelegationFailed Text
     deriving (Show)
 
 instance ToText TestsLog where
@@ -120,6 +121,11 @@ instance ToText TestsLog where
             | isAsyncException (SomeException e)
                 -> "Server thread cancelled: " <> T.pack (show e)
             | otherwise -> T.pack (show e)
+        MsgRewardWalletDelegationFailed msg ->
+            T.unlines
+                [ "Reward wallet delegation failed."
+                , "Details: " <> toText msg
+                ]
 
 instance HasPrivacyAnnotation TestsLog
 instance HasSeverityAnnotation TestsLog where
@@ -132,6 +138,7 @@ instance HasSeverityAnnotation TestsLog where
         MsgServerError e
             | isAsyncException e -> Notice
             | otherwise -> Warning
+        MsgRewardWalletDelegationFailed{} -> Error
 
 withTracers
     :: DirOf "cluster"

lib/integration/framework/Test/Integration/Framework/Setup.hs

@@ -123,6 +123,9 @@ import Cardano.Wallet.TokenMetadata.MockServer
     ( queryServerStatic
     , withMetadataServer
     )
+import Control.Concurrent
+    ( threadDelay
+    )
 import Control.Lens
     ( view
     )
@@ -530,18 +533,29 @@ withContext testingCtx@TestingCtx{..} action = do
                     ctx
                     (Link.listStakePools arbitraryStake)
                     Empty
-        -- Having 'runResourceT' /inside/ the loop ensures the wallets are
-        -- deleted as quickly as possible, not to deplete file descriptors or
-        -- resources for unnecessary restoration.
-        forM_ mnemonics $ \mw -> runResourceT $ do
-            w <- walletFromMnemonic ctx mw
-            _ <-
-                joinStakePool
-                    @('Testnet 0) -- protocol magic doesn't matter
-                    ctx
-                    (SpecificPool pool)
-                    (w, fixturePassphrase)
-            pure ()
+
+        -- We only delete the wallets together at the end of the loop by
+        -- using this /outer/ 'runResourceT'. While this may be taxing on
+        -- resources, it enables tx submission to ensure all txs make it into
+        -- the chain.
+        runResourceT $ do
+            forM_ mnemonics $ \mw -> do
+                w <- walletFromMnemonic ctx mw
+                (httpStatus, res) <-
+                    joinStakePool
+                        @('Testnet 0) -- protocol magic doesn't matter
+                        ctx
+                        (SpecificPool pool)
+                        (w, fixturePassphrase)
+                liftIO $ case res of
+                    Left err -> traceWith tr
+                        $ MsgRewardWalletDelegationFailed
+                        $ T.pack $ show (httpStatus, err)
+                    Right _ -> return ()
+
+            -- Extra time to ensure the final txs make it into the chain
+            let second = 1_000_000
+            liftIO $ threadDelay $ 10 * second
 
 bracketTracer' :: Tracer IO TestsLog -> Text -> IO a -> IO a
 bracketTracer' tr name = bracketTracer $ contramap (MsgBracket name) tr

lib/local-cluster/lib/Cardano/Wallet/Faucet.hs

@@ -322,7 +322,7 @@ preregKeyWallet tag = do
 
 rewardWalletMnemonicRange :: MnemonicRange
 rewardWalletMnemonicRange =
-    MnemonicRange 0 30
+    MnemonicRange 0 25
     -- As few as possible; the cardano-wallet integration tests need to call
     -- postWallet for each one at the beginning of the tests.