Challenges update.

Added RKC-response by BBA
Better reporting of deals that should be trained

UCBC 2024/Boards/BEN/deals.txt

@@ -0,0 +1,87 @@
+# These deals are samples, where BEN have had problems during simulation
+# Pavlicek
+Sampling this auction: 1H-P-2D-P-2H-P-3D-P-3H-P-4H-P with this deal 9.AK9864.3.KQJT2 82.Q75.T762.9754 K4.JT.AKQJ954.83 AQJT7653.32.8.A6
+Please add deal to training to avoid this aucction 1H-P-2D-P-2H-P-3D-P-3H-P-4H-P-4N-P-5D-P-5H-P-P
+Sampling this auction: 2D-P with this deal J5.95.KJ9543.Q84 72.86.QT82.J9765 T93.AK72.A76.KT3 AKQ864.QJT43..A2
+Please add deal to training to avoid this aucction 2D-P-2N-3D-P-P
+
+# BW
+Sampling this auction: PAD_START-P with this deal 7.K2.KT52.QJT754 954.JT874.96.932 AQ863.A653.873.8 KJT2.Q9.AQJ4.AK6
+Please add deal to training to avoid this auction P-1S-X-2C-P-2H-X-P
+Please add deal to training to avoid this auction P-1S-P-3N-P
+Sample: 85, Hand AKQxxx.Kx.KTx.xx Bid not valid: 3N insta_score: 0.344
+Sampling this auction: PAD_START-P-1S-P with this deal J92.J85.A52.AK83 T4.T4.JT9763.652 K8653.AKQ6.K8.Q7 AQ7.9732.Q4.JT94
+Please add deal to training to avoid this auction P-1S-P-3N-P
+Sample: 132, Hand Kxxxx.AKQx.Kx.Qx Bid not valid: 3N insta_score: 0.256
+Sampling this auction: PAD_START-P-1S-P with this deal J94.J42.A85.AK72 T6.863.QT9643.J8 KQ8753.AK9.J2.Q6 A2.QT75.K7.T9543
+Please add deal to training to avoid this auction P-1S-P-3N-P
+Sample: 134, Hand KQxxxx.AKx.Jx.Qx Bid not valid: 3N insta_score: 0.284
+Sampling this auction: PAD_START-P-1S-P with this deal J53.J63.A42.AK52 92.9.8653.QT9863 AKT764.AKT.JT.J4 Q8.Q87542.KQ97.7
+Please add deal to training to avoid this auction P-1S-P-3N-P
+Sample: 138, Hand AKTxxx.AKT.JT.Jx Bid not valid: 3N insta_score: 0.363
+Sampling this auction: PAD_START-P-1S-P with this deal J43.J98.A73.AK74 T72.KQ42.854.Q95 AKQ865.A3.K96.82 9.T765.QJT2.JT63
+Please add deal to training to avoid this auction P-1S-P-3N-P
+Sample: 147, Hand AKQxxx.Ax.Kxx.xx Bid not valid: 3N insta_score: 0.488
+Sampling this auction: PAD_START-P-1S-P with this deal J73.J87.A82.AK82 9.KT643.73.T9764 AKT652.A95.K4.Q5 Q84.Q2.QJT965.J3
+Please add deal to training to avoid this auction P-1S-P-3N-P
+Sample: 174, Hand AKTxxx.Axx.Kx.Qx Bid not valid: 3N insta_score: 0.339
+Sampling this auction: PAD_START-P with this deal AQT43.K974.KJ.75 9872.6.T942.KQ82 J.Q83.Q87653.J94 K65.AJT52.A.AT63
+Please add deal to training to avoid this auction P-2D-2H-3D-P-P-X-P
+Sample: 97, Hand xxxx.x.Txxx.KQxx Bid not valid: 3C insta_score: 0.445
+Sampling this auction: PAD_START-PAD_START-1H-P-2C-P-2N-P-3H-P-3S-P with this deal K2.KJT.AQ8.K9843 8765.AQ762.T2.65 AJT.98543.K97.AJ Q943..J6543.QT72
+Please add deal to training to avoid this auction 1H-P-2C-P-2N-P-3H-P-3S-P-3N-P-4C-P-4D-P-4N-P-5H-X-P-P
+Sample: 18, Hand Kx.KJT.AQx.Kxxxx Bid not valid: 5H insta_score: 0.474
+
+# BW2
+Sampling this auction: 1D-1H-X-P with this deal 8.Q.AK76543.A752 JT62.AJT9852..K3 K974.643.9.QJ864 AQ53.K7.QJT82.T9
+Please add deal to training to avoid this auction 1D-1H-X-P-2C-2H-3C-3H-4D-P
+Sample: 96, Hand Kxxx.xxx.x.QJxxx Bid not valid: 4C insta_score: 0.303
+Sampling this auction: 1D-1H-X-P-2D-P with this deal T8.K9.AKQ8743.85 KJ73.A7532.2.T74 AQ95.86.JT9.KQ63 642.QJT4.65.AJ92
+Please add deal to training to avoid this auction 1D-1H-X-P-2D-P-2H
+Sample: 38, Hand xxx.QJTx.xx.AJxx Bid not valid: 2H insta_score: 0.424
+Sampling this auction: 1D-1H-X-P-2D-P with this deal T2.Q3.AQ96542.A7 973.AJ752.K3.952 AQ65.86.JT7.KQ43 KJ84.KT94.8.JT86
+Please add deal to training to avoid this auction 1D-1H-X-P-2D-P-2H
+Sample: 96, Hand KJxx.KTxx.x.JTxx Bid not valid: 2H insta_score: 0.295
+Sampling this auction: 1D-1H-X-P-2D-P with this deal K7.42.AKQ653.T53 JT82.AKT87.42.74 AQ96.96.JT9.KQ96 543.QJ53.87.AJ82
+Please add deal to training to avoid this auction 1D-1H-X-P-2D-P-2H
+Sample: 146, Hand xxx.QJxx.xx.AJxx Bid not valid: 2H insta_score: 0.423
+Sampling this auction: 1D-1H-X-P-2D-P with this deal K96.95.AK7652.43 JT8.AQ842.Q9.JT2 AQ32.76.JT3.KQ86 754.KJT3.84.A975
+Please add deal to training to avoid this auction 1D-1H-X-P-2D-P-2H
+Sample: 179, Hand xxx.KJTx.xx.Axxx Bid not valid: 2H insta_score: 0.395
+Sampling this auction: 1H-P-1S-P-3D-P-4N-P-5C-P with this deal .AKQJT93.KT876.A K754.74.Q532.T84 AJ963.82.AJ.KQJ3 QT82.65.94.97652
+Please add deal to training to avoid this auction 1H-P-1S-P-3D-P-4N-P-5C-P-6N-P
+Sample: 56, Hand .AKQJTxx.KTxxx.A Bid not valid: 6H insta_score: 0.500
+Sampling this auction: 1H-P-1S-P-3D-P-4N-P-5C-P with this deal .AKQJT82.KT873.A QT7.53.9642.8532 AJ984.64.AJ.KQJ6 K6532.97.Q5.T974
+Please add deal to training to avoid this auction 1H-P-1S-P-3D-P-4N-P-5C-P-6N-P
+Sample: 105, Hand .AKQJTxx.KTxxx.A Bid not valid: 6H insta_score: 0.500
+Sampling this auction: 1H-P-1S-P-3D-P-4N-P-5C-P with this deal .AKQJ873.K9876.A KQ742.T6.QT.T853 AJ983.54.AJ.KQJ2 T65.92.5432.9764
+Please add deal to training to avoid this auction 1H-P-1S-P-3D-P-4N-P-5C-P-6N-P
+Sample: 128, Hand .AKQJxxx.Kxxxx.A Bid not valid: 6H insta_score: 0.517
+Sampling this auction: 1H-P-1S-P-3D-P-4N-P-5C-P with this deal .AKQJ987.KT987.A KT73.T4.432.7643 AJ986.32.AJ.KQJ5 Q542.65.Q65.T982
+Please add deal to training to avoid this auction 1H-P-1S-P-3D-P-4N-P-5C-P-6N-P
+Sample: 156, Hand .AKQJxxx.KTxxx.A Bid not valid: 6H insta_score: 0.521
+Sampling this auction: 1H-P-1S-P-3D-P-4N-P-5C-P with this deal .AKQJ865.K8765.A 87.92.QT92.98765 AJ965.43.AJ.KQJ2 KQT432.T7.43.T43
+Please add deal to training to avoid this auction 1H-P-1S-P-3D-P-4N-P-5C-P-6N-P
+Sample: 186, Hand .AKQJxxx.Kxxxx.A Bid not valid: 6H insta_score: 0.517
+Sampling this auction: 1H-P-1S-P-3D-P-4N-P-5C-P with this deal .AKQJ743.KT872.A T832.T.964.T7542 AJ974.65.AJ.KQJ8 KQ65.982.Q53.963
+Please add deal to training to avoid this auction 1H-P-1S-P-3D-P-4N-P-5C-P-6N-P
+Sample: 191, Hand .AKQJxxx.KTxxx.A Bid not valid: 6H insta_score: 0.521
+...l_start", "dealer": 2, "vuln": [false, true], "hand": ["932.A93.865432.K", "Q74.8652.T.QJ654", "AKJT5.KQ.AK97.A8", "86.JT74.QJ.T9732"], "name": "default (GIB) Keras (24 card deck)", "board_no": 22}
+Please create samples for PAD_START-PAD_START-2C-P-3D-P-4D-P-4S-P-4N-P-5D-P-5N-P-6D-P
+
+# oct13_feb23_nov23_jun24
+Sampling this auction: 1H-P-2N-P-3D-P with this deal A.AJT9852.3.Q953 T94.7.AQJ6.T8642 Q.KQ64.T9854.AK7 KJ876532.3.K72.J
+Please add deal to training to avoid this auction 1H-P-2N-P-3D-P-3H-P-4N-P-5S
+Sample: 117, Hand KJxxxxxx.x.Kxx.J Bid not valid: 4S insta_score: 0.755
+Sampling this auction: 1H-P-2N-P-3D-P with this deal A.AJT8632.4.Q963 T82.9.AQJ5.T8754 Q.KQ74.T8732.AK2 KJ976543.5.K96.J
+Please add deal to training to avoid this auction 1H-P-2N-P-3D-P-4H-P-4N-P-5S
+Sample: 117, Hand KJxxxxxx.x.Kxx.J Bid not valid: 4S insta_score: 0.837
+Sampling this auction: 1H-P-2N-P-3D-P-4C-P-4D-P with this deal AK.AT874.9.J6542 T2.J962.J54.QT93 Q.KQ53.T8763.AK7 J9876543..AKQ2.8
+Please add deal to training to avoid this auction 1H-P-2N-P-3D-P-4C-P-4D-P-4H-P-4N-P-5S
+Sample: 29, Hand Jxxxxxxx..AKQx.x Bid not valid: 4S insta_score: 0.761
+Sampling this auction: P-1C-P with this deal AT7.KQJ.AK643.A6 QJ864.2.QJ8.Q742 K93.A874.9.KJT85 52.T9653.T752.93
+Please add deal to training to avoid this auction P-1C-P-1D-1S-P-P
+Sample: 111, Hand ATx.KQJ.AKxxx.Ax Bid not valid: 1S insta_score: 0.783
+Sampling this auction: 1H-1S-P-2S-P with this deal K643.87.KQ9.8643 Q7.AQJT93.J74.Q9 AJT952.4.AT.AKT5 8.K652.86532.J72
+Please add deal to training to avoid this auction 1H-1S-P-2S-P-4H
+Sample: 12, Hand x.Kxxx.xxxxx.Jxx Bid not valid: 4H insta_score: 0.389

scripts/training/data/Better bidding.txt

@@ -0,0 +1,63 @@
+P-P-P-1C-1S-P-1N
+P-P-P-1D-P-1S-P-2N-P-3N-P-P-4S
+P-P-P-1N-P-2H-2S-X-P
+P-P-1C-1N-P-2C-3C-P-P
+P-P-1D-P-P-X-1H-2C-P-P-X-P
+P-P-1D-P-P-1S-X
+P-P-1D-P-P-1S-2D
+P-P-1D-1N-P-2D-P-2H-P-P-X-P
+P-1C-P-1D-P-2D-P-P-X-P-3C-5D-P-P
+P-1C-P-1H-P-2H-P-4H-P-P-5D
+P-1C-P-1N-P-P-X-P
+P-1C-1S-P-1N-2C-3N-P-P-X-P
+P-1D-P-1H-P-2C-P-3N-P-P-X
+P-1D-P-1H-X-XX-1N-P-P
+P-1D-1S-P-2C-X-XX-P
+P-1D-1S-X-P-1N-P-5D-P-P
+P-1H-P-P-1S-2H
+P-1H-P-P-2C-X-P
+P-1H-P-2N-P-3C
+P-1H-X-P-2C-X-P
+P-1H-X-P-2C-X-XX
+P-1H-X-P-2C-X-XX-3C-P-P
+P-1H-X-P-2D-2H-X-P-P
+P-1H-2C-3C
+P-1S-P-3H-P-4S-P-P-5C
+P-1S-P-3N-P
+P-1S-2H-P-2S-3S-P-4H
+P-1S-2H-X-P-P
+P-2D-P-P-2H-P-3D
+P-2D-P-2N-P-3C-P-3D-P-P-X-P
+P-2H-X-P-2S-P-3C-X-P-P
+1C-P-1D-P-1H-1S
+1C-P-1H-P-1S-P-2D-P-3N-P
+1C-1S-2C-P-P-2H-P-3C
+1C-1S-2C-2D-P-3C
+1D-P-1H-P-1S-P-1N-P-2H-P-P-X-P
+1D-P-1N-P-P
+1D-P-2H-P-3H-P-4H-P-4N-P-5S
+1D-X-P-2C-X-XX-P
+1D-1H-X-P-2D-P-2H
+1D-1S-1N
+1D-2H-3N-P-P-X-XX-P
+1H-P-1S-2D-P-P-X-P
+1H-P-1S-P-3D-P-4N-P-5C-P-6N-P
+1H-P-2C-P-2H-P-2S-P-4S-P-P-X-P
+1H-P-2D-P-3D-P-4C-P-4H-P-4S
+1H-P-2D-P-3D-P-4S
+1H-P-2H-5D
+1H-1S-P-2S-P-4H
+1H-2H-2S-X-3H-3S
+1H-2H-2S-X-P-3S
+1H-2H-2S-X-XX-3S
+1H-2S-3S-P-4H-P-4N-P-5S
+1S-P-2H-P-3H-P-4N-P-5C-P-5N
+1S-P-2C-P-3C-P-4D-P-4S-P-4N-P-5D-P-5H-P-6H-X
+P-1S-2H-P-P
+1N-P-2C-P-2D-P-3S-P-4H-P-4N-P-5H-X-XX-P
+1N-P-2D-P-2H-P-3N-P-P-X-P
+1N-P-2H-P-2S-P-3N-P-P-X
+1N-P-2S-P-3D-P-4C-P-4H-P-4S
+2C-2S-P-P-3H-P-4C-P-4N
+2S-3S
+3D-4S-X-XX-P

scripts/training/data/findbiddings.py

@@ -0,0 +1,45 @@
+
+# Function to load and list all entries from the pickle file, including averages and counts
+from collections import Counter
+
+
+def list_matching_entries():
+    # Load data from the pickle file
+    with open('GIB-Thorvald-8663-Optimum_OK.ben', 'r') as f:
+        hands_data = f.readlines()
+
+    with open('../data/Better bidding.txt', 'r') as f:
+        sequences = f.readlines()
+
+    print("Bid boards:",len(hands_data))
+    print("Missing bidding sequences:",len(sequences))
+    sequences = [sequence.replace('-', ' ').replace('\n', '') for sequence in sequences]
+
+    print(sequences[0])
+
+    key_counts = Counter()
+
+    # Iterate through the list two lines at a time
+    for i in range(0, len(hands_data), 2):
+        # Read two lines at a time
+        line1 = hands_data[i].strip()
+        if i + 1 < len(hands_data):  # Ensure there's a second line
+            line2 = hands_data[i + 1].strip()
+        else:
+            line2 = None  # Handle case where there's an odd number of lines
+
+        key = ' '.join(line2.split()[2:]).replace("*","")
+        if i == 101984:
+            print(key)
+
+        for sequence in sequences:  
+            if key.startswith(sequence):
+                #print(f"{line1} found for {key}")
+                key_counts[sequence] += 1
+
+    # Iterate through the Counter and print each key and count on a separate line
+    for key, count in key_counts.items():
+        print(f"{key}: {count}")
+
+# Call the function to list entries
+list_matching_entries()

scripts/training/data/listbiddingstats.py

@@ -12,7 +12,7 @@ def list_pickle_entries(pickle_db_path):
 
         # Iterate and print all entries
         for key, hands in hands_data.items():
-            if "N-S 1H P 1S 2D P P X" in key:
+            if "N-S P P 1C 1D X 2C" in key:
                 print(f"Key: {key}")
                 for hand_key, stats in hands.items():
                     print(f"  {hand_key}:")

src/bba/BBA.py

@@ -138,6 +138,17 @@ def load_ccs(self, ns_system, ew_system):
                     conventions_ns[key] = bool(int(value))
         return conventions_ns, conventions_ew
 
+
+    def is_key_card_ask(self, auction):
+        # Did partner ask for keycards
+        if len(auction) > 1:
+            if auction[-2] == "4N":
+                explanation = self.explain(auction[:-1])
+                if self.verbose:
+                    print(explanation)
+                if "Blackwood" in explanation:
+                    return self.bid(auction)
+        return None
 
     def explain(self, auction):
         #print("new_hand", self.position, self.hand_str, self.dealer, self.vuln)

src/bots.py

@@ -17,6 +17,7 @@
 from util import hand_to_str, expected_tricks_sd, p_defeat_contract, follow_suit, calculate_seed
 from colorama import Fore, Back, Style, init
 
+init()
 class BotBid:
 
     def __init__(self, vuln, hand_str, models, sampler, seat, dealer, ddsolver, verbose):
@@ -43,6 +44,9 @@ def __init__(self, vuln, hand_str, models, sampler, seat, dealer, ddsolver, verb
             self.state = models.bidder_model.zero_state
         self.ddsolver = ddsolver
         self.my_bid_no = 1
+        if self.models.use_bba_to_count_aces:
+            from bba.BBA import BBABotBid
+            self.bbabot = BBABotBid(self.models.bba_ns, self.models.bba_ew, self.seat, self.hand_str, self.vuln, self.dealer, self.models.matchpoint, self.verbose)
 
     def get_random_generator(self):
         return self.rng
@@ -180,7 +184,7 @@ def bid(self, auction):
                     sorted_score[i]
                 ))
 
-        if self.do_rollout(auction, candidates, self.get_max_candidate_score(self.my_bid_no)):
+        if self.do_rollout(auction, candidates, self.get_max_candidate_score(self.my_bid_no), hands_np.shape[0]):
             ev_candidates = []
             ev_scores = {}
             for candidate in candidates:
@@ -358,7 +362,7 @@ def bid(self, auction):
                     for bid, score in expected_score.items():
                         for candidate in ev_candidates:
                             if candidate.bid == bid:
-                                adjust = candidate.adjust / 10
+                                adjust = candidate.adjust / self.models.factor_to_translate_to_mp
                                 ev_c = candidate.with_expected_score_mp(score, adjust)
                                 #print("ev_c", ev_c)
                                 ev_candidates_mp_imp.append(ev_c)
@@ -367,7 +371,7 @@ def bid(self, auction):
                     for bid, score in expected_score.items():
                         for candidate in ev_candidates:
                             if candidate.bid == bid:
-                                adjust = candidate.adjust / 25
+                                adjust = candidate.adjust / self.models.factor_to_translate_to_imp
                                 ev_c = candidate.with_expected_score_imp(score, adjust)
                                 ev_candidates_mp_imp.append(ev_c)
 
@@ -581,7 +585,7 @@ def bid(self, auction):
         # We return the bid with the highest expected score or highest adjusted score 
         return BidResp(bid=candidates[0].bid, candidates=candidates, samples=samples[:self.sample_hands_for_review], shape=p_shp, hcp=p_hcp, who=who, quality=quality, alert = bool(candidates[0].alert))
 
-    def do_rollout(self, auction, candidates, max_candidate_score):
+    def do_rollout(self, auction, candidates, max_candidate_score, sample_count):
         if candidates[0].insta_score > max_candidate_score:
             if self.verbose:
                 print(f"A candidate above threshold {max_candidate_score}, so no need for rolling out the bidding")
@@ -601,17 +605,28 @@ def do_rollout(self, auction, candidates, max_candidate_score):
                 print("Only simulate opening bid if enabled in configuration")
             return self.models.eval_opening_bid
 
+        if sample_count == 0:
+            if self.verbose:
+                print("We found no samples, so will just trust the NN")
+            return False
         return True
 
     def get_bid_candidates(self, auction):
+        if self.models.use_bba_to_count_aces:
+            kc_resp = self.bbabot.is_key_card_ask(auction)
+            if kc_resp != None:
+                if self.verbose:
+                    print("Keycards: ", kc_resp)
+                return [CandidateBid(bid=kc_resp.bid, insta_score=1, alert = True)], False
+
         bid_softmax, alerts = self.next_bid_np(auction)
 
         #print(bid_softmax, alerts)
 
         if self.verbose:
             index_highest = np.argmax(bid_softmax)
-            if self.models.alert_supported and alerts[0] > 0.5:
-                print(f"bid {bidding.ID2BID[index_highest]} value {bid_softmax[index_highest]:.4f} is recommended by NN with alert")
+            if self.models.alert_supported and alerts[0] > self.models.alert_threshold:
+                print(f"bid {bidding.ID2BID[index_highest]} value {bid_softmax[index_highest]:.4f} is recommended by NN with alert {alerts[0]:.2f}")
             else:
                 print(f"bid {bidding.ID2BID[index_highest]} value {bid_softmax[index_highest]:.4f} is recommended by NN")
 
@@ -623,7 +638,7 @@ def get_bid_candidates(self, auction):
                 # We have to loop to avoid returning an invalid bid
                 bid_i = np.argmax(bid_softmax)
                 if self.models.alert_supported:
-                    alert = alerts[0]  > 0.5
+                    alert = alerts[0]  > self.models.alert_threshold
                 else:
                     alert = None
                 #print("bid_i",bid_i)
@@ -639,7 +654,8 @@ def get_bid_candidates(self, auction):
 
                     if bid_softmax[bid_i] >= self.get_min_candidate_score(self.my_bid_no):
                         # Seems to be an error in the training that needs to be solved
-                        sys.stderr.write(f"{Fore.GREEN}Please create samples for {auction}\n{Style.RESET_ALL}")
+                        sys.stderr.write(f"{Fore.GREEN}Please create samples for {"-".join(auction).replace("PASS", "P").replace("PAD_START-", "")}\n{Style.RESET_ALL}")
+                        sys.stderr.write(f"{Fore.GREEN}Hand {self.hand_str}\n{Style.RESET_ALL}")
                         sys.stderr.write(f"Bid not valid {bidding.ID2BID[bid_i]} insta_score: {bid_softmax[bid_i]}\n")
 
                         #assert(bid_i > 1)
@@ -682,15 +698,16 @@ def get_bid_candidates(self, auction):
                     break
             if bidding.can_bid(bidding.ID2BID[bid_i], auction):
                 if self.models.alert_supported:
-                    alert = alerts[0]  > 0.5
+                    alert = alerts[0]  > self.models.alert_threshold
                 else:
                     alert = None
                 candidates.append(CandidateBid(bid=bidding.ID2BID[bid_i], insta_score=bid_softmax[bid_i], alert = alert))
             else:
                 # Seems to be an error in the training that needs to be solved
                 # Only report it if above threshold
                 if bid_softmax[bid_i] >= self.get_min_candidate_score(self.my_bid_no) and self.get_min_candidate_score(self.my_bid_no) != -1:
-                    sys.stderr.write(f"{Fore.GREEN}Please create samples for {auction}\n{Style.RESET_ALL}")
+                    sys.stderr.write(f"{Fore.GREEN}Please create samples for {"-".join(auction).replace("PASS", "P").replace("PAD_START-", "")}\n{Style.RESET_ALL}")
+                    sys.stderr.write(f"{Fore.GREEN}Hand {self.hand_str}\n{Style.RESET_ALL}")
                     sys.stderr.write(f"Bid not valid: {bidding.ID2BID[bid_i]} insta_score: {bid_softmax[bid_i]:.3f} {self.get_min_candidate_score(self.my_bid_no)}\n")
                 if len(candidates) > 0:
                     break
@@ -829,7 +846,11 @@ def bidding_rollout(self, auction_so_far, candidate_bid, hands_np):
                         # Pass is always allowed
                         if (bid > 2 and not bidding.can_bid(bidding.ID2BID[bid], auction)):
                             invalid_bids = True
-                            sys.stderr.write(f"{Fore.GREEN}Please create samples for {auction}\n{Style.RESET_ALL}")
+                            deal = ' '.join(deck52.handxxto52str(hand,self.models.n_cards_bidding) for hand in hands_np[i,:,:])
+                            deal = deck52.convert_cards(deal,0, "", self.get_random_generator(), self.models.n_cards_bidding)
+                            deal = deck52.reorder_hand(deal)
+                            sys.stderr.write(f"{Fore.GREEN}Sampling this auction: {'-'.join(auction_so_far).replace('PASS', 'P').replace("PAD_START-", "")} with this deal {deal}\n")
+                            sys.stderr.write(f"{Fore.GREEN}Please add deal to training to avoid this auction {"-".join(auction).replace("PASS", "P")}\n{Style.RESET_ALL}")
                             sys.stderr.write(f"Sample: {i}, Hand {hand_to_str(hands_np[i,turn_i,:], self.models.n_cards_bidding)} Bid not valid: {bidding.ID2BID[bid]} insta_score: {bid_np[i][bid]:.3f}\n")
                             bid_np[i][bid] = 0