Real-time loss plots during solving through solver callback functions

data/solverData.csv

@@ -107,3 +107,147 @@ Uni-criterion: clusterCenters,greedySwap,1000,2,10,0.13878262508660555,0.7724079
 "Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,7,0.1945001189596951,0.0
 "Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,7,0.18785801599733531,0.0
 "Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,7,0.234673575963825,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,11,0.29593419295269996,0.0
+"Uni-criterion: sum, outlierness",greedySwap,200,2,40,0.11445782298687845,-77.13353928048035
+"Uni-criterion: distinctness, distances",greedySwap,200,2,60,0.5462852430064231,-74.18261659032585
+"Multi-criterion: 100*(earthMoversDistance) + 1*(distinctness, distances)",greedySwap,200,2,80,16.019615472992882,-39.956777992395615
+"Multi-criterion: 10*(earthMoversDistance) + 1*(distinctness, distances)",greedySwap,200,2,80,15.961098090047017,-71.3535755814987
+"Multi-criterion: 1*(earthMoversDistance) + 1*(distinctness, distances)",greedySwap,200,2,80,17.308373406063765,-76.92932448798969
+"Uni-criterion: distinctness, distances",greedyMixed,200,2,16,0.10311844199895859,-36.98853938692782
+"Uni-criterion: distinctness, distances",greedyMixed,200,2,40,0.3425103520276025,-55.53023413167705
+"Uni-criterion: distinctness, distances",greedyMixed,200,2,102,0.7111876929411665,-77.72499108526614
+"Uni-criterion: sum, outlierness",greedySwap,200,2,40,0.17156568495556712,-107.85032842123348
+"Uni-criterion: sum, outlierness",greedySwap,200,2,40,0.009137309971265495,-59.99163302732708
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,10,0.28689464705530554,0.0
+"Uni-criterion: sum, outlierness",greedySwap,200,2,40,0.07565713091753423,-76.90150167654883
+"Uni-criterion: distinctness, distances",greedySwap,200,2,60,0.33772983809467405,-82.65893651396253
+"Multi-criterion: 100*(earthMoversDistance) + 1*(distinctness, distances)",greedySwap,200,2,80,15.733736847061664,-37.72502949783789
+"Multi-criterion: 10*(earthMoversDistance) + 1*(distinctness, distances)",greedySwap,200,2,80,16.34970149002038,-79.21528178454805
+"Multi-criterion: 1*(earthMoversDistance) + 1*(distinctness, distances)",greedySwap,200,2,80,16.09934757091105,-87.1484395780551
+"Uni-criterion: distinctness, distances",greedyMixed,200,2,21,0.09933742694556713,-48.916226463106035
+"Uni-criterion: distinctness, distances",greedyMixed,200,2,18,0.07778958394192159,-52.25319459589197
+"Uni-criterion: distinctness, distances",greedyMixed,200,2,92,0.4698902699165046,-86.11815292551535
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,11,0.3114050959702581,0.0
+"Uni-criterion: sum, outlierness",greedySwap,200,2,40,0.1782841000240296,-94.00493289823834
+"Uni-criterion: distinctness, distances",greedySwap,200,2,60,0.6240650089457631,-81.66286493321189
+"Multi-criterion: 100*(earthMoversDistance) + 1*(distinctness, distances)",greedySwap,200,2,80,15.76467304804828,-36.356196781059154
+"Multi-criterion: 10*(earthMoversDistance) + 1*(distinctness, distances)",greedySwap,200,2,80,17.084750175010413,-78.89219470658081
+"Multi-criterion: 1*(earthMoversDistance) + 1*(distinctness, distances)",greedySwap,200,2,80,18.41927482408937,-86.71818903727858
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,11,0.9344452220248058,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,10,0.8697499780682847,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,10,0.4994492910336703,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,10,0.4910243459744379,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,9,0.4479509260272607,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,9,0.7013822080334648,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,9,0.7932263310067356,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,10,0.925686597940512,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,10,0.9141074389917776,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,10,0.9515322380466387,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,10,0.8977055459981784,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,10,0.8896270530531183,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,10,1.3545968279941007,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,1.1011312199989334,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,0.8075686269439757,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,0.7807131060399115,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,0.9366903710179031,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,1.150744732003659,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,1.1362005930859596,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,0.6776644670171663,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,0.8283384050009772,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,0.7767721880227327,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,1.1211024429649115,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,1.1975002819672227,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,0.8234556920360774,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,0.7040190550033003,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,0.7956955870613456,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,0.6690607069758698,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,0.6585186719894409,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,0.6739458279917017,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,0.39736516401171684,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,0.8139416560297832,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,0.4658096390776336,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,0.5328066080110148,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,0.40911367000080645,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,0.42375491897109896,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,0.4381504019256681,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,1.0618235199945047,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.858548079035245,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.9225411330116913,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.8708883179351687,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.59070663806051,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.6839401100296527,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.401693034917116,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.6874031629413366,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.7578133980277926,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.5088547869818285,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,1.169300428009592,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.232280447962694,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.1939020979916677,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.139898219029419,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.007311812019907,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.013121727039106,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,1.9959770779823884,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.232953047961928,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.2974324349779636,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.105868001934141,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.038483888027258,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.2182905660010874,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.2297344179823995,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.181866207974963,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.459757408942096,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.0248638410121202,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,1.8204639760078862,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.1470516480039805,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,1.907505503972061,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,1.873534856014885,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,1.7429639380425215,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,1.8942721990169957,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.134770376025699,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,1.9120049610501155,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.98742147104349,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.976412029005587,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.9450863489182666,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,3.081287116976455,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.7693268969887868,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.8417827270459384,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.855016679968685,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.8780118320137262,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.8611697829328477,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.961038665031083,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,3.2737037639599293,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,3.3527815849520266,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,3.2911710800835863,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,3.5750179030001163,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,3.1241388369817287,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,3.108717106981203,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,2.9237201779615134,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,3.1779711049748585,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,3.0755136230727658,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,3.065400000079535,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,8,3.2507090859580785,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,10,3.2701174030080438,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,10,3.5799345769919455,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,10,3.3197142069693655,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,10,3.446158067905344,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,10,3.5796150349779055,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,11,3.578337203944102,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,11,3.711465527070686,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,11,4.2906140179838985,0.0
+"Uni-criterion: preserveMetric, hull",greedyMinSubset,200,2,10,3.549799660919234,0.0
+"Uni-criterion: sum, outlierness",greedySwap,200,2,40,0.06814886396750808,-89.60162497757574
+"Uni-criterion: sum, outlierness",greedySwap,200,2,40,36.6902008849429,-89.60162497757574
+"Uni-criterion: distinctness, distances",greedySwap,200,2,60,0.3224127379944548,-79.90049127593495
+"Uni-criterion: distinctness, distances",greedySwap,200,2,60,36.83346623403486,-80.79567585790252
+"Multi-criterion: 100*(earthMoversDistance) + 1*(distinctness, distances)",greedySwap,200,2,80,15.037519195000641,-37.37670531499123
+"Multi-criterion: 100*(earthMoversDistance) + 1*(distinctness, distances)",greedySwap,200,2,80,50.97084774100222,-39.22812364018983
+"Multi-criterion: 10*(earthMoversDistance) + 1*(distinctness, distances)",greedySwap,200,2,80,53.80880820809398,-76.04688048633925
+"Multi-criterion: 1*(earthMoversDistance) + 1*(distinctness, distances)",greedySwap,200,2,80,53.854620625963435,-85.06005136858656
+"Uni-criterion: distinctness, distances",greedyMixed,200,2,10,0.04771749395877123,-38.18019263929493
+"Uni-criterion: distinctness, distances",greedyMixed,200,2,52,0.22991829796228558,-58.48640809913543
+"Uni-criterion: distinctness, distances",greedyMixed,200,2,87,0.42976776498835534,-81.60528780506851
+"Uni-criterion: distinctness, distances",greedyMixed,200,2,23,4.512196074007079,-47.91799174414664
+"Uni-criterion: distinctness, distances",greedyMixed,200,2,27,5.162200314924121,-55.62985637521753
+"Uni-criterion: distinctness, distances",greedyMixed,200,2,88,17.476347186020575,-81.49231730696414
+"Uni-criterion: distinctness, distances",greedyMixed,200,2,19,3.7761652149492875,-46.45876245484051
+"Uni-criterion: distinctness, distances",greedyMixed,200,2,31,6.059022648027167,-56.28867409761743
+"Uni-criterion: distinctness, distances",greedyMixed,200,2,52,9.988744632923044,-68.26740153361762

flexibleSubsetSelection/algorithm.py

@@ -140,7 +140,7 @@ def worstOfRandom(dataset, lossFunction, subsetSize, minLoss=0,
 
 
 def greedySwap(dataset, lossFunction, subsetSize, minLoss=0, maxIterations=None,
-               seed=None):
+               seed=None, callback=None):
     """
     A greedy algorithm with a greedy swap heuristic for subset selection.
 
@@ -170,6 +170,9 @@ def greedySwap(dataset, lossFunction, subsetSize, minLoss=0, maxIterations=None,
 
     for i in range(maxIterations):
         log.debug("Iteration %s/%s: Loss %s.", i, maxIterations, loss)
+        if callback:
+            callback(iterations, loss)
+
         if i not in indices:
             zSwapBest = np.copy(z)
             lossSwapBest = loss
@@ -199,7 +202,7 @@ def greedySwap(dataset, lossFunction, subsetSize, minLoss=0, maxIterations=None,
     return z, loss # return indicator and final loss
 
 def greedyMinSubset(dataset, lossFunction, epsilon, minError=0, 
-                    maxIterations=None, seed=None, initialSize=1):
+                    maxIterations=None, seed=None, initialSize=1, callback=None):
     """
     A greedy algorithm for subset selection to minimize the size of the subset 
     such that lossFunction(subset) <= epsilon.
@@ -214,6 +217,7 @@ def greedyMinSubset(dataset, lossFunction, epsilon, minError=0,
         seed (int, rng, optional): The random seed or NumPy rng for random 
             generation and reproducibility
         initialSize (int, optional): Initial size of the subset
+        callback (function, optional): A callback function for loss values
 
     Returns:
         z (array): Indicator vector of included items in the subset
@@ -252,6 +256,8 @@ def greedyMinSubset(dataset, lossFunction, epsilon, minError=0,
 
     while iterations < maxIterations:
         log.debug("Iteration: %s, Loss: %s, Error: %s, Subset Size: %s.", iterations, current_loss, error, np.sum(z))
+        if callback:
+            callback(iterations, current_loss, subsetSize=np.sum(z))
 
         # Check if error is less than or equal to epsilon
         if error <= epsilon:
@@ -314,7 +320,7 @@ def greedyMinSubset(dataset, lossFunction, epsilon, minError=0,
     return z, error
 
 def greedyMixed(dataset, lossFunction, weight=1.0, minError=0, 
-                maxIterations=None, seed=None, initialSize=1):
+                maxIterations=None, seed=None, initialSize=1, callback=None):
     """
     A greedy algorithm to minimize the total 
     loss = weight * subsetSize + lossFunction.calculate().
@@ -363,6 +369,8 @@ def greedyMixed(dataset, lossFunction, weight=1.0, minError=0,
 
     while iterations < maxIterations:
         log.debug("Iteration %s: Total Loss %s, Subset Size %s", iterations, total_loss, np.sum(z))
+        if callback:
+            callback(iterations, total_loss, subsetSize=np.sum(z))
 
         # Check if error is less than or equal to minError
         if error <= minError:

flexibleSubsetSelection/plot.py

@@ -4,6 +4,8 @@
 from typing import Callable
 
 # Third party
+from IPython.display import display, clear_output
+
 import matplotlib
 from matplotlib.axes import Axes
 from matplotlib.colors import to_rgb, to_hex
@@ -346,4 +348,58 @@ def histogram(ax: Axes, color: Color, dataset: (Dataset | None) = None,
             positions = barPositions[i] + np.arange(numBins)
 
             ax.bar(positions, subsetHeights, width=1, 
-                   color=color.palette["darkGreen"], alpha=0.5)
+                   color=color.palette["darkGreen"], alpha=0.5)
+
+class RealTimePlotter:
+    def __init__(self, color):
+        # Initialize the figure and axis
+        self.fig, self.ax = plt.subplots(figsize=(4, 4))
+        self.iterations = []
+        self.losses = []
+        self.subsetSizes = []
+        self.color = color
+
+        # Set up the initial plot
+        self.ax.set_xlabel('Iteration')
+        self.ax.set_ylabel('Loss')
+        self.ax.set_title('Real-Time Loss During Solver')
+        self.ax.set_ylim(bottom=0)  # Set the lower y-axis limit to 0
+
+        # Display the figure initially
+        display(self.fig)
+
+    def update(self, iteration, loss, subsetSize=None):
+        # Append the current iteration and loss to lists
+        self.iterations.append(iteration)
+        self.losses.append(loss)
+        self.subsetSizes.append(subsetSize)
+
+        # Clear the previous output (but don't clear the figure)
+        clear_output(wait=True)
+        self.ax.clear()
+
+        self.ax.set_xlabel('Iteration')
+        # self.ax.set_ylabel('Loss')
+        self.ax.set_title('Minimum Loss')
+
+        # Plot the updated loss values
+        self.ax.plot(self.iterations, 
+                     self.losses, 
+                     c=self.color["orange"], 
+                     label="Loss", 
+                     lw=2)
+        if subsetSize is not None:
+            self.ax.plot(self.iterations, 
+                        self.subsetSizes, 
+                        c=self.color["green"], 
+                        label="Subset Size", 
+                        lw=2)
+        self.ax.legend()
+
+        # Display the updated plot
+        display(self.fig)
+        plt.close(self.fig)
+
+    def close(self):
+        # Close the plot when done
+        plt.close(self.fig)