Merge pull request #38 from jimilp7/feature/progress-bar

Issue#17 - Add Progress Bar to show progress on perturbations and scores

auditor/evaluation/discriminative.py

@@ -15,6 +15,7 @@
 )
 from auditor.reporting import generate_robustness_report
 from auditor.utils.logging import get_logger
+from auditor.utils.progress_logger import ProgressLogger
 
 LOG = get_logger(__name__)
 
@@ -90,6 +91,12 @@ def evaluate(
             f'Started model evaluation with perturbation type '
             f'{self.perturbed_dataset.perturbation_type}'
         )
+        progress_bar = ProgressLogger(
+            total_steps=min(len(self.perturbed_dataset.data),
+                            len(self.perturbed_dataset.metadata)),
+            description="Starting Model Evaluation"
+        )
+
         for perturbed_samples, metadata_samples in zip(
             self.perturbed_dataset.data, self.perturbed_dataset.metadata
         ):
@@ -126,12 +133,14 @@ def evaluate(
                         metadata=mdata,
                     )
                 )
+            progress_bar.update()
         robust_accuracy = self.compute_accuracy(test_results)
         LOG.info(f'Robust Accuracy: {robust_accuracy*100.}')
         LOG.info(
             'Completed model evaluation with perturbation type '
             f'{self.perturbed_dataset.perturbation_type}'
         )
+        progress_bar.close()
         self.test_results = TestSummary(
             results=test_results,
             robust_accuracy=robust_accuracy,

auditor/evaluation/expected_behavior.py

@@ -4,6 +4,7 @@
 import numpy as np
 from sentence_transformers.SentenceTransformer import SentenceTransformer
 
+from auditor.utils.progress_logger import ProgressLogger
 from auditor.utils.similarity import compute_similarity
 from auditor.utils.logging import get_logger
 
@@ -163,6 +164,9 @@ def check(
         reference_generation: str,
     ) -> List[Tuple[bool, Dict[str, float]]]:
         test_results = []
+        progress_bar = ProgressLogger(total_steps=len(perturbed_generations),
+                                      description="Fetching Scores")
+
         for peturbed_gen in perturbed_generations:
             try:
                 score = compute_similarity(
@@ -178,9 +182,13 @@ def check(
                     self.similarity_metric_key: round(score, ndigits=2)
                 }
                 test_results.append((test_status, score_dict))
+                progress_bar.update()
             except Exception as e:
                 LOG.error('Unable to complete semanatic similarity checks')
+                progress_bar.close()
                 raise e
+
+        progress_bar.close()
         return test_results
 
     def behavior_description(self):

auditor/evaluation/generative.py

@@ -12,6 +12,7 @@
 from auditor.utils.logging import get_logger
 from auditor.perturbations import Paraphrase
 from auditor.perturbations import TransformBase
+from auditor.utils.progress_logger import ProgressLogger
 
 LOG = get_logger(__name__)
 
@@ -98,6 +99,9 @@ def _evaluate_generations(
         else:
             evaluate_prompts = prompt_perturbations
 
+        progress_bar = ProgressLogger(total_steps=len(evaluate_prompts),
+                                      description="Applying Perturbations")
+
         # generations for each of the perturbed prompts
         alternative_generations = []
         for alt_prompt in evaluate_prompts:
@@ -107,6 +111,9 @@ def _evaluate_generations(
                 post_context,
             )
             alternative_generations.append(resp)
+            progress_bar.update()
+
+        progress_bar.close()
 
         # create test result
         metric = self.expected_behavior.check(

auditor/utils/progress_logger.py

@@ -0,0 +1,17 @@
+import tqdm
+
+
+class ProgressLogger:
+    """class to show progress bar"""
+
+    def __init__(self, total_steps, description="Logging..."):
+        self.total_steps = total_steps
+        self.description = description
+
+        self.pbar = tqdm.tqdm(total=total_steps, desc=description)
+
+    def update(self, incremental=1):
+        self.pbar.update(incremental)
+
+    def close(self):
+        self.pbar.close()

pyproject.toml

@@ -26,6 +26,7 @@ dependencies = [
     "langchain >=0.0.158",
     "openai >=0.27.0",
     "sentence-transformers>=2.2.2",
+    "tqdm>=4.66.1"
 ]
 
 [project.license]