added progress bar on classifier and featurizer

kairon/shared/nlu/classifier/openai.py

@@ -13,6 +13,7 @@
 from rasa.shared.nlu.constants import TEXT, INTENT
 import openai
 import numpy as np
+from tqdm import tqdm
 
 logger = logging.getLogger(__name__)
 
@@ -86,7 +87,7 @@ def train(
         """Train the intent classifier on a data set."""
         data_map = []
         vector_map = []
-        for example in training_data.intent_examples:
+        for example in tqdm(training_data.intent_examples):
             vector_map.append(self.get_embeddings(example.get(TEXT)))
             data_map.append({'text': example.get(TEXT), 'intent': example.get(INTENT)})
         np_vector = np.asarray(vector_map, dtype=np.float32)

kairon/shared/nlu/featurizer/openai.py

@@ -24,6 +24,7 @@
 from rasa.shared.nlu.training_data.training_data import TrainingData
 from rasa.nlu.components import Component
 import os
+from tqdm import tqdm
 
 logger = logging.getLogger(__name__)
 
@@ -99,7 +100,7 @@ def _get_model_features_for_batch(
         """
         sentence_embeddings = []
         sequence_embeddings = []
-        for example in batch_examples:
+        for example in tqdm(batch_examples):
             text = example.get(attribute)
             tokens = example.get(TOKENS_NAMES[attribute])
 
@@ -165,7 +166,6 @@ def train(
             )
 
             batch_start_index = 0
-
             while batch_start_index < len(non_empty_examples):
 
                 batch_end_index = min(
@@ -179,6 +179,7 @@ def train(
                 batch_docs = self._get_docs_for_batch(batch_messages, attribute)
 
                 for index, ex in enumerate(batch_messages):
+                    print(index)
                     self._set_lm_features(batch_docs[index], ex, attribute)
                 batch_start_index += batch_size