Updates can be optionally sorted by word frequency or score

smoothnlp/algorithm/phrase/ngram_utils.py

@@ -206,7 +206,8 @@ def get_scores(corpus,
                  left_right_entropy[word][0],   #left_entropy
                  left_right_entropy[word][1],   #right_entropy
                  min(left_right_entropy[word][0],left_right_entropy[word][1]),    #branch entropy  BE=min{left_entropy,right_entropy}
-                 word_liberalization(left_right_entropy[word][0],left_right_entropy[word][1])+mi[word][1]   #our score
+                 word_liberalization(left_right_entropy[word][0],left_right_entropy[word][1])+mi[word][1],   #our score
+                 ngram_freq[word]  # word frequency
                      )
               for word in joint_phrase}
 

smoothnlp/algorithm/phrase/phrase_extraction.py

@@ -39,14 +39,16 @@ def extract_phrase(corpus,
                    chunk_size: int = 1000000,
                    min_n:int = 2,
                    max_n:int=4,
-                   min_freq:int = 5):
+                   min_freq:int = 5,
+                   order_by: str = 'score'):
     '''
-    取前k个new words或前k%的new words
+    按score或者freq取前k个new words或前k%的new words
     :param corpus:
     :param top_k:
     :param chunk_size:
     :param max_n:
     :param min_freq:
+    :param order_by:
     :return:
     '''
     if isinstance(corpus,str):
@@ -57,7 +59,10 @@ def extract_phrase(corpus,
     else:
         corpus_splits = chunk_generator_adapter(corpus, chunk_size)
     word_info_scores = get_scores(corpus_splits,min_n,max_n,chunk_size,min_freq)
-    new_words = [item[0] for item in sorted(word_info_scores.items(),key=lambda item:item[1][-1],reverse = True)]
+    if order_by == 'score':
+        new_words = [item[0] for item in sorted(word_info_scores.items(), key=lambda item: item[1][-2], reverse=True)]
+    elif order_by == 'freq':
+        new_words = [item[0] for item in sorted(word_info_scores.items(), key=lambda item: item[1][-1], reverse=True)]
     if top_k > 1:              #输出前k个词
         return new_words[:top_k]
     elif top_k < 1:            #输出前k%的词