Visualize hero.describe with plots

.travis.yml

@@ -20,7 +20,7 @@ jobs:
       env: PATH=/c/Python38:/c/Python38/Scripts:$PATH
 install: 
     - pip3 install --upgrade pip  # all three OSes agree about 'pip3'
-    - pip3 install black
+    - pip3 install black==19.10b0
     - pip3 install ".[dev]" .
 # 'python' points to Python 2.7 on macOS but points to Python 3.8 on Linux and Windows
 # 'python3' is a 'command not found' error on Windows but 'py' works on Windows only

setup.cfg

@@ -41,7 +41,7 @@ install_requires =
 # TODO pick the correct version.
 [options.extras_require]
 dev =
-    black>=19.10b0
+    black==19.10b0
     pytest>=4.0.0
     Sphinx>=3.0.3
     sphinx-markdown-builder>=0.5.4

tests/test_indexes.py

@@ -56,21 +56,9 @@
 ]
 
 test_cases_representation = [
-    [
-        "count",
-        lambda x: representation.flatten(representation.count(x)),
-        (s_tokenized_lists,),
-    ],
-    [
-        "term_frequency",
-        lambda x: representation.flatten(representation.term_frequency(x)),
-        (s_tokenized_lists,),
-    ],
-    [
-        "tfidf",
-        lambda x: representation.flatten(representation.tfidf(x)),
-        (s_tokenized_lists,),
-    ],
+    ["count", representation.count, (s_tokenized_lists,),],
+    ["term_frequency", representation.term_frequency, (s_tokenized_lists,),],
+    ["tfidf", representation.tfidf, (s_tokenized_lists,),],
     ["pca", representation.pca, (s_numeric_lists, 0)],
     ["nmf", representation.nmf, (s_numeric_lists,)],
     ["tsne", representation.tsne, (s_numeric_lists,)],

tests/test_preprocessing.py

@@ -381,3 +381,60 @@ def test_remove_hashtags(self):
         s_true = pd.Series("Hi  , we will remove you")
 
         self.assertEqual(preprocessing.remove_hashtags(s), s_true)
+
+    """
+    Test describe DataFrame
+    """
+
+    def test_describe(self):
+        df = pd.DataFrame(
+            [
+                ["here here here here go", "sport"],
+                ["There There There", "sport"],
+                ["Test, Test, Test, Test, Test, Test, Test, Test", "sport"],
+                [np.nan, "music"],
+                ["super super", pd.NA],
+                [pd.NA, pd.NA],
+                ["great great great great great", "music"],
+            ],
+            columns=["text", "topics"],
+        )
+        df_description = preprocessing.describe(df["text"], df["topics"])
+        df_true = pd.DataFrame(
+            [
+                7,
+                7,
+                2,
+                ["Test", "great", "here", "There", "super", "go"],
+                ["test", "great", "super", "go"],
+                6.0,
+                2.0,
+                15.0,
+                5.196152422706632,
+                3.0,
+                5.0,
+                5.0,
+                0.6,
+                0.4,
+            ],
+            columns=["Value"],
+            index=pd.MultiIndex.from_tuples(
+                [
+                    ("number of documents", ""),
+                    ("number of unique documents", ""),
+                    ("number of missing documents", ""),
+                    ("most common words", ""),
+                    ("most common words excluding stopwords", ""),
+                    ("average document length", ""),
+                    ("length of shortest document", ""),
+                    ("length of longest document", ""),
+                    ("standard deviation of document lengths", ""),
+                    ("25th percentile document lengths", ""),
+                    ("50th percentile document lengths", ""),
+                    ("75th percentile document lengths", ""),
+                    ("label distribution", "sport"),
+                    ("label distribution", "music"),
+                ]
+            ),
+        )
+        pd.testing.assert_frame_equal(df_description, df_true, check_less_precise=True)