Now can op to calculate advanced stats

eapframework · Nov 11, 2019 · 23e7f35 · 23e7f35
1 parent e8e4d95
commit 23e7f35
Show file tree

Hide file tree

Showing 3 changed files with 245 additions and 258 deletions.
diff --git a/optimus/dataframe/extension.py b/optimus/dataframe/extension.py
@@ -462,7 +462,7 @@ def reset(self):
 
 
 @add_method(DataFrame)
-def send(self, name=None, infer=True, mismatch=None, stats=True):
+def send(self, name=None, infer=True, mismatch=None, stats=True, advanced_stats=True):
     """
     Profile and send the data to the queue
     :param self:
@@ -476,12 +476,14 @@ def send(self, name=None, infer=True, mismatch=None, stats=True):
     if name is not None:
         df.set_name(name)
 
-    columns, output = Profiler.instance.dataset(df, columns="*", buckets=35, infer=infer, relative_error=RELATIVE_ERROR,
-                                                approx_count=True,
-                                                sample=10000,
-                                                stats=stats,
-                                                format="json",
-                                                mismatch=mismatch)
+    output = Profiler.instance.dataset(df, columns="*", buckets=35, infer=infer, relative_error=RELATIVE_ERROR,
+                                       approx_count=True,
+                                       sample=10000,
+                                       stats=stats,
+                                       format="json",
+                                       mismatch=mismatch,
+                                       advanced_stats=advanced_stats
+                                       )
 
     if Comm.instance:
         Comm.instance.send(output)

diff --git a/optimus/helpers/constants.py b/optimus/helpers/constants.py
@@ -75,6 +75,8 @@
 
 # Profiler
 PROFILER_COLUMN_TYPES = {"categorical", "numeric", "date", "null", "array", "binary"}
+PYTHON_TO_PROFILER = {"string": "categorical", "boolean": "categorical", "int": "numeric", "decimal": "numeric",
+                         "date": "date", "array": "array", "binaty": "binary", "null": "null"}
 
 SPARK_DTYPES_TO_PROFILER = {"int": ["smallint", "tinyint", "bigint", "int"], "decimal": ["float", "double"],
                             "string": "string", "date": {"date", "timestamp"}, "boolean": "boolean", "binary": "binary",