Skip to content

Commit

Permalink
Fix: Rename actions was firing a profling
Browse files Browse the repository at this point in the history
  • Loading branch information
argenisleon committed Nov 6, 2019
1 parent a7255d0 commit a2de19e
Showing 1 changed file with 6 additions and 8 deletions.
14 changes: 6 additions & 8 deletions optimus/profiler/profiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -247,9 +247,6 @@ def dataset(self, df, columns="*", buckets=10, infer=False, relative_error=RELAT
# So process the dataframe's metadata to be sure which columns need to be profiled
is_cached = len(self.output_columns) > 0
actions = df.get_meta("transformations.actions")
# print(actions)
# are_actions = None
# actions = actions.get("actions")
are_actions = actions is not None and len(actions) > 0

# Process actions to check if any column must be processed
Expand All @@ -264,13 +261,13 @@ def match_actions_names(_actions):
:return:
"""

_actions = df.get_meta("transformations.actions")
_actions_json = df.get_meta("transformations.actions")

modified = []
for action in _actions:
if _actions.get(action):
if _actions_json.get(action):
# Check if was renamed
col = _actions.get(action)
col = _actions_json.get(action)
if len(match_names(col)) == 0:
_result = col
else:
Expand Down Expand Up @@ -325,16 +322,18 @@ def match_names(_col_names):
# Actions applied to current columns

modified_columns = match_actions_names(Actions.list())
# print(modified_columns, new_columns)
calculate_columns = modified_columns + new_columns

# Remove duplicated.
calculate_columns = list(set(calculate_columns))

elif is_cached and not are_actions:
calculate_columns = None
else:
elif not is_cached:
calculate_columns = columns

# print ("calculate_columns",calculate_columns)
# Get the stats for all the columns
if stats is True:
# Are there column to process?
Expand Down Expand Up @@ -386,7 +385,6 @@ def match_names(_col_names):
result = output_columns

self.output_columns = output_columns
# print(result)
df = df.set_meta("transformations.actions", {})

return col_names, result
Expand Down

0 comments on commit a2de19e

Please sign in to comment.