From e902e452e25946989222beb157d4223b1bc0ce8c Mon Sep 17 00:00:00 2001 From: Jaume Amores Date: Fri, 13 Sep 2024 05:30:42 +0100 Subject: [PATCH] Removed the need to use get_ipython when the function code is not run and we don't require the function to be in the notebook namespace. Could add a detection of whether the code is run from notebook, if this is available --- nbmodular/cell2func.py | 143 ++++++++++++++++++++++++++++++++--------- nbmodular/export.py | 1 + 2 files changed, 112 insertions(+), 32 deletions(-) diff --git a/nbmodular/cell2func.py b/nbmodular/cell2func.py index cbb8ea6..8f9c97d 100644 --- a/nbmodular/cell2func.py +++ b/nbmodular/cell2func.py @@ -1,14 +1,36 @@ # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/cell2func.ipynb. # %% auto 0 -__all__ = ['bunch_io', 'get_non_callable_ipython', 'get_non_callable', 'get_ast', 'remove_duplicates_from_list', - 'VariableClassifier', 'add_dict_values', 'run_cell_and_cache', 'FunctionProcessor', 'update_cell_code', - 'add_function_to_list', 'get_args_and_defaults', 'get_args_and_defaults_from_ast', - 'get_args_and_defaults_from_function_in_cell', 'derive_paths', 'CellProcessor', 'CellProcessorMagic', - 'load_ipython_extension', 'retrieve_function_values_through_disk', 'retrieve_function_values_through_memory', - 'copy_values_and_run_code_in_nb', 'copy_values_in_nb', 'transfer_variables_to_nb', - 'retrieve_nb_locals_through_disk', 'retrieve_nb_locals_through_memory', 'remove_name_from_nb', - 'acceptable_variable', 'store_variables'] +__all__ = [ + "bunch_io", + "get_non_callable_ipython", + "get_non_callable", + "get_ast", + "remove_duplicates_from_list", + "VariableClassifier", + "add_dict_values", + "run_cell_and_cache", + "FunctionProcessor", + "update_cell_code", + "add_function_to_list", + "get_args_and_defaults", + "get_args_and_defaults_from_ast", + "get_args_and_defaults_from_function_in_cell", + "derive_paths", + "CellProcessor", + "CellProcessorMagic", + "load_ipython_extension", + "retrieve_function_values_through_disk", + "retrieve_function_values_through_memory", + "copy_values_and_run_code_in_nb", + "copy_values_in_nb", + "transfer_variables_to_nb", + "retrieve_nb_locals_through_disk", + "retrieve_nb_locals_through_memory", + "remove_name_from_nb", + "acceptable_variable", + "store_variables", +] # %% ../nbs/cell2func.ipynb 2 import pdb @@ -51,6 +73,7 @@ from . import function_io from .utils import set_handle_and_log_level, get_config + # %% ../nbs/cell2func.ipynb 6 def bunch_io(func): def bunch_wrapper(*args, **kwargs): @@ -81,6 +104,7 @@ def bunch_wrapper(*args, **kwargs): return bunch_wrapper + # %% ../nbs/cell2func.ipynb 9 # import pdb @@ -114,6 +138,7 @@ def get_non_callable_ipython(variables_to_inspect, locals_, self=None): self[non_callable_variables].append(name) self[variables_to_inspect] = self[non_callable_variables].copy() + # %% ../nbs/cell2func.ipynb 11 def get_non_callable(variables): non_callable = [] @@ -126,6 +151,7 @@ def get_non_callable(variables): non_callable.append(name) return non_callable + # %% ../nbs/cell2func.ipynb 13 def get_ast(code): print(ast.dump(ast.parse(code), indent=2)) @@ -138,6 +164,7 @@ def remove_duplicates_from_list(list_with_potential_duplicates): list_without_duplicates.append(x) return list_without_duplicates + # %% ../nbs/cell2func.ipynb 15 class VariableClassifier(NodeVisitor): def __init__(self, *args, **kwargs): @@ -159,10 +186,12 @@ def generic_visit(self, node): self.previous_variables.append(node.id) super().generic_visit(node) + # %% ../nbs/cell2func.ipynb 17 def add_dict_values(d: dict): return reduce(lambda x, y: ("+", x[1] + y[1]), d.items())[1] + # %% ../nbs/cell2func.ipynb 19 def run_cell_and_cache( cell: str, @@ -209,6 +238,7 @@ def run_cell_and_cache( output.show() return output + # %% ../nbs/cell2func.ipynb 44 class FunctionProcessor(Bunch): """ @@ -273,6 +303,7 @@ def update_code( return_values=None, display=False, run=True, + ns_not_run=False, # create function in name space if not run ) -> None: """ Updates the original code found in the cell. @@ -434,7 +465,8 @@ def update_code( # assemble function parts into single text self.code = add_dict_values(self.code_parts) - get_ipython().run_cell(self.code) + if run or ns_not_run: + get_ipython().run_cell(self.code) if display: print(self.code) @@ -685,11 +717,14 @@ def run_code_and_collect_locals( if not is_test_function: # pdb.no_set_trace() if first_call: - self.run_code_and_store_its_local_values( - "previous_values", code="", store_values=store_values - ) - if self.copy_locals: - self.previous_values = copy.deepcopy(self.previous_values) + if run: + self.run_code_and_store_its_local_values( + "previous_values", code="", store_values=store_values + ) + if self.copy_locals: + self.previous_values = copy.deepcopy(self.previous_values) + else: + self.previous_values = {k: None for k in self.previous_variables} if function_in_previous_cells is not None: # if previous_values is in current_values of previous cells of this function, it cannot be a previous value self.previous_values = { @@ -704,16 +739,17 @@ def run_code_and_collect_locals( else: self.current_values = {k: None for k in self.created_variables} else: - get_ipython().run_cell( - 'from nbmodular.cell2func import get_non_callable_ipython\nget_non_callable_ipython ("previous_variables", locals())' - ) - get_ipython().run_cell( - 'from nbmodular.cell2func import get_non_callable_ipython\nget_non_callable_ipython ("created_variables", locals())' - ) + if run: + get_ipython().run_cell( + 'from nbmodular.cell2func import get_non_callable_ipython\nget_non_callable_ipython ("previous_variables", locals())' + ) + get_ipython().run_cell( + 'from nbmodular.cell2func import get_non_callable_ipython\nget_non_callable_ipython ("created_variables", locals())' + ) self.previous_values = {k: None for k in self.previous_variables} self.current_values = {k: None for k in self.created_variables} - self._create_function_info_object() + self._create_function_info_object() self.match_variables_and_locals( function_in_previous_cells=function_in_previous_cells ) @@ -834,7 +870,7 @@ def add_to_signature(self, input=None, output=None, **kwargs): self.posterior_variables += output if input is not None or output is not None: self.signature = None - self.update_code() + self.update_code(run=self.run) def __str__(self): name = None if not hasattr(self, "name") else self.name @@ -996,6 +1032,7 @@ def restore_locals(self): field="previous_values", code=restore_previous_values_code ) + # %% ../nbs/cell2func.ipynb 50 def update_cell_code(cell, defined=False): original_code = "" @@ -1016,6 +1053,7 @@ def update_cell_code(cell, defined=False): return cell + # %% ../nbs/cell2func.ipynb 52 def add_function_to_list(function, function_list, idx=None, position=None): if idx is None: @@ -1038,6 +1076,7 @@ def add_function_to_list(function, function_list, idx=None, position=None): function.idx = idx return function_list + # %% ../nbs/cell2func.ipynb 55 def get_args_and_defaults(list_args, list_defaults): if len(list_defaults) == 0: @@ -1080,6 +1119,7 @@ def get_args_and_defaults(list_args, list_defaults): return args_without_defaults, args_with_defaults, default_values + # %% ../nbs/cell2func.ipynb 57 def get_args_and_defaults_from_ast(root): args_without_defaults, args_with_defaults1, default_values1 = get_args_and_defaults( @@ -1103,6 +1143,7 @@ def get_args_and_defaults_from_function_in_cell(): root = ast.parse(cell) return get_args_and_defaults_from_ast(root) + # %% ../nbs/cell2func.ipynb 59 def derive_paths( original_path: Path, @@ -1159,6 +1200,7 @@ def derive_paths( ) cell_processor.path_to_code_cells_file.parent.mkdir(parents=True, exist_ok=True) + # %% ../nbs/cell2func.ipynb 63 class CellProcessor: """ @@ -1459,6 +1501,12 @@ def __init__( default=None, help="execute the contents of the cell", ) + self.parser.add_argument( + "--ns-not-run", + action="store_true", + default=None, + help="create function in namespace regardless of whether the function's code is run", + ) self.parser.add_argument( "--copy-locals", action="store_true", @@ -2058,7 +2106,6 @@ def create_function( input = arguments output = return_values defined = True - self._evaluate_kwargs_defaults(cell) else: defined = False arguments = [] @@ -2108,6 +2155,12 @@ def create_function( this_function.code = cell this_function.parse_variables() self.set_function_action_and_io_args(this_function, test=test, **kwargs) + if defined and this_function.run: + # create variables with the same name as the function keyword parameters, + # and assign default values to those variables. + # This is only required if the function's code is run + # (i.e., when this_function.run is True) + self._evaluate_kwargs_defaults(cell) return this_function @@ -2325,7 +2378,7 @@ def create_function_and_run_code( if self.current_function.run: self.current_function._update_function_info_object() - else: + elif self.current_function.ns_not_run: self.current_function._create_function_info_object_with_only_parsed_variables() if self.current_function.save and self.current_function.run: @@ -2418,6 +2471,7 @@ def update_pipeline( kwarguments=current_function.kwarguments, return_values=return_values, display=show, + run=run, ) current_function = self.update_return_values( @@ -2435,7 +2489,7 @@ def update_pipeline( ) if current_function.test and not current_function.data: - current_function.update_code() + current_function.update_code(run=run) if current_function.defined: # potential_arguments = current_function.loaded_names @@ -2507,7 +2561,12 @@ def update_return_values( return_values = function.return_values + [ x for x in return_values if x not in function.return_values ] - function.update_code(return_values=return_values, display=False) + function.update_code( + return_values=return_values, + display=False, + run=function.run, + ns_not_run=function.ns_not_run, + ) if current_function.test and function.test and function.data: current_function.add_function_call(function) @@ -2617,12 +2676,16 @@ def function( if this_function.method: self.add_method(this_function.name) - get_ipython().run_cell(self.imports) - get_ipython().run_cell(self.test_imports) + if this_function.run: + get_ipython().run_cell(self.imports) + get_ipython().run_cell(self.test_imports) if this_function.pipe: this_function.pipeline_name_or_default = pipe_name - self.register_pipeline(pipeline_name_or_default=pipe_name) + self.register_pipeline( + pipeline_name_or_default=pipe_name, + pipeline_in_namespace=this_function.run or this_function.ns_not_run, + ) else: this_function.pipeline_name_or_default = None @@ -2931,7 +2994,9 @@ def test_{pipeline_name} (test=True, prev_result=None, result_file_name="{pipeli """ return code, f"test_{pipeline_name}" - def register_pipeline(self, pipeline_name_or_default="default_pipeline"): + def register_pipeline( + self, pipeline_name_or_default="default_pipeline", pipeline_in_namespace=False + ): if ( pipeline_name_or_default is not None and pipeline_name_or_default != "None" @@ -2954,7 +3019,8 @@ def register_pipeline(self, pipeline_name_or_default="default_pipeline"): name=name, is_pipeline=True, ) - get_ipython().run_cell(code) + if pipeline_in_namespace: + get_ipython().run_cell(code) # test pipeline test_pipeline_name = self.get_pipeline_name( @@ -2970,7 +3036,8 @@ def register_pipeline(self, pipeline_name_or_default="default_pipeline"): is_pipeline=True, ) - get_ipython().run_cell(code) + if pipeline_in_namespace: + get_ipython().run_cell(code) def print_pipeline(self, test=False, **kwargs): code = ( @@ -3039,6 +3106,7 @@ def load_data(self, **kwargs): def save_data(self, **kwargs): self.run_io(io_action="save", **kwargs) + # %% ../nbs/cell2func.ipynb 72 @magics_class class CellProcessorMagic(Magics): @@ -3224,6 +3292,7 @@ def set(self, line): attr, value = values self.processor.set_value(attr, value) + # %% ../nbs/cell2func.ipynb 74 def load_ipython_extension(ipython): """ @@ -3232,6 +3301,7 @@ def load_ipython_extension(ipython): magics = CellProcessorMagic(ipython) ipython.register_magics(magics) + # %% ../nbs/cell2func.ipynb 78 def retrieve_function_values_through_disk(filename="variable_values.pk"): """ @@ -3247,6 +3317,7 @@ def retrieve_function_values_through_disk(filename="variable_values.pk"): } return variable_values + # %% ../nbs/cell2func.ipynb 80 def retrieve_function_values_through_memory(field): """ @@ -3279,6 +3350,7 @@ def retrieve_function_values_through_memory(field): return variable_values return None + # %% ../nbs/cell2func.ipynb 82 def copy_values_and_run_code_in_nb(self, field="shared_variables", code=""): """ @@ -3313,6 +3385,7 @@ def copy_values_and_run_code_in_nb(self, field="shared_variables", code=""): if "retrieve_function_values_through_memory" not in self: get_ipython().run_cell(code_to_run2) + # %% ../nbs/cell2func.ipynb 83 def copy_values_in_nb(self, field="shared_variables"): copy_values_code = """ @@ -3324,6 +3397,7 @@ def copy_values_in_nb(self, field="shared_variables"): """ copy_values_and_run_code_in_nb(self, field=field, code=copy_values_code) + # %% ../nbs/cell2func.ipynb 84 def transfer_variables_to_nb(**kwargs): communicator = Bunch() @@ -3331,6 +3405,7 @@ def transfer_variables_to_nb(**kwargs): communicator.tab_size = 4 copy_values_in_nb(communicator) + # %% ../nbs/cell2func.ipynb 90 def retrieve_nb_locals_through_disk(variable_values, filename="variable_values.pk"): """ @@ -3345,6 +3420,7 @@ def retrieve_nb_locals_through_disk(variable_values, filename="variable_values.p } joblib.dump(variable_values, filename) + # %% ../nbs/cell2func.ipynb 92 def retrieve_nb_locals_through_memory(field, variable_values): """ @@ -3372,10 +3448,12 @@ def retrieve_nb_locals_through_memory(field, variable_values): self[field] = variable_values.copy() # del variable_values['created_current_values'] + # %% ../nbs/cell2func.ipynb 94 def remove_name_from_nb(name): get_ipython().run_cell(f'exec("del {name}")') + # %% ../nbs/cell2func.ipynb 96 def acceptable_variable(variable_values, k): return ( @@ -3386,6 +3464,7 @@ def acceptable_variable(variable_values, k): and k not in ["variable_values", "In", "Out"] ) + # %% ../nbs/cell2func.ipynb 98 def store_variables( path_variables, diff --git a/nbmodular/export.py b/nbmodular/export.py index c5576f7..c01e496 100644 --- a/nbmodular/export.py +++ b/nbmodular/export.py @@ -300,6 +300,7 @@ def cell(self, cell): words = line.split() command = words[0][2:] if command in self.cell_processor.magic_commands_list: + # run %%function magic command with --not-run flag. This will store the code cell, but not run it. self.cell_processor.process_function_call( line=" ".join(*words[1:], ["--not-run"]), cell="\n".join(source_lines[1:]) if len(source_lines) > 1 else "",