From 7a119085d958305f6a5acdbce6fd7f10682c57eb Mon Sep 17 00:00:00 2001 From: Jaume Amores Date: Wed, 4 Sep 2024 14:08:52 +0100 Subject: [PATCH] Passed until nbm_update. To complete that test --- nbmodular/cell2func.py | 550 ++++++++++++++++++++-------------------- nbmodular/export.py | 298 +++++++++++++++++++--- nbmodular/test_utils.py | 185 ++++++++++++-- nbs/cell2func.ipynb | 208 +++++++-------- 4 files changed, 807 insertions(+), 434 deletions(-) diff --git a/nbmodular/cell2func.py b/nbmodular/cell2func.py index 75c3b4d..42847d5 100644 --- a/nbmodular/cell2func.py +++ b/nbmodular/cell2func.py @@ -16,15 +16,15 @@ # %% [markdown] # # cell2func # -# > Convert desired notebook cells to functions. +# > Convert desired notebook cells to functions. # # Detects function inputs automatically and function outputs semi-automatically. In the latter case, hints are provided to the developer to refine the list of outputs per each cell. # %% -#| default_exp cell2func +# | default_exp cell2func # %% -#| export +# | export import pdb from typing import List, Optional, cast from xxlimited import Str @@ -79,8 +79,9 @@ # %% [markdown] # ## bunch_io + # %% -#| export +# | export def bunch_io(func): def bunch_wrapper(*args, **kwargs): if (len(args) > 1) or ((len(args) == 1) and not isinstance(args[0], Bunch)): @@ -111,7 +112,6 @@ def bunch_wrapper(*args, **kwargs): return bunch_wrapper - # %% [markdown] # ## get_non_callable @@ -119,8 +119,8 @@ def bunch_wrapper(*args, **kwargs): # ### get_non_callable_ipython # %% -#| export -import pdb +# | export +#import pdb def get_non_callable_ipython(variables_to_inspect, locals_, self=None): @@ -153,12 +153,12 @@ def get_non_callable_ipython(variables_to_inspect, locals_, self=None): self[variables_to_inspect] = self[non_callable_variables].copy() - # %% [markdown] # ### get_non_callable + # %% -#| export +# | export def get_non_callable(variables): non_callable = [] for name in variables: @@ -171,12 +171,12 @@ def get_non_callable(variables): return non_callable - # %% [markdown] # ## get_ast + # %% -#| export +# | export def get_ast(code): print(ast.dump(ast.parse(code), indent=2)) @@ -189,12 +189,12 @@ def remove_duplicates_from_list(list_with_potential_duplicates): return list_without_duplicates - # %% [markdown] # ## VariableClassifier + # %% -#| export +# | export class VariableClassifier(NodeVisitor): def __init__(self, *args, **kwargs): self.created_variables = [] @@ -216,22 +216,22 @@ def generic_visit(self, node): super().generic_visit(node) - # %% [markdown] # ## add_dict_values + # %% -#| export +# | export def add_dict_values(d: dict): return reduce(lambda x, y: ("+", x[1] + y[1]), d.items())[1] - # %% [markdown] # ## cache output + # %% -#| export +# | export def run_cell_and_cache( cell: str, load_disk: bool = False, @@ -278,148 +278,144 @@ def run_cell_and_cache( return output - # %% [markdown] # ### Example usage # %% -memory={} -cell=( -""" -import matplotlib.pyplot as plt -plt.plot ([1, 100, 1000]) -""" -) -run_cell_and_cache ( - cell, - save_memory=True, - memory=memory, - memory_key="previous_plot", -); +def test_run_cell_and_cache(): + memory = {} + cell = """ + import matplotlib.pyplot as plt + plt.plot ([1, 100, 1000]) + """ + run_cell_and_cache( + cell, + save_memory=True, + memory=memory, + memory_key="previous_plot", + ) -# %% [markdown] -# or a simple computation: + # %% [markdown] + # or a simple computation: -# %% -cell=( -""" -3+4 -""" -) -output=run_cell_and_cache ( - cell, - save_memory=True, - memory=memory, - memory_key="previous_sum", -) + # %% + cell = """ + 3+4 + """ + output = run_cell_and_cache( + cell, + save_memory=True, + memory=memory, + memory_key="previous_sum", + ) -# %% [markdown] -# The returned object is of class CapturedIO: + # %% [markdown] + # The returned object is of class CapturedIO: -# %% -assert output.__class__.__name__ == "CapturedIO" + # %% + assert output.__class__.__name__ == "CapturedIO" -# %% [markdown] -# When printing to console or notebook, the returned object contains information about the printed text: + # %% [markdown] + # When printing to console or notebook, the returned object contains information about the printed text: -# %% -assert len(output.outputs)==1 and output.outputs[0].data["text/plain"]=="7" -assert sorted(memory) == sorted (["previous_sum", "previous_plot"]) + # %% + assert len(output.outputs) == 1 and output.outputs[0].data["text/plain"] == "7" + assert sorted(memory) == sorted(["previous_sum", "previous_plot"]) -# %% [markdown] -# Subsequent calls can load a previous capture from memory, without needing to run the code in the cell. We check this by passing a code that should not be run: + # %% [markdown] + # Subsequent calls can load a previous capture from memory, without needing to run the code in the cell. We check this by passing a code that should not be run: -# %% -should_not_run_cell=( -""" -raise RuntimeError("cell should not run") -""" -) -run_cell_and_cache ( - should_not_run_cell, - load_memory=True, - memory=memory, - memory_key="previous_plot", -); + # %% + should_not_run_cell = """ + raise RuntimeError("cell should not run") + """ + run_cell_and_cache( + should_not_run_cell, + load_memory=True, + memory=memory, + memory_key="previous_plot", + ) -# %% -assert memory.get("previous_plot").__class__.__name__ == "CapturedIO" + # %% + assert memory.get("previous_plot").__class__.__name__ == "CapturedIO" -# %% [markdown] -# Raises error if couln't load and error_if_not_loaded is True: + # %% [markdown] + # Raises error if couln't load and error_if_not_loaded is True: -# %% -with pytest.raises (RuntimeError): - run_cell_and_cache ( - cell, - load_memory=True, - memory=memory, - memory_key="previous_plot_bis", - error_if_not_loaded=True, - ); + # %% + with pytest.raises(RuntimeError): + run_cell_and_cache( + cell, + load_memory=True, + memory=memory, + memory_key="previous_plot_bis", + error_if_not_loaded=True, + ) -# %% [markdown] -# Can also save to disk: + # %% [markdown] + # Can also save to disk: -# %% -memory={} -test_result_folder=Path("test_run_cell_and_cache") -output_path=test_result_folder / "previous_sum.pk" -run_cell_and_cache ( - cell, - save_memory=True, - save_disk=True, - output_path=output_path, - memory=memory, - memory_key="previous_sum", -); + # %% + memory = {} + test_result_folder = Path("test_run_cell_and_cache") + output_path = test_result_folder / "previous_sum.pk" + run_cell_and_cache( + cell, + save_memory=True, + save_disk=True, + output_path=output_path, + memory=memory, + memory_key="previous_sum", + ) -# %% -assert "previous_sum" in memory -assert output_path.exists() -shutil.rmtree (output_path.parent, ignore_errors=True) + # %% + assert "previous_sum" in memory + assert output_path.exists() + shutil.rmtree(output_path.parent, ignore_errors=True) -# %% [markdown] -# or save only to disk but not to memory: + # %% [markdown] + # or save only to disk but not to memory: -# %% -memory={} -run_cell_and_cache ( - cell, - save_disk=True, - output_path=output_path, - memory=memory, - memory_key="previous_sum", -); + # %% + memory = {} + run_cell_and_cache( + cell, + save_disk=True, + output_path=output_path, + memory=memory, + memory_key="previous_sum", + ) -# %% -assert "previous_sum" not in memory -assert output_path.exists() + # %% + assert "previous_sum" not in memory + assert output_path.exists() -# %% [markdown] -# It doesn't save into disk if loaded from disk... + # %% [markdown] + # It doesn't save into disk if loaded from disk... -# %% -# run 2+8 computation -_=run_cell_and_cache (cell="2+8", load_disk=True, save_disk=True, output_path=output_path ) + # %% + # run 2+8 computation + _ = run_cell_and_cache( + cell="2+8", load_disk=True, save_disk=True, output_path=output_path + ) -# check that saved capture is still 7 -output=joblib.load (output_path) -assert len(output.outputs)==1 and output.outputs[0].data["text/plain"]=="7" + # check that saved capture is still 7 + output = joblib.load(output_path) + assert len(output.outputs) == 1 and output.outputs[0].data["text/plain"] == "7" -# %% [markdown] -# But it does save if not loaded, even if the output existed before + # %% [markdown] + # But it does save if not loaded, even if the output existed before -# %% -# run 2+8 computation -_=run_cell_and_cache (cell="2+8", save_disk=True, output_path=output_path ) + # %% + # run 2+8 computation + _ = run_cell_and_cache(cell="2+8", save_disk=True, output_path=output_path) -# check that saved capture is 10 -output=joblib.load (output_path) -assert len(output.outputs)==1 and output.outputs[0].data["text/plain"]=="10" + # check that saved capture is 10 + output = joblib.load(output_path) + assert len(output.outputs) == 1 and output.outputs[0].data["text/plain"] == "10" -# clean results -shutil.rmtree (output_path.parent, ignore_errors=True) + # clean results + shutil.rmtree(output_path.parent, ignore_errors=True) # %% [markdown] @@ -427,8 +423,9 @@ def run_cell_and_cache( # ## FunctionProcessor # ::: + # %% -#|export +# |export class FunctionProcessor(Bunch): """ Function processor. @@ -1216,47 +1213,46 @@ def restore_locals(self): ) - -# %% -# Trick used for forcing FunctionProcessor class be the same as the one imported from cell2func -CurrentFunctionProcessor=FunctionProcessor -import nbmodular.cell2func as cf -cf.FunctionProcessor = CurrentFunctionProcessor - # %% [markdown] # ### Example -# %% -cell_captures_path_to_folder=Path('.cell_captures/core/cell2func') -fp=FunctionProcessor ( - name="myf", - cell_processor=Bunch(cell_captures_path_to_folder=cell_captures_path_to_folder), - save_capture_disk=True, -) -fp.run_cell_and_cache ("1+3") -cell_captures_path_to_file = cell_captures_path_to_folder / f"{fp.name}.pk" -assert cell_captures_path_to_file.exists() -output = joblib.load (cell_captures_path_to_file) -assert len(output.outputs)==1 and output.outputs[0].data["text/plain"]=="4" -# clean -shutil.rmtree (cell_captures_path_to_folder, ignore_errors=True) # %% -# Set attributes in fp object that are used by the method -# `run_code_and_store_its_local_values`: -logger=logging.getLogger("testing_function_processor_logger") -fp.update(previous_values={}, current_values={}, logger=logger) - -# Call the method without and with code: -fp.run_code_and_store_its_local_values("previous_values", code="", store_values=True) -fp.run_code_and_store_its_local_values("current_values", code="1+3\n", store_values=True) +def test_captures_cell(): + cell_captures_path_to_folder = Path(".cell_captures/core/cell2func") + fp = FunctionProcessor( + name="myf", + cell_processor=Bunch(cell_captures_path_to_folder=cell_captures_path_to_folder), + save_capture_disk=True, + ) + fp.run_cell_and_cache("1+3") + cell_captures_path_to_file = cell_captures_path_to_folder / f"{fp.name}.pk" + assert cell_captures_path_to_file.exists() + output = joblib.load(cell_captures_path_to_file) + assert len(output.outputs) == 1 and output.outputs[0].data["text/plain"] == "4" + # clean + shutil.rmtree(cell_captures_path_to_folder, ignore_errors=True) + + # %% + # Set attributes in fp object that are used by the method + # `run_code_and_store_its_local_values`: + logger = logging.getLogger("testing_function_processor_logger") + fp.update(previous_values={}, current_values={}, logger=logger) + + # Call the method without and with code: + fp.run_code_and_store_its_local_values( + "previous_values", code="", store_values=True + ) + fp.run_code_and_store_its_local_values( + "current_values", code="1+3\n", store_values=True + ) -# Check results: -assert cell_captures_path_to_file.exists() -output = joblib.load (cell_captures_path_to_file) -assert len(output.outputs)==1 and output.outputs[0].data["text/plain"]=="4" -# clean -shutil.rmtree (cell_captures_path_to_folder, ignore_errors=True) + # Check results: + assert cell_captures_path_to_file.exists() + output = joblib.load(cell_captures_path_to_file) + assert len(output.outputs) == 1 and output.outputs[0].data["text/plain"] == "4" + # clean + shutil.rmtree(cell_captures_path_to_folder, ignore_errors=True) # %% [markdown] @@ -1267,8 +1263,9 @@ def restore_locals(self): # %% [markdown] # ### update_cell_code + # %% -#| export +# | export def update_cell_code(cell, defined=False): original_code = "" for line in cell.splitlines(): @@ -1289,12 +1286,12 @@ def update_cell_code(cell, defined=False): return cell - # %% [markdown] # ### add_function_to_list + # %% -#|export +# |export def add_function_to_list(function, function_list, idx=None, position=None): if idx is None: function_list.append(function) @@ -1317,15 +1314,15 @@ def add_function_to_list(function, function_list, idx=None, position=None): return function_list - # %% [markdown] # ### get_args_and_defaults_from_function_in_cell # %% [markdown] # #### get_args_and_defaults + # %% -#| export +# | export def get_args_and_defaults(list_args, list_defaults): if len(list_defaults) == 0: args_without_defaults = [arg.arg for arg in list_args] @@ -1368,12 +1365,12 @@ def get_args_and_defaults(list_args, list_defaults): return args_without_defaults, args_with_defaults, default_values - # %% [markdown] # #### get_args_and_defaults_from_function_in_cell + # %% -#| export +# | export def get_args_and_defaults_from_ast(root): args_without_defaults, args_with_defaults1, default_values1 = get_args_and_defaults( root.body[0].args.posonlyargs + root.body[0].args.args, @@ -1397,12 +1394,12 @@ def get_args_and_defaults_from_function_in_cell(): return get_args_and_defaults_from_ast(root) - # %% [markdown] # ### derive_paths + # %% -#| export +# | export def derive_paths( original_path: Path, folder: str, @@ -1459,30 +1456,34 @@ def derive_paths( cell_processor.path_to_code_cells_file.parent.mkdir(parents=True, exist_ok=True) - # %% [markdown] # #### Example + # %% -cell_processor=Bunch() -derive_paths ( - original_path=Path("root/folder_with_nb/subfolder/myfile.ipynb"), - folder="folder_with_nb", - lib_folder="mylib", - file_name="hello.py", - cell_processor=cell_processor, - code_cells_path=Path(".nbmodular"), -) -assert cell_processor=={'file_path': Path('root/mylib/subfolder/hello.py'), - 'test_file_path': Path('root/tests/subfolder/test_hello.py'), - 'cell_captures_path_to_folder': Path('root/.cell_captures/subfolder/hello'), - 'path_to_code_cells_file': Path('root/.nbmodular/subfolder/hello.pk'), - 'code_cells_path': Path('root/.nbmodular/subfolder')} +def test_derive_paths(): + cell_processor = Bunch() + derive_paths( + original_path=Path("root/folder_with_nb/subfolder/myfile.ipynb"), + folder="folder_with_nb", + lib_folder="mylib", + file_name="hello.py", + cell_processor=cell_processor, + code_cells_path=Path(".nbmodular"), + ) + assert cell_processor == { + "file_path": Path("root/mylib/subfolder/hello.py"), + "test_file_path": Path("root/tests/subfolder/test_hello.py"), + "cell_captures_path_to_folder": Path("root/.cell_captures/subfolder/hello"), + "path_to_code_cells_file": Path("root/.nbmodular/subfolder/hello.pk"), + "code_cells_path": Path("root/.nbmodular/subfolder"), + } # %% [markdown] # ### CellProcessor + # %% # | export class CellProcessor: @@ -3365,73 +3366,67 @@ def save_data(self, **kwargs): self.run_io(io_action="save", **kwargs) - # %% [markdown] # #### Examples of usage # # > See README.md for more examples # %% -cp = CellProcessor() -cp.set_file_path ("nbmodular/core/cell2func.py") - -assert cp.file_path==Path('nbmodular/core/cell2func.py') -assert cp.test_file_path==Path('tests/core/test_cell2func.py') -assert cp.cell_captures_path_to_folder==Path('.cell_captures/core/cell2func') -assert cp.path_to_code_cells_file==Path('.nbmodular/core/cell2func.pk') -assert cp.code_cells_path==Path('.nbmodular/core') - -# %% -function_name, kwargs = cp.parse_signature ("example") -assert kwargs['write'] is False -function_name, kwargs = cp.parse_signature ("example --write") -assert kwargs['write'] is True - -# %% - -# %% -# run simple computation -cell=( -""" -1+2 -""" -) -cp.process_function_call ("plus_1_2 --save-capture-disk", cell) +def test_cell_processor (): + cp = CellProcessor() + cp.set_file_path("nbmodular/core/cell2func.py") + + assert cp.file_path == Path("nbmodular/core/cell2func.py") + assert cp.test_file_path == Path("tests/core/test_cell2func.py") + assert cp.cell_captures_path_to_folder == Path(".cell_captures/core/cell2func") + assert cp.path_to_code_cells_file == Path(".nbmodular/core/cell2func.pk") + assert cp.code_cells_path == Path(".nbmodular/core") + + # %% + function_name, kwargs = cp.parse_signature("example") + assert kwargs["write"] is False + function_name, kwargs = cp.parse_signature("example --write") + assert kwargs["write"] is True + + # %% + + # %% + # run simple computation + cell = """ + 1+2 + """ + cp.process_function_call("plus_1_2 --save-capture-disk", cell) -#check outputs -cell_captures_path_to_file = cell_captures_path_to_folder / "plus_1_2.pk" -assert cell_captures_path_to_file.exists() -output = joblib.load (cell_captures_path_to_file) -assert len(output.outputs)==1 and output.outputs[0].data["text/plain"]=="3" + # check outputs + cell_captures_path_to_file = cell_captures_path_to_folder / "plus_1_2.pk" + assert cell_captures_path_to_file.exists() + output = joblib.load(cell_captures_path_to_file) + assert len(output.outputs) == 1 and output.outputs[0].data["text/plain"] == "3" -# check that the following code is not run -# because a previous run has been cached in disk -should_not_run_cell=( -""" -raise RuntimeError ("should not run") -""" -) -cp.process_function_call ("plus_1_2 --load-capture-disk", should_not_run_cell) + # check that the following code is not run + # because a previous run has been cached in disk + should_not_run_cell = """ + raise RuntimeError ("should not run") + """ + cp.process_function_call("plus_1_2 --load-capture-disk", should_not_run_cell) -# clean -shutil.rmtree (cell_captures_path_to_folder, ignore_errors=True) + # clean + shutil.rmtree(cell_captures_path_to_folder, ignore_errors=True) -# %% [markdown] -# We can also use cache the output to and load it from memory: + # %% [markdown] + # We can also use cache the output to and load it from memory: -# %% -cell=( -""" -4+5 -""" -) -cp.process_function_call ("plus_4_5 --save-capture-memory", cell) -output = cp.function_info["plus_4_5"].capture -assert len(output.outputs)==1 and output.outputs[0].data["text/plain"]=="9" + # %% + cell = """ + 4+5 + """ + cp.process_function_call("plus_4_5 --save-capture-memory", cell) + output = cp.function_info["plus_4_5"].capture + assert len(output.outputs) == 1 and output.outputs[0].data["text/plain"] == "9" -# load it: we make sure the cell doesn't run: -cp.process_function_call ("plus_4_5 --load-capture-memory", should_not_run_cell) + # load it: we make sure the cell doesn't run: + cp.process_function_call("plus_4_5 --load-capture-memory", should_not_run_cell) # %% [markdown] @@ -3439,8 +3434,9 @@ def save_data(self, **kwargs): # ## CellProcessorMagic # ::: + # %% -#| export +# | export @magics_class class CellProcessorMagic(Magics): """ @@ -3626,15 +3622,15 @@ def set(self, line): self.processor.set_value(attr, value) - # %% [markdown] # ::: {.content-hidden} # ## load_ipython_extension # ::: + # %% -#| export -#| hide +# | export +# | hide def load_ipython_extension(ipython): """ This module can be loaded via `%load_ext cell2func` or be configured to be autoloaded by IPython at startup time. @@ -3643,7 +3639,6 @@ def load_ipython_extension(ipython): ipython.register_magics(magics) - # %% [markdown] # ## managing and sharing variables with notebook @@ -3655,8 +3650,9 @@ def load_ipython_extension(ipython): # #### retrieve_function_values_through_disk # ::: + # %% -#| export +# | export def retrieve_function_values_through_disk(filename="variable_values.pk"): """ Store `variables` in disk @@ -3672,14 +3668,14 @@ def retrieve_function_values_through_disk(filename="variable_values.pk"): return variable_values - # %% [markdown] # ::: {.content-hidden} # #### retrieve_function_values_through_memory # ::: + # %% -#| export +# | export def retrieve_function_values_through_memory(field): """ Store `variables` in dictionary entry `self[field]` @@ -3712,14 +3708,14 @@ def retrieve_function_values_through_memory(field): return None - # %% [markdown] # ::: {.content-hidden} # #### copy_values_in_nb # ::: + # %% -#| export +# | export def copy_values_and_run_code_in_nb(self, field="shared_variables", code=""): """ Makes desired variables available in notebook context. @@ -3754,9 +3750,8 @@ def copy_values_and_run_code_in_nb(self, field="shared_variables", code=""): get_ipython().run_cell(code_to_run2) - # %% -#| export +# | export def copy_values_in_nb(self, field="shared_variables"): copy_values_code = """ for k, v in variables_to_insert.items(): @@ -3768,9 +3763,8 @@ def copy_values_in_nb(self, field="shared_variables"): copy_values_and_run_code_in_nb(self, field=field, code=copy_values_code) - # %% -#| export +# | export def transfer_variables_to_nb(**kwargs): communicator = Bunch() communicator.shared_variables = kwargs @@ -3778,31 +3772,33 @@ def transfer_variables_to_nb(**kwargs): copy_values_in_nb(communicator) - # %% [markdown] # #### Example use + # %% -def transfer_value_vector(): - vector = ["hello", 1] - transfer_variables_to_nb(vector=vector) +def test_transfer_variables_to_nb (): + def transfer_value_vector(): + vector = ["hello", 1] + transfer_variables_to_nb(vector=vector) -# %% -transfer_value_vector() -assert vector == ["hello", 1] + # %% + transfer_value_vector() + assert vector == ["hello", 1] -# %% [markdown] -# ### 2. From notebook to function + # %% [markdown] + # ### 2. From notebook to function # %% [markdown] # ::: {.content-hidden} # #### retrieve_nb_locals_through_disk # ::: + # %% -#| export +# | export def retrieve_nb_locals_through_disk(variable_values, filename="variable_values.pk"): """ Store `variables` in disk @@ -3817,14 +3813,14 @@ def retrieve_nb_locals_through_disk(variable_values, filename="variable_values.p joblib.dump(variable_values, filename) - # %% [markdown] # ::: {.content-hidden} # #### retrieve_nb_locals_through_memory # ::: + # %% -#| export +# | export def retrieve_nb_locals_through_memory(field, variable_values): """ Store `variables` in dictionary entry `self[field]` @@ -3852,24 +3848,24 @@ def retrieve_nb_locals_through_memory(field, variable_values): # del variable_values['created_current_values'] - # %% [markdown] # ### remove_name_from_nb + # %% -#| export +# | export def remove_name_from_nb(name): get_ipython().run_cell(f'exec("del {name}")') - # %% [markdown] # ::: {.content-hidden} # ### acceptable_variable # ::: + # %% -#|export +# |export def acceptable_variable(variable_values, k): return ( not k.startswith("_") @@ -3880,12 +3876,12 @@ def acceptable_variable(variable_values, k): ) - # %% [markdown] # ## store_variables + # %% -#| export +# | export def store_variables( path_variables, locals_, diff --git a/nbmodular/export.py b/nbmodular/export.py index ccd4e9b..31bc8f8 100644 --- a/nbmodular/export.py +++ b/nbmodular/export.py @@ -234,8 +234,8 @@ def replace_folder_in_path( # %% # | export def set_paths_nb_processor( - nb_processor: "NbMagicExporter", - path: None, + nb_processor: "NbMagicExporter | Bunch", + path: str | Path, ) -> None: """ Set the paths for the notebook processor. @@ -673,14 +673,18 @@ def end(self): nb_export(self.test_dest_nb_path, lib_path=lib_path) # step 2 (beginning) in diagram - self.tmp_nb_path.rename(self.duplicate_tmp_path) + if self.tmp_nb_path.exists(): + self.tmp_nb_path.rename(self.duplicate_tmp_path) # step 3 in diagram - self.dest_nb_path.rename(self.tmp_dest_nb_path) - self.test_dest_nb_path.rename(self.tmp_test_dest_nb_path) + if self.dest_nb_path.exists(): + self.dest_nb_path.rename(self.tmp_dest_nb_path) + if self.test_dest_nb_path.exists(): + self.test_dest_nb_path.rename(self.tmp_test_dest_nb_path) # step 2 (end) in diagram - self.duplicate_tmp_path.rename(self.dest_nb_path) + if self.duplicate_tmp_path.exists(): + self.duplicate_tmp_path.rename(self.dest_nb_path) # %% [markdown] @@ -797,7 +801,6 @@ def second(): __all__ = ['first'] # @%% ../../nbs/mixed/mixed_cells.ipynb 1 - #@@function def first(): pass @@ -820,8 +823,8 @@ def second(): expected_py_modules=expected_py_modules, current_root=current_root, new_root=new_root, - clean=False, - keep_cwd=True, + clean=True, + keep_cwd=False, ) @@ -876,14 +879,163 @@ def nbm_export_cli(): # %% [markdown] # #### Checks & Cleaning + # %% # check original content -# nbs, py_modules = tst.read_content_in_repo (nb_paths, "./", print_as_list=True) +# nbs, py_modules = tst.read_content_in_repo(nb_paths, "./", print_as_list=True) + +# %% +expected_nbs = [ + # nbm/first_folder/first.ipynb + """ +[markdown] +# First notebook + +[code] +%%function hello +print ('hello') + +[code] +%%function one_plus_one --test +a=1+1 +print (a) +""", + # nbs/first_folder/first.ipynb + """ +[markdown] +# First notebook + +[code] +#|export +def hello(): + print ('hello') + +[code] +a=1+1 +print (a) +""", + # .nbs/first_folder/first.ipynb + """ +[code] +#|default_exp first_folder.first + +[code] +#|export +#@@function hello +def hello(): + print ('hello') +""", + # .nbs/first_folder/test_first.ipynb + """ +[code] +#|default_exp tests.first_folder.test_first + +[code] +#|export +#@@function one_plus_one --test +def one_plus_one(): + a=1+1 + print (a) +""", + # nbm/second_folder/second.ipynb + """ +[markdown] +# Second notebook + +[code] +%%function bye +print ('bye') + +[markdown] +%%function two_plus_two --test +a=2+2 +print (a) +""", + # nbs/second_folder/second.ipynb + """ +[markdown] +# Second notebook + +[code] +#|export +def bye(): + print ('bye') + +[markdown] +%%function two_plus_two --test +a=2+2 +print (a) +""", + # .nbs/second_folder/second.ipynb + """ +[code] +#|default_exp second_folder.second + +[code] +#|export +#@@function bye +def bye(): + print ('bye') +""", +] +expected_py_modules = [ + # nbmodular/first_folder/first.py + """ + + +# @%% auto 0 +__all__ = ['hello'] + +# @%% ../../nbs/first_folder/first.ipynb 1 +#@@function hello +def hello(): + print ('hello') + + +""", + # nbmodular/tests/first_folder/test_first.py + """ + + +# @%% auto 0 +__all__ = ['one_plus_one'] + +# @%% ../../../nbs/first_folder/test_first.ipynb 1 +#@@function one_plus_one --test +def one_plus_one(): + a=1+1 + print (a) + + +""", + # nbmodular/second_folder/second.py + """ + + +# @%% auto 0 +__all__ = ['bye'] + +# @%% ../../nbs/second_folder/second.ipynb 1 +#@@function bye +def bye(): + print ('bye') + + +""", +] # %% -# clean -shutil.rmtree(new_root, ignore_errors=True) +# Finally we check that the result is the same as the expected +tst.check_test_repo_content( + nb_paths, + expected_nbs=expected_nbs, + expected_py_modules=expected_py_modules, + current_root=current_root, + new_root=new_root, + clean=True, + keep_cwd=False, +) # %% [markdown] @@ -938,12 +1090,13 @@ def process_cell_for_nbm_update(cell: NbCell): # %% # | export def nbm_update( - path, - code_cells_path=".nbmodular", - logger=None, - log_level="INFO", + path: str | Path, + code_cells_path: str | Path = ".nbmodular", + logger: logging.Logger | None = None, + log_level: str = "INFO", ): nb_processor = Bunch() + path = Path(path) nb_processor.code_cells_path = Path(code_cells_path) nb_processor.logger = logging.getLogger("nb_importer") if logger is None else logger @@ -952,18 +1105,25 @@ def nbm_update( # prior to step 5 in diagram: # nbs/nb.ipynb => nbs/_nb.ipynb - nb_processor.dest_nb_path.rename(nb_processor.duplicate_dest_nb_path) + if nb_processor.dest_nb_path.exists(): + nb_processor.dest_nb_path.rename(nb_processor.duplicate_dest_nb_path) # step 5 in diagram: # .nbs/nb.ipynb => nbs/nb.ipynb - nb_processor.tmp_dest_nb_path.rename(nb_processor.dest_nb_path) + if nb_processor.tmp_dest_nb_path.exists(): + nb_processor.tmp_dest_nb_path.rename(nb_processor.dest_nb_path) # .nbs/test_nb.ipynb => nbs/test_nb.ipynb - nb_processor.tmp_test_dest_nb_path.rename(nb_processor.test_dest_nb_path) + if nb_processor.tmp_test_dest_nb_path.exists(): + nb_processor.tmp_test_dest_nb_path.rename(nb_processor.test_dest_nb_path) # step 5 in diagram: nbdev_update - _update_mod(nb_processor.dest_python_path, lib_dir=nb_processor.lib_path.parent) _update_mod( - nb_processor.test_dest_python_path, lib_dir=nb_processor.lib_path.parent + nb_processor.dest_python_path, + lib_dir=Path(nb_processor.lib_path).parent.resolve(), + ) + _update_mod( + nb_processor.test_dest_python_path, + lib_dir=Path(nb_processor.lib_path).parent.resolve(), ) # obtain cell types and read them from notebooks @@ -997,36 +1157,102 @@ def nbm_update( # %% [markdown] # ### Example usage + +# %% +new_root = "test_nbm_update/" +nb_folder = "nbm" +nb_path = "mixed/mixed_cells.ipynb" +# Create notebook in "new repo", and cd to it +current_root, nb_paths = tst.create_test_content( + nbs=tst.mixed_nb1, + nb_paths=nb_path, + nb_folder=nb_folder, + new_root=new_root, +) +nbm_export(path=f"{nb_folder}/{nb_path}") + +# %% +# Simulate the exporting +if False: + new_root = "test_nbm_export" + nb_folder = "nbm" + nb_path = "mixed/mixed_cells.ipynb" + # Create notebook in "new repo", and cd to it + current_root, nb_paths = tst.create_test_content( + nbs=tst.mixed_nb1, + nb_paths=nb_path, + nb_folder=nb_folder, + new_root=new_root, + ) + + nbm_export(path=f"{nb_folder}/{nb_path}") + + exported_nbs, updated_py_modules = tst.read_content_in_repo( + [nb_path], "./", print_as_list=True + ) + # joblib.load("test_nbm_update/.nbmodular/cell_types.pk") + + # manually updated the py modules + # updated_py_modules = [...] + + # copy and paste into test_utils module: + # - content of exported_nbs and paths + # - content of updated_py_modules and paths + # %% [markdown] # #### Set up before running example + # %% -os.makedirs("nbmodular/test_nbs", exist_ok=True) -os.makedirs("nbmodular/tests/test_nbs", exist_ok=True) -shutil.copy("test_data/nb.py", "nbmodular/test_nbs/nb.py") -shutil.copy("test_data/test_nb.py", "nbmodular/tests/test_nbs/test_nb.py") -shutil.copy("nbm/test_nbs/nb.ipynb", "nbm/test_nbs/_nb.ipynb") +updated_py_modules = [x.replace("@%%", "%%") for x in tst.updated_py_modules] +new_root = "test_nbm_update" +nb_folder = "nbm" +lib_folder = "nbmodular" +# Create notebook in "new repo", and cd to it +current_root, nb_paths = tst.create_test_content( + nbs=tst.exported_nbs, + nb_paths=tst.exported_nb_paths, + py_modules=updated_py_modules, + py_paths=tst.updated_py_paths, + nb_folder="", + lib_folder="", + new_root=new_root, +) +cell_types = ["code", "original", "test"] +joblib.dump(cell_types, f"{new_root}/.nbmodular/cell_types.pk") # %% [markdown] # #### Example usage # %% -nbm_update(path) + +nb_path = "mixed/mixed_cells.ipynb" +nbm_update(path=f"{nb_folder}/{nb_path}") # %% [markdown] # #### Checks # %% -[c["source"] for c in read_nb(path).cells] -assert [c["source"] for c in read_nb(path).cells] == [ - "%%function\ndef first():\n x = 3 + 1", - "comment", - '%%function --test\ndef second():\n print("hello")', -] - -# %% -shutil.move("nbm/test_nbs/_nb.ipynb", "nbm/test_nbs/nb.ipynb") +if False: + # 3) read result and manually check if it's correct + expected_nbs, expected_py_modules = tst.read_content_in_repo( + [nb_path], "./", print_as_list=True + ) + # copy and paste the result: + # expected_nbs = [...] + # expected_py_modules = [...] + + # Finally we check that the result is the same as the expected + tst.check_test_repo_content( + nb_paths, + expected_nbs=expected_nbs, + expected_py_modules=expected_py_modules, + current_root=current_root, + new_root=new_root, + clean=False, + keep_cwd=True, + ) # %% [markdown] # ## nbm_update_cli diff --git a/nbmodular/test_utils.py b/nbmodular/test_utils.py index f9af940..37075a2 100644 --- a/nbmodular/test_utils.py +++ b/nbmodular/test_utils.py @@ -31,13 +31,14 @@ import shutil from pathlib import Path from token import OP -from typing import List, Tuple, Optional, Union +from typing import List, Tuple, Optional, Union, Dict import re # 3rd party from execnb.nbio import new_nb, write_nb, mk_cell, read_nb from plum import Val from requests import post +from fastcore.basics import AttrDict # ours from nbmodular.utils import cd_root @@ -136,6 +137,99 @@ def two_plus_two (): print (a) """ + +# %% [markdown] +# ### Updated example 1 + +# %% +exported_nbs = [ + # nbm/mixed/mixed_cells.ipynb + """ +[code] +%%function +def first(): + pass + +[markdown] +comment + +[code] +%%function --test +def second (): + pass +""", + # nbs/mixed/mixed_cells.ipynb + """ +[code] +#|export +def first(): + pass + +[markdown] +comment + +[code] +pass +""", + # .nbs/mixed/mixed_cells.ipynb + """ +[code] +#|default_exp mixed.mixed_cells + +[code] +#|export +#@@function +def first(): + pass +""", + # .nbs/mixed/test_mixed_cells.ipynb + """ +[code] +#|default_exp tests.mixed.test_mixed_cells + +[code] +#|export +#@@function --test +def second(): + pass +""", +] + +exported_nb_paths = [ + "nbm/mixed/mixed_cells.ipynb", + "nbs/mixed/mixed_cells.ipynb", + ".nbs/mixed/mixed_cells.ipynb", + ".nbs/mixed/test_mixed_cells.ipynb", +] + +updated_py_modules = [ + # nbmodular/mixed/mixed_cells.py + """ +# @%% auto 0 +__all__ = ['first'] + +# @%% ../../nbs/mixed/mixed_cells.ipynb 1 +#@@function +def first(): + x = 3 + 1 +""", + # nbmodular/tests/mixed/test_mixed_cells.py + """ +# @%% auto 0 +__all__ = ['second'] + +# @%% ../../../nbs/mixed/test_mixed_cells.ipynb 1 +#@@function --test +def second(): + print("hello") +""", +] +updated_py_paths = [ + "nbmodular/mixed/mixed_cells.py", + "nbmodular/tests/mixed/test_mixed_cells.py", +] + + # %% [markdown] # ## Notebook structure # @@ -202,11 +296,11 @@ def parse_nb_sections(nb): # %% # | export -def text2nb(nb: str): +def text2nb(nb: str) -> dict | AttrDict: cells = [ mk_cell(text, cell_type=cell_type) for cell_type, text in parse_nb_sections(nb) ] - return new_nb(cells) + return new_nb(cells) # type: ignore # %% [markdown] @@ -622,7 +716,10 @@ def derive_nb_paths( if nbm_folder is not None: all_nb_paths.append(Path(new_root) / nbm_folder / nb_path) if nbs_folder is not None: - all_nb_paths.append(Path(new_root) / nbs_folder / nb_path) + nb_code_path = Path(new_root) / nbs_folder / nb_path + all_nb_paths.append(nb_code_path) + nb_test_path = nb_code_path.parent / f"test_{nb_code_path.name}" + all_nb_paths.append(nb_test_path) if tmp_folder is not None: tmp_nb = Path(new_root) / tmp_folder / nb_path all_nb_paths.append(tmp_nb) @@ -721,7 +818,9 @@ def derive_all_paths( # %% # | export -def read_nbs(paths: List[str], as_text: bool = True) -> List[str] | List[dict]: +def read_nbs( + paths: List[str] | List[Path], must_exist: dict = {}, as_text: bool = True +) -> List[str] | List[dict]: """ Read notebooks from disk. @@ -737,11 +836,14 @@ def read_nbs(paths: List[str], as_text: bool = True) -> List[str] | List[dict]: returned as dictionaries. """ nbs_in_disk = [] + paths = [Path(path) for path in paths] for path in paths: # Check that file exists. useful for being called inside a test utility # to see where it fails. - assert os.path.exists(path) - nbs_in_disk.append(read_nb(path)) + if path.exists(): + nbs_in_disk.append(read_nb(path)) + elif must_exist.get(path, False): + raise FileNotFoundError(f"File {path} does not exist") return [strip_nb(nb2text(nb)) for nb in nbs_in_disk] if as_text else nbs_in_disk @@ -788,7 +890,9 @@ def compare_nbs(nbs1: List[str], nbs2: List[str]) -> bool: # %% # | export -def read_text_files(paths: List[str]) -> List[str]: +def read_text_files( + paths: List[str | Path], must_exist: Dict[str | Path, bool] = {} +) -> List[str]: """ Read the contents of Python modules from the given paths. @@ -809,12 +913,15 @@ def read_text_files(paths: List[str]) -> List[str]: """ text_files = [] + paths = [Path(path) for path in paths] for path in paths: # Check that file exists. useful for being called inside a test utility # to see where it fails. - assert os.path.exists(path) - with open(path, "rt") as file: - text_files.append(file.read()) + if path.exists(): + text_files.append(path.read_text()) + elif must_exist.get(path, False): + raise FileNotFoundError(f"File {path} does not exist") + return text_files @@ -993,7 +1100,12 @@ def check_py_modules( AssertionError If the actual Python modules do not match the expected modules. """ - actual = read_pymodules_in_repo(nb_paths, new_root, lib_folder=lib_folder, interactive_notebook=interactive_notebook) + actual = read_pymodules_in_repo( + nb_paths, + new_root, + lib_folder=lib_folder, + interactive_notebook=interactive_notebook, + ) assert compare_texts(actual, expected) @@ -1052,27 +1164,46 @@ def check_test_repo_content( ValueError Raised when both clean and keep_cwd are set to True. """ + changed_dir = False if current_root is not None: assert Path(current_root).name == "nbmodular" new_wd = os.getcwd() assert Path(new_wd).resolve() == Path(f"{current_root}/{new_root}").resolve() os.chdir(current_root) + changed_dir = True if new_root is not None: - assert (Path(new_root) / "settings.ini").exists() + if not (Path(new_root) / "settings.ini").exists(): + if changed_dir: + os.chdir(new_wd) + raise FileNotFoundError(f"settings.ini not found in {new_root}") use_new_root = True else: new_root = "./" use_new_root = False if expected_nbs is not None: - check_nbs(nb_paths, expected_nbs, new_root, nbm_folder, tmp_folder, nbs_folder) + try: + check_nbs( + nb_paths, expected_nbs, new_root, nbm_folder, tmp_folder, nbs_folder + ) + except AssertionError as e: + if changed_dir: + os.chdir(new_wd) + raise e if expected_py_modules is not None: - check_py_modules(nb_paths, expected_py_modules, new_root, lib_folder) + try: + check_py_modules(nb_paths, expected_py_modules, new_root, lib_folder) + except AssertionError as e: + if changed_dir: + os.chdir(new_wd) + raise e if clean and use_new_root: shutil.rmtree(new_root) if keep_cwd and use_new_root: if clean: + if changed_dir: + os.chdir(new_wd) raise ValueError("keep_cwd can't be True if clean is True") os.chdir(new_root) @@ -1133,9 +1264,12 @@ def create_and_cd_to_new_root_folder( # %% # | export def create_test_content( - nbs: List[str] | str, + nbs: List[str] | str | None = None, nb_paths: Optional[List[str] | List[Path] | str | Path] = None, + py_modules: List[str] | str | None = None, + py_paths: Optional[List[str] | List[Path] | str | Path] = None, nb_folder: str = "nbm", + lib_folder: Optional[str] = "nbmodular", new_root: str = "new_test", config_path: str = "settings.ini", ) -> Tuple[str, List[str]]: @@ -1159,7 +1293,7 @@ def create_test_content( current_root = os.getcwd() # Convert input texts into corresponding dicts with notebook structure - nbs = texts2nbs(nbs) + nbs = texts2nbs(nbs) if nbs is not None else [] # Generate list of nb_paths if None if nb_paths is None: @@ -1175,6 +1309,23 @@ def create_test_content( full_nb_path.parent.mkdir(parents=True, exist_ok=True) write_nb(nb, full_nb_path) + if py_modules is not None: + py_modules = [py_modules] if isinstance(py_modules, str) else py_modules + if py_paths is None: + py_paths = [f"f{idx}" for idx in range(len(py_modules))] + else: + if not isinstance(py_paths, list): + py_paths = [py_paths] + if len(py_paths) != len(py_modules): + raise ValueError( + "py_paths must have same number of items as py_modules" + ) + + for py_module, py_path in zip(py_modules, py_paths): + full_py_path = Path(new_root) / lib_folder / py_path + full_py_path.parent.mkdir(parents=True, exist_ok=True) + full_py_path.write_text(py_module) + # Copy settings.ini in new root folder, so that this file # can be read later on by our export / import functions. # Also, cd to new root folder. diff --git a/nbs/cell2func.ipynb b/nbs/cell2func.ipynb index d87e99a..44d6b66 100644 --- a/nbs/cell2func.ipynb +++ b/nbs/cell2func.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "ecf95f77", + "id": "4776c0d9", "metadata": {}, "source": [ "# cell2func\n", @@ -15,7 +15,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c2644607", + "id": "3c260fd8", "metadata": {}, "outputs": [], "source": [ @@ -25,7 +25,7 @@ { "cell_type": "code", "execution_count": null, - "id": "eb466d15", + "id": "e92e48d8", "metadata": { "lines_to_next_cell": 2 }, @@ -69,14 +69,14 @@ "from fastcore.all import argnames\n", "import nbdev\n", "\n", - "from nbmodular.core import function_io\n", + "from nbmodular import function_io\n", "from nbmodular.utils import set_log_level, get_config" ] }, { "cell_type": "code", "execution_count": null, - "id": "b539ea35", + "id": "9b722328", "metadata": {}, "outputs": [], "source": [ @@ -89,7 +89,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ceec741f", + "id": "12cf90ca", "metadata": {}, "outputs": [], "source": [ @@ -98,7 +98,7 @@ }, { "cell_type": "markdown", - "id": "ae3bd682", + "id": "66cc6de7", "metadata": {}, "source": [ "## bunch_io" @@ -107,7 +107,7 @@ { "cell_type": "code", "execution_count": null, - "id": "e162b381", + "id": "caafab7f", "metadata": {}, "outputs": [], "source": [ @@ -145,7 +145,7 @@ }, { "cell_type": "markdown", - "id": "6b6b7a13", + "id": "a1e8cb6f", "metadata": {}, "source": [ "## get_non_callable" @@ -153,7 +153,7 @@ }, { "cell_type": "markdown", - "id": "93eb1908", + "id": "938e6dcd", "metadata": {}, "source": [ "### get_non_callable_ipython" @@ -162,7 +162,7 @@ { "cell_type": "code", "execution_count": null, - "id": "a6c8ce46", + "id": "a39a761b", "metadata": {}, "outputs": [], "source": [ @@ -203,7 +203,7 @@ }, { "cell_type": "markdown", - "id": "c954955e", + "id": "2226b3d0", "metadata": {}, "source": [ "### get_non_callable" @@ -212,7 +212,7 @@ { "cell_type": "code", "execution_count": null, - "id": "2d661793", + "id": "26961f1a", "metadata": {}, "outputs": [], "source": [ @@ -232,7 +232,7 @@ }, { "cell_type": "markdown", - "id": "7ce7f49a", + "id": "55330fff", "metadata": {}, "source": [ "## get_ast" @@ -241,7 +241,7 @@ { "cell_type": "code", "execution_count": null, - "id": "3fee6c5c", + "id": "fa4aeb41", "metadata": {}, "outputs": [], "source": [ @@ -261,7 +261,7 @@ }, { "cell_type": "markdown", - "id": "2cc95eff", + "id": "efc89600", "metadata": {}, "source": [ "## VariableClassifier" @@ -270,7 +270,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8a39c531", + "id": "90e0f1d5", "metadata": {}, "outputs": [], "source": [ @@ -299,7 +299,7 @@ }, { "cell_type": "markdown", - "id": "92fa5e43", + "id": "ae21fc70", "metadata": {}, "source": [ "## add_dict_values" @@ -308,7 +308,7 @@ { "cell_type": "code", "execution_count": null, - "id": "6f2a91ce", + "id": "ef632b8c", "metadata": {}, "outputs": [], "source": [ @@ -320,7 +320,7 @@ }, { "cell_type": "markdown", - "id": "d35ca6e3", + "id": "45e2d6a3", "metadata": {}, "source": [ "## cache output" @@ -329,7 +329,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8123f02f", + "id": "3198ac09", "metadata": {}, "outputs": [], "source": [ @@ -383,7 +383,7 @@ }, { "cell_type": "markdown", - "id": "cd6e183c", + "id": "de78f85b", "metadata": {}, "source": [ "### Example usage" @@ -392,7 +392,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8ab01bbe", + "id": "9ea77add", "metadata": {}, "outputs": [], "source": [ @@ -413,7 +413,7 @@ }, { "cell_type": "markdown", - "id": "084ec57d", + "id": "55760532", "metadata": {}, "source": [ "or a simple computation:" @@ -422,7 +422,7 @@ { "cell_type": "code", "execution_count": null, - "id": "613dcba2", + "id": "d7e42b60", "metadata": {}, "outputs": [], "source": [ @@ -441,7 +441,7 @@ }, { "cell_type": "markdown", - "id": "47175317", + "id": "8cdbfdd6", "metadata": {}, "source": [ "The returned object is of class CapturedIO:" @@ -450,7 +450,7 @@ { "cell_type": "code", "execution_count": null, - "id": "f7df63cf", + "id": "e81b0745", "metadata": {}, "outputs": [], "source": [ @@ -459,7 +459,7 @@ }, { "cell_type": "markdown", - "id": "108f7448", + "id": "e8f646f7", "metadata": {}, "source": [ "When printing to console or notebook, the returned object contains information about the printed text:" @@ -468,7 +468,7 @@ { "cell_type": "code", "execution_count": null, - "id": "69cc9f54", + "id": "6a43359a", "metadata": {}, "outputs": [], "source": [ @@ -478,7 +478,7 @@ }, { "cell_type": "markdown", - "id": "67e6f619", + "id": "8216f10d", "metadata": {}, "source": [ "Subsequent calls can load a previous capture from memory, without needing to run the code in the cell. We check this by passing a code that should not be run:" @@ -487,7 +487,7 @@ { "cell_type": "code", "execution_count": null, - "id": "9b9d44cb", + "id": "e9074302", "metadata": {}, "outputs": [], "source": [ @@ -507,7 +507,7 @@ { "cell_type": "code", "execution_count": null, - "id": "77b0ba26", + "id": "da8ae705", "metadata": {}, "outputs": [], "source": [ @@ -516,7 +516,7 @@ }, { "cell_type": "markdown", - "id": "93f29b26", + "id": "e4c05f46", "metadata": {}, "source": [ "Raises error if couln't load and error_if_not_loaded is True:" @@ -525,7 +525,7 @@ { "cell_type": "code", "execution_count": null, - "id": "e6deb159", + "id": "32637e7a", "metadata": {}, "outputs": [], "source": [ @@ -541,7 +541,7 @@ }, { "cell_type": "markdown", - "id": "3c76a02e", + "id": "a763f209", "metadata": {}, "source": [ "Can also save to disk:" @@ -550,7 +550,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d257fd4f", + "id": "190805eb", "metadata": {}, "outputs": [], "source": [ @@ -570,7 +570,7 @@ { "cell_type": "code", "execution_count": null, - "id": "f6c0f3f4", + "id": "2f51ab28", "metadata": {}, "outputs": [], "source": [ @@ -581,7 +581,7 @@ }, { "cell_type": "markdown", - "id": "ffbad7f9", + "id": "3e0ecde6", "metadata": {}, "source": [ "or save only to disk but not to memory:" @@ -590,7 +590,7 @@ { "cell_type": "code", "execution_count": null, - "id": "b1ce4a30", + "id": "be37b0b4", "metadata": {}, "outputs": [], "source": [ @@ -607,7 +607,7 @@ { "cell_type": "code", "execution_count": null, - "id": "2f8eb948", + "id": "f61cd014", "metadata": {}, "outputs": [], "source": [ @@ -617,7 +617,7 @@ }, { "cell_type": "markdown", - "id": "60e362ec", + "id": "b53e31d9", "metadata": {}, "source": [ "It doesn't save into disk if loaded from disk..." @@ -626,7 +626,7 @@ { "cell_type": "code", "execution_count": null, - "id": "aaf90691", + "id": "26c8c5a5", "metadata": {}, "outputs": [], "source": [ @@ -640,7 +640,7 @@ }, { "cell_type": "markdown", - "id": "475dc4eb", + "id": "f0ca60e1", "metadata": {}, "source": [ "But it does save if not loaded, even if the output existed before" @@ -649,7 +649,7 @@ { "cell_type": "code", "execution_count": null, - "id": "0e1ba99c", + "id": "f4f86199", "metadata": {}, "outputs": [], "source": [ @@ -666,7 +666,7 @@ }, { "cell_type": "markdown", - "id": "032e67c2", + "id": "dcdb8c55", "metadata": {}, "source": [ "::: {.content-hidden}\n", @@ -677,7 +677,7 @@ { "cell_type": "code", "execution_count": null, - "id": "64f2df49", + "id": "e8a66198", "metadata": {}, "outputs": [], "source": [ @@ -1473,7 +1473,7 @@ { "cell_type": "code", "execution_count": null, - "id": "3485ecc9", + "id": "baa44eaa", "metadata": {}, "outputs": [], "source": [ @@ -1485,7 +1485,7 @@ }, { "cell_type": "markdown", - "id": "0fbf7e11", + "id": "fbbf0bd9", "metadata": {}, "source": [ "### Example" @@ -1494,7 +1494,7 @@ { "cell_type": "code", "execution_count": null, - "id": "e8771b45", + "id": "9166ffb6", "metadata": {}, "outputs": [], "source": [ @@ -1516,7 +1516,7 @@ { "cell_type": "code", "execution_count": null, - "id": "38d93e3c", + "id": "b6e6d67c", "metadata": {}, "outputs": [], "source": [ @@ -1539,7 +1539,7 @@ }, { "cell_type": "markdown", - "id": "32fd1bad", + "id": "0a972874", "metadata": {}, "source": [ "::: {.content-hidden}\n", @@ -1549,7 +1549,7 @@ }, { "cell_type": "markdown", - "id": "320e1b5e", + "id": "86ce8a61", "metadata": {}, "source": [ "### update_cell_code" @@ -1558,7 +1558,7 @@ { "cell_type": "code", "execution_count": null, - "id": "3bfb28ae", + "id": "d3cf6706", "metadata": {}, "outputs": [], "source": [ @@ -1586,7 +1586,7 @@ }, { "cell_type": "markdown", - "id": "cfe7559e", + "id": "0571f90a", "metadata": {}, "source": [ "### add_function_to_list" @@ -1595,7 +1595,7 @@ { "cell_type": "code", "execution_count": null, - "id": "b74097a5", + "id": "095325a8", "metadata": {}, "outputs": [], "source": [ @@ -1625,7 +1625,7 @@ }, { "cell_type": "markdown", - "id": "f64b92a8", + "id": "1b280e02", "metadata": {}, "source": [ "### get_args_and_defaults_from_function_in_cell" @@ -1633,7 +1633,7 @@ }, { "cell_type": "markdown", - "id": "b620c7d8", + "id": "f29722b5", "metadata": {}, "source": [ "#### get_args_and_defaults" @@ -1642,7 +1642,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8d317198", + "id": "bb89be70", "metadata": {}, "outputs": [], "source": [ @@ -1692,7 +1692,7 @@ }, { "cell_type": "markdown", - "id": "14579706", + "id": "dd0eeea2", "metadata": {}, "source": [ "#### get_args_and_defaults_from_function_in_cell" @@ -1701,7 +1701,7 @@ { "cell_type": "code", "execution_count": null, - "id": "cf0c24b5", + "id": "79be5cad", "metadata": {}, "outputs": [], "source": [ @@ -1732,7 +1732,7 @@ }, { "cell_type": "markdown", - "id": "79a0222e", + "id": "3ccf01e7", "metadata": {}, "source": [ "### derive_paths" @@ -1741,7 +1741,7 @@ { "cell_type": "code", "execution_count": null, - "id": "f87846aa", + "id": "d2ab5279", "metadata": {}, "outputs": [], "source": [ @@ -1805,7 +1805,7 @@ }, { "cell_type": "markdown", - "id": "8db4d705", + "id": "19d70522", "metadata": {}, "source": [ "#### Example" @@ -1814,7 +1814,7 @@ { "cell_type": "code", "execution_count": null, - "id": "1d9ebb6b", + "id": "036ff45d", "metadata": {}, "outputs": [], "source": [ @@ -1836,7 +1836,7 @@ }, { "cell_type": "markdown", - "id": "39ce88d8", + "id": "0aa4eca0", "metadata": {}, "source": [ "### CellProcessor" @@ -1845,7 +1845,7 @@ { "cell_type": "code", "execution_count": null, - "id": "fdd599cd", + "id": "09b1cad0", "metadata": {}, "outputs": [], "source": [ @@ -2873,10 +2873,10 @@ " if this_function.io_code and not self._added_io_imports:\n", " if this_function.test:\n", " self.test_imports += \"from pathlib import Path\\n\"\n", - " self.test_imports += \"from nbmodular.core import function_io\\n\"\n", + " self.test_imports += \"from nbmodular import function_io\\n\"\n", " else:\n", " self.imports += \"from pathlib import Path\\n\"\n", - " self.imports += \"from nbmodular.core import function_io\\n\"\n", + " self.imports += \"from nbmodular import function_io\\n\"\n", " self._added_io_imports = True\n", "\n", " def get_function_kwargs(self, kwargs: dict, test: bool = False) -> dict:\n", @@ -3733,7 +3733,7 @@ }, { "cell_type": "markdown", - "id": "a43ee9d0", + "id": "2e4560b3", "metadata": {}, "source": [ "#### Examples of usage\n", @@ -3744,7 +3744,7 @@ { "cell_type": "code", "execution_count": null, - "id": "e69f96b9", + "id": "860de004", "metadata": {}, "outputs": [], "source": [ @@ -3761,7 +3761,7 @@ { "cell_type": "code", "execution_count": null, - "id": "fa9b0c13", + "id": "92727b9c", "metadata": {}, "outputs": [], "source": [ @@ -3774,7 +3774,7 @@ { "cell_type": "code", "execution_count": null, - "id": "5ba27256", + "id": "03b78a36", "metadata": {}, "outputs": [], "source": [] @@ -3782,7 +3782,7 @@ { "cell_type": "code", "execution_count": null, - "id": "cd307127", + "id": "00b50278", "metadata": {}, "outputs": [], "source": [ @@ -3816,7 +3816,7 @@ }, { "cell_type": "markdown", - "id": "bd3f35c3", + "id": "6b123e27", "metadata": {}, "source": [ "We can also use cache the output to and load it from memory:" @@ -3825,7 +3825,7 @@ { "cell_type": "code", "execution_count": null, - "id": "4a46702a", + "id": "ed4481ac", "metadata": {}, "outputs": [], "source": [ @@ -3844,7 +3844,7 @@ }, { "cell_type": "markdown", - "id": "cf69f1e4", + "id": "d77e6675", "metadata": {}, "source": [ "::: {.content-hidden}\n", @@ -3855,7 +3855,7 @@ { "cell_type": "code", "execution_count": null, - "id": "6d39ff23", + "id": "6591b89a", "metadata": {}, "outputs": [], "source": [ @@ -4048,7 +4048,7 @@ }, { "cell_type": "markdown", - "id": "9d881191", + "id": "294c3ed7", "metadata": {}, "source": [ "::: {.content-hidden}\n", @@ -4059,7 +4059,7 @@ { "cell_type": "code", "execution_count": null, - "id": "80610aa3", + "id": "5e5b775e", "metadata": {}, "outputs": [], "source": [ @@ -4076,7 +4076,7 @@ }, { "cell_type": "markdown", - "id": "c018818d", + "id": "666bbb5a", "metadata": {}, "source": [ "## managing and sharing variables with notebook" @@ -4084,7 +4084,7 @@ }, { "cell_type": "markdown", - "id": "86b7866c", + "id": "90e126c9", "metadata": {}, "source": [ "### 1. From function to notebook" @@ -4092,7 +4092,7 @@ }, { "cell_type": "markdown", - "id": "3bb00963", + "id": "0a7cac87", "metadata": {}, "source": [ "::: {.content-hidden}\n", @@ -4103,7 +4103,7 @@ { "cell_type": "code", "execution_count": null, - "id": "09bc50df", + "id": "7a592a9e", "metadata": {}, "outputs": [], "source": [ @@ -4126,7 +4126,7 @@ }, { "cell_type": "markdown", - "id": "8d9fef2d", + "id": "c17a3bfe", "metadata": {}, "source": [ "::: {.content-hidden}\n", @@ -4137,7 +4137,7 @@ { "cell_type": "code", "execution_count": null, - "id": "b3986ec8", + "id": "d7c75c60", "metadata": {}, "outputs": [], "source": [ @@ -4177,7 +4177,7 @@ }, { "cell_type": "markdown", - "id": "c91a7091", + "id": "470d39a1", "metadata": {}, "source": [ "::: {.content-hidden}\n", @@ -4188,7 +4188,7 @@ { "cell_type": "code", "execution_count": null, - "id": "65e97dc7", + "id": "e7504728", "metadata": {}, "outputs": [], "source": [ @@ -4231,7 +4231,7 @@ { "cell_type": "code", "execution_count": null, - "id": "02fb97e9", + "id": "817adf8f", "metadata": {}, "outputs": [], "source": [ @@ -4251,7 +4251,7 @@ { "cell_type": "code", "execution_count": null, - "id": "f9638b9e", + "id": "36c54c8a", "metadata": {}, "outputs": [], "source": [ @@ -4266,7 +4266,7 @@ }, { "cell_type": "markdown", - "id": "b8027cf1", + "id": "ba473263", "metadata": {}, "source": [ "#### Example use" @@ -4275,7 +4275,7 @@ { "cell_type": "code", "execution_count": null, - "id": "f3aaf6cc", + "id": "93ec62fe", "metadata": {}, "outputs": [], "source": [ @@ -4287,7 +4287,7 @@ { "cell_type": "code", "execution_count": null, - "id": "6fea368b", + "id": "0b68e273", "metadata": {}, "outputs": [], "source": [ @@ -4297,7 +4297,7 @@ }, { "cell_type": "markdown", - "id": "eb0e8573", + "id": "56e317f6", "metadata": {}, "source": [ "### 2. From notebook to function" @@ -4305,7 +4305,7 @@ }, { "cell_type": "markdown", - "id": "4fbfa6b7", + "id": "fa6f9071", "metadata": {}, "source": [ "::: {.content-hidden}\n", @@ -4316,7 +4316,7 @@ { "cell_type": "code", "execution_count": null, - "id": "361651c0", + "id": "123e39a0", "metadata": {}, "outputs": [], "source": [ @@ -4338,7 +4338,7 @@ }, { "cell_type": "markdown", - "id": "5421935a", + "id": "5b0a2bdf", "metadata": {}, "source": [ "::: {.content-hidden}\n", @@ -4349,7 +4349,7 @@ { "cell_type": "code", "execution_count": null, - "id": "76925b41", + "id": "02648ecd", "metadata": {}, "outputs": [], "source": [ @@ -4384,7 +4384,7 @@ }, { "cell_type": "markdown", - "id": "3a069927", + "id": "6fb23f94", "metadata": {}, "source": [ "### remove_name_from_nb" @@ -4393,7 +4393,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ffd801c8", + "id": "7ec20ca1", "metadata": {}, "outputs": [], "source": [ @@ -4405,7 +4405,7 @@ }, { "cell_type": "markdown", - "id": "c025d033", + "id": "71dff879", "metadata": {}, "source": [ "::: {.content-hidden}\n", @@ -4416,7 +4416,7 @@ { "cell_type": "code", "execution_count": null, - "id": "cbeb0375", + "id": "1953de1a", "metadata": {}, "outputs": [], "source": [ @@ -4434,7 +4434,7 @@ }, { "cell_type": "markdown", - "id": "2b74a1b6", + "id": "b2af79f0", "metadata": {}, "source": [ "## store_variables" @@ -4443,7 +4443,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ebebf728", + "id": "23802dd6", "metadata": {}, "outputs": [], "source": [ @@ -4461,7 +4461,7 @@ " Store `variables` in dictionary entry `self.variables_field[function]`\n", " \"\"\"\n", " ##pdb.no_set_trace()\n", - " from nbmodular.core import function_io\n", + " from nbmodular import function_io\n", "\n", " current_values = function_io.load(path_variables, io_type=io_type, **load_args)\n", " if not io_locals:\n",