From 74b152b25890303da224bae5223e926370e1ec62 Mon Sep 17 00:00:00 2001 From: Elijah ben Izzy Date: Thu, 15 Aug 2024 14:40:42 -0700 Subject: [PATCH 01/33] Update graph_functions.py Describes what to do in `graph_functions.py` --- hamilton/execution/graph_functions.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/hamilton/execution/graph_functions.py b/hamilton/execution/graph_functions.py index 3a86d8b60..4cdc1cf13 100644 --- a/hamilton/execution/graph_functions.py +++ b/hamilton/execution/graph_functions.py @@ -205,6 +205,15 @@ def dfs_traverse( result = None success = True pre_node_execute_errored = False + # TODO -- take everything from HERE to THERE + # Put it in a function + # That function should take an adapter as well as a node + other params (run_id, kwargs, etc...) + # And output result + # Then call the lifecycle method you created called do_remote_execute using the recipe below (call_lifecycle_method) + # And delegate to that + # only under if adapter.does_method("do_remote_execute") + # Otherwise just call the function we just defined + ##### HERE ###### try: if adapter.does_hook("pre_node_execute", is_async=False): try: @@ -254,6 +263,7 @@ def dfs_traverse( message = create_error_message(kwargs, node_, "[post-node-execute]") logger.exception(message) raise + ##### THERE ##### computed[node_.name] = result # > pruning the graph From 41decaa9ecbf020207ab58719c1fca087680fd43 Mon Sep 17 00:00:00 2001 From: Elijah ben Izzy Date: Thu, 15 Aug 2024 14:43:13 -0700 Subject: [PATCH 02/33] Adds comments to lifecycle base --- hamilton/lifecycle/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hamilton/lifecycle/base.py b/hamilton/lifecycle/base.py index ef9d07387..48c24fee7 100644 --- a/hamilton/lifecycle/base.py +++ b/hamilton/lifecycle/base.py @@ -496,7 +496,10 @@ async def pre_node_execute( """ pass - +# TODO -- copy this + name it `BaseDoRemoteExecute` and `do_remote_execute` (for the string/function) +# Look at the comments in graph_functions.py for what parameters to add +# Add the right parameters +# Add some docstrings @lifecycle.base_method("do_node_execute") class BaseDoNodeExecute(abc.ABC): @abc.abstractmethod From 5b73b8f41c39e7d915f0d1c8d049c67c47a20906 Mon Sep 17 00:00:00 2001 From: Elijah ben Izzy Date: Thu, 15 Aug 2024 14:46:21 -0700 Subject: [PATCH 03/33] Update h_ray.py with comments for ray tracking compatibility --- hamilton/plugins/h_ray.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hamilton/plugins/h_ray.py b/hamilton/plugins/h_ray.py index 0f0692803..55fbb5c95 100644 --- a/hamilton/plugins/h_ray.py +++ b/hamilton/plugins/h_ray.py @@ -49,7 +49,11 @@ def parse_ray_remote_options_from_tags(tags: typing.Dict[str, str]) -> typing.Di return ray_options - +# TODO -- change the base classes here to be the underlying ones in HamiltonGraphAdapter +# BaseDoRemoteExecute, # Change this one -- add it in +# BaseDoValidateInput, +# BaseDoCheckEdgeTypesMatch, +# Then, implement do_remote_execute, kill do_node_execute class RayGraphAdapter(base.HamiltonGraphAdapter, base.ResultMixin): """Class representing what's required to make Hamilton run on Ray. From aa3ac053bdc5bd054e158a237a8f2ab8a2d4ca3d Mon Sep 17 00:00:00 2001 From: JFrank Date: Mon, 19 Aug 2024 15:42:27 +0100 Subject: [PATCH 04/33] Replicate previous error --- hamilton/execution/graph_functions.py | 68 +++++++++++++++++++++++++++ hamilton/lifecycle/base.py | 28 +++++++++-- z_test_implementation.py | 55 ++++++++++++++++++++++ 3 files changed, 148 insertions(+), 3 deletions(-) create mode 100644 z_test_implementation.py diff --git a/hamilton/execution/graph_functions.py b/hamilton/execution/graph_functions.py index 4cdc1cf13..f5f0a39f1 100644 --- a/hamilton/execution/graph_functions.py +++ b/hamilton/execution/graph_functions.py @@ -144,6 +144,74 @@ def create_error_message(kwargs: dict, node_: node.Node, step: str) -> str: message = "\n" + border + "\n" + message + "\n" + border return message +def apply_adapters( + adapter: LifecycleAdapterSet, + node_: node.Node, + run_id: str, + kwargs: Dict[str,Any], + task_id:str, + + ): +# TODO -- take everything from HERE to THERE + # Put it in a function + # That function should take an adapter as well as a node + other params (run_id, kwargs, etc...) + # And output result + # Then call the lifecycle method you created called do_remote_execute using the recipe below (call_lifecycle_method) + # And delegate to that + # only under if adapter.does_method("do_remote_execute") + # Otherwise just call the function we just defined + ##### HERE ###### + try: + if adapter.does_hook("pre_node_execute", is_async=False): + try: + adapter.call_all_lifecycle_hooks_sync( + "pre_node_execute", + run_id=run_id, + node_=node_, + kwargs=kwargs, + task_id=task_id, + ) + except Exception as e: + pre_node_execute_errored = True + raise e + if adapter.does_method("do_node_execute", is_async=False): + result = adapter.call_lifecycle_method_sync( + "do_node_execute", + run_id=run_id, + node_=node_, + kwargs=kwargs, + task_id=task_id, + ) + else: + result = node_(**kwargs) + except Exception as e: + success = False + error = e + step = "[pre-node-execute]" if pre_node_execute_errored else "" + message = create_error_message(kwargs, node_, step) + logger.exception(message) + raise + finally: + if not pre_node_execute_errored and adapter.does_hook( + "post_node_execute", is_async=False + ): + try: + adapter.call_all_lifecycle_hooks_sync( + "post_node_execute", + run_id=run_id, + node_=node_, + kwargs=kwargs, + success=success, + error=error, + result=result, + task_id=task_id, + ) + except Exception: + message = create_error_message(kwargs, node_, "[post-node-execute]") + logger.exception(message) + raise + ##### THERE ##### + def execute_subdag( nodes: Collection[node.Node], diff --git a/hamilton/lifecycle/base.py b/hamilton/lifecycle/base.py index 48c24fee7..3bce16e65 100644 --- a/hamilton/lifecycle/base.py +++ b/hamilton/lifecycle/base.py @@ -496,14 +496,36 @@ async def pre_node_execute( """ pass + +@lifecycle.base_method("do_node_execute") +class BaseDoNodeExecute(abc.ABC): + @abc.abstractmethod + def do_node_execute( + self, + *, + run_id: str, + node_: "node.Node", + kwargs: Dict[str, Any], + task_id: Optional[str] = None, + ) -> Any: + """Method that is called to implement node execution. This can replace the execution of a node + with something all together, augment it, or delegate it. + + :param run_id: ID of the run, unique in scope of the driver. + :param node_: Node that is being executed + :param kwargs: Keyword arguments that are being passed into the node + :param task_id: ID of the task, defaults to None if not in a task setting + """ + pass + # TODO -- copy this + name it `BaseDoRemoteExecute` and `do_remote_execute` (for the string/function) # Look at the comments in graph_functions.py for what parameters to add # Add the right parameters # Add some docstrings -@lifecycle.base_method("do_node_execute") -class BaseDoNodeExecute(abc.ABC): +@lifecycle.base_method("do_remote_execute") +class BaseDoRemoteExecute(abc.ABC): @abc.abstractmethod - def do_node_execute( + def do_remote_execute( self, *, run_id: str, diff --git a/z_test_implementation.py b/z_test_implementation.py new file mode 100644 index 000000000..3bf6586b5 --- /dev/null +++ b/z_test_implementation.py @@ -0,0 +1,55 @@ +import time + + +def node_5s()->float: + start = time.time() + time.sleep(5) + return time.time() - start + +def node_5s_error()->float: + start = time.time() + time.sleep(5) + raise ValueError("Does not break telemetry if executed through ray") + return time.time() - start + +if __name__ == "__main__": + import __main__ + from hamilton import base, driver + from hamilton.plugins.h_ray import RayGraphAdapter + from hamilton_sdk import adapters + import ray + + username = 'jernejfrank' + + tracker_ray = adapters.HamiltonTracker( + project_id=1, # modify this as needed + username=username, + dag_name="ray_telemetry_bug", + ) + + try: + ray.init() + rga = RayGraphAdapter(result_builder=base.PandasDataFrameResult()) + dr_ray = ( driver.Builder() + .with_modules(__main__) + .with_adapters(rga,tracker_ray) + .build() + ) + result_ray = dr_ray.execute(final_vars=['node_5s','node_5s_error']) + print(result_ray) + ray.shutdown() + except ValueError: + print("UI displays no problem") + finally: + tracker = adapters.HamiltonTracker( + project_id=1, # modify this as needed + username=username, + dag_name="telemetry_okay", + ) + dr_without_ray = ( driver.Builder() + .with_modules(__main__) + .with_adapters(tracker) + .build() + ) + + result_without_ray = dr_without_ray.execute(final_vars=['node_5s','node_5s_error']) \ No newline at end of file From e519180c527832b7b613519f06323d64510c4200 Mon Sep 17 00:00:00 2001 From: JFrank Date: Tue, 20 Aug 2024 16:22:30 +0100 Subject: [PATCH 05/33] Inline function, unsure if catching errors and exceptions to be handadled differently --- hamilton/execution/graph_functions.py | 206 ++++++++++++-------------- 1 file changed, 92 insertions(+), 114 deletions(-) diff --git a/hamilton/execution/graph_functions.py b/hamilton/execution/graph_functions.py index f5f0a39f1..2fe62a9e1 100644 --- a/hamilton/execution/graph_functions.py +++ b/hamilton/execution/graph_functions.py @@ -144,74 +144,6 @@ def create_error_message(kwargs: dict, node_: node.Node, step: str) -> str: message = "\n" + border + "\n" + message + "\n" + border return message -def apply_adapters( - adapter: LifecycleAdapterSet, - node_: node.Node, - run_id: str, - kwargs: Dict[str,Any], - task_id:str, - - ): -# TODO -- take everything from HERE to THERE - # Put it in a function - # That function should take an adapter as well as a node + other params (run_id, kwargs, etc...) - # And output result - # Then call the lifecycle method you created called do_remote_execute using the recipe below (call_lifecycle_method) - # And delegate to that - # only under if adapter.does_method("do_remote_execute") - # Otherwise just call the function we just defined - ##### HERE ###### - try: - if adapter.does_hook("pre_node_execute", is_async=False): - try: - adapter.call_all_lifecycle_hooks_sync( - "pre_node_execute", - run_id=run_id, - node_=node_, - kwargs=kwargs, - task_id=task_id, - ) - except Exception as e: - pre_node_execute_errored = True - raise e - if adapter.does_method("do_node_execute", is_async=False): - result = adapter.call_lifecycle_method_sync( - "do_node_execute", - run_id=run_id, - node_=node_, - kwargs=kwargs, - task_id=task_id, - ) - else: - result = node_(**kwargs) - except Exception as e: - success = False - error = e - step = "[pre-node-execute]" if pre_node_execute_errored else "" - message = create_error_message(kwargs, node_, step) - logger.exception(message) - raise - finally: - if not pre_node_execute_errored and adapter.does_hook( - "post_node_execute", is_async=False - ): - try: - adapter.call_all_lifecycle_hooks_sync( - "post_node_execute", - run_id=run_id, - node_=node_, - kwargs=kwargs, - success=success, - error=error, - result=result, - task_id=task_id, - ) - except Exception: - message = create_error_message(kwargs, node_, "[post-node-execute]") - logger.exception(message) - raise - ##### THERE ##### - def execute_subdag( nodes: Collection[node.Node], @@ -273,6 +205,7 @@ def dfs_traverse( result = None success = True pre_node_execute_errored = False + # TODO -- take everything from HERE to THERE # Put it in a function # That function should take an adapter as well as a node + other params (run_id, kwargs, etc...) @@ -282,57 +215,102 @@ def dfs_traverse( # only under if adapter.does_method("do_remote_execute") # Otherwise just call the function we just defined ##### HERE ###### - try: - if adapter.does_hook("pre_node_execute", is_async=False): - try: - adapter.call_all_lifecycle_hooks_sync( - "pre_node_execute", - run_id=run_id, - node_=node_, - kwargs=kwargs, - task_id=task_id, - ) - except Exception as e: - pre_node_execute_errored = True - raise e - if adapter.does_method("do_node_execute", is_async=False): - result = adapter.call_lifecycle_method_sync( - "do_node_execute", - run_id=run_id, - node_=node_, - kwargs=kwargs, - task_id=task_id, - ) - else: - result = node_(**kwargs) - except Exception as e: - success = False - error = e - step = "[pre-node-execute]" if pre_node_execute_errored else "" - message = create_error_message(kwargs, node_, step) - logger.exception(message) - raise - finally: - if not pre_node_execute_errored and adapter.does_hook( - "post_node_execute", is_async=False - ): - try: - adapter.call_all_lifecycle_hooks_sync( - "post_node_execute", + def execute_lifecycle_for_node( + node_: node.Node = node_, + adapter: LifecycleAdapterSet = adapter, + run_id:str = run_id, + task_id: str = task_id, + kwargs:Dict[str,Any] = kwargs, + success: bool = True, + pre_node_execute_errored: bool = False, + error = None, + result = None + ): + try: + if adapter.does_hook("pre_node_execute", is_async=False): + try: + adapter.call_all_lifecycle_hooks_sync( + "pre_node_execute", + run_id=run_id, + node_=node_, + kwargs=kwargs, + task_id=task_id, + ) + except Exception as e: + pre_node_execute_errored = True + raise e + if adapter.does_method("do_node_execute", is_async=False): + result = adapter.call_lifecycle_method_sync( + "do_node_execute", run_id=run_id, node_=node_, kwargs=kwargs, - success=success, - error=error, - result=result, task_id=task_id, ) - except Exception: - message = create_error_message(kwargs, node_, "[post-node-execute]") - logger.exception(message) - raise - ##### THERE ##### - + else: + result = node_(**kwargs) + + return error, result, success, pre_node_execute_errored + + except Exception as e: + success = False + error = e + step = "[pre-node-execute]" if pre_node_execute_errored else "" + message = create_error_message(kwargs, node_, step) + logger.exception(message) + raise + finally: + if not pre_node_execute_errored and adapter.does_hook( + "post_node_execute", is_async=False + ): + try: + adapter.call_all_lifecycle_hooks_sync( + "post_node_execute", + run_id=run_id, + node_=node_, + kwargs=kwargs, + success=success, + error=error, + result=result, + task_id=task_id, + ) + except Exception: + message = create_error_message(kwargs, node_, "[post-node-execute]") + logger.exception(message) + raise + return error, result, success, pre_node_execute_errored + + ##### THERE ##### + if adapter.does_method("do_remote_execute", is_async=False): + error, result, success, pre_node_execute_errored = adapter.call_lifecycle_method_sync( + "do_remote_execute", + execute_lifecycle_for_node = execute_lifecycle_for_node( + node_ = node_, + adapter = adapter, + run_id = run_id, + task_id = task_id, + kwargs = kwargs, + success=success, + pre_node_execute_errored=pre_node_execute_errored, + error=error, + result=result), + run_id=run_id, + node =node_, + kwargs=kwargs, + task_id=task_id, + ) + else: + error, result, success, pre_node_execute_errored = execute_lifecycle_for_node( + node_ = node_, + adapter = adapter, + run_id = run_id, + task_id = task_id, + kwargs = kwargs, + success=success, + pre_node_execute_errored=pre_node_execute_errored, + error=error, + result=result) + computed[node_.name] = result # > pruning the graph # This doesn't narrow it down to the entire space of the graph From 2dca334ffb5ff7087fae869031e7ab0e67c875ff Mon Sep 17 00:00:00 2001 From: JFrank Date: Tue, 20 Aug 2024 16:23:39 +0100 Subject: [PATCH 06/33] BaseDoRemoteExecute has the added Callable function that snadwisched lifecycle hooks --- hamilton/lifecycle/base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hamilton/lifecycle/base.py b/hamilton/lifecycle/base.py index 3bce16e65..3b0d751f8 100644 --- a/hamilton/lifecycle/base.py +++ b/hamilton/lifecycle/base.py @@ -529,15 +529,15 @@ def do_remote_execute( self, *, run_id: str, - node_: "node.Node", + node: "node.Node", kwargs: Dict[str, Any], + execute_lifecycle_for_node : Callable = None, task_id: Optional[str] = None, ) -> Any: - """Method that is called to implement node execution. This can replace the execution of a node - with something all together, augment it, or delegate it. + """Method that is called to implement correct remote execution of hooks. This makes sure that all the pre-node and post-node hooks get executed in the remote environment which is necessary for some adapters. Node execution is called the same as before through "do_node_execute". + :param execute_lifecycle_for_node: Function executing lifecycle_hooks and lifecycle_methods :param run_id: ID of the run, unique in scope of the driver. - :param node_: Node that is being executed :param kwargs: Keyword arguments that are being passed into the node :param task_id: ID of the task, defaults to None if not in a task setting """ From 04f1a1b9529e4e469127576d6c26fead57945dcf Mon Sep 17 00:00:00 2001 From: JFrank Date: Tue, 20 Aug 2024 16:24:36 +0100 Subject: [PATCH 07/33] method fails, says AssertionError about ray.remote decorator --- hamilton/plugins/h_ray.py | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/hamilton/plugins/h_ray.py b/hamilton/plugins/h_ray.py index 55fbb5c95..bf3accf79 100644 --- a/hamilton/plugins/h_ray.py +++ b/hamilton/plugins/h_ray.py @@ -1,3 +1,4 @@ +import abc import functools import json import logging @@ -6,7 +7,7 @@ import ray from ray import workflow -from hamilton import base, htypes, node +from hamilton import base, htypes, node, lifecycle from hamilton.execution import executors from hamilton.execution.executors import TaskFuture from hamilton.execution.grouping import TaskImplementation @@ -54,7 +55,12 @@ def parse_ray_remote_options_from_tags(tags: typing.Dict[str, str]) -> typing.Di # BaseDoValidateInput, # BaseDoCheckEdgeTypesMatch, # Then, implement do_remote_execute, kill do_node_execute -class RayGraphAdapter(base.HamiltonGraphAdapter, base.ResultMixin): +class RayGraphAdapter( + lifecycle.base.BaseDoRemoteExecute, + # base.LegacyResultMixin, + lifecycle.base.BaseDoValidateInput, + lifecycle.base.BaseDoCheckEdgeTypesMatch, + abc.ABC,): """Class representing what's required to make Hamilton run on Ray. This walks the graph and translates it to run onto `Ray `__. @@ -105,25 +111,36 @@ def __init__(self, result_builder: base.ResultMixin): ) @staticmethod - def check_input_type(node_type: typing.Type, input_value: typing.Any) -> bool: + def do_validate_input(node_type: typing.Type, input_value: typing.Any) -> bool: # NOTE: the type of a raylet is unknown until they are computed if isinstance(input_value, ray._raylet.ObjectRef): return True return htypes.check_input_type(node_type, input_value) @staticmethod - def check_node_type_equivalence(node_type: typing.Type, input_type: typing.Type) -> bool: - return node_type == input_type - - def execute_node(self, node: node.Node, kwargs: typing.Dict[str, typing.Any]) -> typing.Any: + def do_check_edge_types_match(type_from: typing.Type, type_to: typing.Type) -> bool: + return type_from == type_to + + # def execute_node(self, node: node.Node, kwargs: typing.Dict[str, typing.Any]) -> typing.Any: + # """Function that is called as we walk the graph to determine how to execute a hamilton function. + + # :param node: the node from the graph. + # :param kwargs: the arguments that should be passed to it. + # :return: returns a ray object reference. + # """ + # ray_options = parse_ray_remote_options_from_tags(node.tags) + # return ray.remote(raify(node.callable)).options(**ray_options).remote(**kwargs) + + def do_remote_execute(self, *, execute_lifecycle_for_node : typing.Callable, node: node.Node, kwargs: typing.Dict[str, typing.Any],**future_kwargs: typing.Any) -> typing.Any: """Function that is called as we walk the graph to determine how to execute a hamilton function. - :param node: the node from the graph. + :param execute_lifecycle_for_node: wrapper function that executes lifecycle hooks and methods :param kwargs: the arguments that should be passed to it. :return: returns a ray object reference. """ ray_options = parse_ray_remote_options_from_tags(node.tags) - return ray.remote(raify(node.callable)).options(**ray_options).remote(**kwargs) + return ray.remote(raify(execute_lifecycle_for_node))#.options(**ray_options).remote()#**kwargs) + def build_result(self, **outputs: typing.Dict[str, typing.Any]) -> typing.Any: """Builds the result and brings it back to this running process. From b77860e8feff1dcf8cb46cb7f254432ba5f71751 Mon Sep 17 00:00:00 2001 From: JFrank Date: Tue, 20 Aug 2024 16:25:23 +0100 Subject: [PATCH 08/33] simple script for now to check telemetry, execution yield the ray.remote AssertionError --- z_test_implementation.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/z_test_implementation.py b/z_test_implementation.py index 3bf6586b5..0e3739db5 100644 --- a/z_test_implementation.py +++ b/z_test_implementation.py @@ -19,8 +19,8 @@ def node_5s_error()->float: from hamilton_sdk import adapters import ray - username = 'jernejfrank' - + username = 'jf' + tracker_ray = adapters.HamiltonTracker( project_id=1, # modify this as needed username=username, @@ -35,21 +35,27 @@ def node_5s_error()->float: .with_adapters(rga,tracker_ray) .build() ) - result_ray = dr_ray.execute(final_vars=['node_5s','node_5s_error']) + result_ray = dr_ray.execute(final_vars=[ + 'node_5s', + # 'node_5s_error' + ]) print(result_ray) ray.shutdown() except ValueError: print("UI displays no problem") - finally: - tracker = adapters.HamiltonTracker( - project_id=1, # modify this as needed - username=username, - dag_name="telemetry_okay", - ) - dr_without_ray = ( driver.Builder() - .with_modules(__main__) - .with_adapters(tracker) - .build() - ) + # finally: + # tracker = adapters.HamiltonTracker( + # project_id=1, # modify this as needed + # username=username, + # dag_name="telemetry_okay", + # ) + # dr_without_ray = ( driver.Builder() + # .with_modules(__main__) + # .with_adapters(tracker) + # .build() + # ) - result_without_ray = dr_without_ray.execute(final_vars=['node_5s','node_5s_error']) \ No newline at end of file + # result_without_ray = dr_without_ray.execute(final_vars=[ + # 'node_5s', + # # 'node_5s_error' + # ]) \ No newline at end of file From c8358f8abdf7e9433da5c4c3ddac42d2628bc773 Mon Sep 17 00:00:00 2001 From: JFrank Date: Tue, 20 Aug 2024 19:17:02 +0100 Subject: [PATCH 09/33] passing pointer through and arguments to lifecycle wrapper into ray.remote --- hamilton/execution/graph_functions.py | 46 +++++++++++++-------------- hamilton/lifecycle/base.py | 6 ++-- hamilton/plugins/h_ray.py | 13 ++++++-- 3 files changed, 37 insertions(+), 28 deletions(-) diff --git a/hamilton/execution/graph_functions.py b/hamilton/execution/graph_functions.py index 2fe62a9e1..fa01ff26e 100644 --- a/hamilton/execution/graph_functions.py +++ b/hamilton/execution/graph_functions.py @@ -206,6 +206,14 @@ def dfs_traverse( success = True pre_node_execute_errored = False + lifecycle_kwargs = { + "adapter": adapter, + "error" : error, + "result" : result, + "success": success, + "pre_node_execute_errored": pre_node_execute_errored + } + # TODO -- take everything from HERE to THERE # Put it in a function # That function should take an adapter as well as a node + other params (run_id, kwargs, etc...) @@ -221,10 +229,10 @@ def execute_lifecycle_for_node( run_id:str = run_id, task_id: str = task_id, kwargs:Dict[str,Any] = kwargs, - success: bool = True, - pre_node_execute_errored: bool = False, error = None, - result = None + result = None, + success: bool = True, + pre_node_execute_errored: bool = False ): try: if adapter.does_hook("pre_node_execute", is_async=False): @@ -250,7 +258,7 @@ def execute_lifecycle_for_node( else: result = node_(**kwargs) - return error, result, success, pre_node_execute_errored + return result#, success, pre_node_execute_errored except Exception as e: success = False @@ -278,38 +286,28 @@ def execute_lifecycle_for_node( message = create_error_message(kwargs, node_, "[post-node-execute]") logger.exception(message) raise - return error, result, success, pre_node_execute_errored + # return error, result, success, pre_node_execute_errored ##### THERE ##### if adapter.does_method("do_remote_execute", is_async=False): - error, result, success, pre_node_execute_errored = adapter.call_lifecycle_method_sync( + # error, result, success, pre_node_execute_errored = adapter.call_lifecycle_method_sync( + result = adapter.call_lifecycle_method_sync( "do_remote_execute", - execute_lifecycle_for_node = execute_lifecycle_for_node( - node_ = node_, - adapter = adapter, - run_id = run_id, - task_id = task_id, - kwargs = kwargs, - success=success, - pre_node_execute_errored=pre_node_execute_errored, - error=error, - result=result), run_id=run_id, node =node_, kwargs=kwargs, task_id=task_id, + execute_lifecycle_for_node = execute_lifecycle_for_node, + lifecycle_kwargs=lifecycle_kwargs ) else: - error, result, success, pre_node_execute_errored = execute_lifecycle_for_node( - node_ = node_, - adapter = adapter, + result = execute_lifecycle_for_node( + # error, result, success, pre_node_execute_errored = execute_lifecycle_for_node( run_id = run_id, - task_id = task_id, + node_ = node_, kwargs = kwargs, - success=success, - pre_node_execute_errored=pre_node_execute_errored, - error=error, - result=result) + task_id = task_id, + **lifecycle_kwargs) computed[node_.name] = result # > pruning the graph diff --git a/hamilton/lifecycle/base.py b/hamilton/lifecycle/base.py index 3b0d751f8..1441cfbf8 100644 --- a/hamilton/lifecycle/base.py +++ b/hamilton/lifecycle/base.py @@ -43,6 +43,7 @@ # python, which (our usage of) leans type-hinting trigger-happy, this will suffice. if TYPE_CHECKING: from hamilton import graph, node + from hamilton.lifecycle.base import LifecycleAdapterSet # All of these are internal APIs. Specifically, structure required to manage a set of # hooks/methods/validators that we will likely expand. We store them in constants (rather than, say, a more complex single object) @@ -531,8 +532,9 @@ def do_remote_execute( run_id: str, node: "node.Node", kwargs: Dict[str, Any], - execute_lifecycle_for_node : Callable = None, - task_id: Optional[str] = None, + execute_lifecycle_for_node : Callable, + lifecycle_kwargs: Dict[str, Any], + task_id: Optional[str] = None ) -> Any: """Method that is called to implement correct remote execution of hooks. This makes sure that all the pre-node and post-node hooks get executed in the remote environment which is necessary for some adapters. Node execution is called the same as before through "do_node_execute". diff --git a/hamilton/plugins/h_ray.py b/hamilton/plugins/h_ray.py index bf3accf79..3e830dcf0 100644 --- a/hamilton/plugins/h_ray.py +++ b/hamilton/plugins/h_ray.py @@ -131,7 +131,15 @@ def do_check_edge_types_match(type_from: typing.Type, type_to: typing.Type) -> b # ray_options = parse_ray_remote_options_from_tags(node.tags) # return ray.remote(raify(node.callable)).options(**ray_options).remote(**kwargs) - def do_remote_execute(self, *, execute_lifecycle_for_node : typing.Callable, node: node.Node, kwargs: typing.Dict[str, typing.Any],**future_kwargs: typing.Any) -> typing.Any: + def do_remote_execute( + self, + *, + execute_lifecycle_for_node : typing.Callable, + node: node.Node, + kwargs: typing.Dict[str, typing.Any], + lifecycle_kwargs: typing.Dict[str, typing.Any], + **future_kwargs: typing.Any) -> typing.Any: + """Function that is called as we walk the graph to determine how to execute a hamilton function. :param execute_lifecycle_for_node: wrapper function that executes lifecycle hooks and methods @@ -139,7 +147,8 @@ def do_remote_execute(self, *, execute_lifecycle_for_node : typing.Callable, nod :return: returns a ray object reference. """ ray_options = parse_ray_remote_options_from_tags(node.tags) - return ray.remote(raify(execute_lifecycle_for_node))#.options(**ray_options).remote()#**kwargs) + kwargs.update(lifecycle_kwargs) + return ray.remote(raify(execute_lifecycle_for_node)).options(**ray_options).remote(**kwargs) def build_result(self, **outputs: typing.Dict[str, typing.Any]) -> typing.Any: From e77f6f76e6c5d0bf29998a428603b80b54943d27 Mon Sep 17 00:00:00 2001 From: JFrank Date: Wed, 21 Aug 2024 19:52:46 +0100 Subject: [PATCH 10/33] post-execute hook for node not called --- hamilton/execution/graph_functions.py | 46 ++++++++++++++++----------- hamilton/plugins/h_ray.py | 9 ++++-- z_test_implementation.py | 4 +-- 3 files changed, 35 insertions(+), 24 deletions(-) diff --git a/hamilton/execution/graph_functions.py b/hamilton/execution/graph_functions.py index fa01ff26e..5ee3494ae 100644 --- a/hamilton/execution/graph_functions.py +++ b/hamilton/execution/graph_functions.py @@ -208,10 +208,10 @@ def dfs_traverse( lifecycle_kwargs = { "adapter": adapter, - "error" : error, - "result" : result, - "success": success, - "pre_node_execute_errored": pre_node_execute_errored + # "error" : error, + # "result" : result, + # "success": success, + # "pre_node_execute_errored": pre_node_execute_errored } # TODO -- take everything from HERE to THERE @@ -223,17 +223,18 @@ def dfs_traverse( # only under if adapter.does_method("do_remote_execute") # Otherwise just call the function we just defined ##### HERE ###### + # TODO: name better def execute_lifecycle_for_node( node_: node.Node = node_, adapter: LifecycleAdapterSet = adapter, run_id:str = run_id, task_id: str = task_id, - kwargs:Dict[str,Any] = kwargs, - error = None, - result = None, - success: bool = True, - pre_node_execute_errored: bool = False + kwargs:Dict[str,Any] = kwargs ): + error = None, + result = None, + success = True, + pre_node_execute_errored = False try: if adapter.does_hook("pre_node_execute", is_async=False): try: @@ -258,7 +259,7 @@ def execute_lifecycle_for_node( else: result = node_(**kwargs) - return result#, success, pre_node_execute_errored + return result except Exception as e: success = False @@ -268,9 +269,12 @@ def execute_lifecycle_for_node( logger.exception(message) raise finally: + print("finally called") + if not pre_node_execute_errored and adapter.does_hook( "post_node_execute", is_async=False ): + print("running post hook") try: adapter.call_all_lifecycle_hooks_sync( "post_node_execute", @@ -283,15 +287,16 @@ def execute_lifecycle_for_node( task_id=task_id, ) except Exception: + print("running post hook exception") message = create_error_message(kwargs, node_, "[post-node-execute]") logger.exception(message) raise - # return error, result, success, pre_node_execute_errored + print("success post hook") ##### THERE ##### + # TODO: Catching correctly errors and knowing if the node function failed or ray if adapter.does_method("do_remote_execute", is_async=False): - # error, result, success, pre_node_execute_errored = adapter.call_lifecycle_method_sync( - result = adapter.call_lifecycle_method_sync( + remote_object = adapter.call_lifecycle_method_sync( "do_remote_execute", run_id=run_id, node =node_, @@ -300,14 +305,17 @@ def execute_lifecycle_for_node( execute_lifecycle_for_node = execute_lifecycle_for_node, lifecycle_kwargs=lifecycle_kwargs ) + # if isinstance(remote_object, tuple): + # error, result, success, pre_node_execute_errored = remote_object + # else: + result = remote_object else: result = execute_lifecycle_for_node( - # error, result, success, pre_node_execute_errored = execute_lifecycle_for_node( - run_id = run_id, - node_ = node_, - kwargs = kwargs, - task_id = task_id, - **lifecycle_kwargs) + run_id = run_id, + node_ = node_, + kwargs = kwargs, + task_id = task_id, + **lifecycle_kwargs) computed[node_.name] = result # > pruning the graph diff --git a/hamilton/plugins/h_ray.py b/hamilton/plugins/h_ray.py index 3e830dcf0..ee8925072 100644 --- a/hamilton/plugins/h_ray.py +++ b/hamilton/plugins/h_ray.py @@ -58,6 +58,7 @@ def parse_ray_remote_options_from_tags(tags: typing.Dict[str, str]) -> typing.Di class RayGraphAdapter( lifecycle.base.BaseDoRemoteExecute, # base.LegacyResultMixin, + lifecycle.base.BaseDoBuildResult, lifecycle.base.BaseDoValidateInput, lifecycle.base.BaseDoCheckEdgeTypesMatch, abc.ABC,): @@ -147,11 +148,13 @@ def do_remote_execute( :return: returns a ray object reference. """ ray_options = parse_ray_remote_options_from_tags(node.tags) - kwargs.update(lifecycle_kwargs) - return ray.remote(raify(execute_lifecycle_for_node)).options(**ray_options).remote(**kwargs) + # kwargs = {"kwargs":kwargs, + # "adapter": lifecycle_kwargs["adapter"],} + # # kwargs.update(lifecycle_kwargs) + return ray.remote(raify(execute_lifecycle_for_node)).options(**ray_options).remote(kwargs=kwargs) - def build_result(self, **outputs: typing.Dict[str, typing.Any]) -> typing.Any: + def do_build_result(self, outputs: typing.Dict[str, typing.Any]) -> typing.Any: """Builds the result and brings it back to this running process. :param outputs: the dictionary of key -> Union[ray object reference | value] diff --git a/z_test_implementation.py b/z_test_implementation.py index 0e3739db5..f753aafdd 100644 --- a/z_test_implementation.py +++ b/z_test_implementation.py @@ -37,7 +37,7 @@ def node_5s_error()->float: ) result_ray = dr_ray.execute(final_vars=[ 'node_5s', - # 'node_5s_error' + 'node_5s_error' ]) print(result_ray) ray.shutdown() @@ -57,5 +57,5 @@ def node_5s_error()->float: # result_without_ray = dr_without_ray.execute(final_vars=[ # 'node_5s', - # # 'node_5s_error' + # 'node_5s_error' # ]) \ No newline at end of file From f7e81a04e57dbcf8f487a0fbffaae4089f22ab2e Mon Sep 17 00:00:00 2001 From: JFrank Date: Wed, 21 Aug 2024 23:35:20 +0100 Subject: [PATCH 11/33] finally executed only when exception occurs, hamilton tracker not executed --- hamilton/execution/graph_functions.py | 69 +++++++++++---------------- hamilton/lifecycle/base.py | 5 -- hamilton/plugins/h_ray.py | 22 +-------- ui/sdk/src/hamilton_sdk/adapters.py | 1 + z_test_implementation.py | 4 +- 5 files changed, 32 insertions(+), 69 deletions(-) diff --git a/hamilton/execution/graph_functions.py b/hamilton/execution/graph_functions.py index 5ee3494ae..8fc90480f 100644 --- a/hamilton/execution/graph_functions.py +++ b/hamilton/execution/graph_functions.py @@ -201,29 +201,9 @@ def dfs_traverse( for dependency in node_.dependencies: if dependency.name in computed: kwargs[dependency.name] = computed[dependency.name] - error = None - result = None - success = True - pre_node_execute_errored = False - - lifecycle_kwargs = { - "adapter": adapter, - # "error" : error, - # "result" : result, - # "success": success, - # "pre_node_execute_errored": pre_node_execute_errored - } - - # TODO -- take everything from HERE to THERE - # Put it in a function - # That function should take an adapter as well as a node + other params (run_id, kwargs, etc...) - # And output result - # Then call the lifecycle method you created called do_remote_execute using the recipe below (call_lifecycle_method) - # And delegate to that - # only under if adapter.does_method("do_remote_execute") - # Otherwise just call the function we just defined - ##### HERE ###### - # TODO: name better + + + # TODO: better function name def execute_lifecycle_for_node( node_: node.Node = node_, adapter: LifecycleAdapterSet = adapter, @@ -231,13 +211,19 @@ def execute_lifecycle_for_node( task_id: str = task_id, kwargs:Dict[str,Any] = kwargs ): - error = None, - result = None, - success = True, + """Sandwich function that guarantees the pre_node and post_node lifecycle hooks are executed in the correct environment (local or remote).""" + + error = None + result = None + success = True pre_node_execute_errored = False + try: + print("try lifecycle") if adapter.does_hook("pre_node_execute", is_async=False): + print("pre-hook") try: + print("try pre-hook") adapter.call_all_lifecycle_hooks_sync( "pre_node_execute", run_id=run_id, @@ -246,9 +232,11 @@ def execute_lifecycle_for_node( task_id=task_id, ) except Exception as e: + print("exception to pre-hook") pre_node_execute_errored = True raise e if adapter.does_method("do_node_execute", is_async=False): + print("node execute by lifecycle method") result = adapter.call_lifecycle_method_sync( "do_node_execute", run_id=run_id, @@ -257,11 +245,13 @@ def execute_lifecycle_for_node( task_id=task_id, ) else: + print("node execute as callable") result = node_(**kwargs) return result except Exception as e: + print("exception to node execute") success = False error = e step = "[pre-node-execute]" if pre_node_execute_errored else "" @@ -270,7 +260,13 @@ def execute_lifecycle_for_node( raise finally: print("finally called") - + print(f"run_id={run_id}", + f"node_={node_}", + f"kwargs={kwargs}", + f"success={success}", + f"error={error}", + f"result={result}", + f"task_id={task_id}") if not pre_node_execute_errored and adapter.does_hook( "post_node_execute", is_async=False ): @@ -286,37 +282,26 @@ def execute_lifecycle_for_node( result=result, task_id=task_id, ) + print("post-hook complete") except Exception: print("running post hook exception") message = create_error_message(kwargs, node_, "[post-node-execute]") logger.exception(message) raise - print("success post hook") - ##### THERE ##### # TODO: Catching correctly errors and knowing if the node function failed or ray if adapter.does_method("do_remote_execute", is_async=False): - remote_object = adapter.call_lifecycle_method_sync( + result = adapter.call_lifecycle_method_sync( "do_remote_execute", run_id=run_id, node =node_, kwargs=kwargs, task_id=task_id, execute_lifecycle_for_node = execute_lifecycle_for_node, - lifecycle_kwargs=lifecycle_kwargs ) - # if isinstance(remote_object, tuple): - # error, result, success, pre_node_execute_errored = remote_object - # else: - result = remote_object else: - result = execute_lifecycle_for_node( - run_id = run_id, - node_ = node_, - kwargs = kwargs, - task_id = task_id, - **lifecycle_kwargs) - + result = execute_lifecycle_for_node() + computed[node_.name] = result # > pruning the graph # This doesn't narrow it down to the entire space of the graph diff --git a/hamilton/lifecycle/base.py b/hamilton/lifecycle/base.py index 1441cfbf8..3c45b1f0b 100644 --- a/hamilton/lifecycle/base.py +++ b/hamilton/lifecycle/base.py @@ -519,10 +519,6 @@ def do_node_execute( """ pass -# TODO -- copy this + name it `BaseDoRemoteExecute` and `do_remote_execute` (for the string/function) -# Look at the comments in graph_functions.py for what parameters to add -# Add the right parameters -# Add some docstrings @lifecycle.base_method("do_remote_execute") class BaseDoRemoteExecute(abc.ABC): @abc.abstractmethod @@ -533,7 +529,6 @@ def do_remote_execute( node: "node.Node", kwargs: Dict[str, Any], execute_lifecycle_for_node : Callable, - lifecycle_kwargs: Dict[str, Any], task_id: Optional[str] = None ) -> Any: """Method that is called to implement correct remote execution of hooks. This makes sure that all the pre-node and post-node hooks get executed in the remote environment which is necessary for some adapters. Node execution is called the same as before through "do_node_execute". diff --git a/hamilton/plugins/h_ray.py b/hamilton/plugins/h_ray.py index ee8925072..caebdc97c 100644 --- a/hamilton/plugins/h_ray.py +++ b/hamilton/plugins/h_ray.py @@ -50,14 +50,8 @@ def parse_ray_remote_options_from_tags(tags: typing.Dict[str, str]) -> typing.Di return ray_options -# TODO -- change the base classes here to be the underlying ones in HamiltonGraphAdapter -# BaseDoRemoteExecute, # Change this one -- add it in -# BaseDoValidateInput, -# BaseDoCheckEdgeTypesMatch, -# Then, implement do_remote_execute, kill do_node_execute class RayGraphAdapter( lifecycle.base.BaseDoRemoteExecute, - # base.LegacyResultMixin, lifecycle.base.BaseDoBuildResult, lifecycle.base.BaseDoValidateInput, lifecycle.base.BaseDoCheckEdgeTypesMatch, @@ -121,16 +115,6 @@ def do_validate_input(node_type: typing.Type, input_value: typing.Any) -> bool: @staticmethod def do_check_edge_types_match(type_from: typing.Type, type_to: typing.Type) -> bool: return type_from == type_to - - # def execute_node(self, node: node.Node, kwargs: typing.Dict[str, typing.Any]) -> typing.Any: - # """Function that is called as we walk the graph to determine how to execute a hamilton function. - - # :param node: the node from the graph. - # :param kwargs: the arguments that should be passed to it. - # :return: returns a ray object reference. - # """ - # ray_options = parse_ray_remote_options_from_tags(node.tags) - # return ray.remote(raify(node.callable)).options(**ray_options).remote(**kwargs) def do_remote_execute( self, @@ -138,8 +122,7 @@ def do_remote_execute( execute_lifecycle_for_node : typing.Callable, node: node.Node, kwargs: typing.Dict[str, typing.Any], - lifecycle_kwargs: typing.Dict[str, typing.Any], - **future_kwargs: typing.Any) -> typing.Any: + **other_kwargs: typing.Any) -> typing.Any: """Function that is called as we walk the graph to determine how to execute a hamilton function. @@ -148,9 +131,6 @@ def do_remote_execute( :return: returns a ray object reference. """ ray_options = parse_ray_remote_options_from_tags(node.tags) - # kwargs = {"kwargs":kwargs, - # "adapter": lifecycle_kwargs["adapter"],} - # # kwargs.update(lifecycle_kwargs) return ray.remote(raify(execute_lifecycle_for_node)).options(**ray_options).remote(kwargs=kwargs) diff --git a/ui/sdk/src/hamilton_sdk/adapters.py b/ui/sdk/src/hamilton_sdk/adapters.py index 995143e64..2c05dec41 100644 --- a/ui/sdk/src/hamilton_sdk/adapters.py +++ b/ui/sdk/src/hamilton_sdk/adapters.py @@ -252,6 +252,7 @@ def post_node_execute( result: Optional[Any], task_id: Optional[str] = None, ): + print("I got to hamilton tracker") """Captures end of node execution.""" logger.debug("post_node_execute %s %s", run_id, task_id) task_run: TaskRun = self.task_runs[run_id][node_.name] diff --git a/z_test_implementation.py b/z_test_implementation.py index f753aafdd..e61d3d7b1 100644 --- a/z_test_implementation.py +++ b/z_test_implementation.py @@ -2,11 +2,13 @@ def node_5s()->float: + print("5s executed") start = time.time() time.sleep(5) return time.time() - start def node_5s_error()->float: + print("5s error executed") start = time.time() time.sleep(5) raise ValueError("Does not break telemetry if executed through ray") @@ -36,7 +38,7 @@ def node_5s_error()->float: .build() ) result_ray = dr_ray.execute(final_vars=[ - 'node_5s', + # 'node_5s', 'node_5s_error' ]) print(result_ray) From 3a1cccd73800a3c4da0d6a7af946de2acc522eb2 Mon Sep 17 00:00:00 2001 From: JFrank Date: Fri, 23 Aug 2024 23:28:24 +0100 Subject: [PATCH 12/33] atexit.register does not work, node keeps running inui --- ui/sdk/src/hamilton_sdk/api/clients.py | 4 ++++ z_test_implementation.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ui/sdk/src/hamilton_sdk/api/clients.py b/ui/sdk/src/hamilton_sdk/api/clients.py index 87e687027..0f5ff66b4 100644 --- a/ui/sdk/src/hamilton_sdk/api/clients.py +++ b/ui/sdk/src/hamilton_sdk/api/clients.py @@ -196,6 +196,10 @@ def __init__( ).start() self.worker_thread.start() + import atexit + + atexit.register(self.stop) + def __getstate__(self): # Copy the object's state from self.__dict__ which contains # all our instance attributes. Always use the dict.copy() diff --git a/z_test_implementation.py b/z_test_implementation.py index e61d3d7b1..cce5cdd1c 100644 --- a/z_test_implementation.py +++ b/z_test_implementation.py @@ -38,8 +38,8 @@ def node_5s_error()->float: .build() ) result_ray = dr_ray.execute(final_vars=[ - # 'node_5s', - 'node_5s_error' + 'node_5s', + # 'node_5s_error' ]) print(result_ray) ray.shutdown() From 09b47ded995a03e0a5a760c3e267fb9aff4636c3 Mon Sep 17 00:00:00 2001 From: JFrank Date: Fri, 23 Aug 2024 23:39:27 +0100 Subject: [PATCH 13/33] added stop() method, but doesn't get called --- hamilton/execution/graph_functions.py | 6 ++++++ ui/sdk/src/hamilton_sdk/adapters.py | 5 ++++- z_test_implementation.py | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/hamilton/execution/graph_functions.py b/hamilton/execution/graph_functions.py index 8fc90480f..aa4d60fc8 100644 --- a/hamilton/execution/graph_functions.py +++ b/hamilton/execution/graph_functions.py @@ -288,6 +288,12 @@ def execute_lifecycle_for_node( message = create_error_message(kwargs, node_, "[post-node-execute]") logger.exception(message) raise + + print("stopping adapters") + for a in adapter._adapters: + if hasattr(a, "stop"): + print("stopping!") + a.stop() # TODO: Catching correctly errors and knowing if the node function failed or ray if adapter.does_method("do_remote_execute", is_async=False): diff --git a/ui/sdk/src/hamilton_sdk/adapters.py b/ui/sdk/src/hamilton_sdk/adapters.py index 2c05dec41..3c511af40 100644 --- a/ui/sdk/src/hamilton_sdk/adapters.py +++ b/ui/sdk/src/hamilton_sdk/adapters.py @@ -102,7 +102,10 @@ def __init__( # set this to some constant value if you want to generate the same sample each time. # if you're using a float value. self.seed = None - + + def stop(self): + self.client.stop() + def post_graph_construct( self, graph: h_graph.FunctionGraph, modules: List[ModuleType], config: Dict[str, Any] ): diff --git a/z_test_implementation.py b/z_test_implementation.py index cce5cdd1c..b15b4a5f7 100644 --- a/z_test_implementation.py +++ b/z_test_implementation.py @@ -39,7 +39,7 @@ def node_5s_error()->float: ) result_ray = dr_ray.execute(final_vars=[ 'node_5s', - # 'node_5s_error' + 'node_5s_error' ]) print(result_ray) ray.shutdown() From 933991f2f95ad9e2071f818b1837ceda98a1f1c6 Mon Sep 17 00:00:00 2001 From: JFrank Date: Sat, 24 Aug 2024 00:32:59 +0100 Subject: [PATCH 14/33] Ray telemtry works for single node, problem with connected nodes --- ui/sdk/.pre-commit-config.yaml | 12 ++++++------ ui/sdk/src/hamilton_sdk/adapters.py | 6 +++--- ui/sdk/src/hamilton_sdk/api/clients.py | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ui/sdk/.pre-commit-config.yaml b/ui/sdk/.pre-commit-config.yaml index 75a5287c3..e5fdcd2f3 100644 --- a/ui/sdk/.pre-commit-config.yaml +++ b/ui/sdk/.pre-commit-config.yaml @@ -5,12 +5,12 @@ # $ pre-commit install repos: -- repo: https://github.com/charliermarsh/ruff-pre-commit - # Ruff version. - rev: v0.0.265 - hooks: - - id: ruff - args: [ --fix, --exit-non-zero-on-fix ] +# - repo: https://github.com/charliermarsh/ruff-pre-commit +# # Ruff version. +# rev: v0.0.265 +# hooks: +# - id: ruff +# args: [ --fix , --exit-non-zero-on-fix ] - repo: https://github.com/ambv/black rev: 23.3.0 hooks: diff --git a/ui/sdk/src/hamilton_sdk/adapters.py b/ui/sdk/src/hamilton_sdk/adapters.py index 3c511af40..23cbb3a2c 100644 --- a/ui/sdk/src/hamilton_sdk/adapters.py +++ b/ui/sdk/src/hamilton_sdk/adapters.py @@ -102,10 +102,11 @@ def __init__( # set this to some constant value if you want to generate the same sample each time. # if you're using a float value. self.seed = None - + def stop(self): + """Initiates stop if run in remote environment""" self.client.stop() - + def post_graph_construct( self, graph: h_graph.FunctionGraph, modules: List[ModuleType], config: Dict[str, Any] ): @@ -255,7 +256,6 @@ def post_node_execute( result: Optional[Any], task_id: Optional[str] = None, ): - print("I got to hamilton tracker") """Captures end of node execution.""" logger.debug("post_node_execute %s %s", run_id, task_id) task_run: TaskRun = self.task_runs[run_id][node_.name] diff --git a/ui/sdk/src/hamilton_sdk/api/clients.py b/ui/sdk/src/hamilton_sdk/api/clients.py index 0f5ff66b4..580762774 100644 --- a/ui/sdk/src/hamilton_sdk/api/clients.py +++ b/ui/sdk/src/hamilton_sdk/api/clients.py @@ -1,5 +1,6 @@ import abc import asyncio +import atexit import datetime import functools import logging @@ -196,8 +197,7 @@ def __init__( ).start() self.worker_thread.start() - import atexit - + # Makes sure the process stops even if in remote environment atexit.register(self.stop) def __getstate__(self): From d1e6ea0c9a6784018132c91a088927a7cec1f1fd Mon Sep 17 00:00:00 2001 From: JFrank Date: Sat, 24 Aug 2024 00:34:15 +0100 Subject: [PATCH 15/33] Ray telemtry works for single node, problem with connected nodes --- ui/sdk/.pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ui/sdk/.pre-commit-config.yaml b/ui/sdk/.pre-commit-config.yaml index e5fdcd2f3..6cddba381 100644 --- a/ui/sdk/.pre-commit-config.yaml +++ b/ui/sdk/.pre-commit-config.yaml @@ -5,12 +5,12 @@ # $ pre-commit install repos: -# - repo: https://github.com/charliermarsh/ruff-pre-commit -# # Ruff version. -# rev: v0.0.265 -# hooks: -# - id: ruff -# args: [ --fix , --exit-non-zero-on-fix ] +- repo: https://github.com/charliermarsh/ruff-pre-commit + # Ruff version. + rev: v0.0.265 + hooks: + - id: ruff + args: [ --fix , --exit-non-zero-on-fix ] - repo: https://github.com/ambv/black rev: 23.3.0 hooks: From b528a4533050827b45bad99d1ee8cbea3cc27289 Mon Sep 17 00:00:00 2001 From: JFrank Date: Sat, 24 Aug 2024 00:38:38 +0100 Subject: [PATCH 16/33] Ray telemtry works for single node, problem with connected nodes --- .pre-commit-config.yaml | 4 +- hamilton/execution/graph_functions.py | 55 ++++++++--------------- hamilton/lifecycle/base.py | 6 ++- hamilton/plugins/h_ray.py | 28 +++++++----- z_test_implementation.py | 64 ++++++++++++++------------- 5 files changed, 75 insertions(+), 82 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 183541c4f..0cee73c6a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,12 +9,12 @@ repos: # Ruff version. rev: v0.5.7 hooks: - # Run the linter. + Run the linter. - id: ruff args: [ --fix ] # Run the formatter. - id: ruff-format -# args: [ --diff ] # Use for previewing changes + # args: [ --diff ] # Use for previewing changes - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 hooks: diff --git a/hamilton/execution/graph_functions.py b/hamilton/execution/graph_functions.py index aa4d60fc8..658f90813 100644 --- a/hamilton/execution/graph_functions.py +++ b/hamilton/execution/graph_functions.py @@ -202,28 +202,25 @@ def dfs_traverse( if dependency.name in computed: kwargs[dependency.name] = computed[dependency.name] - # TODO: better function name def execute_lifecycle_for_node( - node_: node.Node = node_, - adapter: LifecycleAdapterSet = adapter, - run_id:str = run_id, - task_id: str = task_id, - kwargs:Dict[str,Any] = kwargs + node_: node.Node = node_, + adapter: LifecycleAdapterSet = adapter, + run_id: str = run_id, + task_id: str = task_id, + kwargs: Dict[str, Any] = kwargs, + remote_execute=False, ): """Sandwich function that guarantees the pre_node and post_node lifecycle hooks are executed in the correct environment (local or remote).""" - + error = None result = None success = True pre_node_execute_errored = False try: - print("try lifecycle") if adapter.does_hook("pre_node_execute", is_async=False): - print("pre-hook") try: - print("try pre-hook") adapter.call_all_lifecycle_hooks_sync( "pre_node_execute", run_id=run_id, @@ -232,11 +229,9 @@ def execute_lifecycle_for_node( task_id=task_id, ) except Exception as e: - print("exception to pre-hook") pre_node_execute_errored = True raise e if adapter.does_method("do_node_execute", is_async=False): - print("node execute by lifecycle method") result = adapter.call_lifecycle_method_sync( "do_node_execute", run_id=run_id, @@ -245,13 +240,11 @@ def execute_lifecycle_for_node( task_id=task_id, ) else: - print("node execute as callable") result = node_(**kwargs) - + return result - + except Exception as e: - print("exception to node execute") success = False error = e step = "[pre-node-execute]" if pre_node_execute_errored else "" @@ -259,18 +252,9 @@ def execute_lifecycle_for_node( logger.exception(message) raise finally: - print("finally called") - print(f"run_id={run_id}", - f"node_={node_}", - f"kwargs={kwargs}", - f"success={success}", - f"error={error}", - f"result={result}", - f"task_id={task_id}") if not pre_node_execute_errored and adapter.does_hook( "post_node_execute", is_async=False ): - print("running post hook") try: adapter.call_all_lifecycle_hooks_sync( "post_node_execute", @@ -282,32 +266,31 @@ def execute_lifecycle_for_node( result=result, task_id=task_id, ) - print("post-hook complete") except Exception: - print("running post hook exception") message = create_error_message(kwargs, node_, "[post-node-execute]") logger.exception(message) raise - - print("stopping adapters") - for a in adapter._adapters: - if hasattr(a, "stop"): - print("stopping!") - a.stop() + + if remote_execute: + for a in adapter._adapters: + if hasattr(a, "stop"): + a.stop() # TODO: Catching correctly errors and knowing if the node function failed or ray + if adapter.does_method("do_remote_execute", is_async=False): result = adapter.call_lifecycle_method_sync( "do_remote_execute", run_id=run_id, - node =node_, + node=node_, kwargs=kwargs, task_id=task_id, - execute_lifecycle_for_node = execute_lifecycle_for_node, + execute_lifecycle_for_node=execute_lifecycle_for_node, + remote_execute=True, ) else: result = execute_lifecycle_for_node() - + computed[node_.name] = result # > pruning the graph # This doesn't narrow it down to the entire space of the graph diff --git a/hamilton/lifecycle/base.py b/hamilton/lifecycle/base.py index 3c45b1f0b..d150e9f47 100644 --- a/hamilton/lifecycle/base.py +++ b/hamilton/lifecycle/base.py @@ -519,6 +519,7 @@ def do_node_execute( """ pass + @lifecycle.base_method("do_remote_execute") class BaseDoRemoteExecute(abc.ABC): @abc.abstractmethod @@ -528,8 +529,9 @@ def do_remote_execute( run_id: str, node: "node.Node", kwargs: Dict[str, Any], - execute_lifecycle_for_node : Callable, - task_id: Optional[str] = None + execute_lifecycle_for_node: Callable, + remote_execute: bool, + task_id: Optional[str] = None, ) -> Any: """Method that is called to implement correct remote execution of hooks. This makes sure that all the pre-node and post-node hooks get executed in the remote environment which is necessary for some adapters. Node execution is called the same as before through "do_node_execute". diff --git a/hamilton/plugins/h_ray.py b/hamilton/plugins/h_ray.py index caebdc97c..30dc00985 100644 --- a/hamilton/plugins/h_ray.py +++ b/hamilton/plugins/h_ray.py @@ -50,12 +50,14 @@ def parse_ray_remote_options_from_tags(tags: typing.Dict[str, str]) -> typing.Di return ray_options + class RayGraphAdapter( lifecycle.base.BaseDoRemoteExecute, lifecycle.base.BaseDoBuildResult, lifecycle.base.BaseDoValidateInput, lifecycle.base.BaseDoCheckEdgeTypesMatch, - abc.ABC,): + abc.ABC, +): """Class representing what's required to make Hamilton run on Ray. This walks the graph and translates it to run onto `Ray `__. @@ -115,15 +117,16 @@ def do_validate_input(node_type: typing.Type, input_value: typing.Any) -> bool: @staticmethod def do_check_edge_types_match(type_from: typing.Type, type_to: typing.Type) -> bool: return type_from == type_to - - def do_remote_execute( - self, - *, - execute_lifecycle_for_node : typing.Callable, - node: node.Node, - kwargs: typing.Dict[str, typing.Any], - **other_kwargs: typing.Any) -> typing.Any: + def do_remote_execute( + self, + *, + execute_lifecycle_for_node: typing.Callable, + node: node.Node, + kwargs: typing.Dict[str, typing.Any], + remote_execute: bool, + **other_kwargs: typing.Any, + ) -> typing.Any: """Function that is called as we walk the graph to determine how to execute a hamilton function. :param execute_lifecycle_for_node: wrapper function that executes lifecycle hooks and methods @@ -131,8 +134,11 @@ def do_remote_execute( :return: returns a ray object reference. """ ray_options = parse_ray_remote_options_from_tags(node.tags) - return ray.remote(raify(execute_lifecycle_for_node)).options(**ray_options).remote(kwargs=kwargs) - + return ( + ray.remote(raify(execute_lifecycle_for_node)) + .options(**ray_options) + .remote(kwargs=kwargs, remote_execute=remote_execute) + ) def do_build_result(self, outputs: typing.Dict[str, typing.Any]) -> typing.Any: """Builds the result and brings it back to this running process. diff --git a/z_test_implementation.py b/z_test_implementation.py index b15b4a5f7..de1da76a4 100644 --- a/z_test_implementation.py +++ b/z_test_implementation.py @@ -1,19 +1,28 @@ import time -def node_5s()->float: +def node_5s() -> float: print("5s executed") start = time.time() time.sleep(5) return time.time() - start -def node_5s_error()->float: + +def at_1_to_previous(node_5s: float) -> float: + print("1s executed") + start = time.time() + time.sleep(5) + return node_5s # + (time.time()-start) + + +def node_5s_error() -> float: print("5s error executed") start = time.time() time.sleep(5) raise ValueError("Does not break telemetry if executed through ray") return time.time() - start + if __name__ == "__main__": import __main__ from hamilton import base, driver @@ -21,43 +30,36 @@ def node_5s_error()->float: from hamilton_sdk import adapters import ray - username = 'jf' - + username = "jf" + tracker_ray = adapters.HamiltonTracker( project_id=1, # modify this as needed username=username, dag_name="ray_telemetry_bug", - ) - + ) + try: ray.init() rga = RayGraphAdapter(result_builder=base.PandasDataFrameResult()) - dr_ray = ( driver.Builder() - .with_modules(__main__) - .with_adapters(rga,tracker_ray) - .build() - ) - result_ray = dr_ray.execute(final_vars=[ - 'node_5s', - 'node_5s_error' - ]) + dr_ray = driver.Builder().with_modules(__main__).with_adapters(rga, tracker_ray).build() + result_ray = dr_ray.execute( + final_vars=[ + "node_5s", + # 'node_5s_error' + # 'at_1_to_previous', + ] + ) print(result_ray) ray.shutdown() + print(result_ray) except ValueError: print("UI displays no problem") - # finally: - # tracker = adapters.HamiltonTracker( - # project_id=1, # modify this as needed - # username=username, - # dag_name="telemetry_okay", - # ) - # dr_without_ray = ( driver.Builder() - # .with_modules(__main__) - # .with_adapters(tracker) - # .build() - # ) - - # result_without_ray = dr_without_ray.execute(final_vars=[ - # 'node_5s', - # 'node_5s_error' - # ]) \ No newline at end of file + finally: + tracker = adapters.HamiltonTracker( + project_id=1, # modify this as needed + username=username, + dag_name="telemetry_okay", + ) + dr_without_ray = driver.Builder().with_modules(__main__).with_adapters(tracker).build() + + result_without_ray = dr_without_ray.execute(final_vars=["node_5s", "node_5s_error"]) From 7acdd3836b36b8354ff57636b72c53a9cd656fc2 Mon Sep 17 00:00:00 2001 From: Stefan Krawczyk Date: Sun, 25 Aug 2024 09:39:37 -0700 Subject: [PATCH 17/33] Fixes ray object dereferencing Ray does not resolve nested arguments: https://docs.ray.io/en/latest/ray-core/objects.html#passing-object-arguments So one option is to make them all top level: - one way to do that is to make the other arguments not clash with any possible user parameters -- hence the `__` prefix. This is what I did. - another way would be in the ray adapter, wrap the incoming function, and explicitly do a ray.get() on any ray object references in the kwargs arguments. i.e. keep the nested structure, but when the ray task starts way for all inputs... not sure which is best, but this now works correctly. --- .pre-commit-config.yaml | 4 +-- hamilton/execution/graph_functions.py | 48 ++++++++++++--------------- hamilton/plugins/h_ray.py | 12 ++----- 3 files changed, 26 insertions(+), 38 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0cee73c6a..d15c7ece4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,12 +9,12 @@ repos: # Ruff version. rev: v0.5.7 hooks: - Run the linter. + # Run the linter. - id: ruff args: [ --fix ] # Run the formatter. - id: ruff-format - # args: [ --diff ] # Use for previewing changes + # args: [ --diff ] # Use for previewing changes - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 hooks: diff --git a/hamilton/execution/graph_functions.py b/hamilton/execution/graph_functions.py index 658f90813..17f49cfcc 100644 --- a/hamilton/execution/graph_functions.py +++ b/hamilton/execution/graph_functions.py @@ -204,12 +204,12 @@ def dfs_traverse( # TODO: better function name def execute_lifecycle_for_node( - node_: node.Node = node_, - adapter: LifecycleAdapterSet = adapter, - run_id: str = run_id, - task_id: str = task_id, - kwargs: Dict[str, Any] = kwargs, - remote_execute=False, + __node_: node.Node = node_, + __adapter: LifecycleAdapterSet = adapter, + __run_id: str = run_id, + __task_id: str = task_id, + # __kwargs: Dict[str, Any] = kwargs, + **__kwargs: Dict[str, Any], ): """Sandwich function that guarantees the pre_node and post_node lifecycle hooks are executed in the correct environment (local or remote).""" @@ -223,10 +223,10 @@ def execute_lifecycle_for_node( try: adapter.call_all_lifecycle_hooks_sync( "pre_node_execute", - run_id=run_id, - node_=node_, - kwargs=kwargs, - task_id=task_id, + run_id=__run_id, + node_=__node_, + kwargs=__kwargs, + task_id=__task_id, ) except Exception as e: pre_node_execute_errored = True @@ -234,13 +234,13 @@ def execute_lifecycle_for_node( if adapter.does_method("do_node_execute", is_async=False): result = adapter.call_lifecycle_method_sync( "do_node_execute", - run_id=run_id, - node_=node_, - kwargs=kwargs, - task_id=task_id, + run_id=__run_id, + node_=__node_, + kwargs=__kwargs, + task_id=__task_id, ) else: - result = node_(**kwargs) + result = node_(**__kwargs) return result @@ -258,9 +258,9 @@ def execute_lifecycle_for_node( try: adapter.call_all_lifecycle_hooks_sync( "post_node_execute", - run_id=run_id, - node_=node_, - kwargs=kwargs, + run_id=__run_id, + node_=__node_, + kwargs=__kwargs, success=success, error=error, result=result, @@ -271,22 +271,16 @@ def execute_lifecycle_for_node( logger.exception(message) raise - if remote_execute: - for a in adapter._adapters: - if hasattr(a, "stop"): - a.stop() - # TODO: Catching correctly errors and knowing if the node function failed or ray if adapter.does_method("do_remote_execute", is_async=False): result = adapter.call_lifecycle_method_sync( "do_remote_execute", - run_id=run_id, + __run_id=run_id, node=node_, - kwargs=kwargs, - task_id=task_id, + __task_id=task_id, execute_lifecycle_for_node=execute_lifecycle_for_node, - remote_execute=True, + **kwargs, ) else: result = execute_lifecycle_for_node() diff --git a/hamilton/plugins/h_ray.py b/hamilton/plugins/h_ray.py index 30dc00985..d52b4189d 100644 --- a/hamilton/plugins/h_ray.py +++ b/hamilton/plugins/h_ray.py @@ -7,7 +7,7 @@ import ray from ray import workflow -from hamilton import base, htypes, node, lifecycle +from hamilton import base, htypes, lifecycle, node from hamilton.execution import executors from hamilton.execution.executors import TaskFuture from hamilton.execution.grouping import TaskImplementation @@ -123,9 +123,7 @@ def do_remote_execute( *, execute_lifecycle_for_node: typing.Callable, node: node.Node, - kwargs: typing.Dict[str, typing.Any], - remote_execute: bool, - **other_kwargs: typing.Any, + **kwargs: typing.Dict[str, typing.Any], ) -> typing.Any: """Function that is called as we walk the graph to determine how to execute a hamilton function. @@ -134,11 +132,7 @@ def do_remote_execute( :return: returns a ray object reference. """ ray_options = parse_ray_remote_options_from_tags(node.tags) - return ( - ray.remote(raify(execute_lifecycle_for_node)) - .options(**ray_options) - .remote(kwargs=kwargs, remote_execute=remote_execute) - ) + return ray.remote(raify(execute_lifecycle_for_node)).options(**ray_options).remote(**kwargs) def do_build_result(self, outputs: typing.Dict[str, typing.Any]) -> typing.Any: """Builds the result and brings it back to this running process. From 377dc71dbc6f7fe0a0c15810e5cd277bc4ed043f Mon Sep 17 00:00:00 2001 From: JFrank Date: Sun, 25 Aug 2024 19:30:34 +0100 Subject: [PATCH 18/33] ray works checkpoint, pre-commit fixed --- z_test_implementation.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/z_test_implementation.py b/z_test_implementation.py index de1da76a4..d7ae60ed6 100644 --- a/z_test_implementation.py +++ b/z_test_implementation.py @@ -11,24 +11,23 @@ def node_5s() -> float: def at_1_to_previous(node_5s: float) -> float: print("1s executed") start = time.time() - time.sleep(5) - return node_5s # + (time.time()-start) + time.sleep(1) + return node_5s + (time.time() - start) def node_5s_error() -> float: print("5s error executed") - start = time.time() time.sleep(5) raise ValueError("Does not break telemetry if executed through ray") - return time.time() - start if __name__ == "__main__": + import ray + import __main__ from hamilton import base, driver from hamilton.plugins.h_ray import RayGraphAdapter from hamilton_sdk import adapters - import ray username = "jf" @@ -46,7 +45,7 @@ def node_5s_error() -> float: final_vars=[ "node_5s", # 'node_5s_error' - # 'at_1_to_previous', + "at_1_to_previous", ] ) print(result_ray) @@ -54,12 +53,12 @@ def node_5s_error() -> float: print(result_ray) except ValueError: print("UI displays no problem") - finally: - tracker = adapters.HamiltonTracker( - project_id=1, # modify this as needed - username=username, - dag_name="telemetry_okay", - ) - dr_without_ray = driver.Builder().with_modules(__main__).with_adapters(tracker).build() + # finally: + # tracker = adapters.HamiltonTracker( + # project_id=1, # modify this as needed + # username=username, + # dag_name="telemetry_okay", + # ) + # dr_without_ray = driver.Builder().with_modules(__main__).with_adapters(tracker).build() - result_without_ray = dr_without_ray.execute(final_vars=["node_5s", "node_5s_error"]) + # result_without_ray = dr_without_ray.execute(final_vars=["node_5s", "node_5s_error"]) From 8e3eacd9fa9867e799dcc06a6cb2e9046be76d05 Mon Sep 17 00:00:00 2001 From: JFrank Date: Sun, 25 Aug 2024 19:41:46 +0100 Subject: [PATCH 19/33] fixed graph level telemtry proposal --- hamilton/driver.py | 45 +++++++++++++++++++++++++--------------- z_test_implementation.py | 2 +- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/hamilton/driver.py b/hamilton/driver.py index a2ba558d5..cff4aa7bc 100644 --- a/hamilton/driver.py +++ b/hamilton/driver.py @@ -583,7 +583,9 @@ def execute( error = None _final_vars = self._create_final_vars(final_vars) try: - outputs = self.raw_execute(_final_vars, overrides, display_graph, inputs=inputs) + outputs, run_id, function_graph = self.raw_execute( + _final_vars, overrides, display_graph, inputs=inputs + ) if self.adapter.does_method("do_build_result", is_async=False): # Build the result if we have a result builder return self.adapter.call_lifecycle_method_sync("do_build_result", outputs=outputs) @@ -595,6 +597,15 @@ def execute( error = telemetry.sanitize_error(*sys.exc_info()) raise e finally: + if self.adapter.does_hook("post_graph_execute", is_async=False): + self.adapter.call_all_lifecycle_hooks_sync( + "post_graph_execute", + run_id=run_id, + graph=function_graph, + success=run_successful, + error=error, + results=outputs, + ) duration = time.time() - start_time self.capture_execute_telemetry( error, _final_vars, inputs, overrides, run_successful, duration @@ -698,8 +709,8 @@ def raw_execute( overrides=overrides, ) results = None - error = None - success = False + # error = None + # success = False try: results = self.graph_executor.execute( function_graph, @@ -708,22 +719,22 @@ def raw_execute( inputs if inputs is not None else {}, run_id, ) - success = True + # success = True except Exception as e: - error = e - success = False + # error = e + # success = False raise e - finally: - if self.adapter.does_hook("post_graph_execute", is_async=False): - self.adapter.call_all_lifecycle_hooks_sync( - "post_graph_execute", - run_id=run_id, - graph=function_graph, - success=success, - error=error, - results=results, - ) - return results + # finally: + # if self.adapter.does_hook("post_graph_execute", is_async=False): + # self.adapter.call_all_lifecycle_hooks_sync( + # "post_graph_execute", + # run_id=run_id, + # graph=function_graph, + # success=success, + # error=error, + # results=results, + # ) + return results, run_id, function_graph @capture_function_usage def list_available_variables( diff --git a/z_test_implementation.py b/z_test_implementation.py index d7ae60ed6..51ab06b86 100644 --- a/z_test_implementation.py +++ b/z_test_implementation.py @@ -11,7 +11,7 @@ def node_5s() -> float: def at_1_to_previous(node_5s: float) -> float: print("1s executed") start = time.time() - time.sleep(1) + time.sleep(10) return node_5s + (time.time() - start) From 5aa592b6f08feae9cc7f81b8cfb4e767647aa8d8 Mon Sep 17 00:00:00 2001 From: JFrank Date: Mon, 26 Aug 2024 08:16:42 +0100 Subject: [PATCH 20/33] pinned ruff --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1a61e8aae..dbc756987 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ dask-distributed = ["dask[distributed]"] datadog = ["ddtrace"] dev = [ "pre-commit", - "ruff", + "ruff==0.5.7", ] diskcache = ["diskcache"] docs = [ From 36e8fcbda7850d073725fea7a935ac20aef887c7 Mon Sep 17 00:00:00 2001 From: JFrank Date: Wed, 28 Aug 2024 22:51:49 +0100 Subject: [PATCH 21/33] Correct output, added option to start ray cluster --- hamilton/driver.py | 15 --- hamilton/execution/graph_functions.py | 162 ++++++++++++++----------- hamilton/lifecycle/base.py | 9 +- hamilton/plugins/h_ray.py | 31 ++++- ui/sdk/src/hamilton_sdk/api/clients.py | 1 + z_test_implementation.py | 21 ++-- 6 files changed, 132 insertions(+), 107 deletions(-) diff --git a/hamilton/driver.py b/hamilton/driver.py index cff4aa7bc..157862ae2 100644 --- a/hamilton/driver.py +++ b/hamilton/driver.py @@ -709,8 +709,6 @@ def raw_execute( overrides=overrides, ) results = None - # error = None - # success = False try: results = self.graph_executor.execute( function_graph, @@ -719,21 +717,8 @@ def raw_execute( inputs if inputs is not None else {}, run_id, ) - # success = True except Exception as e: - # error = e - # success = False raise e - # finally: - # if self.adapter.does_hook("post_graph_execute", is_async=False): - # self.adapter.call_all_lifecycle_hooks_sync( - # "post_graph_execute", - # run_id=run_id, - # graph=function_graph, - # success=success, - # error=error, - # results=results, - # ) return results, run_id, function_graph @capture_function_usage diff --git a/hamilton/execution/graph_functions.py b/hamilton/execution/graph_functions.py index 17f49cfcc..49cdea2e1 100644 --- a/hamilton/execution/graph_functions.py +++ b/hamilton/execution/graph_functions.py @@ -1,5 +1,6 @@ import logging import pprint +from functools import partial from typing import Any, Collection, Dict, List, Optional, Set, Tuple from hamilton import node @@ -202,88 +203,23 @@ def dfs_traverse( if dependency.name in computed: kwargs[dependency.name] = computed[dependency.name] - # TODO: better function name - def execute_lifecycle_for_node( - __node_: node.Node = node_, - __adapter: LifecycleAdapterSet = adapter, - __run_id: str = run_id, - __task_id: str = task_id, - # __kwargs: Dict[str, Any] = kwargs, - **__kwargs: Dict[str, Any], - ): - """Sandwich function that guarantees the pre_node and post_node lifecycle hooks are executed in the correct environment (local or remote).""" - - error = None - result = None - success = True - pre_node_execute_errored = False - - try: - if adapter.does_hook("pre_node_execute", is_async=False): - try: - adapter.call_all_lifecycle_hooks_sync( - "pre_node_execute", - run_id=__run_id, - node_=__node_, - kwargs=__kwargs, - task_id=__task_id, - ) - except Exception as e: - pre_node_execute_errored = True - raise e - if adapter.does_method("do_node_execute", is_async=False): - result = adapter.call_lifecycle_method_sync( - "do_node_execute", - run_id=__run_id, - node_=__node_, - kwargs=__kwargs, - task_id=__task_id, - ) - else: - result = node_(**__kwargs) - - return result - - except Exception as e: - success = False - error = e - step = "[pre-node-execute]" if pre_node_execute_errored else "" - message = create_error_message(kwargs, node_, step) - logger.exception(message) - raise - finally: - if not pre_node_execute_errored and adapter.does_hook( - "post_node_execute", is_async=False - ): - try: - adapter.call_all_lifecycle_hooks_sync( - "post_node_execute", - run_id=__run_id, - node_=__node_, - kwargs=__kwargs, - success=success, - error=error, - result=result, - task_id=task_id, - ) - except Exception: - message = create_error_message(kwargs, node_, "[post-node-execute]") - logger.exception(message) - raise - - # TODO: Catching correctly errors and knowing if the node function failed or ray + execute_lifecycle_for_node_partial = partial( + execute_lifecycle_for_node, + node_=node_, + adapter=adapter, + run_id=run_id, + task_id=task_id, + ) if adapter.does_method("do_remote_execute", is_async=False): result = adapter.call_lifecycle_method_sync( "do_remote_execute", - __run_id=run_id, node=node_, - __task_id=task_id, - execute_lifecycle_for_node=execute_lifecycle_for_node, + execute_lifecycle_for_node=execute_lifecycle_for_node_partial, **kwargs, ) else: - result = execute_lifecycle_for_node() + result = execute_lifecycle_for_node_partial(**kwargs) computed[node_.name] = result # > pruning the graph @@ -315,6 +251,84 @@ def execute_lifecycle_for_node( return computed +# TODO: better function name +def execute_lifecycle_for_node( + node_: node.Node, + adapter: LifecycleAdapterSet, + run_id: str, + task_id: str, + **kwargs: Dict[str, Any], +): + """Helper function to properly execute node lifecycle. + + Firstly, we execute the pre-node-execute hooks if supplied adapters have any, then we execute the node function, and lastly, we execute the post-node-execute hooks if present in the adapters. + + For local runtime gets execute directy. Otherwise, serves as a sandwich function that guarantees the pre_node and post_node lifecycle hooks are executed in the remote environment. + + :param node_: Node that is being executed + :param adapter: Adapter to use to compute + :param run_id: ID of the run, unique in scope of the driver. + :param task_id: ID of the task, defaults to None if not in a task setting + :param kwargs: Keyword arguments that are being passed into the node + """ + + error = None + result = None + success = True + pre_node_execute_errored = False + + try: + if adapter.does_hook("pre_node_execute", is_async=False): + try: + adapter.call_all_lifecycle_hooks_sync( + "pre_node_execute", + run_id=run_id, + node_=node_, + kwargs=kwargs, + task_id=task_id, + ) + except Exception as e: + pre_node_execute_errored = True + raise e + if adapter.does_method("do_node_execute", is_async=False): + result = adapter.call_lifecycle_method_sync( + "do_node_execute", + run_id=run_id, + node_=node_, + kwargs=kwargs, + task_id=task_id, + ) + else: + result = node_(**kwargs) + + return result + + except Exception as e: + success = False + error = e + step = "[pre-node-execute]" if pre_node_execute_errored else "" + message = create_error_message(kwargs, node_, step) + logger.exception(message) + raise + finally: + if not pre_node_execute_errored and adapter.does_hook("post_node_execute", is_async=False): + try: + adapter.call_all_lifecycle_hooks_sync( + "post_node_execute", + run_id=run_id, + node_=node_, + kwargs=kwargs, + success=success, + error=error, + result=result, + task_id=task_id, + ) + except Exception: + message = create_error_message(kwargs, node_, "[post-node-execute]") + logger.exception(message) + raise + + def nodes_between( end_node: node.Node, search_condition: lambda node_: bool, diff --git a/hamilton/lifecycle/base.py b/hamilton/lifecycle/base.py index d150e9f47..ef4f17ea4 100644 --- a/hamilton/lifecycle/base.py +++ b/hamilton/lifecycle/base.py @@ -43,7 +43,6 @@ # python, which (our usage of) leans type-hinting trigger-happy, this will suffice. if TYPE_CHECKING: from hamilton import graph, node - from hamilton.lifecycle.base import LifecycleAdapterSet # All of these are internal APIs. Specifically, structure required to manage a set of # hooks/methods/validators that we will likely expand. We store them in constants (rather than, say, a more complex single object) @@ -526,19 +525,17 @@ class BaseDoRemoteExecute(abc.ABC): def do_remote_execute( self, *, - run_id: str, node: "node.Node", kwargs: Dict[str, Any], execute_lifecycle_for_node: Callable, - remote_execute: bool, - task_id: Optional[str] = None, ) -> Any: """Method that is called to implement correct remote execution of hooks. This makes sure that all the pre-node and post-node hooks get executed in the remote environment which is necessary for some adapters. Node execution is called the same as before through "do_node_execute". + + :param node: Node that is being executed :param execute_lifecycle_for_node: Function executing lifecycle_hooks and lifecycle_methods - :param run_id: ID of the run, unique in scope of the driver. :param kwargs: Keyword arguments that are being passed into the node - :param task_id: ID of the task, defaults to None if not in a task setting + """ pass diff --git a/hamilton/plugins/h_ray.py b/hamilton/plugins/h_ray.py index d52b4189d..179ba1bee 100644 --- a/hamilton/plugins/h_ray.py +++ b/hamilton/plugins/h_ray.py @@ -2,6 +2,7 @@ import functools import json import logging +import time import typing import ray @@ -56,6 +57,7 @@ class RayGraphAdapter( lifecycle.base.BaseDoBuildResult, lifecycle.base.BaseDoValidateInput, lifecycle.base.BaseDoCheckEdgeTypesMatch, + lifecycle.base.BasePostGraphExecute, abc.ABC, ): """Class representing what's required to make Hamilton run on Ray. @@ -93,13 +95,21 @@ class RayGraphAdapter( DISCLAIMER -- this class is experimental, so signature changes are a possibility! """ - def __init__(self, result_builder: base.ResultMixin): + def __init__( + self, + result_builder: base.ResultMixin, + ray_init_config: typing.Dict[str, typing.Any] = None, + keep_cluster_open: bool = False, + ): """Constructor You have the ability to pass in a ResultMixin object to the constructor to control the return type that gets \ produce by running on Ray. :param result_builder: Required. An implementation of base.ResultMixin. + :param ray_init_config: allows to connect to an existing cluster or start a new one with custom configuration (https://docs.ray.io/en/latest/ray-core/api/doc/ray.init.html) + :param keep_cluster_open: to access Ray dashboard and logs for the cluster run + """ self.result_builder = result_builder if not self.result_builder: @@ -107,6 +117,17 @@ def __init__(self, result_builder: base.ResultMixin): "Error: ResultMixin object required. Please pass one in for `result_builder`." ) + self.keep_cluster_open = keep_cluster_open + + if ray_init_config: + ray.init(**ray_init_config) + + # If the cluster is already open we don't want to close it with Hamilton + if "address" in ray_init_config: + self.keep_cluster_open = True + else: + ray.init() + @staticmethod def do_validate_input(node_type: typing.Type, input_value: typing.Any) -> bool: # NOTE: the type of a raylet is unknown until they are computed @@ -148,6 +169,14 @@ def do_build_result(self, outputs: typing.Dict[str, typing.Any]) -> typing.Any: result = ray.get(remote_combine) # this materializes the object locally return result + def post_graph_execute(self, *args, **kwargs): + """When we create a Ray cluster with Hamilton we tear it down after execution, unless manual overwrite.""" + + if not self.keep_cluster_open: + # In case we have Hamilton Tracker to have enough time to properly flush + time.sleep(5) + ray.shutdown() + class RayWorkflowGraphAdapter(base.HamiltonGraphAdapter, base.ResultMixin): """Class representing what's required to make Hamilton run Ray Workflows diff --git a/ui/sdk/src/hamilton_sdk/api/clients.py b/ui/sdk/src/hamilton_sdk/api/clients.py index 580762774..551216495 100644 --- a/ui/sdk/src/hamilton_sdk/api/clients.py +++ b/ui/sdk/src/hamilton_sdk/api/clients.py @@ -221,6 +221,7 @@ def __setstate__(self, state): target=lambda: threading.main_thread().join() or self.data_queue.put(None) ).start() self.worker_thread.start() + atexit.register(self.stop) def worker(self): """Worker thread to process the queue.""" diff --git a/z_test_implementation.py b/z_test_implementation.py index 51ab06b86..5668df90b 100644 --- a/z_test_implementation.py +++ b/z_test_implementation.py @@ -8,22 +8,20 @@ def node_5s() -> float: return time.time() - start -def at_1_to_previous(node_5s: float) -> float: +def add_1_to_previous(node_5s: float) -> float: print("1s executed") start = time.time() - time.sleep(10) + time.sleep(1) return node_5s + (time.time() - start) -def node_5s_error() -> float: +def node_1s_error() -> float: print("5s error executed") - time.sleep(5) + time.sleep(1) raise ValueError("Does not break telemetry if executed through ray") if __name__ == "__main__": - import ray - import __main__ from hamilton import base, driver from hamilton.plugins.h_ray import RayGraphAdapter @@ -38,19 +36,20 @@ def node_5s_error() -> float: ) try: - ray.init() + # ray.init() rga = RayGraphAdapter(result_builder=base.PandasDataFrameResult()) dr_ray = driver.Builder().with_modules(__main__).with_adapters(rga, tracker_ray).build() result_ray = dr_ray.execute( final_vars=[ "node_5s", - # 'node_5s_error' - "at_1_to_previous", + # "node_1s_error", + "add_1_to_previous", ] ) print(result_ray) - ray.shutdown() - print(result_ray) + time.sleep(5) + # ray.shutdown() + except ValueError: print("UI displays no problem") # finally: From f17d4f2433a932d6542358cd600185c3415c738f Mon Sep 17 00:00:00 2001 From: JFrank Date: Sat, 31 Aug 2024 20:20:57 +0100 Subject: [PATCH 22/33] Unit test mimicks the DoNodeExecute unit test --- hamilton/lifecycle/base.py | 3 +-- .../lifecycle_adapters_for_testing.py | 18 ++++++++++++++++++ .../test_lifecycle_adapters_end_to_end.py | 11 +++++++++++ z_test_implementation.py | 4 ++-- 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/hamilton/lifecycle/base.py b/hamilton/lifecycle/base.py index ef4f17ea4..12ea36b2a 100644 --- a/hamilton/lifecycle/base.py +++ b/hamilton/lifecycle/base.py @@ -533,9 +533,8 @@ def do_remote_execute( :param node: Node that is being executed - :param execute_lifecycle_for_node: Function executing lifecycle_hooks and lifecycle_methods :param kwargs: Keyword arguments that are being passed into the node - + :param execute_lifecycle_for_node: Function executing lifecycle_hooks and lifecycle_methods """ pass diff --git a/tests/lifecycle/lifecycle_adapters_for_testing.py b/tests/lifecycle/lifecycle_adapters_for_testing.py index a151c4ecc..37f3b3961 100644 --- a/tests/lifecycle/lifecycle_adapters_for_testing.py +++ b/tests/lifecycle/lifecycle_adapters_for_testing.py @@ -9,6 +9,7 @@ from hamilton.lifecycle.base import ( BaseDoBuildResult, BaseDoNodeExecute, + BaseDoRemoteExecute, BaseDoValidateInput, BasePostGraphConstruct, BasePostGraphExecute, @@ -187,6 +188,23 @@ def do_node_execute( return node_(**kwargs) +class TrackingDoRemoteExecuteHook(ExtendToTrackCalls, BaseDoRemoteExecute): + def __init__(self, name: str, additional_value: int): + super().__init__(name) + self._additional_value = additional_value + + def do_remote_execute( + self, + node: "node.Node", + execute_lifecycle_for_node: Callable, + **kwargs: Dict[str, Any], + ) -> Any: + node_ = node + if node_.type == int and node_.name != "n_iters": + return execute_lifecycle_for_node(**kwargs) + self._additional_value + return execute_lifecycle_for_node(**kwargs) + + class TrackingDoBuildResultMethod(ExtendToTrackCalls, BaseDoBuildResult): def __init__(self, name: str, result: Any): super().__init__(name) diff --git a/tests/lifecycle/test_lifecycle_adapters_end_to_end.py b/tests/lifecycle/test_lifecycle_adapters_end_to_end.py index cf57c02c0..bd41d29f9 100644 --- a/tests/lifecycle/test_lifecycle_adapters_end_to_end.py +++ b/tests/lifecycle/test_lifecycle_adapters_end_to_end.py @@ -22,6 +22,7 @@ SentinelException, TrackingDoBuildResultMethod, TrackingDoNodeExecuteHook, + TrackingDoRemoteExecuteHook, TrackingDoValidateInputMethod, TrackingPostGraphConstructHook, TrackingPostGraphExecuteHook, @@ -378,6 +379,16 @@ def test_individual_do_node_execute_method(): assert res == {"d": 17**3 + 1} # adding one to each one +def test_individual_do_remote_execute_method(): + method_name = "do_remote_execute" + method = TrackingDoRemoteExecuteHook(name=method_name, additional_value=1) + dr = _sample_driver(method) + res = dr.execute(["d"], inputs={"input": 1}) + relevant_calls = [item for item in method.calls if item.name == method_name] + assert len(relevant_calls) == 4 + assert res == {"d": 17**3 + 1} # adding one to each one + + def test_individual_do_build_results_method(): method_name = "do_build_result" method = TrackingDoBuildResultMethod(name=method_name, result=-1) diff --git a/z_test_implementation.py b/z_test_implementation.py index 5668df90b..91125c59c 100644 --- a/z_test_implementation.py +++ b/z_test_implementation.py @@ -16,7 +16,7 @@ def add_1_to_previous(node_5s: float) -> float: def node_1s_error() -> float: - print("5s error executed") + print("1s error executed") time.sleep(1) raise ValueError("Does not break telemetry if executed through ray") @@ -42,7 +42,7 @@ def node_1s_error() -> float: result_ray = dr_ray.execute( final_vars=[ "node_5s", - # "node_1s_error", + "node_1s_error", "add_1_to_previous", ] ) From abd87f750773383f475431c0790e506cfced8377 Mon Sep 17 00:00:00 2001 From: JFrank Date: Sat, 31 Aug 2024 21:10:55 +0100 Subject: [PATCH 23/33] Refactored driver so all tests pass --- dag_example_module.png | Bin 0 -> 24324 bytes hamilton/driver.py | 51 ++++++++++++++++++++++++++--------------- 2 files changed, 32 insertions(+), 19 deletions(-) create mode 100644 dag_example_module.png diff --git a/dag_example_module.png b/dag_example_module.png new file mode 100644 index 0000000000000000000000000000000000000000..52d85a429dd7e7cef30a1e0a489a235e58af5ddd GIT binary patch literal 24324 zcmc$`1yo$yn=e>|0Kp|#(BKJ9kl^kbG`I$LcMC3o;E>=1cXxLQ9^45Q9D>{Iy8rk3 zz1Kb6z1FN*Gxx4=>r!=2owLv0-}kefun+PQs7M4z5C{ZSN>Wq_0)c4;f0ACngWu31 zJSv0#z#Ga)h(eyA|7N!4$3Y;kAyT43DsHI<%kJ88H%|yBv!ylRSeV9Ap_HsxL7zT- z53Bsqhrg(gKdjz=dGTP_ryfFYluxfgJ1Bn{FRf7(PxZ+hKQ4Z77i0L#?AdLA~6gmrlPx16$Lr@#Q$4Al`$>4 z0}#-0EvA0F&Qr)hv$V9#>VBO~D_D`f+!=s?NleTzy)C7WfKJjAfyWex77$wWqSM0# z!fd_3#>vHn`05#=D0$2a;%AtuWG$u=qN3UTpZ@*Nk5qBXITF#YVkRQ6bx>Oc-(}l`PkjdlrC*7jF zJnD}#Gcz0?mwLa5y%0|KB`*Y&K9GoDjIO@*s?i>~Cow<5jQt>YYK?4Jl zlr%IHuU|t@2|0SUhLh~~XBadZ6^yuCkBGogr@seZ-rt{(x%UZ+QhEsIYN&ip*QvKz z{xLK(G?~PtH&lv@j-K@F>u5Tk7Hxubu9}*fQkW=ttLIgj_B;un$G}Bi_>_zMSefZq zx`Xl4?LO06$X=OF^QU2Mr`@l%^vlhzrejf|MFf`zNLVB!Q6nR=_^)0GiHX61FZiIO z^f@*bYh+{u0vXE?K!EW2+~cDXuzh8CQYqJMmHy>|#>2zY=o0zm3k)PIEUd~nnuKrY zed0>1C!YOrSZI&PxCT15j+z=S*~9`y}hcZ(PO#C$uan<_#f7g@$vDmbaB-` zTk3>FMd#JRQa}}H~L~YPwqQ8H#Y$$CZ@w-DxYVaiZxzRSQz5t zO>>tf`##&jR34 zyKc^RU?AYdXD}mkawr&engW0Ppyc(uM48Hy>rP=c1$Sk?)$`75{Izz29nA4+yM&4g zHnn`J`UXc>C~a+cyFxN560=}#X8Wq|lg_t<1S(=Khfn@n@@d>u-~mkj{{6}PJBRfd z(UE}Li9Q(Z_3jX37VP}PR2W)DMn;Fb*lL@H+lwk=ufKkwZ}8AQybTp4_y7K#ke8Pi zEGl#z9i4)rqOQ3){D-@%km%^>1LH{o0)hs&Q}Vtqgb3c=-UY?QIXcZQoF0GQXx3W5 zLB@amVo+S{%`Gd70t?#^x|Fo)8K-~!I_<^N>*((ffmI1DkdBqi?TWMlYxU&h#4j!` z4&!?{vxbI7MNN&Ux;nn0p^K^-)q2?yXdKsodn}Aeh5cZse zgv8Flfi~$f;PdASkBdLTo}84CXZ%m1q8TSx)?-J#5qSU9Jp&BPm}eN)(26(I`5#K= zzg?;WJUWT?uE-AfvM{r_v@c&!dHMK)Kp76j_5=T)l||-BBP#k_uxBtTH1rKZ95E(l z7|H+5PaPPre&NGJKL;V>ArVUlot>R+OiPMV(q_xK2nh+PZcGID`9UNpvFLSM)OSCC z(9k79&|<@{`~>0z&ub zD4M3G=J?c9cPJ*s$>rgqwzZe$Oq#`1dR za_ys$@}of1TijThoc5kUu8)16kU=rJqae?YPCk%+!^Vc~;^K00eh!^YD{bBpp!yhf z_`$d*qNAXQf{KEHjU5;tj}6YNe|LFUQCr)4cXdq1=|P@d6f{^C2reWB~Vt;dHywv0@Q`VA{gcpoT7z|2ki|-Q&D2~C6ja)~o?a`pDCcOy;E5Cbn zl@xMvcib*Qn=tek8oEnC-Hi2TzM6)^dg1xYmoLrC%rcep<&)VhaUs7`4oXT&*8coK zg1%vHo+XpS$mw;>T%=m8_Q#)1R8)l-99&gh9j9etWrYHfl$DLwnyoffX!JDi>+3UH zY2hLf@X=;uNyOf%rXa7xw63B%zBvFL?swuAW%Z=CxQfycKNVPl4*N<Cs$VzpgZ>-pkOLeQ7pVKctZe|>%mGZp^c4AW8(L3-wZDg=BviQo#AuZ zy#~!R&}O-5FtNCxK!2{nU=G}O*)2G~2q>*~>s@fuWjb1QsTA+RLir8H%04^&nMTZ) zPy3vZfCC2LL$ix@Bnck@ctqkCe||Y8ELR#K_}rgL?C$Le>F5xHuFDD9pOv-s)`F^< zS|30!6OE3h;323^9|4d-doYjly4=+WbYY%VU1gztNTx(I$?imsMAqW3iHXiAVx9`4 z{wVF?W|zZo7Nfp_gyXr&oC9|-;bZ&L|5;a_E|ejT^O!Mnp>?JoRky(d%I*pSFLBT3 z6N0*qHUT`ld1(zg^wPIiN2QC2Ic5b)0Ty$_J^+ zzi~g^5Nq4FwzASUGaI&T6g~b{z@VuB5(b?o% zQrPl-1Vg+2Hby!q#Y&DmrlY$XR$g9S*ed;b-?@S)&OpQ!z+sYqlmCDEsS=2m+43Vd zmyDN}=;`r}2|$@3wEW2r%E|#{WsCq(d?;39$m%8|r+t|%?P70lKiBH{9*y`+NPi8- z>-t1!_bV-XoXO7<>vkV}b_;B9KpX|a#E?k zRPoGsu@*Xk=C2^M%E^mtJyaUiDBK|XnLYLxIafv zuC@8HHvqX~y7Qc%$H0$w4qI?k<7+}7DRLHq)CiA|9r{|e5&5- z#-XRLwYQ?MItVR@-(dcN&S}>q8u2B~8JQ2cl6nI9d@mpZTN%DxH#=_&R3-Fw?RHsn zcIX8^eWFAo;(YD&MpV?j2z$`x>S+1vo44!x$H9oiNLyJxIn)(LE64zrU~*%VXY;Ct ze{$XZiq7qF+>6kMldRM3{{jiAXMeUFd~K*?R2NP2^t1{9A{)bu%YkS?NaHhk$no*< zTpxl$v17rzfH9C!P{7J*ot55&%gX7%iucLc?~D>>iI9=U=9~;3fhh#>OKV3WcHDFvB;#UTOag%$XR5rWRnX^g@_DPv5-#)uYALbSZ# zxvN==ERnZgzXemT<_U`;Nv+4F0e1NH;qqbfWZ@RC#nq~7e|KkKXoNmud`P9|0)sP| zV51|iRyah-5GzddpRbh4%@`{y_&jzeETcu}OTvlNO)W>JsC3QCG*#kENYqZs;Disr zhFXG@i|wO(5WL*Ct^#vb=xt;pskt}3r>b!td@57f*WH`-;8EwE^ODfiuGNdNQild#Xrn8K3?W%D6h4tx2J}hKXN!f1w zMCktR3YmU!lYx#VLAuhgPxLy2_gnMh@?vNU53)1r7wMV@^jEKvpEfZ>mtG_=3`;8X zm#)z@c(Y@?#HAO?7+c{~o=yf7Mf@*cbH+A9*XiBF_~eW@as-pzy$@T|YII+Z*s%1k zquk991A*~Zi=)riKFm>+%O zgAqX?dv=ZNL!lb#>H{wv-6hF0$UXDcM~z6h<$Ru_qu@}&tOBEP$kSrW%k^fv{j+%|H8nA8 z|6+1#((a-w1*=yUbT3&Ks_DFd+NCNF@h=d_pcH8adxX!~i(i)0qFUb>jhh~l#xjJp zB=GWZ0o$zpV}cHGyErf}+Y|&4+&<*wECpxLB-5-GFTRs_W4K*n zZ=;dYYn7Fhd@gSz4n&raeoxunFKDWL)gDOKMpNgpBo%`R);!TFvr>hBV|@5wxq-uzqiSON66^R)xTD z7g<;^TQ8#d9`Hg~hhx9Q-^YteAv(@j_5EH|>zLkI zsHE1nEnPt4JuxLnyoUHQ7KkKjV3=rB1Xb_%}(`-DV-NUc=z=nK)jQ|;q2_} z>RO9y!B1jYB3aLtkdOq`)Tn_d1giMI&JO|D6aoq&d64b)M|Hetx9)`nc<22Y9MBlZ zxj=_OCFLiYETK>3t&rIL@CKbJLyYXY>s2JFfIO5aovFNyO#|@68I)D;t2C&{wNEX_ z80k$+#V&B;)lFsQnC$nq!qGP{ZPyeuG@7iVn15K8UJV?zvmKcIU#EId! zJ<@smG}?7o><@Z#ZZ2hO{|2H!*FXhKo%J%U6Q`&sT#?Ek8U&X~H`3?f)?hx~{zbcs zD(K8VdagO?3EmUEJ>IatpJSQVvH=}QHCM((T3Xs}wHo1axxMRNQN&HE`m>mTdtl1w z!&i)R0wCQ5#l*ynlVM`w6irg`K-HKIAtt7gR4!P{ZTbF%rx8IftH^PKFin$f1ftdb zV7>M`Tygjd3<9iYK+nnQ22deZI_1FcU>+lpUgMdqt?d#zVpoH1-N{O27H|s@R+k!&c*_P*?y;)>n)x+%a;}waZ_E&(pZVKv~MRUG?Gg+`yNWl?_CT z2`!kT^{#@gB9&2Zup`TpP5BJ?Wp{r+0=Sv(5OmT&v{EHjbMPE_xD2NY6wQ7WC5*3Q zU?wELiPb>^vJ1e0aLReILYkTcnddGO%<
wwNg#t5D`%o zYSDxP{@lF0V5YWf;az3`wP92iU}!}}Ma6pma|z(zX{t24B3nQbIyu|G<^z1P?YWFg zR8*9(^*3xkAWiLbQDh@g#7O5VWd*zjBQ2r(pQ*C{CNccG^Z)O^r@68j+1)jJ4i7)X z&%?!qGv}9|Z|c9Vk#P}JV*BaSb4$zpXTumV*MqHuQ&UqfM@L7i#)4Y8mC0kU8`3W{ z^agIhrKM^FgtnZeU1SyI5)wZGS()a<-h}j}-Pw zYlMWFS|WJX30zcERMRDe0n~kR`txPPKurna_r4Vr5y??GMB)a7Y3mp~EcGU*w{=$Y z;ekERlk3bQwyhVla=Kre%})D^hSe`gdHvAPko3Zf22Y;+^yD@X^GkONc+<#Xk?=?f1FBa@=Bm62 zwRLnVo%WO>3D}XOd+#gzGlE5tD1o93gsi@X(@lfrW+sk|rpQhKIm7H6S{aFQH25br z#P13^!Oq8R`0ssq!F_zWzP>iBrI@W|1+v7?Rsq;+DlLSoV zZQ&5nAIwFw1z{PF=Y&n@_>sLY#dC}#EP*gdB=?&tb?IXMg7AdW>0MoH?Qo}TW`O%N!lveVBW!T|&T;JcDr z;oHIF_g>6j)5D89JI5y`-WD|A9nWJTzM-Q-j*1F61s4U*ZJbR(OAZbhSzQZna76?< z0sUV^B5EFTn8wD&_1IOak=2HPB*w7t!bBgPk=5OB86O`zM@PB}dL|<*L`pI;n4jP4 z1^ZXf@OfObz(P{hTuh`FTr13RyFM}SNn&+>6({*dCXrrfAp`{_g;NexSlU{$xI799 zI^^L*hT(dYIN}%x5F;v#ht<4z4C#wz-^>$|g}%|cAYeZ|46U4NawY_CpV3n!9c1U^ zAbZF0DjH)D5k;=8>1TBhOo7Sv^=q;pNkA$xldNdaAC`ev$5<=tfA^-`%aUYV0CjFO z;d@NRm7eC{^_WZTg;~6A*b~v%gQoasY?URC`{Th zg311&?+foN3mCJt`6<(a)r!h-gEuvx!>{REyt&J<%11khA&w&>zL9QJ_+>l&Tu~wY zcViTIB0vU594}H70VP3HtXLeCl9KX&yg|zEe^>%HtQGc7SSPqy?M)^pgXqZODZ<^B`bX`>(1`I+Dmu$dV~GV!8$z#c0?+8 z$(N3byz>lF1P^P zxVTt|zIb}Tfn?^EqDGH<=5mu7ORaBL11X1Kq24wc4jET}y({Q*Z`MF;Z0yMnpo@!( zgur+B93GBDz-|EwnI#G#2TB@`>*$5Ck{az|W1-dgJWhX~*!kLiBhzeE@<(5a?O&@e zkpD`kXA1Z%w!I%~qBE5h?%3O`8XF_!>j9rUIlEDgOnDH12zpr1`g~_bGVp(F{6pQ> zzYcT&DA;?X=rh@6Jt8_J#v_oE#9Wj`2`bNx7j2Uko zEw=>U2%rf?`^$^*5(E{0^-h$o^ja8u!GElEa4qe~??~Cx+B$&8v-i}h^1m~ZVlrG zw1Gg&HM;$~Cmf>4$Vd!S!#8i~j2fHYqpgN@**p=d*Yy*wM&ZLQFWkD%$V5hW1C&#F zC=0oSTMq6XkEwDFS+3hyKV*h(DW zE6;>tbmdJv>E-jKz{X$wq$Cn9hhHDA7?X^vMDw#$iIF*uKhIv=CD*$kGZOP?cjZuw z^S4JxI87Us%z3`xug_Qm^ML%~k6&!Yn8Klh2DOaSJnORqznxmo)(vge5z+3D2^ zQt{ye2RQ{hIVEqGAUHBIGLpHI(i4stT?@6AP+|@YtIpwJ957`%uO0+Bw&!7n}=jtm)-Gt1~$g1?9RZrbQM?VKQ`W%KUz&YcA;j_L<6)t zORE{p1Lan}?*3wx>1)ajj7CbziGaIoT}$*ix|!`PG0>$ZCfZ!fxN}t%?|7{Xku__p zh1peq8TNksc8(}`q<-l_5+1eeDFh%r_r)#zG6Hs~>Bl*)J8daXm!P!hvkp&}1lbQC zFd+J%cJ5B)M_ajr7ZU(9M-ou^7W(((0~+FXJog-|_nDFJFpv*wYC@phnprNtT8$!# z0IJOIKdOM(h{SDIjnV!p%F2>*?CEcVf!q^c;6JrRid#6umW(vrv>Ptp>zzkZ{ON4v zP5P*;8e}iSI=_>ej`{VWH#H_P_(zW@WxGN=gRR93_eVlGt&}W!m1+KBmEvGfRH(u$ zf9$YIEPYSb?w|bOe8l;!)+RtXsOLkxV_u2IZMeBCSl6W{vUDL7x@YItl69b9Wr zT^$Fwe3;+9HsK3C8YCe_Hoqeqlh67yuB#kB&X;eDjG((c?vh0v+4q(lBfLJ<@oMZp zHUE4a-;}3TruRVAxRhIPwrC3EHL8`dD*g;Fgp-+I)C<+bxoTfkES+%l%n&dHEaHF(4YVn2r(w2^Izd zRZKmvYfEZ_nJm1@-w9nO}wKhk>Y=4QU~ ziFJ0|;w~Da5;GC(Uqcua}PUM59)4s=~^~9NN zb;L(E$-k2iopq($w@?X3$?&q$%RzvsZq6(vS{>h!S z7#HbQR+odz?IKR>up#0p}-L^C`oyFaX~={0f$xSa1ztzOsQ7x3Jntz6SNP3x1sR>M!hx^ z5Exi$_ay}%VT$v9TKY;R!tA&$3n)V-h%x|}CRD6e%Ixr45&+5pDr#3(*MsF|yzT96 zPWLlPLn9-oI(vO{)7jPa4AlEL9fAj^_F#00SM^11p2z`26Lh;NkopPwoD7U^w7}mWWUhmO?(KCFjJZYVEBM=ZYzU4TU%5H5Xb!az^2Z=iV1egvW|VAXekW^ZCL zKN}Am=P1DJ*B>A5I6W_|(l0zN55fSB9^`igUQ@Nn&)5sj@T$Yl>KYoIzkaFCTpUm4 zB18R2Wo2a|UN?;Pr$-*Y$wNO{F zqmHdU-Xg$5psCkx&O@`9Df+^~XZ3sLf{>qpfxZm_iA?1}A0w66Q2PMcFfuy13nV>X zl4m`yqgq9Tbm#vRQrCy*K0Q2pD|K9HcK-r0A6C&XNOSFR$fSDbw{hsGEA7EAz;1X; zw1o1v9|*>h(vlsYD+M5L7L5=ghldA>vlUrZ*oWb-Ucx{u{t)c43}_r-hL7P~c*WNu zeIFa`?S%sdB=G1v!HrkgZ;CIqxK~R$lL8U~k4i9rwVV$aPJ%5TH0EL4*Eg^8yQgPT zPwQyDI$v9unS(?8eEa9=^=be2lHB>Ldp4^%3_zee{b3PKfH^5DCYGhw?(+f>v9m;j zlym?U9bLl2ggP=ZGBZ2-nX$3)o40R;q@<)=cryfi(~GS=J@3e)t7zr#w(N!(3k_ZXIw79)f>UFf{?Yr9VtgpF!;`i<#PNV)>hxwOLg{=1&`J(=28X zAbROf*SzX`a%J)PRxharRq!KR54u4i`E(TnER{?k*7h-IW_vz$Cbt_Pv=Q?+soW`6 zCz1f4<=LJ8XhD`1^8XOuB6O}cheusdvz0%VoNN z3j13qzsB}gRD7yea3##ITHg@7w?P4QVt03U?e}0@)fU%XyrUiq6NuR~^6&5M;kY;V znzxfjb>};5SDl}o(N4MG_=EX~a&ONI|H#=k=d7xt!t8X0f|ACY)8PPh)Arw23|y5V zd3;GoNNYVPCC(@YccPs1o9|fP$2nXk)SEGCBX#U^U182g1m(wTWt*G)BwfP#>*Q-` zkI_E_96d|5TE7$D+e~1!!lS=s_Q~Ac$)Q)>%|W20r3DZ*sI-*9?*N8^L9?a<__df= zSh2RP(0~u!wI1oyzZZY zoRe)nmDf0w=(HNK38aGokXK?~n4X@-es8nX0cvan`{(KcP!Gk35VDIbCay7euT`09 z>i&cR{T>7FoFJd?iu`)1A5pHIM2UU1Si@j&w66~y6w(lIjo;YJKmYyflq%p$0`gr! zm+4?N1b+RB!O6{SxY_r`Y_*MVrNupf-EwAaB!x9Mih;(zOrsi}nA=GZ_-0V)s6?-w zAE?q9$YAz^J7uw5;l;t$P|L5 zq@r2}&^DRp{oS%72y|7Q0aG8^U=9usEA=~{Iqi?V(SD^ zH50zK8TfKL3=$o4&-Ufhd0znstoLxSexg+C{k#JRngIX38vtzZaOmu<{#ODZzStWF z2M5~ZJTDIhtJ8sI<#N1&`u=^r_OP6s91AGR^E;CVcRK)K+5H~CZ0v_~FtfDW91{@G z?Zam=d=4^m6^=Xdf5CUj8eXf|FE9;n3W}cjEFMM0AzZG3ny4$$E||!M3^+)4jWy3- zms^<9MOO@N(M}chs%DQ1+B)mSV32ttr=>*#ZQN`k8xDl(RCl!mK+3=lIO`+d-=p*M z^Gixf`lY12!e!JBwVW-3fjBuk%iyenm_TMuP5^~O)a$oz5%s)I;Sw2j!UqNhyyA2~ zSm9(!LBIf5rcka`Q&V$tSifqxH<{ZQJeg#&Do_tV6DKF<+CU6NgYOeR@L6Gi#KmSY z{Q^wJ{>WFX<39%bvpPX|5p)lL*U_Pu3J?Va_^ROGV6&+_RPjiB7*O+`)u~r()aWJl z`-jWRDtBS;kff6zIBG0A($k6<#W@iFR#XZJ6`|JqyrQ+#L z26yn*l+P%gue<6u*J}_2fjv4pa=Scu1&|^pE-pL`4Gm}nFF{xbCvg|VInNXuZVt}Q z*E(TKbQzvHjDf&^t}RcweRB|6;PyKz&sV!TO6>9*` z77UL+nMiP^@o*xLz(m~(*g&KK`xk&>l~(ibYAk*snV6WUR~iN_(anWQs>`~F8o#EX zxcl-K_`}^Gt29xdNCAzR9xl|nJ>1rIOkiTh5i7ommCmi|aRsyydfR}5gw)ou*V!zG z2L=WTYrf7t`(S=FkD1Ujm9KzFK%i-J*zSH#z3Qw~sB-!F08{|*G5xdOgH`W;etLZR z-5{#+n0z2P4Pt~UWwxd_PH63)e~+L6&=9(T$O~-A;F-ZrV=U> zp92C6EG&=;RWxmU0heK9Lxc80Iyw=Mx&*<^K0sX|kYxlElWdBj*A9b>ru;nFLpEUb zRdF5CeglDms;YOCScp(H1jt(J0JB38B7lO7lCq+zuYfw2kOe|LP!k&@a=N1h9z#I} z5${Hc1~Y*a=u7V=TihMzySFk%d4)u~HggcRKK)yYS3jjb4aA3_tq6dr@#$&NF9huR zHa1ZiKDQup`M7oIk_48J5P1w#h1r_)F`lk45CJt10pQ_2fYF#ip#w2GxSn(!s<^LT z{YKMxHo>39#8nUqbv<6`1?#{LjD8za&1WxeMMliGjMUB2Cg34K*BQVDAZcd-5SFl( z7NJ50zZi)9f}RLRm3y{Vp{1#LmX;~1@|I;@^&J3f5I+X3x=iq?sx9zB6!2lL&G(p! zB)^*=!ax5Tj+ypv$>L5B3^On?f(0dOcYojLV6HMWEG)(n#A&;J{lekk;258o0V@`s zo}RwKpgUNbFF|1u4Gk?gEsYqI1(2Ey!K4sh12xRSy1&2QX!3^?Gc&Uv^Z*71`}^zD zGcHMplamt!0%|tBLt5-XdqxInlglBJwY9anrslJOr*s{~MW#PojCA4LIyecfrZMOh zHjm}_Q{C_nKfkYe?sFt_ToWFwu6=%AuA8{%vON2M2<&s-s0L*>1FZP2k^ht&I+P-# zp!9&JV_a?|4di*i*)TCM{Km&Wfc!48;3eKExT6a2RJ1Pu1^EKv$?Jf;0NvHH%sDG7 zE1A_4ZE|uFn8JgD(0oohzc-SAfPh9L2v9if&!9pXr{U3TIcaKYs3(_pM)yxGdK4&xX>{CyN^mp^>1#lU3=RpAXe7!9HA^2{N@!@PAYchupu+$G+5kjh zPfkx`WMrx4eWBD47?J2CydB_1>;Rzf+I%a2*UQ&08sv`cL+Hzp3`=E96`9PBusEvS znWYt7d9L&ZD3ws!i#}C%IAnbX%?^Yd^FvMDtvXa%LL^5(oW?OApz+pFLNc%WJ1;LUXrvPW4E@-Nh9_F&?bsH+f_!MI=KIN z_gs~>@2lRive>V``%AzpTG!&j2RmJ6>IZah9A<7MN~Ec3M|*oA;O_v-4;Vk3R!a?- z?`>B`>b<~~Znc3V`j-h%-9S)Xy^bDa&w#|i{O%o88wNL-_2b8L(4;j!-+E=sK`G7Q zD?&n82*{=x4}+|HAe8bf+^8I{v?8zWDVfbx{9_`*BA~5-`WYG?F5K?(fPsbeS)E8@**@3GY!yiI$>Ep4$0Dj3o>_TySx5q1Xe^2TXkY7rwr}^qMs@2KfwJSd^`hDxgtObCb@yL3{vxMH9UaSq4c4&(uvYu0w z7;yl6FE4=#Mv#;X1WF0P>;skm*#HUtWVRSGG$BexM%Fl#&gBT-{&1;P<~yg5&f5tf z`b3Sn^87o%|GZ1OH*}(x@LC#@8cuG*Dm>aj5L@+;HZ%5h*a_A zh(P2wL2G+=HxwYr?w>yyW*xu`837w(F5P=VF+uFS8vqewR#Cj%hiqtdg2u@Ke0Dls z(W%o0($f5Vqqg{`zMdW^sCS4){tX)c=mk>o+0xTSo!*k8wq|E%C!`8t>8o2|QDN5b z?8Erh{sF0H*s1;8{hcpVC%bH*?`%AHoxd-g+|m5^KITd0KCB?ARr_sCkx8Sp@J`GW@qp13400MwKOlXCU|q? zYrn{J#Kp)!214)aX8XGvcZ2g|>TO?c@E|8-6tId{)d_#whcuKEIF0^_(!pDP* zjC_AFtJhmJl_P=v7cgu|5*+LY00;-Z(HW3DdL^q~MQHN; z6)U_^U*uc0*2yD*4=2qm5tBYrqb!P2fpE955FjqWKqO_P{8ebhni*ZxYkszl-&S?f zn7Cin@RO!0EnUJ0YG}Y&bI)bUArL}3TEEYp?0Z|*s2zz2o8#ocY5&O6`snm<)6s+FX?hjkSxC+ELJ z<^S~Q_3a=9(Q+#+%ZfrU1+y?{Hn^D!C8`1lX`y53NC4cNy~d8G~l_4Bz3deh^j zK_WaH>%dt2K}jQ|&`uI5`>peG-_~G?)e|=cFPc0iJu&f^YNYT5fo8Gn}#`F(5Re#h?#^Byxn-b0gi`rIu>pyQJSO_3vKb# z0?`Ojva?Z!`y{7HHNn@p1edncvAC^eo&WHlF zL9MKt$r-6|hL_L*k9$FkrI^cRd#!Iq3*+zq@$Bv?v>cfNEBqfHP;z|x7F)yhyi4b(9SrYk1C1ukQMvAXZLKI{}ha5X0WHEvVd4xfts?8WoD&4@RuQW|`ZK zlsoYE56WFaHfX-JKsSe)Qo4m%WFsyiCG1Df^A-=>AYZ-Kz`R6UA_UYfAnqENn`=y( zw1%fbqc8yOB_DfGP~<%Gm8$07N7iVsl97`W0}`RlQiFd%!JD4mUNR8K-K_{6N(DPO z0sYhmrJkw%9t#K#!A(sb=HD+q*MtHi>j&iQfsogYdB3`?YlbG9_GerYOIWDn!F3Ze zIXRX97&LM$zW*CbIKu=s2ik$>1NgtL2)xl~R; zh|DeH1g$)sueZJYF&GRnJStJBp3eHEtJNGREJHRx%1!1v|2RE9 z)g9gbh&RLMfehAJ*5}dm;QgW17!1M)fhX;-)6M>*oo%q(X^X+eTiNd%sJ;&uc&WbW z&+_f#vWoJkozz9e64Mjm#~AjZ$w+FsZ_n}Y@PJm7)Sm}EEC-0xe~$$qcuYDV$zTb! z9)MP+VhGkuhw-S2Q}upucZfGwESfXlPd11xv!%@dWNtV(j;3mEN!~4(+@Ibjq@Ujk z8h(7dHeYIRn1=@%8C5k1#T~3vBd|VdM{ip%Ac57NVz!dUU{XF0^&9B5>PkiycZ_AT zrGEn_x}vJ8E4dMQ!gekiWFu9!p+5IzE;11JKLhC|LA4s>7F9GPbCu>6T*S1cX%ibs zKZ}xMa#LnIu>4!y8Tzxz1u|-?xIlas)Bo_g@2B2~s6;l0$&qkiV6Z;mWX3bpo-U#Z zqI9gRY-SS#%1LFo+}PO;^(IG7EnfIhC(W+I*<2u=Od9}%@eN(k=R`jEZ^5P!oQ>~U z6quMKQD47)eP>=iH#iFNjUZld`7v2xc>ALNwnQ)eqIcu^bd#Cc;>F8q4oMHA4hV!K z3;Op0fM~A@88pP;5iHILwmhcPeB97L2kU02hG@Uq$9)H)N%&KrsyGA6q#j(s(FjN&id#a^r^@YsOR*!#$&wDrml0!=bx+0&yVkO@8cA< z$PHK~&81cUaQhRZwYqNVSX2R)+ubFD#h+i`f`nQ;9_l(9j zIPl!y9E+o2^)E{_V4{ESLq0`^J94?b`?#!dZ2GIr6f%3>`9Vod+{+5ZrMX0Kpto95 z-Xl7XLCa`vS_2yHufF4a1&Gka{ads~n?#1<)dIu?mnt|2KvV&5VebYmCn(sS0Xr1p z_Y~>W2eyTP7uG9?bUdCAzpKkHc6W}5kWA;hDD}D>{kBB~IjO#5f#}#;<^D_VDf@1J=Et+K+5_baUVNW8|2g`_E;L%F*-Dg^G`ht-8y?QbQ&ht#S>yze;Mz}t z@{{e1k1u#Vk=p5X!$1lH7V_7H1gYGb@GIyr1I_iP^k*;t>yj0Fj_z!tN9qQ&hs0LB z!`fIe+;&}s#siG^|5s=&_sQQ2%-51{7cynWz$Qe&zj>M3|MU!`Mgu8_0xOV5iQe?wqs6^5REWW4(K>Fl0#E!4mr_e(WlZ;krUMrac8V$eaJ* z0;*R10Be`6HXC|ddZT9{OZlVtNO3zR?rwC2pznPh-*J6#Nh%8>z?j4Q0=|DS2=RV) z5AP_jm|Q+VAbW*{*VbKc0CL|c`i@T;vj3Q52l58D(L@m}j<~-GH~(-c+k-2g-+_v% zd-YF(v_C(0k1Im_z#fz2^?L?giKMYcMs#R}f`e}L+q}v#_Lb<)Fi2WuM0wn{j@V@AfxveVM#9aVWS$wM~KQ0=U7wD;40+|o+m`#WqWW|dqf_&b zYYE=S=8!u?0he@SgAT&YgfTQXD@t<87k;IBOsZIExe4sUb)M_Q*n?Qn%lO}uf7h=5 z8MW z|4}aXAkYnMZ}*Q<^(Ar(%LA;jj*>9%gn@$Y&-kaaIVORJP|Y;djzO=Srcv+I%9=8{ zD9xWKISM3?hXQA)+*f-N7X;K!`&YEyx_3LihwW@;Z$i3l*PK~Wx6)VUcsQSZP5;(q zQIF1cyHe(F^_wx*Ysv~BuzcUDRXlQq>RZ+qos9q!^&^h#(Q7R@sl zbL6}G!0*p4mx7XaibpT_9Q+PS=;hr6+!-%^zH+qNoeQ#n_LjY<^OK1oQwdjVCI^8j zJcfLqms<1|8gD7Q{hHrD373vC+Lh^uw+%+L-sykR4jA>mv$Rr<88DV5ONX7;&>&i= z#CeM2zsDgKlQw)dKR*w*z}MufaMLJqr8`}=`nCDJVVEyQe=lqMnc>)zS>z-MO^?LY zrlg#_(#&G5%XFyaY7=U|^qzWTq~-Wf;mZuKoz8}I;(?z(souX&QbxEkxO)mK@@|{Z z_`QWOsjU10aiCdc-gWl{7_J6Sip1faG5f|*BUKl>f8w>PHGjU@jmUM$crK8j*^;1K+vm9wn zlX=P?U)$E?+qai3#$g)x+`QBWgMwzgs+7$iow3yyNu@W@=E;>vE`oWr#^!hR*VfqM+Q)h&0< zQ03-s;kClfuwL-ru@wS6XUIXpmgU}IEUHW!_c(_BLS`$7!Lc|^tDJ^l;k zrkF|=_+KcEp65(6K4%9?_N>F-oE&{{2JYZIPe;lj_$tNX8D6Tcw8X?74Jb;9(TAtM zH&XPZWTdw1HznAP&dn{imagV+FO<3h7_pw#_+Rar2O{Wv8mtK;z-i>uhPSXv60jH3 z8I-9*BW})ADvbZ+x<6(K^rmOy9W#$9z0?7HExmzU=LyEqI5H;-4f4fC8Yh>82lp;i z9|M>$pI)Q!1z5o8Iwq#2sqWUTGB|9G>Qz$mdklaXWKMajTaG5Apr@CE<#2vBhREss zIsOObq*83ym++3O0hq+L_hA`6Z{`QE&yKDx?OtWllP+!!2I(oprhohVG`M*`=II({ zltfj^#68^1A||%JwWYeEQ-BRtIdr-_dpKKRuG$*zwYqU&fjo#u`po1(S~j@hS1G>X zxPY58Ff=Kvv{^!MY;?LrakcGsJlD^Eb&T|& zovXCYZB1R8%57fa>Q>1LR+scU*m%m1psG$=PX}uG>MHYFe(a$dxeu00l~xmnPha(d z>!?c@9vD<)JxhCf{F&$uDjUuCSW?<5E|}{!8DbK$XUQJ3*s^5%J#&87_qw{S{$YGR z?|I(aeZQXjm)BBXeX~EOlxNR1=e4bFGv+|qrzBLGTs4E`KFP05{Vqq~=eIj2JOU3X zrr{LAWR`2$FVmzlsYMR2p|`M*T@n+*s8ORC4tMLsHE)!2&d9Cy6`pR8wK`R`BBL_Z z{6hXTucX!1?RUZDbM7^+^Iz}h+@f!8_ttvpBfP*B*tp0q!NwbovD0QLG|2HAXT3IA z9wkgPMUvMoDA!^~MTc&5k@e2Y`(znre1JF!tEc)IjMZD40Taok+vlylnhN^~EG(y) zFE`-K0L|8NE4MbyDx~ z5AS=k99gQl)}M!0o@K~#7RmqoGBBL?vRIKCSpHYn2G><$;W?f%-whr6pxqd>1>^&; zae6ZI+;si}FHpL%_JPQ9QK9VdfmkFwy&*~I-c**LWYs@kZw-^{w-4`@bw~F2^P_~I zYImLv{-RG^Mn<&{Om!SN@bOFdiEnR;FJ3eVys<)`F6Z-lUMep|y(!+9Eew(C(5EoF zw2u$2q>P+{XY&31`B{9ULK5nPU#eR8PR;#tPSvC|Wx*0B{a_{6&akdce;M}=xMReX zBiDw$0p%;39Cm7Y$+5YG3AQ3(S7h+6>vKyKJ|?9_!US(fd?Gd!_I2X8Pv+6q0Ytq_oF{uNz$i;ar5||zz2Ov?PaymJfYoJ>gG&M{^rqR5p@Nb2R?&Cz6f z_#}aZg!arK1P_k}Xx>)VL(s;Dl@>{HTRZGrRu_^S5j8?ojoxvn)hZbSjAkgFYxj8LW2{u zKC!B+tHWRO*jfF~5yG~x;78?^mJ&VuJ5ESW2`n61haL1HK_-Yw-!>_d7fhU0KOX=u z;CZycA$a;VpB>Pg>9@+K@TOW!5B*BQf5Vx3 zG89syqgm)+OlTIuaH$1hc-Y7>6^Hw*Dba{}@E}edf8sQc{>;B`z_%=S9u}YUTf6`$ zmIl0kvdvYk`>|BgXa$tUUCh8W5s^Io8Rr02yb&WKqpiI?2QP2;S&P6H@ZV#B$Z&61 zqlx=1LqpXK^;b~IdRFQ_Cs@xnkBo3bck0aSER-Guw7nj!a5(#`d$PMb3i4qP=g!@S z&Ji%)I$By@uC6{yk+3MK+}PZ-CB$9;nfBS{(nD~!Rp4g;2nGglhC(TMA|L1i>XX)A z*Iz>75WL*l#zqeCAwXfJL4#WE1R#7TPo2sJvzQj(22dY6Cub{IZSbdc-!z>tGBSc8 z)ds*E4Cc0P-_ldWOm775`)*OEEchjp=XM$z8ffhaMy2LL3JO?I86vZ_16_NwUc7(b z@Z$-nfX2`*35Zsu>v#$jJD`9QXi5gc;4avyz$oMana{!_%><3Qg}_QfX+evvCGaT6 z$DWf$;sKZeJ|m*6Oj6N*bGrE@1F$d7NqYglzL`Zu-5^XsCQA#%McLTc09oAy&e<_q z*p<2qNbN(V7H?s!p#XsbiD6Uj!ETF&ly3k;4?7N-tPQDsw8GF%leU`p;0@)P*EZE2 zc}5ht8Y@7@fPF|Okz8OBc7VzaNFDRqny&o~@vb#*$~O!Y7tP)UT^&Dq)|K{Ew{t>6 zqQ(r6lr#e2Yq2TBihTH;8^bJ62@A}4{qxd}N^-6p08(>vbMR7xq^o*6lzJ*;9X*OC zL+b+bW@|^sE5J-SE*#i$O^OX6brV{Y>E{}cE6Yh{^)%@U z6qeHxM#+^ITww;u6aSf`pdEbmBw=M|)7^Ju2&dz)l zETJo_L1P1ETr>cJunL`5^w&4GNSG~x;w#%O(%vtZkLZOkmlS$}Q9OCHBEq!B?Z8z5 zbX1`~jXJOKou&w-E~{H1DS z9rUlE(P-&H)$a8j4Gs&e=VKmPCfc%OcR$of)V>z9I_Krxg_|lYTVVgEL86n5on5#_ zuIg0tv<0aEVW0`-+HjS#k*v#z3QQeGM@Q(eoSvPfC0{~$UHt;U=7v`Kt#}Z@X*pLL z8?#lz=7OT4QGjrWk|D?Mo|6+YXm4wN^?NI*cf%`LNQCk6wBM*WdtM zHNL*SPv5&kdKh2OW_{S1X!S;$+AtT(fFz_R*L$yY0I0kKW=enXOxqh+J0W=B_&XzVdtLIzayjMF@HCZMixPmlXfp z(>6e!;VU-LWlR0{?y#!9nnUab>)oD$+<5l}WDo5keZm`jbE-Jg+LMG`5-L|{WAvmz zq^7!0c7KBMhcHCD!H)#9qr-#w)O3VuD?Z*xuGW=P;MhkGxIPvRU(fP6VDutQq^3s1 z{tc&3yK;+O_a^J@=S9XFvyv{RDTkVzs2>GhT{=tx>k44!t#EzFGIS7K;owueyw-OI z)P-HCb)=_}`<28XLYSF4%s}-mV)x~yD~ce9Ad(HgCL@~6Zk+`Oaxz^qm2z-`rjNkw zN(X-im)0X!oEslRm`Hrh z3feZGA5nlDslOsVg(Fl=hf7}_`%XAo&RC|?;Upu?!Y1grxXPeF}?o-jd?{w(+r@x+LlrHUEYgEwuU0F~urg5rtzER+F zG-)IuUZUD^VWkvpJtdCDt<88pyaC64Reh6`AqhGGmjnNEIg>&7S3cT!xgdjYWbCzc z=~pB9Off(EtWu4YJdupAUU`u+w$(H9MYHI3j!3Pg5o}jn$tH471Koyy-M7;4m@OI{ z?cr07T+NK)A;T!XVLAQItOoaZupn=~MgXU_Vrw+%{(yZ(QbMA1o$}`uhgpX@53$?P zQUq3yg%N>hZLNPC`*;ZA6CM{Sk-5oL{jySmq2Y-g&sO|dN(SuvotqKwSB6A{uEq;0 z)RgVj51!xJ>`W!8(lN2Iwvn+hw;D#y=`m{xnZx(9x%o4B=6BZ_i=BA~xTE&33<}ih z1mMSh@+Kr9bWxo`tbYFYvPBvGe+W*ykZH2mi$uc7^=-jr^p(^QHvY|hx!`XNFMDOF znRPY+bg`5%9r%8a%o^wyzw1({=kzOPh)16nQt`oahZ*B^`GccRnAuu8YZ;?zNDJYN zr6}RFtAnxvX?hVBX zMY=oM5kK5fo~>R8SeACuy4UPp8xxeV{pyxz5Y)W7kLqir$P(!s9o*U&_yxscU>TR+ zTq0h2-urRF{Hl_p!Op#vAMg#$GQiH?2SU!6*Yc@mzSLlc*cC)9b>ihpC8j-6AX1n1 zQ5quhq#JzE+|2^IY}SXhs-Qw*e*E}W%S9~lE_!D10rA4<34z|y#VHbQ#=rgAovE`w zezTMihr#Kft?B!$vV2|lQw%)um;cHa72w9EJ<7AnQQbcB@`#jOWAk;f{n=prqzR$2 zAG&AjVCNwy9Eq=XXUu$s@xuhU_e!0E)d!ie^v{D!s(-#_I*Hu$Ox9=*8{J5H$k_}U z#x8nD+Nac_mGaa1l#A4K%vy8eOcEE_;u`u$_iWx%&ZD=PQTsmoOv*>Kc`DcrZS2PS z158)WwR^PLC*R1(l9d+E9;_5m7c7&$_uMV3zprIE{6l4M8E0v2tk|mc!bNcgxS!4u zh%0ZHPF@m|2~Qi6_|p6thbv`yS5(1F7ow(?fQf%NC)5+MP2swt(uIgrC+HJc2Dzin)K?6qtw=X#p?lfPw~|KgzM24$KAc5 zSBK~fgh8K4gz^YO7~n1|2q2FExrj#3%{R~NCM4#l>y~+YEW9`TS!AoLb2*CIy4>Jgm#<%V=}568 zDUocqS3VXDfNrmgUh*yPB^A^&OTXuLUWXXib_$KvI2$3$Gy)}It12Ol>aAvV5_qJa z@{f0K*)<(ZRTUw@_9VO`VnI@k@Y60@r^F)cD4@ap8!Az@lenA9p^K}4hrFk$nd3I$ z&l2o>s#F(eJ~_dj-Qx!(fxj(_-dH#dwELmQtvUx0ln~th;8>9;S4+&8YYY4D^N@0q zFGI(_VQ%|9a=a|Ba!NleUwWYz|2C`XSbdY3@uV4hwifxxp3E*%3Jjcjb)47^d%>mc7w3IaF+e6z*lC_{g7T^Ja;Y=RQnt(3Tb+zO; zT*{}0veG;D^2ZJj|H7s&Hv~V*yo9o|adZ0|FBswHyT`TB(exQUvb$bWbzhK;IzikE z+1(P0y(sr+>YN1Zz+Vejkf`}YKa$uE?j_HR|nv38Wr5R*J%%L5FZ!U z3`gaW+P;jiFyFK)$Ga8P-jGVSh+Vs~f z1~?26pcC`Xoz{>FUfQCps_(bH^b(=^x= List[str]: @@ -681,11 +682,11 @@ def raw_execute( :param inputs: Runtime inputs to the DAG :return: """ - function_graph = _fn_graph if _fn_graph is not None else self.graph - run_id = str(uuid.uuid4()) - nodes, user_nodes = function_graph.get_upstream_nodes(final_vars, inputs, overrides) + self.function_graph = _fn_graph if _fn_graph is not None else self.graph + self.run_id = str(uuid.uuid4()) + nodes, user_nodes = self.function_graph.get_upstream_nodes(final_vars, inputs, overrides) Driver.validate_inputs( - function_graph, self.adapter, user_nodes, inputs, nodes + self.function_graph, self.adapter, user_nodes, inputs, nodes ) # TODO -- validate within the function graph itself if display_graph: # deprecated flow. logger.warning( @@ -694,7 +695,7 @@ def raw_execute( ) self.visualize_execution(final_vars, "test-output/execute.gv", {"view": True}) if self.has_cycles( - final_vars, function_graph + final_vars, self.function_graph ): # here for backwards compatible driver behavior. raise ValueError("Error: cycles detected in your graph.") all_nodes = nodes | user_nodes @@ -702,8 +703,8 @@ def raw_execute( if self.adapter.does_hook("pre_graph_execute", is_async=False): self.adapter.call_all_lifecycle_hooks_sync( "pre_graph_execute", - run_id=run_id, - graph=function_graph, + run_id=self.run_id, + graph=self.function_graph, final_vars=final_vars, inputs=inputs, overrides=overrides, @@ -711,15 +712,15 @@ def raw_execute( results = None try: results = self.graph_executor.execute( - function_graph, + self.function_graph, final_vars, overrides if overrides is not None else {}, inputs if inputs is not None else {}, - run_id, + self.run_id, ) except Exception as e: raise e - return results, run_id, function_graph + return results @capture_function_usage def list_available_variables( @@ -1513,6 +1514,8 @@ def materialize( start_time = time.time() run_successful = True error = None + execution_error = None + raw_results_output = None final_vars = self._create_final_vars(additional_vars) # This is so the finally logging statement does not accidentally die @@ -1559,9 +1562,19 @@ def materialize( except Exception as e: run_successful = False logger.error(SLACK_ERROR_MESSAGE) + execution_error = e error = telemetry.sanitize_error(*sys.exc_info()) raise e finally: + if self.adapter.does_hook("post_graph_execute", is_async=False): + self.adapter.call_all_lifecycle_hooks_sync( + "post_graph_execute", + run_id=self.run_id, + graph=self.function_graph, + success=run_successful, + error=execution_error, + results=raw_results_output, + ) duration = time.time() - start_time self.capture_execute_telemetry( error, final_vars + materializer_vars, inputs, overrides, run_successful, duration From 4e286772ac94412f0cb6ffd306c8cdcc1cfee84f Mon Sep 17 00:00:00 2001 From: JFrank Date: Sat, 31 Aug 2024 21:50:59 +0100 Subject: [PATCH 24/33] Workaround to not break ray by calling init on an open cluster --- hamilton/plugins/h_ray.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/hamilton/plugins/h_ray.py b/hamilton/plugins/h_ray.py index 179ba1bee..e733ec788 100644 --- a/hamilton/plugins/h_ray.py +++ b/hamilton/plugins/h_ray.py @@ -120,13 +120,16 @@ def __init__( self.keep_cluster_open = keep_cluster_open if ray_init_config: - ray.init(**ray_init_config) - + # Ray breaks if you try to open the cluster twice without this flag + if "ignore_reinit_error" not in ray_init_config: + ray_init_config["ignore_reinit_error"] = True # If the cluster is already open we don't want to close it with Hamilton if "address" in ray_init_config: self.keep_cluster_open = True + + ray.init(**ray_init_config) else: - ray.init() + ray.init(ignore_reinit_error=True) @staticmethod def do_validate_input(node_type: typing.Type, input_value: typing.Any) -> bool: From d9e86a9221e2ed4bed7d1236167c945ca9e9dd8b Mon Sep 17 00:00:00 2001 From: JFrank Date: Sun, 1 Sep 2024 10:08:40 +0100 Subject: [PATCH 25/33] raw_execute does not have post_graph_execute and is private now --- hamilton/dev_utils/deprecation.py | 6 ++-- hamilton/driver.py | 49 +++++++++++++++++++++++++++---- z_test_implementation.py | 43 ++++++++++++++------------- 3 files changed, 69 insertions(+), 29 deletions(-) diff --git a/hamilton/dev_utils/deprecation.py b/hamilton/dev_utils/deprecation.py index e1bbe6d83..2ee2d000d 100644 --- a/hamilton/dev_utils/deprecation.py +++ b/hamilton/dev_utils/deprecation.py @@ -48,8 +48,8 @@ class deprecated: @deprecate( warn_starting=(1,10,0) fail_starting=(2,0,0), - use_instead=parameterize_values, - reason='We have redefined the parameterization decorators to consist of `parametrize`, `parametrize_inputs`, and `parametrize_values` + use_this=parameterize_values, + explanation='We have redefined the parameterization decorators to consist of `parametrize`, `parametrize_inputs`, and `parametrize_values` migration_guide="https://github.com/dagworks-inc/hamilton/..." ) class parameterized(...): @@ -66,7 +66,7 @@ class parameterized(...): explanation: str migration_guide: Optional[ str - ] # If this is None, this means that the use_instead is a drop in replacement + ] # If this is None, this means that the use_this is a drop in replacement current_version: Union[Tuple[int, int, int], Version] = dataclasses.field( default_factory=lambda: CURRENT_VERSION ) diff --git a/hamilton/driver.py b/hamilton/driver.py index 063fb6706..c758c944d 100644 --- a/hamilton/driver.py +++ b/hamilton/driver.py @@ -19,6 +19,7 @@ import pandas as pd from hamilton import common, graph_types, htypes +from hamilton.dev_utils import deprecation from hamilton.execution import executors, graph_functions, grouping, state from hamilton.graph_types import HamiltonNode from hamilton.io import materialization @@ -585,7 +586,7 @@ def execute( outputs = None _final_vars = self._create_final_vars(final_vars) try: - outputs = self.raw_execute(_final_vars, overrides, display_graph, inputs=inputs) + outputs = self.__raw_execute(_final_vars, overrides, display_graph, inputs=inputs) if self.adapter.does_method("do_build_result", is_async=False): # Build the result if we have a result builder return self.adapter.call_lifecycle_method_sync("do_build_result", outputs=outputs) @@ -661,6 +662,13 @@ def capture_execute_telemetry( if logger.isEnabledFor(logging.DEBUG): logger.debug(f"Error caught in processing telemetry: \n{e}") + @deprecation.deprecated( + warn_starting=(1, 0, 0), + fail_starting=(2, 0, 0), + use_this=None, + explanation="This has become a private method and does not guarantee that all the adapters work correctly.", + migration_guide="Don't use this entry point for execution directly. Always go through `.execute()`.", + ) def raw_execute( self, final_vars: List[str], @@ -669,12 +677,43 @@ def raw_execute( inputs: Dict[str, Any] = None, _fn_graph: graph.FunctionGraph = None, ) -> Dict[str, Any]: - """Raw execute function that does the meat of execute. - - Don't use this entry point for execution directly. Always go through `.execute()`. + """Don't use this entry point for execution directly. Always go through `.execute()`. In case you are using `.raw_execute()` directly, please switch to `.execute()` using a `base.DictResult()`. Note: `base.DictResult()` is the default return of execute if you are using the `driver.Builder()` class to create a `Driver()` object. + """ + success = True + error = None + results = None + try: + return self.__raw_execute(final_vars, overrides, display_graph, inputs=inputs) + except Exception as e: + success = False + logger.error(SLACK_ERROR_MESSAGE) + error = e + raise e + finally: + if self.adapter.does_hook("post_graph_execute", is_async=False): + self.adapter.call_all_lifecycle_hooks_sync( + "post_graph_execute", + run_id=self.run_id, + graph=self.function_graph, + success=success, + error=error, + results=results, + ) + + def __raw_execute( + self, + final_vars: List[str], + overrides: Dict[str, Any] = None, + display_graph: bool = False, + inputs: Dict[str, Any] = None, + _fn_graph: graph.FunctionGraph = None, + ) -> Dict[str, Any]: + """Raw execute function that does the meat of execute. + + Private method since the result building and post_graph_execute lifecycle hooks are performed outside and so this returns an incomplete result. :param final_vars: Final variables to compute :param overrides: Overrides to run. @@ -1549,7 +1588,7 @@ def materialize( Driver.validate_inputs(function_graph, self.adapter, user_nodes, inputs, nodes) all_nodes = nodes | user_nodes self.graph_executor.validate(list(all_nodes)) - raw_results = self.raw_execute( + raw_results = self.__raw_execute( final_vars=final_vars + materializer_vars, inputs=inputs, overrides=overrides, diff --git a/z_test_implementation.py b/z_test_implementation.py index 91125c59c..7ac5131bf 100644 --- a/z_test_implementation.py +++ b/z_test_implementation.py @@ -23,8 +23,7 @@ def node_1s_error() -> float: if __name__ == "__main__": import __main__ - from hamilton import base, driver - from hamilton.plugins.h_ray import RayGraphAdapter + from hamilton import driver from hamilton_sdk import adapters username = "jf" @@ -37,27 +36,29 @@ def node_1s_error() -> float: try: # ray.init() - rga = RayGraphAdapter(result_builder=base.PandasDataFrameResult()) - dr_ray = driver.Builder().with_modules(__main__).with_adapters(rga, tracker_ray).build() - result_ray = dr_ray.execute( - final_vars=[ - "node_5s", - "node_1s_error", - "add_1_to_previous", - ] - ) - print(result_ray) + # rga = RayGraphAdapter(result_builder=base.PandasDataFrameResult()) + # dr_ray = driver.Builder().with_modules(__main__).with_adapters(rga, tracker_ray).build() + # result_ray = dr_ray.raw_execute( + # final_vars=[ + # "node_5s", + # # "node_1s_error", + # "add_1_to_previous", + # ] + # ) + # print(result_ray) time.sleep(5) # ray.shutdown() except ValueError: print("UI displays no problem") - # finally: - # tracker = adapters.HamiltonTracker( - # project_id=1, # modify this as needed - # username=username, - # dag_name="telemetry_okay", - # ) - # dr_without_ray = driver.Builder().with_modules(__main__).with_adapters(tracker).build() - - # result_without_ray = dr_without_ray.execute(final_vars=["node_5s", "node_5s_error"]) + finally: + tracker = adapters.HamiltonTracker( + project_id=1, # modify this as needed + username=username, + dag_name="telemetry_okay", + ) + dr_without_ray = driver.Builder().with_modules(__main__).with_adapters(tracker).build() + + result_without_ray = dr_without_ray.raw_execute( + final_vars=["node_5s", "add_1_to_previous"] + ) # ,"node_5s_error"]) From 69534173b7475bfc7b5c728d052c6c650118f25f Mon Sep 17 00:00:00 2001 From: JFrank Date: Sun, 1 Sep 2024 10:09:55 +0100 Subject: [PATCH 26/33] Correct version for depraction warning --- hamilton/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hamilton/driver.py b/hamilton/driver.py index c758c944d..419ff8d26 100644 --- a/hamilton/driver.py +++ b/hamilton/driver.py @@ -663,7 +663,7 @@ def capture_execute_telemetry( logger.debug(f"Error caught in processing telemetry: \n{e}") @deprecation.deprecated( - warn_starting=(1, 0, 0), + warn_starting=(1, 75, 0), fail_starting=(2, 0, 0), use_this=None, explanation="This has become a private method and does not guarantee that all the adapters work correctly.", From e988db550cbe43ef518a17cf5f43080bf0479aa4 Mon Sep 17 00:00:00 2001 From: JFrank Date: Sun, 1 Sep 2024 10:12:57 +0100 Subject: [PATCH 27/33] all tests work --- dag_example_module.png | Bin 24324 -> 24183 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/dag_example_module.png b/dag_example_module.png index 52d85a429dd7e7cef30a1e0a489a235e58af5ddd..a38a056f151ac2f7a1d9db955503691c388552cb 100644 GIT binary patch literal 24183 zcmce;1yq%9*Y3Lz1r!kJ21QDc6a=Lt1e7k3?i8fEr9nXcN=r*gOLv3PCEZ9!cgLCQ zefPJ&v(NbUcg7xPjDy3W;#!O6xntgQUe|9v!Sb>a*mub8AP@-bSCV3i2n0$E{0_!I zgYV!^CRV~9X!=qTVu)+xuavs1&j`eQ#49mjW#_LO(@t**RwmGP-2P&w5xi8e2&!XF z(`^~Uufa$wzGv`}j^hYVZn*V54Vgi39rOJoDn5)clv@ff9>2vE{%d)B?i&&1fP&Jb z7kKWZrP-0V*ZYKzVXF4YcDLjo+bRk%F|mcE)Ccyr;4f`qdJJ+n1}0SUUHo2#Z+8zYhBKnW1H5 zyK#Q-6+(byCw hDA|M}-n zPdvAhnmA45&&01^{~-QQQBnDupa1#w*%q<)Ks4Z|gM))g^k#pmuj2Hr{FfLaR9-Yp z4pZYLT`dJ#HAQa6@44Pjkub=9ePM6UwXnEo^LLPr%W{H%oaen?VIdnr_i*j^&hfFt zbhU%c*3`4swzdZ}G&k_+rM(jpCQZr`jrx)VtZi(BO-&i`@bTw2H^a<-f1lr5>cYUl zkbfLg7hPh~N5jf`7l&G~b*$LHF3%s9g5hF6)3szvPEKAWMiV((=cZ~j4j20UAoI?kFLv?Vpw?{H%QXb(muttQ z!b@6O+PU4uj#w@;yxyv?suh*Kp^|2cx<8E4;;Y!d$?n1Q|4Rxy{)^fI;-TSBJ&_n#q7Q$+&_xCy>( z=AkeZ6cjL>|HEwsYa_(0_Ru>>@-}+*pO-X|y6YqP)^>KHF|1m;78Y2Pl$3AP)zfqu zJb9gWQWo?E6>|uPSv4DDISh1-jnM=I1WGN&@g%}2gniMl5s3Eoc7OAll!Jo<>-pwe zAIW*{5;DK>OG=`O=dlX+MaQxAzJnX}8m~$-HWC{z_~!ff@2&0a3H0}MK1$R3mX&c| zTwdZ565di$Qc9I#!hPMf%aDfGD(+~Z;o^Gu3H+Z$y#K=hnQ^p~4+GX|S05NC zk)u63GD7s3L9W-1l!nHkDT#6^2yRKACK9%h@YBbUjXgcs5Ek18>tR2?$$Pu|i|X#} z^vVb!T!p7pX`AB$9%lsS$J@l!STrE=NT}hTs#RPsW7*Blea0fn;S+&3~IhlH6b5l=O7Xi!RBMHY1tJ!)L^M~hqE4_)niHVd4 zhlhH`#sLHA5)YY}UYniDBs>+SiTpFh+!OD#+W!F2_~902f4zqrqVf9bSV)oU0*$8-CkVu-I}gx8672l_3BknNJw+0OhSLE zD0*v0hu8dCaL{Yw0Idg`<0a^+a6as#E_F&Xk9=v^OCXuqb2OEqD9G#1JFqB80BIHCa_J?%@ugW7UYPjpUP$6E#>B)#vcRW+prF0>B4%1Q zH@7IW=;&yKcV#7SlncaqrK)ISGqd354r}-M`D@>rKYm=_7ar6qUaF@$wO#TV!rjo1 z<_$a9hvue+RCsaHnVf8o9_93J;^9Uar(xIpb0k>v#H2UdZ>}W}XLqg%b@;oA6h}{Y zHx}X^E^g{zJA@>aOe22-S65esXelvTCghuzC(7QDaGMh}H#dL!^a){QWp%JUOKoOm zW^HXP|9CV-7!}dj8O3aajCFK$^DuX@oJJuF(Y_nf($WZowzhUqc=#13pQzY3W3ls_ zQUA0zHKFi1ZZyKcL}iZVs^UB`?v{5iFD-5LMnScnYecbM8-B&1B_)N~5`f(TAvJ2G zP^;$YcjcT9a3-a@pYDbHrHFhbc7WpMs3Qp}g3aEaS7T#iFPNgz;6IVqiku)$qfSPa z8kc=z_v2|Nc5ZHe2w4`EmNrKlYWA!B!Wmj{$ZgCvP;X9@`TF=Y#Y|Ar(3CYNX;Im_ zJ~fV@60qImUgE|Gf+s{4 z#WpzVAeVA;{Ptn{$3vQ8T+6GGfQWQH=Jvtq#4d{x0M3Vl+Mp3FH)%l>TG!U}H zFz(!GhCPo4=`%id_B%_<_a6GM7C!$rv z=e%>DUiRXam4QCH^98NDfHO(U<`dKP)y7hNy4W{)e$kto9JZa0rw*!vEWKF8TExTH zT?T2p6Zwc?^NGZ~7W-ml&iCRaYmC9s#yB#gSQ{T(m>r7YupWMkriJ@ilD258|90-( zq-Lj~72Tc*ht=V~%(b4E>Z^kgK4e&0V32O#^+lWS6S{6@3su!!Xl~FoR2N?TAts}w zgjp9*t$SME;(WG1r#?P*pOH~=^#{D~|!^MTWodE)F_4xy&qK?lR(sQ?Qovraxpw_OYJ@QX?mi zQO+FSF*rE*WGUv=DraM47OPZHh{|{Ug*3vk3MV#J?OI0rXJTSiiU~y;)%K6l3}*Dl z*q->Y`u`~2Iw4edTKizL*s)LV3CGhg&A(Q`MW4_-mm`fdQ|xW6yB>za-Bl3E+v+{e z;Y~q|66Uy$5RtsaL{W8csOaqHvkFV)a6`vIe zThqolmUm$xj&_|Zo&6&*F{anCf*abk&M1g%U2V0w*Q-j>%yG=&#JCEt z6c<`pA3Q)oq%i-!&T~{$QY>;f75TWPA1H{Oo31K=Ny|V_@G`iK_2B(|*WV?LNnUR- zv|cCKXPmjmLsljUp9`6&cX!sd*WC#0>qfi%=d~g3H`K(%6~4Ur_Q6V9w9u7ktm1uC ze+B_;bp;AGnzVn!HfK#@bUasNTN?ZCd2R9C2#qhhKft*AtWds20iOahm;jlSl&P3E zZ}oH$XDgNveINPGf*ghe{~WpF%ui1r$A@YkKE6jx?u+howW`v0qi#)2w`VF2;XOp< zHjtZitKh3(!H@SN{0${p1Vis#wZz~ucxyfIMQ*&;TV_YcvUo$@n@_IS)~$2znDtEt zB@}oi=+hjFlRnnwD%5vb*7W~r8RgxqM<*ry3zturi9?A%jFp*JrM1HkM*Q*YVdZ{C z>UO2(L)}m=aZMj>U0M*KBk_Db`j{?b>5kG9+{oB3@kv?=n01R0mR>9(u(1*M>7LzQ z!P)FsH<2qg%*xV#20ztEsl8>r6?f@y8;#~*%U-!OTRnR-D=R~<*Qpny-a(6lg@v0C z#Z&d3^KbOUWmb{Ch$P<6v+==jC?5#aP#gDk6n~N=At6h9KeWtCgcoe^Ad*DL>z7HH z+wRx{d(t_-Fq2xFMj@=OBMPxw`%IN?mrOJw9|*fHSE_Kd8NwU;f08V&ur-xqjuu@X zp;m4Q`E2=qP0h#;YcgQaAKyYncv~^ZDhed4Zwg%&Q-rCPJsLi$lcVgia9VmkHs<8xNhtq(8aM1D()(LROr$9!<&jgflB_unXp;M>$&+@bCy0h~r%sc~B8G}Bl6 z3=z5zC|qSUvb#(E4&OrTEbKna_x{D+t8fr$+p;HQP-IF>If{ehDf{d69Qz`Fu)){Eq6(>#kZG}_ED#sZU<a%n+mNXf{@&JiYsXdZYKpnN***Jm342OnP-v<-XGe}~?7Ea4RrOl!on>0kMsFR|h z-@pCy^C{oEXx&3bZtp@K!yi~62VrX6!Ho=oABjASTF#&@z>#hLt2 zebsjLVFFQ~C?ua8&O5ruQmQjj-4EolWI@WzcKWd@+noLk`KLUri5+F3D@BeyWTs|O zXDcE23b!+!>SYxcMno}d*&KQE^-}KLtERZIvo@}j7cW2xRn#+f|De}gWw-edB<#8f zFkVwAqq(X|>hRc_1wPP_B;{n&lvG3uh2TZHO;3zH~CE?Mv`rw!JJ z_nL&&Ulkv*K1JK_z>*lOc}v0Ps=WJ!91e%qkRgbQqSiQWiN*Bl=!J(qku&qui^OSR z|DA&~BT?5pUUHoZVv|S=90#w=i5(I5qh1%!wxEKn)YPf=xp-~#QBYVoUiD*zuAh<} z#nT!bmexAT+=YJYqXns;1&PGG6#K$vPU`q#dGa0+i~3DDIXOSg{kGUG%DE~RN-KUg zecL!J?{Gw1q)mG{(9R5O38{rb#k$Q{^0leNscJB1E#Oibn&J~|0!J9X4 zL}Dbxly%Li;bJUrLrvV$8jN3z%qaPt?=hRmh`72|_h;mA=fW(u4xk`2%Kj9e|Cv#S z78i5S%YF?C#%G|Rqr27A)Z~1)M!2xB;KwRsxw^L23~K_L{HY&gI{$jlZ3TZV$Li{8 z)0x_;ek0JVZl{!^DhEMc7W&^xv;Vh5Iw^%Gs;0WS8Orw62TbO_e*M}xI%@Ap;DKtx zXKd{4Peabst~5nSm_jILib_kxLWx-V3$*xF|NiyZeSQ;@4`h_or!sWjzP_ll{;F67 zIt}5!b5&dR8&PQKX&bw1?{w#mzKHv=bbnwjnBr^?R=o(7#KK&pf%D}8=>awtyLBVt z;$pwAsJ#42-vKE~UxZqQ1%>kBt`o72!0`ecECA8YXH zXZLG9f0nN6#`9Ivi_;a`i<8BOwyI|Vl#$h)KYuoUd-Zt-GT=tD4Kd#E)7#Iht6_b8 zvC4V$$okreduk-V+3z;~+}he58JV7iF@lzGNy?1OOwh)zDy?^K(6guomrpm~R@0`5 z)YzW!YMNYqkUdJYhVpTBb(P?dp5M7GUJ^Tu^a)x_U2TzR4Z3gmu=eEEtnPp#j^@!` zGmb$!LDy4|ZOn~#7uv>(?HDpIs>mR=;#X6o(I{lSdeZ&2imJwOv-ux7sgPQaq;^`x zhdfp@eqmwQ^~u~g9swSZB8$D;`q|4eR_aUzg=j!Diw2QONv*NOH&)2)p4+bm#eOq< zJnn$N3l@W%Jfv-7-Z!NX<>W;3(f8}*{dgFOu{t*%cXtnV@hwtFK zPLXB5k|-nNP5X-;d>^*-(;+H^5SXT4VLM{T?{&{FrN${V?b#6we-kVM8#Z2&x_U|o zabh~HJfadbyo?y51zODi9NnRC1icA_`;cV0U-cHULYm3z#4Bg|HXCCcv8wrKxdAee zXs?E5PiNm|Qd4dNks$P2orn;|L{Cq?WX$53W&`gCF(x%NHJ61EGlX>=<%i^aAm5yb+)=-$RtZ zT;-?i2;-GbBWsOo9`9}in}TyW#{7g!*Eg{F#%z0W$l~K6Bsbq3HL7%+7ow$aO@DJK zPIA;`-`dtrFz)cRbXd98c@G10dvKC`Igni z%HM#oRmND=Tpw*wiG_D7^RFjAbs1J#|3UOp@3ywM2}?R(Q=J6_WDiw7swaJ}Slp*3 zIMYj&jG!3%LB(*iDYB8ksQg_WTZa#)W3;y=O8iBKxP;z$NJgdSZr)%qYtC57!?HGE z#bgEn7Hli+`bJ3m=Jsb_UDWD6tG-Ktg;<^EBA_B4eGwh$(_iD`OyNrX``xHYp4-t} zpb&dC4%rQC85!iklH+^6ur$OF;&S|cw(;xs*8zvdm1`gO`JX5k+?OU9W_Z}x2h^vi zUp+67X*R;!=u(`~L_fsIDm^kOI$%y?bYQ9gbx={srtdmae;SVP5S;)y~$xiTb*^o*~ESK4i`u-JZk5*7yV~>mM(@r zEh*7Ao)2@htuW}}6ggg5@O$lH%=VpNBR<7{{W#lE)i~QBWj0+oMuJo(610g2tp1p~ z3)#Id;4X+g#bl*;mU42zdmuP+dByz4medv70R@4)(MPf#yG%cv!yRnqxcSl)Q;gi) zpior4pt$Y&aWm$Zu>+3EHO832)=OL_p9|?_sgFJYF(T=o%}W`qFG(zsc8QpfWA*@kFehTDU9$A z$fS?n+G|ba!yTJ zaS&riqcku3Iw0(a?(n}cn)cDw^jos6^|&p+wX=-=L7lrh6uJ`JqhJIW~S!NAPr$9va4~hY-v4i z_7&ofe{}x07=jzvTa*ZER*k{}nU(kK9Jv^3;%K$9?v?p(}53vna%%Yk2*W6j_|+`y0>ay63xSh zT1#(_?azqvH~2)&<Vqo1v0U3o)6b&8&Jq4C5#HQe%4!JnlHZp zA1uJeBLt^ghl;s+TAU@{0p&9?!dzt)H8~9l^xt`FjDqjX-lfN@P_~LUpdk1|F*?>) zA|+*|FRC>n){N|5@KhZPoF^Q?zX*Z{|MYCnx8p=zl)=Rq|#SXyvBvQKOfU zaf!WZO`=qIRJ4<|U2OKteX7ordtd#{U14+t>3F6T7gv9aP4>cm73$W`@nJKBIfUWe zJ7$BKmf@SG?miM#u*{X#rfXexFIxCZLPD&6Le8Gg97C9;olsgX`gz0=Zo%#K==y}` zDz0&i077&dZDBQ>#s9m@pD${$l{<2!rlSf9ZsOw4ll%l4BNH{)JtVza>q1$SCvQax z5R?oJ@yL_;{iC##%e=dB8+U27Cv*jZ@YjWs{gYQ?PAnK5A~Zs$%Aelf^(8z$)~ef~ zzH2#|9eaE@rx&90k)E*oiz^C_=!Y*8x9iAH#?N<)V|f%plCTjZs`B+SQ64G5=V!kq zrDR?{!&ia`nT}CI4w2lmk)_SHN~@4rshq;|YMMlaiaAn!yR)Jbts!stb`r@%n zP4Rk&(9vvlL`DriVH5G?8r_!mzLFXn=sj{RqJ+sqLxVfC8$JtmOJvdPxfZDh1aXhL zuHID3CS0z(c;W2xu1{aH{>DLuCN zePUtCo9>HSlUM3~_pTFy!aRMy{Gs(bnQd*}P5&bPr+j-{!>n$jiIC_EX?d8X1(wb{W!_MWX8I_b01Nu6Ysy(Rd!+r&We z)%^VY1N+;W_?b!-a!}+Mo3XrU{zx;w5m{Q}-o3E#)7ReDl+`ol0j2$0#y8#3S+5(2^1`=R zagR}Dg|04wxu4WQHdIl|rq?I*tl@Ki-j7@A(@IN0d- zg=Z7Q&R0`iYPF~6M}6-f8Srr_TeFUPPHu2rvGaNTY3ciXF5Vjm*XKA&YQ3NldR0-1 zkNWs3^tKW}iIY8=M@M8a9vXj0#u(k@S^Y#PlT{~dpkz6sc1U13XeYA=AA=Ccgc3{WNPSH%Pc{5cOw*#L!kvml<`}*0_$dhm;|l8p^7;fp ziDI_+__%Y$I!CPoCsJQRDv2mrTqa4{*rQQ0-iTzsJIgAtD#E^(nJ;!+vozwAtS}P| z{r;@qv@8(&W5cS=L0CqHg@lYWU3~g%b;|1Aa^CpI`WA_$+?lB_PdM84JOT%dyz${& zVWeCq+T0Mz-CSs{pt_fX5-j$Cv|wFX_IFO`E&Tex*>NQVXN%omRyiMLrqh#=TaD{~ z6dosXl{4-ISP9%uOM7zNfor<2;>7jC%oi@|9{%B-tSKkv4|jj^v%EgY0P8CIC%>n) zj@h4;jeB86mkmDT>2Bl#rk8o3X!KCL#$uND)3S2XPgO7 z2{@~;Ao|D`s!1JKUl4&XdS(Amtj!zh^q1|;gRjjR#k2)ideG@)Y&N9&np*np*yT=% z2*cZHfyH9B9((ruxh<`+pv7c4l9@O^Ub)iFn*vls_v+%*bnx44Fv1neE!2<{kkG{r zYHe+8$mV&2%_17`O3OV7s*g8`x$89|K^e&6kcvIto=w-PajdkHaEaXor@`iUOUq)a z5*Pdh;dJrua>i_yClbWCOT;Y&erjdrgj-WpzTZ`H_1i)SSNl^@v9PgsR|g)RoSy2P zA8qzCW(Jn1YiK-q5hVGA#F6eJ5;&Th^}2ouP!Zt}m~IhJinp}2z04lW#(1n*dCzL5 z)~BT8iG_s)m^azOj2x)SoSd9Ru7~DGkOX`=c23R@K!O0~-~#|ln{AeC;s?OGkghS% zOJn2X!a6$C#>U1Z00|NsNz2=*-2??VAlk;LeIMy}%a$4JDyR>($__KLqHbW|= zzt}*6gp3S{g1y)p-PoJ-^o-CmJK9^4%G0dE16)KL2%6nVtA_cRy05b=;$kneS`Rji zd*c0djk&!DXLI(%- zQ`HWbRgWakZwUn4{FY%<-%H2OPXQX&2Ourtx!(n-PF@^DTjIt8D3jBZI^?9I!frMsJj& zva**J8-YBe#KzYXrDncB$fU|9KJOnM?kw<#Td!$`~ytCR9~b1uYOROs_kZ<86&2%lp~-#PvNSWRlYY#GS+1u(0z< zV8ssFty>QsK179a^M_-Di<>(&H5CQWJuZu}dx)Jszwk300qIdGH!(B2jR*=2Rgwm% zKs1U;g<#WgG+iP>Zrh#;Vod;up!&1*9?m;+C^R%Q{aJF4U}C55 zZv5p7$Mq3xB%2w7BFm`s^cpoLXw?@F-w~0Lk>P)r@_CAAU^fpM18{<{sF5@ZsCJp9 z8*xuecFwmOm=EQt1l@zsai|!z<_c8{so+Ka@ZrR!eg>W}AM=3Q0NkMXcN(}UrkJuj z+Yrz{{W5%=E)r^YeP_s1I9yL*6)3sL3bn7mxqMyc4;H(@!ou<`E2{-UnrHx4)29cb z@~I-XfRBRw#=YrvaBD`ZW(jL^>doK%Fw`)bmf2TZY2QZ2J9M#ptx$a&iIS-=gATg~viL$iS7UnZ?$? z83<8i9#$7zW+S)(&!0a>YVr-9m)1LT!uEd$Gvv9~|NfnC_QxdQc~8OgCJzJfOfIV# z3Rpjn4-irC6Z96_!&ZOi66fXRrR3&f!eQxpxTbe;vWsMWr*3LbovZdAVY*^TF+X|o zMEWZ)E(9eJ7*o&lO&&B#1&+@@F?ko7r>7}pWo18L5Ay;P?RvCv8v?||UKiVk;TOtp z*|ck4K(2s!;jZ-W9IU_Iq_^%J4aXiRc2W#sEFrbKj2u}U!LLyPl}?qq&lGhc$fu~` zo=gcz2LMs!|4Kh%Cj7MktU8ej5BkD?w=SY(lIOwB8ey%==?zdY6b90mEH-LV))5R~ zDQiBSoIX5GMp~zDD0zd`u|uv>Z!6mmvjV~XgRjJkZKONV|Kmsdw$ft`t6jlm7EmhY zY*szGHLL31mS3`;umWW)U*}k-ZSog+o21{*Vqi{PhMHAR}gAUDwl$4|E)Y6+*43n+WRy|`=TH~j#_q01WSoxq8jBhrCU?#OD!x@bxeHp1MlNUOFW7fVCn-2 zfz;nLpLtLW1s(*7hE|_=tY*N_PvuA?y|pdpcm&uAXL zPtK1;qC0#?bF4w0q$xAfe{=iA{qoB)65o!?59V(i_S`*GhC@wa`CLh$*p`o{qH-sO z$?*>d?rq~qG6n&WJvw5}>4<=pitpRok?(p28K-Z;1hJ%lS)Bt%apLet7s^Lvb=Ct} z1dL{i%j8|)Q8K(|(UvkA^%xHDm4kjbJ3C`&-MDQ5)qN1g?SRo==}6HQEg+yMZx%zw zaxyi+_X$8YkD_H{UNub(wCr3Ah$3r(@Vc`msFUy(W)@G9Hj%R$zBv)kvT*|i@c^ZQ zeSI6-bm}`cGUuPb#JuAdU0$wl{aSdU!E^qx$c0rJ6qL)}6YeLdeC0~VnT~53Cw_Vo!_DWb$jg$3py9(0EM@<-AJnpGc0ACx?7+l8(6+4Qj zb`ixig`no_&_KSol~+;-(mnu}9^nnkNzCsiQfI;}-w*|JUb&pok>?)w+HhcWxuegd z*X0E%0<5IIqN36ph*)KIQJ};IGVPA_Pcnlb^Ea1~P$k}M*z!HnF{!`74EE{z^$9gA zrI24@fmE@z#`2^@FZB!#;>2W2`r?Vxa$t#)ZPmgo6z3=iIT7mn+;q3J+z$IVR@1=3 z?oMDVz51S@uP!`cHCn<0)^afE>|H-_HQsWYe{k(lPjgJCe|h>_{wYO@h*+9FkQ5Ju zWY7?`VXoK+NjWJ|G0A0JB3QeIq@*B@#|e$I{u<8Hpl%FQimeJ;?vB@~Q5X^_vr=rr zdKaG5mZx&vTp3?-}N&`@d28ngdc3>i4D`_EKyjH{m&k3sCj*GAw z!K!-womVYi-PEaZWFMKHBLM36r_bN)|5iqcKpHZ!%TqX6(@rfe7vJ2=gikwwkIYHj znX8H27{i45YXhM^$c-4uTZoDK8EliOT4-_+BX-L6tr6bB<3B)E@zI!xDSG$Ymf!8* zUe2#yL0;E)=bN17isz{-doS*>>k0dS+e5Vc`}<1eWdwQ_!M96radek99w9**{q$L+ zMhM=twPA-b;zgXTc-v83l<~$mZsf#murB~k0!24@(H#!Zn4A*AI(me4+!1rM3~7YN z`WwC3Y89q+(0=h61DhTlYLs?5oFi{k^8EP13Q1+~x;4re#gRFzrLH6Df2H$+ODVZ& zzwsNVTJce>9A!)T`2*+uy=Sy!WZCjLoMdI@VTFl-x%{zJ9(^;6?<_vR;+z*w5&8!Y zU)TK2ZIyiU^p;T%z0rII#GKh%Mqp`d1^iEs4w<8x9^Ep#uK8;xU;J8Ss?wc`?P2l@ zx$ijp=D!E#*XviecyGEYUh1E;XE$0gF)%&)F=GUV8L$v-5mroDgF*1k8?yjppLU=y z{g1{UZb(&b#eKv<&m4S7-{l?sX5g#0;TUd?&kv(68V*)j=q3@-k9G|TS& zxoB{*Jrq)XDhASA_G!#=`x!(xH#X*$GhLMoqaiW0^(fHUSobo2R=he>$M@87<|9}; zByn#5YDm*3-lx9<@u7ImtAXg18?dyJ*uWU4+AcAzo==lto#}l5T?tsI%8waqmhYPa}NW)d15JcO%R-3c&N~8@ zlpB{O{s3zb<_hFqe<%+?gL+TJ(2^6ZPruaE2SkrgP&IUW?jxIgnl3|ch+pmvM`Up3jbX_t^cQf75!c&NJIX@%JWmse+ zZaB42Km}Wp!O@Pe^Z8j+KOL?|bVZ?#(UsR}o84Dl&)RI?y9j8l%E?eKGjHnaBmQTY zK-y-b=biJn@@C&{-4FvSNkyf(m6cHSmj|t1B0&EbR(>}oPAh|!=2>Tr8#!U% z1{CSdTfrtJmm%kVJd%RS`@&yx^Oqr~#dI}ciE)oeymUbQL-F)31OrLT%I7!Pe}Y}` z{Y%iNkN}nGJ5yb-x}o{#QD+n5vv~v|O|kRw79|mz_D5hH7N%?74qFaVI3$$40J%*e z)8d(_srel(nkbPUAF)6fZUbAvdiI)V3w=hs%2xbjK5!tG``w$!8SPK#VCs({5p!tC z;q^G-bM+BFhpy<=fpm0aCL^3OEXpyZih2)qXT|F?Zuk-T829z|#( z30~FUol0D{Ro2wB{Q4Dq{UCCpYwghEyVKi$j0_Fuph;=0#)%Ee#W2V<0m(hn|M>AE zc=1kHq?|W(ye_$+yu+OQ68IXz{_B8;65+aoe|`lypWKsCpKO}%NLDCxky0*rg9@rE z;2(5Oc+^$W|2mMb{@{wdpKU!1H5sU|(itu-^;-vN32r5I!e|UPiXFQ8Ab;iWO}fOP zbUSRxbC}fA&JXA}!u7xf3lyv(VKoE%;dc`^F2%#!VFeVq9qO!(7E)K+{|$kd6=vR%Ad4WeY~(lMv_YzS!}gCH`32*`DucD0d$lw%ab#W zZh(n2zrKF{OX0m3T_R{%-rcxyYr2p$lkj-*PdHqo39(&Z{tFBdQ(pL%`E71nC#&)N zz$QHp0EVS~2S>islVg zgKl5*c-tO(^HecuOwGa*E0RWV4=7yK^eEWvE0I;L=nXJ3bC_`_b&`hnl`mDh&i zvVpuM<>8q$NEVlxo{hoy9k`^Vaf_TxdL3>Xl8-gDwC2~=nCwZPJ$FB02Wc5F=^6;8 z!5=JRsQRN`+klutv6AbjHKKtNG@J;685HY&tPdqt*fd`(jyU!Yjt~I{tpI4 z*}6ETSlQ>Ik3@h{rmNPF3TH@gh|y&L1{PMg4RotYB9JA?t^H1s7=<(~$fY5zX7pqn z`sKk#I2&x2&ZHnK8w-f8F!^N8zFyQySOK_aSDUVYl|2t)R`Xz zXj3MJDmm}iE__A1fiJ{SKRvyu#euXy>qPEv&BxHNriEYwf-XwT;O^r53yKoZ`kO2- zXA#~|P81!D1r9`H2^{GkUT2~pu&}V_9WQ50C?nFo;W@&WwUZNDIksmftn~ixD$K~% z1!UJ)Uid$cK|k=iI%VJ&5WpoOLIZ6=xlo%D;vC6S3lfkwR3Gy&eKh=E(&l?jyi8tjJN#pOeP*5lhR+ zwr?+L0@#pI`d-rl3!Jf5nZnRj9$T*Z?IXj_E^T@MHZ z(!L_dss|BYtaz$!l81eT&%~Z`F1F8Vaj5>MVAs;y=Vp&DdJ|c#J(BJCOa2q%X!eRZ zeP%pPcyui787`rqRAp?Ur`MFwDOCjeJM=>nApyZeKBrG#zW71M)>NGvG4$(-1>U8E zPEuO?IJyt{`S}yIm1QZcRL=m40jhRTD~Z>h;Ravrd%a(>atD;V0w22)mMXR3hGxX; z+S{MZj4hG}0p!B8ObJCL&W%eH1Q>qbPBM^rd#ICh6?t|Vn@yAg{3fXZWw_45_Im*^Wh9@jMU z*YY=Xq-RG}qA-n&jC|@^G9R$J*9=*m^-3?La=tntoOVd&5t@*ZvJ=Q@DbP3t4nx+; z{@51^NR++1eOQrX6}a2ToLD?x-QXb4CAO6In4#lias|oqZ}XJA;@dE%#NwGtg!{;9 z>Y$YC>#n_$^ts~#7gu?)e?bz#ze>-Rdu_G&Zccxy6L0UwG^ZJ}00r$0ZZ30X1Mu9i zi9Z<`HmhbL&Zb3qm`GxGBKODZb@u|cvcAtmHEDo2gJedKUElXC387ryMI|c= z1)kGt42KwCeJOU=7l#{y4&8!p#*(J9!X=Qj3-{vsIws=ec-Cv6#6+&`03^0_&&xAw z8wG*iFBXOzyz%*Y3VmVQcYN?1gb0&j7VVCWsU#kXPce6*K#LzgiVCV+$un@yFA zLzdo8Re~9pHs}@<9NYv7t9*_k9rQQ8)zDakZt}5mOLAJg;K7HzkZIfk!3ZeM=Dt3h zqoX5sc6RTaoJSy_`BYfVGL_Yt4bewE6-&3J{E{1)BAv z&_VlOdqXYuCEKI_Pk*O_zG0Ji=~#Xu>?*qIqGuRDmXoCnw3lACidUG(xz~)O+QD z|A9>3JTG=79;=sPz;K3a{t<%?v47c18m9}6y1Mt_aRuJq-UvtmgpC@&H48X6=pG{a z5pjj1HFgxb=f=1R_2}wz>!Jl^C0qR)P#5U$wsIipfmKyDo8x$Diy9`KC$FVs+J<)I z+jK_$nwdG{9lhXau*H^&RVnvQs`pulIs0Pdm9CVG^0FSMeTn%+!^hM#2|Ox$V#LSNl_0{t&E4_NnW+)42K?#>>lbmP<|>_05MBPw^@(sFNM zafZ34#%pD0bn@S;@7!)3x^t0BB`b>oN~B2WIt3FDhv6~z> zC;X?cM`!9fmYlC~D7>blMDggd=mYUyekaxE{`K0h)sN_5G`m^C{(Q}LUs2C!4BZkt{OgVLsC$B8d?^^2w9v_}+s!k^ttnb$g zCMWW7@bIk6HTnMe^XF4Sf)wcV&wXEYNwyaQ1>%$IWeT@mM2Q6P??^;fxRavxi_QU(m@S z7|qEYf72B%YT3SZ7AI#hITT1b6fk_@ioQ(zYLtw#d3Ryz*CN56rM-8N8|+|r+!gzjeD3{FXrqALWyOW9nyM_dcWVO+>vem%oui|;X!@TbxYf7n^(Umq zkaWXiLpCNWaGsR>W6}xxVhNmCIu@3&WUs5LsY~cg9j~-W-zWiM8f>GOX&Xk{jZW}U ziXGP9jOT+Cad{gDr(<~d3wWnL+)r&&Tpif@;DiD$p8oOUkHEx(;W<|C!3UN~M`ve9 zb~fYqVsB&Pi{anDgF`|@baZrx;Bb$go73&??gkj2@pIW&p^nh_;v_M0(31m4$Me@! zUj+rzJJ-oot!p&sy(qkLwCyTOSnscWDV2%!BU)pHe*hJ~D+wNkQWLvQ8dG(Tyai?_ zy6+@=YSEs!$vv7Jl9$|H^`mXTdnr6~>?LL#L zKJA~UN(@z(=MH;YvnIhSfiHTJq`U zeV)+QJ31P(u%HjYJnpF_8TnI-xr@{NJDUlSk-~tV8)8qFpy3oa3$jDIaFvm`I96*T}hK6Ebll}VXJn>6Cv0C*TxS;N5f3y7j{J@%ⅇG0o;UyxA>r`4 z8X~rZ)Lc?Q0T&EQ31~BbC!Gur56deleNOCy5Ev7@34W4wbQI_AyG4Fw@u-!O+=9aL z&E@pdWP|sIPgxFUUubFhw#Un?jy$O-ZP5zF&ttMWNB2o5o>#k<@O=XpDqbzP#GvvcvwF}^r2_&*mU4KcNlj<|Bx|acH=6tVEGQ0@*VAnczP!j zRgL{|HFfoWqNC{(3^u^{d8}3a1-N}!X&$Jv9Ymp^Yg-*nwtLS z4qH=k(D@5bW%&!6IL?7kpjlabeT0t9!q~{@SN?g%hhPK>*yP=yId*cjLi-Llm_t=| zE1Z*W%CfJii6q=yQp^gKLiAWy9%0G^GYT4Rxcy$YHN_^fOm=SNFNo0EAO6$RbQ=zXD~ zp)K%KjJ8nX^uoeEcw)lt-rfROe;=s?s>Ro?_@P}1);q~FTUvl4GT`^0r6uGu2Z&gh zNZoonJ71UA1qPyXaBzUagY1MpJKB_j!lBAxoe0|6L1K#{ePXO@WmTyB>^oiqeV^}+ ztD>QHjm;C2^vNG8lwX$H2w1i&wNeYI&!0tp&Kgm_5%fK`q#e42g<01N zzhq^ZWSqOYxs}+h$o})sKhQy*Vgb+YLAw2LG!CW+`!dD!{QMcDkS@O4f+O@LxG?qq zs^iMzp=`gm-giO~vZl?HwNcqgw$Ngyj4-ycjV((u_Amy?E0nR7P}XeO$yy=1$Tryt z8M3P(jqjQF`~CCx`^(3ChUa-gtgZg|nz&3H5I@Jw5PS9caM8{8?b~G2aZz37- zb#@0-PcrYkWGayuqyC|9>9@v|J4G)(uRkXFQ{8WC(QNWsft=USNUFRiJ!E)Lzg&k{ zg27;5ABp>JeB*ht3lfWVn7A^j^N|1&X{<4b54xBQlo_U;mr6{*MMbU^9aVOn+@N)V z(2Nuli+QQ!UuxZQ7&5;3T^p>8%|~DhcVHe#Q%~=@wszD}{_8NS|5!3_YcTKkCeRBs zObXOmW>%KRZ0iwtb>Bxy_@TA}Sq_8x$DF+P?2!b>2C!+MW4r+D6TD9_FbNI|I|w5$ z@DkDYn>)^N(B4#fwXvhtF}vM0bViTN;6kp*-~6)}-Hlw75rsEiBdv37qzuanE1!*3q;`t37a<9E^%;MJ~wT zK&Y}=Dx>7mEUwYo;W06LOo@<4FL>7=>pC_`hw$kqYg+Y;7Rolax3fU#0TiP#XYP>JxrTg&36tB%{%2G)A73Q0?I!Z)Eo;Nj`{+IhH0Y~m&x;qrF>7F-k6+0il6Db@%VLx5rg2qBt6 ziCk-fNAATqEF&_)G|K1|>L#b}%iuw>@mNZGKtRr#`Y(kIg9wjhchQ z2jBzbNuaP`+1=f(`@K?2+q`3AW4jk$A?t9KDas0}xsPW*#gxH-AtovsC-UmhBD zLf|(H^qo|B_`>l^_JLRriR3B<`SxrDc0|U{M+yq6Y=~*1@GoIIP(^{j3MEe_AVpq7 z!2KHF_5iLRAS`TLZFx2>Jv%!bjcS1+LGziA1hn_dD=HRU>@Tj65gAVgNC7y!!7$lb zSplF>csRZOOT`ROJkX4k4_^5S7unhMD1F8+$%YfB@p#KUY0BEV?fs2RY>4t70&UM89 z*UE)f1Q%jDOW=5Iq7JMa-E-M{AKKeBE9<}|4B8{QU6c!qcmtLiv*6v7-lE*xM_x-q zefh>>54^nk`uaerF+rhFV$;&ZMlVWA!cf}>*oc?>wnTtokap}RfMQ~HaZwY8t2B4X zLwCThC6$%eVZs;U)cFe+fG`*J%-L{R2KLzoi;V!$5;$r=jM@Z*-r3UN|LY=n;9Ld4 zA$%WBU^ke{IWGtt0Q5!&!V2^0z&E${LC8kWd@Q)KQWmO4delmpp>)Hm;p*(Jmr7<` zj=xf>aXajj50zuig%HRF ze3@Cf!_oZPg{ZP+2q7;Nyq|<0IUxzQL6S=Vl3U<2J9P;zPEIy9wku#*x4ynkvwIWS zIr|dif(qU%=gWdXS_n7(4UU5g%!hWpgWSq}(Yy^PYC1UdSb&q}^?pRSa}o}0-E60t zLI7nZoCDR56mTwu^StLK*QP&`Dx7&>Wfqo-8(VvN^llGtptGo;1G1i|^DPaDhV14n z&`>&D{f=y{tdK!HHBrmdC>SMz@;hsW1K3cr-#clp*U(hanP^fyUyG5)**hJE65zqI zHAZwjEakN%DG5~E4vvl`Zj;I@6ZP7bmZyO;0CuoyOlX9EDJO(4!{@9v(1vj=L; zX9)?d&_c5t1+Tdh$W~q5-SXg`A}=rhZGN5|BtoFLfh}2jBQy(K+L9H0B@@N0gj7`0 zl9G;r+z!ZS;cEA3yh|-?qqw9bpd=1~ciA*lO}6&-8|$C=%3x1oD=$04+gjOE}#5S|`A+xCB|(MvT%duqtJf z@sb(CiZ$=A74Ny93BC7e)|C70Gk{RdTiuowxV`<~t|z9?8TUbVXXwe90cPT2nUuIV zT@#69Doc0@&jGMz&ehG%t_5~2z#2MmQc7D}a|eE(1jruP1O}wII2)AApo0Aj75fi2 zXJ>79cX80hg8x^14HH8I2t3ymo;Cz=rARsE;CVQZyDJd;I3pQIntIJT^tI%x$r>jy=GhVwH zQUBE4y$}hhLF@&8u&wHmFaF$-T{fY-#ap7y3J4Zz2B>JOn!Q3 zvX3NNu}F0d1m`NfH+sDP{q6yO*}{+hnfyh!$31?0i8i&Fm|dH&;M_0vzn&|P^2CDn zQ4@nX4##K)s*M+lzE6RM>Mym^252rWCs%xHtpAhWHR9eHCpZ}H6aWKOrMsBe_$GV8 zn>-K+9UCZLXkL6SzDDHYoSd9B4`QU1*|3REV1>gtw}$nooG9f^EB1l-3TBbdiBv_s z6S;r@gf3R5$S)H{EK|+~5ts={p3Z&FTC=}>kw5O_B;Uo+p^>h-F2C8+`a!N-(N^d! zLXhrmaCx(*U1jZyfRN61p6`w}+Mp2o&D!E8CP;|`htv+f=q9_DpEH$i(KDOq|w zChHL#yqkapgq+J|m#m1C4s#n-pEylV?l@vXN4vU;$Hr%U2S;a{fkK7^x5m&}2RTf+ z-(?DJ`t^o1zR^OFJXSvS3nD}VSgxQ^kMJX>9A>=EY?K2C~`O(NpT*DwTkU@h;a z@*junQ8n(>jurvHg|UJ;A{;8zk)!?Wy%JOPan-(o*H479C{}wXZe?!nadL_re}_qz zjFA8~erKh2RYO-dD?L3J?iG4kP$jYe61hM6sig(t*n6NZ+d4b>g@nQ(d!&KWqZFtZ zNjRXjzb!1xPE9pI_Gs_v$$tAb7U~-hFfW7WeKS@yKQn`elAxs}%TGAgyVs3?bWlzbZuvA(MeBvaM3sU!aGzxW!=sBW)V37hq+S`A-s zZ}?NRueM*RSV}x^`{!wxeyu>}Ez*8>p%!~Zk(FM~S|30C5&yKx9|~=!q!xar?Rg46 zZ&dw0ifA=rbJNoR8TTH8)TKGAY1sj3E&~=P8U=Kkw_ZF34aw1 z=P3HPjGCb#2UsM++y^bUEepEmj_-<|up&8|TVw@>eptF@b|cYaLNN5uUg{qnI{nis zLZXEN$ywF8k7iz-&1`8A+<&nMAKYT)E8PLYlj8~@hcX*ewmL;Yh>Jmd;a}_r#^z2_txFNw5}~d ztXkXL{Mg&8?>Y|l3Y@6tl9)=D_V{y#G~d5lzm2M>20J53kDH4-WVi`$G!(tqgP&OF z(D|SZg3+if;~<5EU50x1Ak&RV(;aV2uW*jY2|nO_{*G!GGTEx2pQ_r#CNIc7%~ewH z*ksq|cS`A?;RU1!3Su3}CSu*t0GZrq@$G$mEvIro+o~z=^2TP%{*oXav)6OR;M-;P zv~WR4a=;H2&BWvZi>V0=k4{4<>gp~2yq$J9W-( zkoGXAD4Y4Xk%Ib>hLx)}!i*oZu@L8^`v(Ms?yErd@>uA7t}!s+F!=Kb;N(t&s>lZ` zmOtyesd^tBvq$lKl`aBG=i=~tid&efta=X~mxw1#UrTJc<4zb~gp_wPEJl2vx>RXS zyRJ&gzWlwf6${IeR+PU(xf##OKxlp^b6(JbE9JA?yQXb$~ zN>ZmD8{rm2j8x=+w9(66y}ppjgQ^C2G-i4=!RMcIjN4bV^UuW`c6x}zsrsbOEzI?m z=izFWXlt;YUii?g)NQK+gnmDzzWF~6JFc{Az+91R)V=Bv<0wCB^S}|Uf=a~rzyTe9vyJ(p23p!9) z0oV!}B4bJ02hKaaA&Q?-ENgF!Az-!kDPYdKj*h0&xEYCTbeVa_&%{1IrOYE8WXG+rxbR*3-Vz0YxH$LV<)ZDwI1Mk0vQ;j{=^KUV=zfp&1X0{<3&7|enB z<>V51EiK3DQrd;B0@We>(x7XN1WA;<)Z^-HboYHae;XM`3p5>gmf@ zt#za~mEDE5&CLUOG}+Kv=0p)dE+|9B2-}0@<>D%#aPC>N6Pj__XVPIfmo^;~M_= c^Vj|WRpOrqoJ$_yBu}S#P48-v+8z9V0Ky8EPyhe` literal 24324 zcmc$`1yo$yn=e>|0Kp|#(BKJ9kl^kbG`I$LcMC3o;E>=1cXxLQ9^45Q9D>{Iy8rk3 zz1Kb6z1FN*Gxx4=>r!=2owLv0-}kefun+PQs7M4z5C{ZSN>Wq_0)c4;f0ACngWu31 zJSv0#z#Ga)h(eyA|7N!4$3Y;kAyT43DsHI<%kJ88H%|yBv!ylRSeV9Ap_HsxL7zT- z53Bsqhrg(gKdjz=dGTP_ryfFYluxfgJ1Bn{FRf7(PxZ+hKQ4Z77i0L#?AdLA~6gmrlPx16$Lr@#Q$4Al`$>4 z0}#-0EvA0F&Qr)hv$V9#>VBO~D_D`f+!=s?NleTzy)C7WfKJjAfyWex77$wWqSM0# z!fd_3#>vHn`05#=D0$2a;%AtuWG$u=qN3UTpZ@*Nk5qBXITF#YVkRQ6bx>Oc-(}l`PkjdlrC*7jF zJnD}#Gcz0?mwLa5y%0|KB`*Y&K9GoDjIO@*s?i>~Cow<5jQt>YYK?4Jl zlr%IHuU|t@2|0SUhLh~~XBadZ6^yuCkBGogr@seZ-rt{(x%UZ+QhEsIYN&ip*QvKz z{xLK(G?~PtH&lv@j-K@F>u5Tk7Hxubu9}*fQkW=ttLIgj_B;un$G}Bi_>_zMSefZq zx`Xl4?LO06$X=OF^QU2Mr`@l%^vlhzrejf|MFf`zNLVB!Q6nR=_^)0GiHX61FZiIO z^f@*bYh+{u0vXE?K!EW2+~cDXuzh8CQYqJMmHy>|#>2zY=o0zm3k)PIEUd~nnuKrY zed0>1C!YOrSZI&PxCT15j+z=S*~9`y}hcZ(PO#C$uan<_#f7g@$vDmbaB-` zTk3>FMd#JRQa}}H~L~YPwqQ8H#Y$$CZ@w-DxYVaiZxzRSQz5t zO>>tf`##&jR34 zyKc^RU?AYdXD}mkawr&engW0Ppyc(uM48Hy>rP=c1$Sk?)$`75{Izz29nA4+yM&4g zHnn`J`UXc>C~a+cyFxN560=}#X8Wq|lg_t<1S(=Khfn@n@@d>u-~mkj{{6}PJBRfd z(UE}Li9Q(Z_3jX37VP}PR2W)DMn;Fb*lL@H+lwk=ufKkwZ}8AQybTp4_y7K#ke8Pi zEGl#z9i4)rqOQ3){D-@%km%^>1LH{o0)hs&Q}Vtqgb3c=-UY?QIXcZQoF0GQXx3W5 zLB@amVo+S{%`Gd70t?#^x|Fo)8K-~!I_<^N>*((ffmI1DkdBqi?TWMlYxU&h#4j!` z4&!?{vxbI7MNN&Ux;nn0p^K^-)q2?yXdKsodn}Aeh5cZse zgv8Flfi~$f;PdASkBdLTo}84CXZ%m1q8TSx)?-J#5qSU9Jp&BPm}eN)(26(I`5#K= zzg?;WJUWT?uE-AfvM{r_v@c&!dHMK)Kp76j_5=T)l||-BBP#k_uxBtTH1rKZ95E(l z7|H+5PaPPre&NGJKL;V>ArVUlot>R+OiPMV(q_xK2nh+PZcGID`9UNpvFLSM)OSCC z(9k79&|<@{`~>0z&ub zD4M3G=J?c9cPJ*s$>rgqwzZe$Oq#`1dR za_ys$@}of1TijThoc5kUu8)16kU=rJqae?YPCk%+!^Vc~;^K00eh!^YD{bBpp!yhf z_`$d*qNAXQf{KEHjU5;tj}6YNe|LFUQCr)4cXdq1=|P@d6f{^C2reWB~Vt;dHywv0@Q`VA{gcpoT7z|2ki|-Q&D2~C6ja)~o?a`pDCcOy;E5Cbn zl@xMvcib*Qn=tek8oEnC-Hi2TzM6)^dg1xYmoLrC%rcep<&)VhaUs7`4oXT&*8coK zg1%vHo+XpS$mw;>T%=m8_Q#)1R8)l-99&gh9j9etWrYHfl$DLwnyoffX!JDi>+3UH zY2hLf@X=;uNyOf%rXa7xw63B%zBvFL?swuAW%Z=CxQfycKNVPl4*N<Cs$VzpgZ>-pkOLeQ7pVKctZe|>%mGZp^c4AW8(L3-wZDg=BviQo#AuZ zy#~!R&}O-5FtNCxK!2{nU=G}O*)2G~2q>*~>s@fuWjb1QsTA+RLir8H%04^&nMTZ) zPy3vZfCC2LL$ix@Bnck@ctqkCe||Y8ELR#K_}rgL?C$Le>F5xHuFDD9pOv-s)`F^< zS|30!6OE3h;323^9|4d-doYjly4=+WbYY%VU1gztNTx(I$?imsMAqW3iHXiAVx9`4 z{wVF?W|zZo7Nfp_gyXr&oC9|-;bZ&L|5;a_E|ejT^O!Mnp>?JoRky(d%I*pSFLBT3 z6N0*qHUT`ld1(zg^wPIiN2QC2Ic5b)0Ty$_J^+ zzi~g^5Nq4FwzASUGaI&T6g~b{z@VuB5(b?o% zQrPl-1Vg+2Hby!q#Y&DmrlY$XR$g9S*ed;b-?@S)&OpQ!z+sYqlmCDEsS=2m+43Vd zmyDN}=;`r}2|$@3wEW2r%E|#{WsCq(d?;39$m%8|r+t|%?P70lKiBH{9*y`+NPi8- z>-t1!_bV-XoXO7<>vkV}b_;B9KpX|a#E?k zRPoGsu@*Xk=C2^M%E^mtJyaUiDBK|XnLYLxIafv zuC@8HHvqX~y7Qc%$H0$w4qI?k<7+}7DRLHq)CiA|9r{|e5&5- z#-XRLwYQ?MItVR@-(dcN&S}>q8u2B~8JQ2cl6nI9d@mpZTN%DxH#=_&R3-Fw?RHsn zcIX8^eWFAo;(YD&MpV?j2z$`x>S+1vo44!x$H9oiNLyJxIn)(LE64zrU~*%VXY;Ct ze{$XZiq7qF+>6kMldRM3{{jiAXMeUFd~K*?R2NP2^t1{9A{)bu%YkS?NaHhk$no*< zTpxl$v17rzfH9C!P{7J*ot55&%gX7%iucLc?~D>>iI9=U=9~;3fhh#>OKV3WcHDFvB;#UTOag%$XR5rWRnX^g@_DPv5-#)uYALbSZ# zxvN==ERnZgzXemT<_U`;Nv+4F0e1NH;qqbfWZ@RC#nq~7e|KkKXoNmud`P9|0)sP| zV51|iRyah-5GzddpRbh4%@`{y_&jzeETcu}OTvlNO)W>JsC3QCG*#kENYqZs;Disr zhFXG@i|wO(5WL*Ct^#vb=xt;pskt}3r>b!td@57f*WH`-;8EwE^ODfiuGNdNQild#Xrn8K3?W%D6h4tx2J}hKXN!f1w zMCktR3YmU!lYx#VLAuhgPxLy2_gnMh@?vNU53)1r7wMV@^jEKvpEfZ>mtG_=3`;8X zm#)z@c(Y@?#HAO?7+c{~o=yf7Mf@*cbH+A9*XiBF_~eW@as-pzy$@T|YII+Z*s%1k zquk991A*~Zi=)riKFm>+%O zgAqX?dv=ZNL!lb#>H{wv-6hF0$UXDcM~z6h<$Ru_qu@}&tOBEP$kSrW%k^fv{j+%|H8nA8 z|6+1#((a-w1*=yUbT3&Ks_DFd+NCNF@h=d_pcH8adxX!~i(i)0qFUb>jhh~l#xjJp zB=GWZ0o$zpV}cHGyErf}+Y|&4+&<*wECpxLB-5-GFTRs_W4K*n zZ=;dYYn7Fhd@gSz4n&raeoxunFKDWL)gDOKMpNgpBo%`R);!TFvr>hBV|@5wxq-uzqiSON66^R)xTD z7g<;^TQ8#d9`Hg~hhx9Q-^YteAv(@j_5EH|>zLkI zsHE1nEnPt4JuxLnyoUHQ7KkKjV3=rB1Xb_%}(`-DV-NUc=z=nK)jQ|;q2_} z>RO9y!B1jYB3aLtkdOq`)Tn_d1giMI&JO|D6aoq&d64b)M|Hetx9)`nc<22Y9MBlZ zxj=_OCFLiYETK>3t&rIL@CKbJLyYXY>s2JFfIO5aovFNyO#|@68I)D;t2C&{wNEX_ z80k$+#V&B;)lFsQnC$nq!qGP{ZPyeuG@7iVn15K8UJV?zvmKcIU#EId! zJ<@smG}?7o><@Z#ZZ2hO{|2H!*FXhKo%J%U6Q`&sT#?Ek8U&X~H`3?f)?hx~{zbcs zD(K8VdagO?3EmUEJ>IatpJSQVvH=}QHCM((T3Xs}wHo1axxMRNQN&HE`m>mTdtl1w z!&i)R0wCQ5#l*ynlVM`w6irg`K-HKIAtt7gR4!P{ZTbF%rx8IftH^PKFin$f1ftdb zV7>M`Tygjd3<9iYK+nnQ22deZI_1FcU>+lpUgMdqt?d#zVpoH1-N{O27H|s@R+k!&c*_P*?y;)>n)x+%a;}waZ_E&(pZVKv~MRUG?Gg+`yNWl?_CT z2`!kT^{#@gB9&2Zup`TpP5BJ?Wp{r+0=Sv(5OmT&v{EHjbMPE_xD2NY6wQ7WC5*3Q zU?wELiPb>^vJ1e0aLReILYkTcnddGO%<
wwNg#t5D`%o zYSDxP{@lF0V5YWf;az3`wP92iU}!}}Ma6pma|z(zX{t24B3nQbIyu|G<^z1P?YWFg zR8*9(^*3xkAWiLbQDh@g#7O5VWd*zjBQ2r(pQ*C{CNccG^Z)O^r@68j+1)jJ4i7)X z&%?!qGv}9|Z|c9Vk#P}JV*BaSb4$zpXTumV*MqHuQ&UqfM@L7i#)4Y8mC0kU8`3W{ z^agIhrKM^FgtnZeU1SyI5)wZGS()a<-h}j}-Pw zYlMWFS|WJX30zcERMRDe0n~kR`txPPKurna_r4Vr5y??GMB)a7Y3mp~EcGU*w{=$Y z;ekERlk3bQwyhVla=Kre%})D^hSe`gdHvAPko3Zf22Y;+^yD@X^GkONc+<#Xk?=?f1FBa@=Bm62 zwRLnVo%WO>3D}XOd+#gzGlE5tD1o93gsi@X(@lfrW+sk|rpQhKIm7H6S{aFQH25br z#P13^!Oq8R`0ssq!F_zWzP>iBrI@W|1+v7?Rsq;+DlLSoV zZQ&5nAIwFw1z{PF=Y&n@_>sLY#dC}#EP*gdB=?&tb?IXMg7AdW>0MoH?Qo}TW`O%N!lveVBW!T|&T;JcDr z;oHIF_g>6j)5D89JI5y`-WD|A9nWJTzM-Q-j*1F61s4U*ZJbR(OAZbhSzQZna76?< z0sUV^B5EFTn8wD&_1IOak=2HPB*w7t!bBgPk=5OB86O`zM@PB}dL|<*L`pI;n4jP4 z1^ZXf@OfObz(P{hTuh`FTr13RyFM}SNn&+>6({*dCXrrfAp`{_g;NexSlU{$xI799 zI^^L*hT(dYIN}%x5F;v#ht<4z4C#wz-^>$|g}%|cAYeZ|46U4NawY_CpV3n!9c1U^ zAbZF0DjH)D5k;=8>1TBhOo7Sv^=q;pNkA$xldNdaAC`ev$5<=tfA^-`%aUYV0CjFO z;d@NRm7eC{^_WZTg;~6A*b~v%gQoasY?URC`{Th zg311&?+foN3mCJt`6<(a)r!h-gEuvx!>{REyt&J<%11khA&w&>zL9QJ_+>l&Tu~wY zcViTIB0vU594}H70VP3HtXLeCl9KX&yg|zEe^>%HtQGc7SSPqy?M)^pgXqZODZ<^B`bX`>(1`I+Dmu$dV~GV!8$z#c0?+8 z$(N3byz>lF1P^P zxVTt|zIb}Tfn?^EqDGH<=5mu7ORaBL11X1Kq24wc4jET}y({Q*Z`MF;Z0yMnpo@!( zgur+B93GBDz-|EwnI#G#2TB@`>*$5Ck{az|W1-dgJWhX~*!kLiBhzeE@<(5a?O&@e zkpD`kXA1Z%w!I%~qBE5h?%3O`8XF_!>j9rUIlEDgOnDH12zpr1`g~_bGVp(F{6pQ> zzYcT&DA;?X=rh@6Jt8_J#v_oE#9Wj`2`bNx7j2Uko zEw=>U2%rf?`^$^*5(E{0^-h$o^ja8u!GElEa4qe~??~Cx+B$&8v-i}h^1m~ZVlrG zw1Gg&HM;$~Cmf>4$Vd!S!#8i~j2fHYqpgN@**p=d*Yy*wM&ZLQFWkD%$V5hW1C&#F zC=0oSTMq6XkEwDFS+3hyKV*h(DW zE6;>tbmdJv>E-jKz{X$wq$Cn9hhHDA7?X^vMDw#$iIF*uKhIv=CD*$kGZOP?cjZuw z^S4JxI87Us%z3`xug_Qm^ML%~k6&!Yn8Klh2DOaSJnORqznxmo)(vge5z+3D2^ zQt{ye2RQ{hIVEqGAUHBIGLpHI(i4stT?@6AP+|@YtIpwJ957`%uO0+Bw&!7n}=jtm)-Gt1~$g1?9RZrbQM?VKQ`W%KUz&YcA;j_L<6)t zORE{p1Lan}?*3wx>1)ajj7CbziGaIoT}$*ix|!`PG0>$ZCfZ!fxN}t%?|7{Xku__p zh1peq8TNksc8(}`q<-l_5+1eeDFh%r_r)#zG6Hs~>Bl*)J8daXm!P!hvkp&}1lbQC zFd+J%cJ5B)M_ajr7ZU(9M-ou^7W(((0~+FXJog-|_nDFJFpv*wYC@phnprNtT8$!# z0IJOIKdOM(h{SDIjnV!p%F2>*?CEcVf!q^c;6JrRid#6umW(vrv>Ptp>zzkZ{ON4v zP5P*;8e}iSI=_>ej`{VWH#H_P_(zW@WxGN=gRR93_eVlGt&}W!m1+KBmEvGfRH(u$ zf9$YIEPYSb?w|bOe8l;!)+RtXsOLkxV_u2IZMeBCSl6W{vUDL7x@YItl69b9Wr zT^$Fwe3;+9HsK3C8YCe_Hoqeqlh67yuB#kB&X;eDjG((c?vh0v+4q(lBfLJ<@oMZp zHUE4a-;}3TruRVAxRhIPwrC3EHL8`dD*g;Fgp-+I)C<+bxoTfkES+%l%n&dHEaHF(4YVn2r(w2^Izd zRZKmvYfEZ_nJm1@-w9nO}wKhk>Y=4QU~ ziFJ0|;w~Da5;GC(Uqcua}PUM59)4s=~^~9NN zb;L(E$-k2iopq($w@?X3$?&q$%RzvsZq6(vS{>h!S z7#HbQR+odz?IKR>up#0p}-L^C`oyFaX~={0f$xSa1ztzOsQ7x3Jntz6SNP3x1sR>M!hx^ z5Exi$_ay}%VT$v9TKY;R!tA&$3n)V-h%x|}CRD6e%Ixr45&+5pDr#3(*MsF|yzT96 zPWLlPLn9-oI(vO{)7jPa4AlEL9fAj^_F#00SM^11p2z`26Lh;NkopPwoD7U^w7}mWWUhmO?(KCFjJZYVEBM=ZYzU4TU%5H5Xb!az^2Z=iV1egvW|VAXekW^ZCL zKN}Am=P1DJ*B>A5I6W_|(l0zN55fSB9^`igUQ@Nn&)5sj@T$Yl>KYoIzkaFCTpUm4 zB18R2Wo2a|UN?;Pr$-*Y$wNO{F zqmHdU-Xg$5psCkx&O@`9Df+^~XZ3sLf{>qpfxZm_iA?1}A0w66Q2PMcFfuy13nV>X zl4m`yqgq9Tbm#vRQrCy*K0Q2pD|K9HcK-r0A6C&XNOSFR$fSDbw{hsGEA7EAz;1X; zw1o1v9|*>h(vlsYD+M5L7L5=ghldA>vlUrZ*oWb-Ucx{u{t)c43}_r-hL7P~c*WNu zeIFa`?S%sdB=G1v!HrkgZ;CIqxK~R$lL8U~k4i9rwVV$aPJ%5TH0EL4*Eg^8yQgPT zPwQyDI$v9unS(?8eEa9=^=be2lHB>Ldp4^%3_zee{b3PKfH^5DCYGhw?(+f>v9m;j zlym?U9bLl2ggP=ZGBZ2-nX$3)o40R;q@<)=cryfi(~GS=J@3e)t7zr#w(N!(3k_ZXIw79)f>UFf{?Yr9VtgpF!;`i<#PNV)>hxwOLg{=1&`J(=28X zAbROf*SzX`a%J)PRxharRq!KR54u4i`E(TnER{?k*7h-IW_vz$Cbt_Pv=Q?+soW`6 zCz1f4<=LJ8XhD`1^8XOuB6O}cheusdvz0%VoNN z3j13qzsB}gRD7yea3##ITHg@7w?P4QVt03U?e}0@)fU%XyrUiq6NuR~^6&5M;kY;V znzxfjb>};5SDl}o(N4MG_=EX~a&ONI|H#=k=d7xt!t8X0f|ACY)8PPh)Arw23|y5V zd3;GoNNYVPCC(@YccPs1o9|fP$2nXk)SEGCBX#U^U182g1m(wTWt*G)BwfP#>*Q-` zkI_E_96d|5TE7$D+e~1!!lS=s_Q~Ac$)Q)>%|W20r3DZ*sI-*9?*N8^L9?a<__df= zSh2RP(0~u!wI1oyzZZY zoRe)nmDf0w=(HNK38aGokXK?~n4X@-es8nX0cvan`{(KcP!Gk35VDIbCay7euT`09 z>i&cR{T>7FoFJd?iu`)1A5pHIM2UU1Si@j&w66~y6w(lIjo;YJKmYyflq%p$0`gr! zm+4?N1b+RB!O6{SxY_r`Y_*MVrNupf-EwAaB!x9Mih;(zOrsi}nA=GZ_-0V)s6?-w zAE?q9$YAz^J7uw5;l;t$P|L5 zq@r2}&^DRp{oS%72y|7Q0aG8^U=9usEA=~{Iqi?V(SD^ zH50zK8TfKL3=$o4&-Ufhd0znstoLxSexg+C{k#JRngIX38vtzZaOmu<{#ODZzStWF z2M5~ZJTDIhtJ8sI<#N1&`u=^r_OP6s91AGR^E;CVcRK)K+5H~CZ0v_~FtfDW91{@G z?Zam=d=4^m6^=Xdf5CUj8eXf|FE9;n3W}cjEFMM0AzZG3ny4$$E||!M3^+)4jWy3- zms^<9MOO@N(M}chs%DQ1+B)mSV32ttr=>*#ZQN`k8xDl(RCl!mK+3=lIO`+d-=p*M z^Gixf`lY12!e!JBwVW-3fjBuk%iyenm_TMuP5^~O)a$oz5%s)I;Sw2j!UqNhyyA2~ zSm9(!LBIf5rcka`Q&V$tSifqxH<{ZQJeg#&Do_tV6DKF<+CU6NgYOeR@L6Gi#KmSY z{Q^wJ{>WFX<39%bvpPX|5p)lL*U_Pu3J?Va_^ROGV6&+_RPjiB7*O+`)u~r()aWJl z`-jWRDtBS;kff6zIBG0A($k6<#W@iFR#XZJ6`|JqyrQ+#L z26yn*l+P%gue<6u*J}_2fjv4pa=Scu1&|^pE-pL`4Gm}nFF{xbCvg|VInNXuZVt}Q z*E(TKbQzvHjDf&^t}RcweRB|6;PyKz&sV!TO6>9*` z77UL+nMiP^@o*xLz(m~(*g&KK`xk&>l~(ibYAk*snV6WUR~iN_(anWQs>`~F8o#EX zxcl-K_`}^Gt29xdNCAzR9xl|nJ>1rIOkiTh5i7ommCmi|aRsyydfR}5gw)ou*V!zG z2L=WTYrf7t`(S=FkD1Ujm9KzFK%i-J*zSH#z3Qw~sB-!F08{|*G5xdOgH`W;etLZR z-5{#+n0z2P4Pt~UWwxd_PH63)e~+L6&=9(T$O~-A;F-ZrV=U> zp92C6EG&=;RWxmU0heK9Lxc80Iyw=Mx&*<^K0sX|kYxlElWdBj*A9b>ru;nFLpEUb zRdF5CeglDms;YOCScp(H1jt(J0JB38B7lO7lCq+zuYfw2kOe|LP!k&@a=N1h9z#I} z5${Hc1~Y*a=u7V=TihMzySFk%d4)u~HggcRKK)yYS3jjb4aA3_tq6dr@#$&NF9huR zHa1ZiKDQup`M7oIk_48J5P1w#h1r_)F`lk45CJt10pQ_2fYF#ip#w2GxSn(!s<^LT z{YKMxHo>39#8nUqbv<6`1?#{LjD8za&1WxeMMliGjMUB2Cg34K*BQVDAZcd-5SFl( z7NJ50zZi)9f}RLRm3y{Vp{1#LmX;~1@|I;@^&J3f5I+X3x=iq?sx9zB6!2lL&G(p! zB)^*=!ax5Tj+ypv$>L5B3^On?f(0dOcYojLV6HMWEG)(n#A&;J{lekk;258o0V@`s zo}RwKpgUNbFF|1u4Gk?gEsYqI1(2Ey!K4sh12xRSy1&2QX!3^?Gc&Uv^Z*71`}^zD zGcHMplamt!0%|tBLt5-XdqxInlglBJwY9anrslJOr*s{~MW#PojCA4LIyecfrZMOh zHjm}_Q{C_nKfkYe?sFt_ToWFwu6=%AuA8{%vON2M2<&s-s0L*>1FZP2k^ht&I+P-# zp!9&JV_a?|4di*i*)TCM{Km&Wfc!48;3eKExT6a2RJ1Pu1^EKv$?Jf;0NvHH%sDG7 zE1A_4ZE|uFn8JgD(0oohzc-SAfPh9L2v9if&!9pXr{U3TIcaKYs3(_pM)yxGdK4&xX>{CyN^mp^>1#lU3=RpAXe7!9HA^2{N@!@PAYchupu+$G+5kjh zPfkx`WMrx4eWBD47?J2CydB_1>;Rzf+I%a2*UQ&08sv`cL+Hzp3`=E96`9PBusEvS znWYt7d9L&ZD3ws!i#}C%IAnbX%?^Yd^FvMDtvXa%LL^5(oW?OApz+pFLNc%WJ1;LUXrvPW4E@-Nh9_F&?bsH+f_!MI=KIN z_gs~>@2lRive>V``%AzpTG!&j2RmJ6>IZah9A<7MN~Ec3M|*oA;O_v-4;Vk3R!a?- z?`>B`>b<~~Znc3V`j-h%-9S)Xy^bDa&w#|i{O%o88wNL-_2b8L(4;j!-+E=sK`G7Q zD?&n82*{=x4}+|HAe8bf+^8I{v?8zWDVfbx{9_`*BA~5-`WYG?F5K?(fPsbeS)E8@**@3GY!yiI$>Ep4$0Dj3o>_TySx5q1Xe^2TXkY7rwr}^qMs@2KfwJSd^`hDxgtObCb@yL3{vxMH9UaSq4c4&(uvYu0w z7;yl6FE4=#Mv#;X1WF0P>;skm*#HUtWVRSGG$BexM%Fl#&gBT-{&1;P<~yg5&f5tf z`b3Sn^87o%|GZ1OH*}(x@LC#@8cuG*Dm>aj5L@+;HZ%5h*a_A zh(P2wL2G+=HxwYr?w>yyW*xu`837w(F5P=VF+uFS8vqewR#Cj%hiqtdg2u@Ke0Dls z(W%o0($f5Vqqg{`zMdW^sCS4){tX)c=mk>o+0xTSo!*k8wq|E%C!`8t>8o2|QDN5b z?8Erh{sF0H*s1;8{hcpVC%bH*?`%AHoxd-g+|m5^KITd0KCB?ARr_sCkx8Sp@J`GW@qp13400MwKOlXCU|q? zYrn{J#Kp)!214)aX8XGvcZ2g|>TO?c@E|8-6tId{)d_#whcuKEIF0^_(!pDP* zjC_AFtJhmJl_P=v7cgu|5*+LY00;-Z(HW3DdL^q~MQHN; z6)U_^U*uc0*2yD*4=2qm5tBYrqb!P2fpE955FjqWKqO_P{8ebhni*ZxYkszl-&S?f zn7Cin@RO!0EnUJ0YG}Y&bI)bUArL}3TEEYp?0Z|*s2zz2o8#ocY5&O6`snm<)6s+FX?hjkSxC+ELJ z<^S~Q_3a=9(Q+#+%ZfrU1+y?{Hn^D!C8`1lX`y53NC4cNy~d8G~l_4Bz3deh^j zK_WaH>%dt2K}jQ|&`uI5`>peG-_~G?)e|=cFPc0iJu&f^YNYT5fo8Gn}#`F(5Re#h?#^Byxn-b0gi`rIu>pyQJSO_3vKb# z0?`Ojva?Z!`y{7HHNn@p1edncvAC^eo&WHlF zL9MKt$r-6|hL_L*k9$FkrI^cRd#!Iq3*+zq@$Bv?v>cfNEBqfHP;z|x7F)yhyi4b(9SrYk1C1ukQMvAXZLKI{}ha5X0WHEvVd4xfts?8WoD&4@RuQW|`ZK zlsoYE56WFaHfX-JKsSe)Qo4m%WFsyiCG1Df^A-=>AYZ-Kz`R6UA_UYfAnqENn`=y( zw1%fbqc8yOB_DfGP~<%Gm8$07N7iVsl97`W0}`RlQiFd%!JD4mUNR8K-K_{6N(DPO z0sYhmrJkw%9t#K#!A(sb=HD+q*MtHi>j&iQfsogYdB3`?YlbG9_GerYOIWDn!F3Ze zIXRX97&LM$zW*CbIKu=s2ik$>1NgtL2)xl~R; zh|DeH1g$)sueZJYF&GRnJStJBp3eHEtJNGREJHRx%1!1v|2RE9 z)g9gbh&RLMfehAJ*5}dm;QgW17!1M)fhX;-)6M>*oo%q(X^X+eTiNd%sJ;&uc&WbW z&+_f#vWoJkozz9e64Mjm#~AjZ$w+FsZ_n}Y@PJm7)Sm}EEC-0xe~$$qcuYDV$zTb! z9)MP+VhGkuhw-S2Q}upucZfGwESfXlPd11xv!%@dWNtV(j;3mEN!~4(+@Ibjq@Ujk z8h(7dHeYIRn1=@%8C5k1#T~3vBd|VdM{ip%Ac57NVz!dUU{XF0^&9B5>PkiycZ_AT zrGEn_x}vJ8E4dMQ!gekiWFu9!p+5IzE;11JKLhC|LA4s>7F9GPbCu>6T*S1cX%ibs zKZ}xMa#LnIu>4!y8Tzxz1u|-?xIlas)Bo_g@2B2~s6;l0$&qkiV6Z;mWX3bpo-U#Z zqI9gRY-SS#%1LFo+}PO;^(IG7EnfIhC(W+I*<2u=Od9}%@eN(k=R`jEZ^5P!oQ>~U z6quMKQD47)eP>=iH#iFNjUZld`7v2xc>ALNwnQ)eqIcu^bd#Cc;>F8q4oMHA4hV!K z3;Op0fM~A@88pP;5iHILwmhcPeB97L2kU02hG@Uq$9)H)N%&KrsyGA6q#j(s(FjN&id#a^r^@YsOR*!#$&wDrml0!=bx+0&yVkO@8cA< z$PHK~&81cUaQhRZwYqNVSX2R)+ubFD#h+i`f`nQ;9_l(9j zIPl!y9E+o2^)E{_V4{ESLq0`^J94?b`?#!dZ2GIr6f%3>`9Vod+{+5ZrMX0Kpto95 z-Xl7XLCa`vS_2yHufF4a1&Gka{ads~n?#1<)dIu?mnt|2KvV&5VebYmCn(sS0Xr1p z_Y~>W2eyTP7uG9?bUdCAzpKkHc6W}5kWA;hDD}D>{kBB~IjO#5f#}#;<^D_VDf@1J=Et+K+5_baUVNW8|2g`_E;L%F*-Dg^G`ht-8y?QbQ&ht#S>yze;Mz}t z@{{e1k1u#Vk=p5X!$1lH7V_7H1gYGb@GIyr1I_iP^k*;t>yj0Fj_z!tN9qQ&hs0LB z!`fIe+;&}s#siG^|5s=&_sQQ2%-51{7cynWz$Qe&zj>M3|MU!`Mgu8_0xOV5iQe?wqs6^5REWW4(K>Fl0#E!4mr_e(WlZ;krUMrac8V$eaJ* z0;*R10Be`6HXC|ddZT9{OZlVtNO3zR?rwC2pznPh-*J6#Nh%8>z?j4Q0=|DS2=RV) z5AP_jm|Q+VAbW*{*VbKc0CL|c`i@T;vj3Q52l58D(L@m}j<~-GH~(-c+k-2g-+_v% zd-YF(v_C(0k1Im_z#fz2^?L?giKMYcMs#R}f`e}L+q}v#_Lb<)Fi2WuM0wn{j@V@AfxveVM#9aVWS$wM~KQ0=U7wD;40+|o+m`#WqWW|dqf_&b zYYE=S=8!u?0he@SgAT&YgfTQXD@t<87k;IBOsZIExe4sUb)M_Q*n?Qn%lO}uf7h=5 z8MW z|4}aXAkYnMZ}*Q<^(Ar(%LA;jj*>9%gn@$Y&-kaaIVORJP|Y;djzO=Srcv+I%9=8{ zD9xWKISM3?hXQA)+*f-N7X;K!`&YEyx_3LihwW@;Z$i3l*PK~Wx6)VUcsQSZP5;(q zQIF1cyHe(F^_wx*Ysv~BuzcUDRXlQq>RZ+qos9q!^&^h#(Q7R@sl zbL6}G!0*p4mx7XaibpT_9Q+PS=;hr6+!-%^zH+qNoeQ#n_LjY<^OK1oQwdjVCI^8j zJcfLqms<1|8gD7Q{hHrD373vC+Lh^uw+%+L-sykR4jA>mv$Rr<88DV5ONX7;&>&i= z#CeM2zsDgKlQw)dKR*w*z}MufaMLJqr8`}=`nCDJVVEyQe=lqMnc>)zS>z-MO^?LY zrlg#_(#&G5%XFyaY7=U|^qzWTq~-Wf;mZuKoz8}I;(?z(souX&QbxEkxO)mK@@|{Z z_`QWOsjU10aiCdc-gWl{7_J6Sip1faG5f|*BUKl>f8w>PHGjU@jmUM$crK8j*^;1K+vm9wn zlX=P?U)$E?+qai3#$g)x+`QBWgMwzgs+7$iow3yyNu@W@=E;>vE`oWr#^!hR*VfqM+Q)h&0< zQ03-s;kClfuwL-ru@wS6XUIXpmgU}IEUHW!_c(_BLS`$7!Lc|^tDJ^l;k zrkF|=_+KcEp65(6K4%9?_N>F-oE&{{2JYZIPe;lj_$tNX8D6Tcw8X?74Jb;9(TAtM zH&XPZWTdw1HznAP&dn{imagV+FO<3h7_pw#_+Rar2O{Wv8mtK;z-i>uhPSXv60jH3 z8I-9*BW})ADvbZ+x<6(K^rmOy9W#$9z0?7HExmzU=LyEqI5H;-4f4fC8Yh>82lp;i z9|M>$pI)Q!1z5o8Iwq#2sqWUTGB|9G>Qz$mdklaXWKMajTaG5Apr@CE<#2vBhREss zIsOObq*83ym++3O0hq+L_hA`6Z{`QE&yKDx?OtWllP+!!2I(oprhohVG`M*`=II({ zltfj^#68^1A||%JwWYeEQ-BRtIdr-_dpKKRuG$*zwYqU&fjo#u`po1(S~j@hS1G>X zxPY58Ff=Kvv{^!MY;?LrakcGsJlD^Eb&T|& zovXCYZB1R8%57fa>Q>1LR+scU*m%m1psG$=PX}uG>MHYFe(a$dxeu00l~xmnPha(d z>!?c@9vD<)JxhCf{F&$uDjUuCSW?<5E|}{!8DbK$XUQJ3*s^5%J#&87_qw{S{$YGR z?|I(aeZQXjm)BBXeX~EOlxNR1=e4bFGv+|qrzBLGTs4E`KFP05{Vqq~=eIj2JOU3X zrr{LAWR`2$FVmzlsYMR2p|`M*T@n+*s8ORC4tMLsHE)!2&d9Cy6`pR8wK`R`BBL_Z z{6hXTucX!1?RUZDbM7^+^Iz}h+@f!8_ttvpBfP*B*tp0q!NwbovD0QLG|2HAXT3IA z9wkgPMUvMoDA!^~MTc&5k@e2Y`(znre1JF!tEc)IjMZD40Taok+vlylnhN^~EG(y) zFE`-K0L|8NE4MbyDx~ z5AS=k99gQl)}M!0o@K~#7RmqoGBBL?vRIKCSpHYn2G><$;W?f%-whr6pxqd>1>^&; zae6ZI+;si}FHpL%_JPQ9QK9VdfmkFwy&*~I-c**LWYs@kZw-^{w-4`@bw~F2^P_~I zYImLv{-RG^Mn<&{Om!SN@bOFdiEnR;FJ3eVys<)`F6Z-lUMep|y(!+9Eew(C(5EoF zw2u$2q>P+{XY&31`B{9ULK5nPU#eR8PR;#tPSvC|Wx*0B{a_{6&akdce;M}=xMReX zBiDw$0p%;39Cm7Y$+5YG3AQ3(S7h+6>vKyKJ|?9_!US(fd?Gd!_I2X8Pv+6q0Ytq_oF{uNz$i;ar5||zz2Ov?PaymJfYoJ>gG&M{^rqR5p@Nb2R?&Cz6f z_#}aZg!arK1P_k}Xx>)VL(s;Dl@>{HTRZGrRu_^S5j8?ojoxvn)hZbSjAkgFYxj8LW2{u zKC!B+tHWRO*jfF~5yG~x;78?^mJ&VuJ5ESW2`n61haL1HK_-Yw-!>_d7fhU0KOX=u z;CZycA$a;VpB>Pg>9@+K@TOW!5B*BQf5Vx3 zG89syqgm)+OlTIuaH$1hc-Y7>6^Hw*Dba{}@E}edf8sQc{>;B`z_%=S9u}YUTf6`$ zmIl0kvdvYk`>|BgXa$tUUCh8W5s^Io8Rr02yb&WKqpiI?2QP2;S&P6H@ZV#B$Z&61 zqlx=1LqpXK^;b~IdRFQ_Cs@xnkBo3bck0aSER-Guw7nj!a5(#`d$PMb3i4qP=g!@S z&Ji%)I$By@uC6{yk+3MK+}PZ-CB$9;nfBS{(nD~!Rp4g;2nGglhC(TMA|L1i>XX)A z*Iz>75WL*l#zqeCAwXfJL4#WE1R#7TPo2sJvzQj(22dY6Cub{IZSbdc-!z>tGBSc8 z)ds*E4Cc0P-_ldWOm775`)*OEEchjp=XM$z8ffhaMy2LL3JO?I86vZ_16_NwUc7(b z@Z$-nfX2`*35Zsu>v#$jJD`9QXi5gc;4avyz$oMana{!_%><3Qg}_QfX+evvCGaT6 z$DWf$;sKZeJ|m*6Oj6N*bGrE@1F$d7NqYglzL`Zu-5^XsCQA#%McLTc09oAy&e<_q z*p<2qNbN(V7H?s!p#XsbiD6Uj!ETF&ly3k;4?7N-tPQDsw8GF%leU`p;0@)P*EZE2 zc}5ht8Y@7@fPF|Okz8OBc7VzaNFDRqny&o~@vb#*$~O!Y7tP)UT^&Dq)|K{Ew{t>6 zqQ(r6lr#e2Yq2TBihTH;8^bJ62@A}4{qxd}N^-6p08(>vbMR7xq^o*6lzJ*;9X*OC zL+b+bW@|^sE5J-SE*#i$O^OX6brV{Y>E{}cE6Yh{^)%@U z6qeHxM#+^ITww;u6aSf`pdEbmBw=M|)7^Ju2&dz)l zETJo_L1P1ETr>cJunL`5^w&4GNSG~x;w#%O(%vtZkLZOkmlS$}Q9OCHBEq!B?Z8z5 zbX1`~jXJOKou&w-E~{H1DS z9rUlE(P-&H)$a8j4Gs&e=VKmPCfc%OcR$of)V>z9I_Krxg_|lYTVVgEL86n5on5#_ zuIg0tv<0aEVW0`-+HjS#k*v#z3QQeGM@Q(eoSvPfC0{~$UHt;U=7v`Kt#}Z@X*pLL z8?#lz=7OT4QGjrWk|D?Mo|6+YXm4wN^?NI*cf%`LNQCk6wBM*WdtM zHNL*SPv5&kdKh2OW_{S1X!S;$+AtT(fFz_R*L$yY0I0kKW=enXOxqh+J0W=B_&XzVdtLIzayjMF@HCZMixPmlXfp z(>6e!;VU-LWlR0{?y#!9nnUab>)oD$+<5l}WDo5keZm`jbE-Jg+LMG`5-L|{WAvmz zq^7!0c7KBMhcHCD!H)#9qr-#w)O3VuD?Z*xuGW=P;MhkGxIPvRU(fP6VDutQq^3s1 z{tc&3yK;+O_a^J@=S9XFvyv{RDTkVzs2>GhT{=tx>k44!t#EzFGIS7K;owueyw-OI z)P-HCb)=_}`<28XLYSF4%s}-mV)x~yD~ce9Ad(HgCL@~6Zk+`Oaxz^qm2z-`rjNkw zN(X-im)0X!oEslRm`Hrh z3feZGA5nlDslOsVg(Fl=hf7}_`%XAo&RC|?;Upu?!Y1grxXPeF}?o-jd?{w(+r@x+LlrHUEYgEwuU0F~urg5rtzER+F zG-)IuUZUD^VWkvpJtdCDt<88pyaC64Reh6`AqhGGmjnNEIg>&7S3cT!xgdjYWbCzc z=~pB9Off(EtWu4YJdupAUU`u+w$(H9MYHI3j!3Pg5o}jn$tH471Koyy-M7;4m@OI{ z?cr07T+NK)A;T!XVLAQItOoaZupn=~MgXU_Vrw+%{(yZ(QbMA1o$}`uhgpX@53$?P zQUq3yg%N>hZLNPC`*;ZA6CM{Sk-5oL{jySmq2Y-g&sO|dN(SuvotqKwSB6A{uEq;0 z)RgVj51!xJ>`W!8(lN2Iwvn+hw;D#y=`m{xnZx(9x%o4B=6BZ_i=BA~xTE&33<}ih z1mMSh@+Kr9bWxo`tbYFYvPBvGe+W*ykZH2mi$uc7^=-jr^p(^QHvY|hx!`XNFMDOF znRPY+bg`5%9r%8a%o^wyzw1({=kzOPh)16nQt`oahZ*B^`GccRnAuu8YZ;?zNDJYN zr6}RFtAnxvX?hVBX zMY=oM5kK5fo~>R8SeACuy4UPp8xxeV{pyxz5Y)W7kLqir$P(!s9o*U&_yxscU>TR+ zTq0h2-urRF{Hl_p!Op#vAMg#$GQiH?2SU!6*Yc@mzSLlc*cC)9b>ihpC8j-6AX1n1 zQ5quhq#JzE+|2^IY}SXhs-Qw*e*E}W%S9~lE_!D10rA4<34z|y#VHbQ#=rgAovE`w zezTMihr#Kft?B!$vV2|lQw%)um;cHa72w9EJ<7AnQQbcB@`#jOWAk;f{n=prqzR$2 zAG&AjVCNwy9Eq=XXUu$s@xuhU_e!0E)d!ie^v{D!s(-#_I*Hu$Ox9=*8{J5H$k_}U z#x8nD+Nac_mGaa1l#A4K%vy8eOcEE_;u`u$_iWx%&ZD=PQTsmoOv*>Kc`DcrZS2PS z158)WwR^PLC*R1(l9d+E9;_5m7c7&$_uMV3zprIE{6l4M8E0v2tk|mc!bNcgxS!4u zh%0ZHPF@m|2~Qi6_|p6thbv`yS5(1F7ow(?fQf%NC)5+MP2swt(uIgrC+HJc2Dzin)K?6qtw=X#p?lfPw~|KgzM24$KAc5 zSBK~fgh8K4gz^YO7~n1|2q2FExrj#3%{R~NCM4#l>y~+YEW9`TS!AoLb2*CIy4>Jgm#<%V=}568 zDUocqS3VXDfNrmgUh*yPB^A^&OTXuLUWXXib_$KvI2$3$Gy)}It12Ol>aAvV5_qJa z@{f0K*)<(ZRTUw@_9VO`VnI@k@Y60@r^F)cD4@ap8!Az@lenA9p^K}4hrFk$nd3I$ z&l2o>s#F(eJ~_dj-Qx!(fxj(_-dH#dwELmQtvUx0ln~th;8>9;S4+&8YYY4D^N@0q zFGI(_VQ%|9a=a|Ba!NleUwWYz|2C`XSbdY3@uV4hwifxxp3E*%3Jjcjb)47^d%>mc7w3IaF+e6z*lC_{g7T^Ja;Y=RQnt(3Tb+zO; zT*{}0veG;D^2ZJj|H7s&Hv~V*yo9o|adZ0|FBswHyT`TB(exQUvb$bWbzhK;IzikE z+1(P0y(sr+>YN1Zz+Vejkf`}YKa$uE?j_HR|nv38Wr5R*J%%L5FZ!U z3`gaW+P;jiFyFK)$Ga8P-jGVSh+Vs~f z1~?26pcC`Xoz{>FUfQCps_(bH^b(=^x= Date: Tue, 3 Sep 2024 00:36:08 +0100 Subject: [PATCH 28/33] this looks better --- dag_example_module.png | Bin 24183 -> 0 bytes .../ray_Hamilton_UI_tracking/ray_lineage.py | 57 +++++++++++ hamilton/driver.py | 94 ++++++++---------- hamilton/execution/graph_functions.py | 72 +++++++------- .../test_lifecycle_adapters_end_to_end.py | 75 +++++++++++++- z_test_implementation.py | 64 ------------ 6 files changed, 209 insertions(+), 153 deletions(-) delete mode 100644 dag_example_module.png create mode 100644 examples/ray/ray_Hamilton_UI_tracking/ray_lineage.py delete mode 100644 z_test_implementation.py diff --git a/dag_example_module.png b/dag_example_module.png deleted file mode 100644 index a38a056f151ac2f7a1d9db955503691c388552cb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24183 zcmce;1yq%9*Y3Lz1r!kJ21QDc6a=Lt1e7k3?i8fEr9nXcN=r*gOLv3PCEZ9!cgLCQ zefPJ&v(NbUcg7xPjDy3W;#!O6xntgQUe|9v!Sb>a*mub8AP@-bSCV3i2n0$E{0_!I zgYV!^CRV~9X!=qTVu)+xuavs1&j`eQ#49mjW#_LO(@t**RwmGP-2P&w5xi8e2&!XF z(`^~Uufa$wzGv`}j^hYVZn*V54Vgi39rOJoDn5)clv@ff9>2vE{%d)B?i&&1fP&Jb z7kKWZrP-0V*ZYKzVXF4YcDLjo+bRk%F|mcE)Ccyr;4f`qdJJ+n1}0SUUHo2#Z+8zYhBKnW1H5 zyK#Q-6+(byCw hDA|M}-n zPdvAhnmA45&&01^{~-QQQBnDupa1#w*%q<)Ks4Z|gM))g^k#pmuj2Hr{FfLaR9-Yp z4pZYLT`dJ#HAQa6@44Pjkub=9ePM6UwXnEo^LLPr%W{H%oaen?VIdnr_i*j^&hfFt zbhU%c*3`4swzdZ}G&k_+rM(jpCQZr`jrx)VtZi(BO-&i`@bTw2H^a<-f1lr5>cYUl zkbfLg7hPh~N5jf`7l&G~b*$LHF3%s9g5hF6)3szvPEKAWMiV((=cZ~j4j20UAoI?kFLv?Vpw?{H%QXb(muttQ z!b@6O+PU4uj#w@;yxyv?suh*Kp^|2cx<8E4;;Y!d$?n1Q|4Rxy{)^fI;-TSBJ&_n#q7Q$+&_xCy>( z=AkeZ6cjL>|HEwsYa_(0_Ru>>@-}+*pO-X|y6YqP)^>KHF|1m;78Y2Pl$3AP)zfqu zJb9gWQWo?E6>|uPSv4DDISh1-jnM=I1WGN&@g%}2gniMl5s3Eoc7OAll!Jo<>-pwe zAIW*{5;DK>OG=`O=dlX+MaQxAzJnX}8m~$-HWC{z_~!ff@2&0a3H0}MK1$R3mX&c| zTwdZ565di$Qc9I#!hPMf%aDfGD(+~Z;o^Gu3H+Z$y#K=hnQ^p~4+GX|S05NC zk)u63GD7s3L9W-1l!nHkDT#6^2yRKACK9%h@YBbUjXgcs5Ek18>tR2?$$Pu|i|X#} z^vVb!T!p7pX`AB$9%lsS$J@l!STrE=NT}hTs#RPsW7*Blea0fn;S+&3~IhlH6b5l=O7Xi!RBMHY1tJ!)L^M~hqE4_)niHVd4 zhlhH`#sLHA5)YY}UYniDBs>+SiTpFh+!OD#+W!F2_~902f4zqrqVf9bSV)oU0*$8-CkVu-I}gx8672l_3BknNJw+0OhSLE zD0*v0hu8dCaL{Yw0Idg`<0a^+a6as#E_F&Xk9=v^OCXuqb2OEqD9G#1JFqB80BIHCa_J?%@ugW7UYPjpUP$6E#>B)#vcRW+prF0>B4%1Q zH@7IW=;&yKcV#7SlncaqrK)ISGqd354r}-M`D@>rKYm=_7ar6qUaF@$wO#TV!rjo1 z<_$a9hvue+RCsaHnVf8o9_93J;^9Uar(xIpb0k>v#H2UdZ>}W}XLqg%b@;oA6h}{Y zHx}X^E^g{zJA@>aOe22-S65esXelvTCghuzC(7QDaGMh}H#dL!^a){QWp%JUOKoOm zW^HXP|9CV-7!}dj8O3aajCFK$^DuX@oJJuF(Y_nf($WZowzhUqc=#13pQzY3W3ls_ zQUA0zHKFi1ZZyKcL}iZVs^UB`?v{5iFD-5LMnScnYecbM8-B&1B_)N~5`f(TAvJ2G zP^;$YcjcT9a3-a@pYDbHrHFhbc7WpMs3Qp}g3aEaS7T#iFPNgz;6IVqiku)$qfSPa z8kc=z_v2|Nc5ZHe2w4`EmNrKlYWA!B!Wmj{$ZgCvP;X9@`TF=Y#Y|Ar(3CYNX;Im_ zJ~fV@60qImUgE|Gf+s{4 z#WpzVAeVA;{Ptn{$3vQ8T+6GGfQWQH=Jvtq#4d{x0M3Vl+Mp3FH)%l>TG!U}H zFz(!GhCPo4=`%id_B%_<_a6GM7C!$rv z=e%>DUiRXam4QCH^98NDfHO(U<`dKP)y7hNy4W{)e$kto9JZa0rw*!vEWKF8TExTH zT?T2p6Zwc?^NGZ~7W-ml&iCRaYmC9s#yB#gSQ{T(m>r7YupWMkriJ@ilD258|90-( zq-Lj~72Tc*ht=V~%(b4E>Z^kgK4e&0V32O#^+lWS6S{6@3su!!Xl~FoR2N?TAts}w zgjp9*t$SME;(WG1r#?P*pOH~=^#{D~|!^MTWodE)F_4xy&qK?lR(sQ?Qovraxpw_OYJ@QX?mi zQO+FSF*rE*WGUv=DraM47OPZHh{|{Ug*3vk3MV#J?OI0rXJTSiiU~y;)%K6l3}*Dl z*q->Y`u`~2Iw4edTKizL*s)LV3CGhg&A(Q`MW4_-mm`fdQ|xW6yB>za-Bl3E+v+{e z;Y~q|66Uy$5RtsaL{W8csOaqHvkFV)a6`vIe zThqolmUm$xj&_|Zo&6&*F{anCf*abk&M1g%U2V0w*Q-j>%yG=&#JCEt z6c<`pA3Q)oq%i-!&T~{$QY>;f75TWPA1H{Oo31K=Ny|V_@G`iK_2B(|*WV?LNnUR- zv|cCKXPmjmLsljUp9`6&cX!sd*WC#0>qfi%=d~g3H`K(%6~4Ur_Q6V9w9u7ktm1uC ze+B_;bp;AGnzVn!HfK#@bUasNTN?ZCd2R9C2#qhhKft*AtWds20iOahm;jlSl&P3E zZ}oH$XDgNveINPGf*ghe{~WpF%ui1r$A@YkKE6jx?u+howW`v0qi#)2w`VF2;XOp< zHjtZitKh3(!H@SN{0${p1Vis#wZz~ucxyfIMQ*&;TV_YcvUo$@n@_IS)~$2znDtEt zB@}oi=+hjFlRnnwD%5vb*7W~r8RgxqM<*ry3zturi9?A%jFp*JrM1HkM*Q*YVdZ{C z>UO2(L)}m=aZMj>U0M*KBk_Db`j{?b>5kG9+{oB3@kv?=n01R0mR>9(u(1*M>7LzQ z!P)FsH<2qg%*xV#20ztEsl8>r6?f@y8;#~*%U-!OTRnR-D=R~<*Qpny-a(6lg@v0C z#Z&d3^KbOUWmb{Ch$P<6v+==jC?5#aP#gDk6n~N=At6h9KeWtCgcoe^Ad*DL>z7HH z+wRx{d(t_-Fq2xFMj@=OBMPxw`%IN?mrOJw9|*fHSE_Kd8NwU;f08V&ur-xqjuu@X zp;m4Q`E2=qP0h#;YcgQaAKyYncv~^ZDhed4Zwg%&Q-rCPJsLi$lcVgia9VmkHs<8xNhtq(8aM1D()(LROr$9!<&jgflB_unXp;M>$&+@bCy0h~r%sc~B8G}Bl6 z3=z5zC|qSUvb#(E4&OrTEbKna_x{D+t8fr$+p;HQP-IF>If{ehDf{d69Qz`Fu)){Eq6(>#kZG}_ED#sZU<a%n+mNXf{@&JiYsXdZYKpnN***Jm342OnP-v<-XGe}~?7Ea4RrOl!on>0kMsFR|h z-@pCy^C{oEXx&3bZtp@K!yi~62VrX6!Ho=oABjASTF#&@z>#hLt2 zebsjLVFFQ~C?ua8&O5ruQmQjj-4EolWI@WzcKWd@+noLk`KLUri5+F3D@BeyWTs|O zXDcE23b!+!>SYxcMno}d*&KQE^-}KLtERZIvo@}j7cW2xRn#+f|De}gWw-edB<#8f zFkVwAqq(X|>hRc_1wPP_B;{n&lvG3uh2TZHO;3zH~CE?Mv`rw!JJ z_nL&&Ulkv*K1JK_z>*lOc}v0Ps=WJ!91e%qkRgbQqSiQWiN*Bl=!J(qku&qui^OSR z|DA&~BT?5pUUHoZVv|S=90#w=i5(I5qh1%!wxEKn)YPf=xp-~#QBYVoUiD*zuAh<} z#nT!bmexAT+=YJYqXns;1&PGG6#K$vPU`q#dGa0+i~3DDIXOSg{kGUG%DE~RN-KUg zecL!J?{Gw1q)mG{(9R5O38{rb#k$Q{^0leNscJB1E#Oibn&J~|0!J9X4 zL}Dbxly%Li;bJUrLrvV$8jN3z%qaPt?=hRmh`72|_h;mA=fW(u4xk`2%Kj9e|Cv#S z78i5S%YF?C#%G|Rqr27A)Z~1)M!2xB;KwRsxw^L23~K_L{HY&gI{$jlZ3TZV$Li{8 z)0x_;ek0JVZl{!^DhEMc7W&^xv;Vh5Iw^%Gs;0WS8Orw62TbO_e*M}xI%@Ap;DKtx zXKd{4Peabst~5nSm_jILib_kxLWx-V3$*xF|NiyZeSQ;@4`h_or!sWjzP_ll{;F67 zIt}5!b5&dR8&PQKX&bw1?{w#mzKHv=bbnwjnBr^?R=o(7#KK&pf%D}8=>awtyLBVt z;$pwAsJ#42-vKE~UxZqQ1%>kBt`o72!0`ecECA8YXH zXZLG9f0nN6#`9Ivi_;a`i<8BOwyI|Vl#$h)KYuoUd-Zt-GT=tD4Kd#E)7#Iht6_b8 zvC4V$$okreduk-V+3z;~+}he58JV7iF@lzGNy?1OOwh)zDy?^K(6guomrpm~R@0`5 z)YzW!YMNYqkUdJYhVpTBb(P?dp5M7GUJ^Tu^a)x_U2TzR4Z3gmu=eEEtnPp#j^@!` zGmb$!LDy4|ZOn~#7uv>(?HDpIs>mR=;#X6o(I{lSdeZ&2imJwOv-ux7sgPQaq;^`x zhdfp@eqmwQ^~u~g9swSZB8$D;`q|4eR_aUzg=j!Diw2QONv*NOH&)2)p4+bm#eOq< zJnn$N3l@W%Jfv-7-Z!NX<>W;3(f8}*{dgFOu{t*%cXtnV@hwtFK zPLXB5k|-nNP5X-;d>^*-(;+H^5SXT4VLM{T?{&{FrN${V?b#6we-kVM8#Z2&x_U|o zabh~HJfadbyo?y51zODi9NnRC1icA_`;cV0U-cHULYm3z#4Bg|HXCCcv8wrKxdAee zXs?E5PiNm|Qd4dNks$P2orn;|L{Cq?WX$53W&`gCF(x%NHJ61EGlX>=<%i^aAm5yb+)=-$RtZ zT;-?i2;-GbBWsOo9`9}in}TyW#{7g!*Eg{F#%z0W$l~K6Bsbq3HL7%+7ow$aO@DJK zPIA;`-`dtrFz)cRbXd98c@G10dvKC`Igni z%HM#oRmND=Tpw*wiG_D7^RFjAbs1J#|3UOp@3ywM2}?R(Q=J6_WDiw7swaJ}Slp*3 zIMYj&jG!3%LB(*iDYB8ksQg_WTZa#)W3;y=O8iBKxP;z$NJgdSZr)%qYtC57!?HGE z#bgEn7Hli+`bJ3m=Jsb_UDWD6tG-Ktg;<^EBA_B4eGwh$(_iD`OyNrX``xHYp4-t} zpb&dC4%rQC85!iklH+^6ur$OF;&S|cw(;xs*8zvdm1`gO`JX5k+?OU9W_Z}x2h^vi zUp+67X*R;!=u(`~L_fsIDm^kOI$%y?bYQ9gbx={srtdmae;SVP5S;)y~$xiTb*^o*~ESK4i`u-JZk5*7yV~>mM(@r zEh*7Ao)2@htuW}}6ggg5@O$lH%=VpNBR<7{{W#lE)i~QBWj0+oMuJo(610g2tp1p~ z3)#Id;4X+g#bl*;mU42zdmuP+dByz4medv70R@4)(MPf#yG%cv!yRnqxcSl)Q;gi) zpior4pt$Y&aWm$Zu>+3EHO832)=OL_p9|?_sgFJYF(T=o%}W`qFG(zsc8QpfWA*@kFehTDU9$A z$fS?n+G|ba!yTJ zaS&riqcku3Iw0(a?(n}cn)cDw^jos6^|&p+wX=-=L7lrh6uJ`JqhJIW~S!NAPr$9va4~hY-v4i z_7&ofe{}x07=jzvTa*ZER*k{}nU(kK9Jv^3;%K$9?v?p(}53vna%%Yk2*W6j_|+`y0>ay63xSh zT1#(_?azqvH~2)&<Vqo1v0U3o)6b&8&Jq4C5#HQe%4!JnlHZp zA1uJeBLt^ghl;s+TAU@{0p&9?!dzt)H8~9l^xt`FjDqjX-lfN@P_~LUpdk1|F*?>) zA|+*|FRC>n){N|5@KhZPoF^Q?zX*Z{|MYCnx8p=zl)=Rq|#SXyvBvQKOfU zaf!WZO`=qIRJ4<|U2OKteX7ordtd#{U14+t>3F6T7gv9aP4>cm73$W`@nJKBIfUWe zJ7$BKmf@SG?miM#u*{X#rfXexFIxCZLPD&6Le8Gg97C9;olsgX`gz0=Zo%#K==y}` zDz0&i077&dZDBQ>#s9m@pD${$l{<2!rlSf9ZsOw4ll%l4BNH{)JtVza>q1$SCvQax z5R?oJ@yL_;{iC##%e=dB8+U27Cv*jZ@YjWs{gYQ?PAnK5A~Zs$%Aelf^(8z$)~ef~ zzH2#|9eaE@rx&90k)E*oiz^C_=!Y*8x9iAH#?N<)V|f%plCTjZs`B+SQ64G5=V!kq zrDR?{!&ia`nT}CI4w2lmk)_SHN~@4rshq;|YMMlaiaAn!yR)Jbts!stb`r@%n zP4Rk&(9vvlL`DriVH5G?8r_!mzLFXn=sj{RqJ+sqLxVfC8$JtmOJvdPxfZDh1aXhL zuHID3CS0z(c;W2xu1{aH{>DLuCN zePUtCo9>HSlUM3~_pTFy!aRMy{Gs(bnQd*}P5&bPr+j-{!>n$jiIC_EX?d8X1(wb{W!_MWX8I_b01Nu6Ysy(Rd!+r&We z)%^VY1N+;W_?b!-a!}+Mo3XrU{zx;w5m{Q}-o3E#)7ReDl+`ol0j2$0#y8#3S+5(2^1`=R zagR}Dg|04wxu4WQHdIl|rq?I*tl@Ki-j7@A(@IN0d- zg=Z7Q&R0`iYPF~6M}6-f8Srr_TeFUPPHu2rvGaNTY3ciXF5Vjm*XKA&YQ3NldR0-1 zkNWs3^tKW}iIY8=M@M8a9vXj0#u(k@S^Y#PlT{~dpkz6sc1U13XeYA=AA=Ccgc3{WNPSH%Pc{5cOw*#L!kvml<`}*0_$dhm;|l8p^7;fp ziDI_+__%Y$I!CPoCsJQRDv2mrTqa4{*rQQ0-iTzsJIgAtD#E^(nJ;!+vozwAtS}P| z{r;@qv@8(&W5cS=L0CqHg@lYWU3~g%b;|1Aa^CpI`WA_$+?lB_PdM84JOT%dyz${& zVWeCq+T0Mz-CSs{pt_fX5-j$Cv|wFX_IFO`E&Tex*>NQVXN%omRyiMLrqh#=TaD{~ z6dosXl{4-ISP9%uOM7zNfor<2;>7jC%oi@|9{%B-tSKkv4|jj^v%EgY0P8CIC%>n) zj@h4;jeB86mkmDT>2Bl#rk8o3X!KCL#$uND)3S2XPgO7 z2{@~;Ao|D`s!1JKUl4&XdS(Amtj!zh^q1|;gRjjR#k2)ideG@)Y&N9&np*np*yT=% z2*cZHfyH9B9((ruxh<`+pv7c4l9@O^Ub)iFn*vls_v+%*bnx44Fv1neE!2<{kkG{r zYHe+8$mV&2%_17`O3OV7s*g8`x$89|K^e&6kcvIto=w-PajdkHaEaXor@`iUOUq)a z5*Pdh;dJrua>i_yClbWCOT;Y&erjdrgj-WpzTZ`H_1i)SSNl^@v9PgsR|g)RoSy2P zA8qzCW(Jn1YiK-q5hVGA#F6eJ5;&Th^}2ouP!Zt}m~IhJinp}2z04lW#(1n*dCzL5 z)~BT8iG_s)m^azOj2x)SoSd9Ru7~DGkOX`=c23R@K!O0~-~#|ln{AeC;s?OGkghS% zOJn2X!a6$C#>U1Z00|NsNz2=*-2??VAlk;LeIMy}%a$4JDyR>($__KLqHbW|= zzt}*6gp3S{g1y)p-PoJ-^o-CmJK9^4%G0dE16)KL2%6nVtA_cRy05b=;$kneS`Rji zd*c0djk&!DXLI(%- zQ`HWbRgWakZwUn4{FY%<-%H2OPXQX&2Ourtx!(n-PF@^DTjIt8D3jBZI^?9I!frMsJj& zva**J8-YBe#KzYXrDncB$fU|9KJOnM?kw<#Td!$`~ytCR9~b1uYOROs_kZ<86&2%lp~-#PvNSWRlYY#GS+1u(0z< zV8ssFty>QsK179a^M_-Di<>(&H5CQWJuZu}dx)Jszwk300qIdGH!(B2jR*=2Rgwm% zKs1U;g<#WgG+iP>Zrh#;Vod;up!&1*9?m;+C^R%Q{aJF4U}C55 zZv5p7$Mq3xB%2w7BFm`s^cpoLXw?@F-w~0Lk>P)r@_CAAU^fpM18{<{sF5@ZsCJp9 z8*xuecFwmOm=EQt1l@zsai|!z<_c8{so+Ka@ZrR!eg>W}AM=3Q0NkMXcN(}UrkJuj z+Yrz{{W5%=E)r^YeP_s1I9yL*6)3sL3bn7mxqMyc4;H(@!ou<`E2{-UnrHx4)29cb z@~I-XfRBRw#=YrvaBD`ZW(jL^>doK%Fw`)bmf2TZY2QZ2J9M#ptx$a&iIS-=gATg~viL$iS7UnZ?$? z83<8i9#$7zW+S)(&!0a>YVr-9m)1LT!uEd$Gvv9~|NfnC_QxdQc~8OgCJzJfOfIV# z3Rpjn4-irC6Z96_!&ZOi66fXRrR3&f!eQxpxTbe;vWsMWr*3LbovZdAVY*^TF+X|o zMEWZ)E(9eJ7*o&lO&&B#1&+@@F?ko7r>7}pWo18L5Ay;P?RvCv8v?||UKiVk;TOtp z*|ck4K(2s!;jZ-W9IU_Iq_^%J4aXiRc2W#sEFrbKj2u}U!LLyPl}?qq&lGhc$fu~` zo=gcz2LMs!|4Kh%Cj7MktU8ej5BkD?w=SY(lIOwB8ey%==?zdY6b90mEH-LV))5R~ zDQiBSoIX5GMp~zDD0zd`u|uv>Z!6mmvjV~XgRjJkZKONV|Kmsdw$ft`t6jlm7EmhY zY*szGHLL31mS3`;umWW)U*}k-ZSog+o21{*Vqi{PhMHAR}gAUDwl$4|E)Y6+*43n+WRy|`=TH~j#_q01WSoxq8jBhrCU?#OD!x@bxeHp1MlNUOFW7fVCn-2 zfz;nLpLtLW1s(*7hE|_=tY*N_PvuA?y|pdpcm&uAXL zPtK1;qC0#?bF4w0q$xAfe{=iA{qoB)65o!?59V(i_S`*GhC@wa`CLh$*p`o{qH-sO z$?*>d?rq~qG6n&WJvw5}>4<=pitpRok?(p28K-Z;1hJ%lS)Bt%apLet7s^Lvb=Ct} z1dL{i%j8|)Q8K(|(UvkA^%xHDm4kjbJ3C`&-MDQ5)qN1g?SRo==}6HQEg+yMZx%zw zaxyi+_X$8YkD_H{UNub(wCr3Ah$3r(@Vc`msFUy(W)@G9Hj%R$zBv)kvT*|i@c^ZQ zeSI6-bm}`cGUuPb#JuAdU0$wl{aSdU!E^qx$c0rJ6qL)}6YeLdeC0~VnT~53Cw_Vo!_DWb$jg$3py9(0EM@<-AJnpGc0ACx?7+l8(6+4Qj zb`ixig`no_&_KSol~+;-(mnu}9^nnkNzCsiQfI;}-w*|JUb&pok>?)w+HhcWxuegd z*X0E%0<5IIqN36ph*)KIQJ};IGVPA_Pcnlb^Ea1~P$k}M*z!HnF{!`74EE{z^$9gA zrI24@fmE@z#`2^@FZB!#;>2W2`r?Vxa$t#)ZPmgo6z3=iIT7mn+;q3J+z$IVR@1=3 z?oMDVz51S@uP!`cHCn<0)^afE>|H-_HQsWYe{k(lPjgJCe|h>_{wYO@h*+9FkQ5Ju zWY7?`VXoK+NjWJ|G0A0JB3QeIq@*B@#|e$I{u<8Hpl%FQimeJ;?vB@~Q5X^_vr=rr zdKaG5mZx&vTp3?-}N&`@d28ngdc3>i4D`_EKyjH{m&k3sCj*GAw z!K!-womVYi-PEaZWFMKHBLM36r_bN)|5iqcKpHZ!%TqX6(@rfe7vJ2=gikwwkIYHj znX8H27{i45YXhM^$c-4uTZoDK8EliOT4-_+BX-L6tr6bB<3B)E@zI!xDSG$Ymf!8* zUe2#yL0;E)=bN17isz{-doS*>>k0dS+e5Vc`}<1eWdwQ_!M96radek99w9**{q$L+ zMhM=twPA-b;zgXTc-v83l<~$mZsf#murB~k0!24@(H#!Zn4A*AI(me4+!1rM3~7YN z`WwC3Y89q+(0=h61DhTlYLs?5oFi{k^8EP13Q1+~x;4re#gRFzrLH6Df2H$+ODVZ& zzwsNVTJce>9A!)T`2*+uy=Sy!WZCjLoMdI@VTFl-x%{zJ9(^;6?<_vR;+z*w5&8!Y zU)TK2ZIyiU^p;T%z0rII#GKh%Mqp`d1^iEs4w<8x9^Ep#uK8;xU;J8Ss?wc`?P2l@ zx$ijp=D!E#*XviecyGEYUh1E;XE$0gF)%&)F=GUV8L$v-5mroDgF*1k8?yjppLU=y z{g1{UZb(&b#eKv<&m4S7-{l?sX5g#0;TUd?&kv(68V*)j=q3@-k9G|TS& zxoB{*Jrq)XDhASA_G!#=`x!(xH#X*$GhLMoqaiW0^(fHUSobo2R=he>$M@87<|9}; zByn#5YDm*3-lx9<@u7ImtAXg18?dyJ*uWU4+AcAzo==lto#}l5T?tsI%8waqmhYPa}NW)d15JcO%R-3c&N~8@ zlpB{O{s3zb<_hFqe<%+?gL+TJ(2^6ZPruaE2SkrgP&IUW?jxIgnl3|ch+pmvM`Up3jbX_t^cQf75!c&NJIX@%JWmse+ zZaB42Km}Wp!O@Pe^Z8j+KOL?|bVZ?#(UsR}o84Dl&)RI?y9j8l%E?eKGjHnaBmQTY zK-y-b=biJn@@C&{-4FvSNkyf(m6cHSmj|t1B0&EbR(>}oPAh|!=2>Tr8#!U% z1{CSdTfrtJmm%kVJd%RS`@&yx^Oqr~#dI}ciE)oeymUbQL-F)31OrLT%I7!Pe}Y}` z{Y%iNkN}nGJ5yb-x}o{#QD+n5vv~v|O|kRw79|mz_D5hH7N%?74qFaVI3$$40J%*e z)8d(_srel(nkbPUAF)6fZUbAvdiI)V3w=hs%2xbjK5!tG``w$!8SPK#VCs({5p!tC z;q^G-bM+BFhpy<=fpm0aCL^3OEXpyZih2)qXT|F?Zuk-T829z|#( z30~FUol0D{Ro2wB{Q4Dq{UCCpYwghEyVKi$j0_Fuph;=0#)%Ee#W2V<0m(hn|M>AE zc=1kHq?|W(ye_$+yu+OQ68IXz{_B8;65+aoe|`lypWKsCpKO}%NLDCxky0*rg9@rE z;2(5Oc+^$W|2mMb{@{wdpKU!1H5sU|(itu-^;-vN32r5I!e|UPiXFQ8Ab;iWO}fOP zbUSRxbC}fA&JXA}!u7xf3lyv(VKoE%;dc`^F2%#!VFeVq9qO!(7E)K+{|$kd6=vR%Ad4WeY~(lMv_YzS!}gCH`32*`DucD0d$lw%ab#W zZh(n2zrKF{OX0m3T_R{%-rcxyYr2p$lkj-*PdHqo39(&Z{tFBdQ(pL%`E71nC#&)N zz$QHp0EVS~2S>islVg zgKl5*c-tO(^HecuOwGa*E0RWV4=7yK^eEWvE0I;L=nXJ3bC_`_b&`hnl`mDh&i zvVpuM<>8q$NEVlxo{hoy9k`^Vaf_TxdL3>Xl8-gDwC2~=nCwZPJ$FB02Wc5F=^6;8 z!5=JRsQRN`+klutv6AbjHKKtNG@J;685HY&tPdqt*fd`(jyU!Yjt~I{tpI4 z*}6ETSlQ>Ik3@h{rmNPF3TH@gh|y&L1{PMg4RotYB9JA?t^H1s7=<(~$fY5zX7pqn z`sKk#I2&x2&ZHnK8w-f8F!^N8zFyQySOK_aSDUVYl|2t)R`Xz zXj3MJDmm}iE__A1fiJ{SKRvyu#euXy>qPEv&BxHNriEYwf-XwT;O^r53yKoZ`kO2- zXA#~|P81!D1r9`H2^{GkUT2~pu&}V_9WQ50C?nFo;W@&WwUZNDIksmftn~ixD$K~% z1!UJ)Uid$cK|k=iI%VJ&5WpoOLIZ6=xlo%D;vC6S3lfkwR3Gy&eKh=E(&l?jyi8tjJN#pOeP*5lhR+ zwr?+L0@#pI`d-rl3!Jf5nZnRj9$T*Z?IXj_E^T@MHZ z(!L_dss|BYtaz$!l81eT&%~Z`F1F8Vaj5>MVAs;y=Vp&DdJ|c#J(BJCOa2q%X!eRZ zeP%pPcyui787`rqRAp?Ur`MFwDOCjeJM=>nApyZeKBrG#zW71M)>NGvG4$(-1>U8E zPEuO?IJyt{`S}yIm1QZcRL=m40jhRTD~Z>h;Ravrd%a(>atD;V0w22)mMXR3hGxX; z+S{MZj4hG}0p!B8ObJCL&W%eH1Q>qbPBM^rd#ICh6?t|Vn@yAg{3fXZWw_45_Im*^Wh9@jMU z*YY=Xq-RG}qA-n&jC|@^G9R$J*9=*m^-3?La=tntoOVd&5t@*ZvJ=Q@DbP3t4nx+; z{@51^NR++1eOQrX6}a2ToLD?x-QXb4CAO6In4#lias|oqZ}XJA;@dE%#NwGtg!{;9 z>Y$YC>#n_$^ts~#7gu?)e?bz#ze>-Rdu_G&Zccxy6L0UwG^ZJ}00r$0ZZ30X1Mu9i zi9Z<`HmhbL&Zb3qm`GxGBKODZb@u|cvcAtmHEDo2gJedKUElXC387ryMI|c= z1)kGt42KwCeJOU=7l#{y4&8!p#*(J9!X=Qj3-{vsIws=ec-Cv6#6+&`03^0_&&xAw z8wG*iFBXOzyz%*Y3VmVQcYN?1gb0&j7VVCWsU#kXPce6*K#LzgiVCV+$un@yFA zLzdo8Re~9pHs}@<9NYv7t9*_k9rQQ8)zDakZt}5mOLAJg;K7HzkZIfk!3ZeM=Dt3h zqoX5sc6RTaoJSy_`BYfVGL_Yt4bewE6-&3J{E{1)BAv z&_VlOdqXYuCEKI_Pk*O_zG0Ji=~#Xu>?*qIqGuRDmXoCnw3lACidUG(xz~)O+QD z|A9>3JTG=79;=sPz;K3a{t<%?v47c18m9}6y1Mt_aRuJq-UvtmgpC@&H48X6=pG{a z5pjj1HFgxb=f=1R_2}wz>!Jl^C0qR)P#5U$wsIipfmKyDo8x$Diy9`KC$FVs+J<)I z+jK_$nwdG{9lhXau*H^&RVnvQs`pulIs0Pdm9CVG^0FSMeTn%+!^hM#2|Ox$V#LSNl_0{t&E4_NnW+)42K?#>>lbmP<|>_05MBPw^@(sFNM zafZ34#%pD0bn@S;@7!)3x^t0BB`b>oN~B2WIt3FDhv6~z> zC;X?cM`!9fmYlC~D7>blMDggd=mYUyekaxE{`K0h)sN_5G`m^C{(Q}LUs2C!4BZkt{OgVLsC$B8d?^^2w9v_}+s!k^ttnb$g zCMWW7@bIk6HTnMe^XF4Sf)wcV&wXEYNwyaQ1>%$IWeT@mM2Q6P??^;fxRavxi_QU(m@S z7|qEYf72B%YT3SZ7AI#hITT1b6fk_@ioQ(zYLtw#d3Ryz*CN56rM-8N8|+|r+!gzjeD3{FXrqALWyOW9nyM_dcWVO+>vem%oui|;X!@TbxYf7n^(Umq zkaWXiLpCNWaGsR>W6}xxVhNmCIu@3&WUs5LsY~cg9j~-W-zWiM8f>GOX&Xk{jZW}U ziXGP9jOT+Cad{gDr(<~d3wWnL+)r&&Tpif@;DiD$p8oOUkHEx(;W<|C!3UN~M`ve9 zb~fYqVsB&Pi{anDgF`|@baZrx;Bb$go73&??gkj2@pIW&p^nh_;v_M0(31m4$Me@! zUj+rzJJ-oot!p&sy(qkLwCyTOSnscWDV2%!BU)pHe*hJ~D+wNkQWLvQ8dG(Tyai?_ zy6+@=YSEs!$vv7Jl9$|H^`mXTdnr6~>?LL#L zKJA~UN(@z(=MH;YvnIhSfiHTJq`U zeV)+QJ31P(u%HjYJnpF_8TnI-xr@{NJDUlSk-~tV8)8qFpy3oa3$jDIaFvm`I96*T}hK6Ebll}VXJn>6Cv0C*TxS;N5f3y7j{J@%ⅇG0o;UyxA>r`4 z8X~rZ)Lc?Q0T&EQ31~BbC!Gur56deleNOCy5Ev7@34W4wbQI_AyG4Fw@u-!O+=9aL z&E@pdWP|sIPgxFUUubFhw#Un?jy$O-ZP5zF&ttMWNB2o5o>#k<@O=XpDqbzP#GvvcvwF}^r2_&*mU4KcNlj<|Bx|acH=6tVEGQ0@*VAnczP!j zRgL{|HFfoWqNC{(3^u^{d8}3a1-N}!X&$Jv9Ymp^Yg-*nwtLS z4qH=k(D@5bW%&!6IL?7kpjlabeT0t9!q~{@SN?g%hhPK>*yP=yId*cjLi-Llm_t=| zE1Z*W%CfJii6q=yQp^gKLiAWy9%0G^GYT4Rxcy$YHN_^fOm=SNFNo0EAO6$RbQ=zXD~ zp)K%KjJ8nX^uoeEcw)lt-rfROe;=s?s>Ro?_@P}1);q~FTUvl4GT`^0r6uGu2Z&gh zNZoonJ71UA1qPyXaBzUagY1MpJKB_j!lBAxoe0|6L1K#{ePXO@WmTyB>^oiqeV^}+ ztD>QHjm;C2^vNG8lwX$H2w1i&wNeYI&!0tp&Kgm_5%fK`q#e42g<01N zzhq^ZWSqOYxs}+h$o})sKhQy*Vgb+YLAw2LG!CW+`!dD!{QMcDkS@O4f+O@LxG?qq zs^iMzp=`gm-giO~vZl?HwNcqgw$Ngyj4-ycjV((u_Amy?E0nR7P}XeO$yy=1$Tryt z8M3P(jqjQF`~CCx`^(3ChUa-gtgZg|nz&3H5I@Jw5PS9caM8{8?b~G2aZz37- zb#@0-PcrYkWGayuqyC|9>9@v|J4G)(uRkXFQ{8WC(QNWsft=USNUFRiJ!E)Lzg&k{ zg27;5ABp>JeB*ht3lfWVn7A^j^N|1&X{<4b54xBQlo_U;mr6{*MMbU^9aVOn+@N)V z(2Nuli+QQ!UuxZQ7&5;3T^p>8%|~DhcVHe#Q%~=@wszD}{_8NS|5!3_YcTKkCeRBs zObXOmW>%KRZ0iwtb>Bxy_@TA}Sq_8x$DF+P?2!b>2C!+MW4r+D6TD9_FbNI|I|w5$ z@DkDYn>)^N(B4#fwXvhtF}vM0bViTN;6kp*-~6)}-Hlw75rsEiBdv37qzuanE1!*3q;`t37a<9E^%;MJ~wT zK&Y}=Dx>7mEUwYo;W06LOo@<4FL>7=>pC_`hw$kqYg+Y;7Rolax3fU#0TiP#XYP>JxrTg&36tB%{%2G)A73Q0?I!Z)Eo;Nj`{+IhH0Y~m&x;qrF>7F-k6+0il6Db@%VLx5rg2qBt6 ziCk-fNAATqEF&_)G|K1|>L#b}%iuw>@mNZGKtRr#`Y(kIg9wjhchQ z2jBzbNuaP`+1=f(`@K?2+q`3AW4jk$A?t9KDas0}xsPW*#gxH-AtovsC-UmhBD zLf|(H^qo|B_`>l^_JLRriR3B<`SxrDc0|U{M+yq6Y=~*1@GoIIP(^{j3MEe_AVpq7 z!2KHF_5iLRAS`TLZFx2>Jv%!bjcS1+LGziA1hn_dD=HRU>@Tj65gAVgNC7y!!7$lb zSplF>csRZOOT`ROJkX4k4_^5S7unhMD1F8+$%YfB@p#KUY0BEV?fs2RY>4t70&UM89 z*UE)f1Q%jDOW=5Iq7JMa-E-M{AKKeBE9<}|4B8{QU6c!qcmtLiv*6v7-lE*xM_x-q zefh>>54^nk`uaerF+rhFV$;&ZMlVWA!cf}>*oc?>wnTtokap}RfMQ~HaZwY8t2B4X zLwCThC6$%eVZs;U)cFe+fG`*J%-L{R2KLzoi;V!$5;$r=jM@Z*-r3UN|LY=n;9Ld4 zA$%WBU^ke{IWGtt0Q5!&!V2^0z&E${LC8kWd@Q)KQWmO4delmpp>)Hm;p*(Jmr7<` zj=xf>aXajj50zuig%HRF ze3@Cf!_oZPg{ZP+2q7;Nyq|<0IUxzQL6S=Vl3U<2J9P;zPEIy9wku#*x4ynkvwIWS zIr|dif(qU%=gWdXS_n7(4UU5g%!hWpgWSq}(Yy^PYC1UdSb&q}^?pRSa}o}0-E60t zLI7nZoCDR56mTwu^StLK*QP&`Dx7&>Wfqo-8(VvN^llGtptGo;1G1i|^DPaDhV14n z&`>&D{f=y{tdK!HHBrmdC>SMz@;hsW1K3cr-#clp*U(hanP^fyUyG5)**hJE65zqI zHAZwjEakN%DG5~E4vvl`Zj;I@6ZP7bmZyO;0CuoyOlX9EDJO(4!{@9v(1vj=L; zX9)?d&_c5t1+Tdh$W~q5-SXg`A}=rhZGN5|BtoFLfh}2jBQy(K+L9H0B@@N0gj7`0 zl9G;r+z!ZS;cEA3yh|-?qqw9bpd=1~ciA*lO}6&-8|$C=%3x1oD=$04+gjOE}#5S|`A+xCB|(MvT%duqtJf z@sb(CiZ$=A74Ny93BC7e)|C70Gk{RdTiuowxV`<~t|z9?8TUbVXXwe90cPT2nUuIV zT@#69Doc0@&jGMz&ehG%t_5~2z#2MmQc7D}a|eE(1jruP1O}wII2)AApo0Aj75fi2 zXJ>79cX80hg8x^14HH8I2t3ymo;Cz=rARsE;CVQZyDJd;I3pQIntIJT^tI%x$r>jy=GhVwH zQUBE4y$}hhLF@&8u&wHmFaF$-T{fY-#ap7y3J4Zz2B>JOn!Q3 zvX3NNu}F0d1m`NfH+sDP{q6yO*}{+hnfyh!$31?0i8i&Fm|dH&;M_0vzn&|P^2CDn zQ4@nX4##K)s*M+lzE6RM>Mym^252rWCs%xHtpAhWHR9eHCpZ}H6aWKOrMsBe_$GV8 zn>-K+9UCZLXkL6SzDDHYoSd9B4`QU1*|3REV1>gtw}$nooG9f^EB1l-3TBbdiBv_s z6S;r@gf3R5$S)H{EK|+~5ts={p3Z&FTC=}>kw5O_B;Uo+p^>h-F2C8+`a!N-(N^d! zLXhrmaCx(*U1jZyfRN61p6`w}+Mp2o&D!E8CP;|`htv+f=q9_DpEH$i(KDOq|w zChHL#yqkapgq+J|m#m1C4s#n-pEylV?l@vXN4vU;$Hr%U2S;a{fkK7^x5m&}2RTf+ z-(?DJ`t^o1zR^OFJXSvS3nD}VSgxQ^kMJX>9A>=EY?K2C~`O(NpT*DwTkU@h;a z@*junQ8n(>jurvHg|UJ;A{;8zk)!?Wy%JOPan-(o*H479C{}wXZe?!nadL_re}_qz zjFA8~erKh2RYO-dD?L3J?iG4kP$jYe61hM6sig(t*n6NZ+d4b>g@nQ(d!&KWqZFtZ zNjRXjzb!1xPE9pI_Gs_v$$tAb7U~-hFfW7WeKS@yKQn`elAxs}%TGAgyVs3?bWlzbZuvA(MeBvaM3sU!aGzxW!=sBW)V37hq+S`A-s zZ}?NRueM*RSV}x^`{!wxeyu>}Ez*8>p%!~Zk(FM~S|30C5&yKx9|~=!q!xar?Rg46 zZ&dw0ifA=rbJNoR8TTH8)TKGAY1sj3E&~=P8U=Kkw_ZF34aw1 z=P3HPjGCb#2UsM++y^bUEepEmj_-<|up&8|TVw@>eptF@b|cYaLNN5uUg{qnI{nis zLZXEN$ywF8k7iz-&1`8A+<&nMAKYT)E8PLYlj8~@hcX*ewmL;Yh>Jmd;a}_r#^z2_txFNw5}~d ztXkXL{Mg&8?>Y|l3Y@6tl9)=D_V{y#G~d5lzm2M>20J53kDH4-WVi`$G!(tqgP&OF z(D|SZg3+if;~<5EU50x1Ak&RV(;aV2uW*jY2|nO_{*G!GGTEx2pQ_r#CNIc7%~ewH z*ksq|cS`A?;RU1!3Su3}CSu*t0GZrq@$G$mEvIro+o~z=^2TP%{*oXav)6OR;M-;P zv~WR4a=;H2&BWvZi>V0=k4{4<>gp~2yq$J9W-( zkoGXAD4Y4Xk%Ib>hLx)}!i*oZu@L8^`v(Ms?yErd@>uA7t}!s+F!=Kb;N(t&s>lZ` zmOtyesd^tBvq$lKl`aBG=i=~tid&efta=X~mxw1#UrTJc<4zb~gp_wPEJl2vx>RXS zyRJ&gzWlwf6${IeR+PU(xf##OKxlp^b6(JbE9JA?yQXb$~ zN>ZmD8{rm2j8x=+w9(66y}ppjgQ^C2G-i4=!RMcIjN4bV^UuW`c6x}zsrsbOEzI?m z=izFWXlt;YUii?g)NQK+gnmDzzWF~6JFc{Az+91R)V=Bv<0wCB^S}|Uf=a~rzyTe9vyJ(p23p!9) z0oV!}B4bJ02hKaaA&Q?-ENgF!Az-!kDPYdKj*h0&xEYCTbeVa_&%{1IrOYE8WXG+rxbR*3-Vz0YxH$LV<)ZDwI1Mk0vQ;j{=^KUV=zfp&1X0{<3&7|enB z<>V51EiK3DQrd;B0@We>(x7XN1WA;<)Z^-HboYHae;XM`3p5>gmf@ zt#za~mEDE5&CLUOG}+Kv=0p)dE+|9B2-}0@<>D%#aPC>N6Pj__XVPIfmo^;~M_= c^Vj|WRpOrqoJ$_yBu}S#P48-v+8z9V0Ky8EPyhe` diff --git a/examples/ray/ray_Hamilton_UI_tracking/ray_lineage.py b/examples/ray/ray_Hamilton_UI_tracking/ray_lineage.py new file mode 100644 index 000000000..e261ccc50 --- /dev/null +++ b/examples/ray/ray_Hamilton_UI_tracking/ray_lineage.py @@ -0,0 +1,57 @@ +import time + + +def node_5s() -> float: + start = time.time() + time.sleep(5) + return time.time() - start + + +def add_1_to_previous(node_5s: float) -> float: + start = time.time() + time.sleep(1) + return node_5s + (time.time() - start) + + +def node_1s_error() -> float: + time.sleep(1) + raise ValueError("Does not break telemetry if executed through ray") + + +if __name__ == "__main__": + import __main__ + from hamilton import base, driver + from hamilton.plugins.h_ray import RayGraphAdapter + from hamilton_sdk import adapters + + username = "admin" + + try: + tracker_ray = adapters.HamiltonTracker( + project_id=1, # modify this as needed + username=username, + dag_name="telemetry_with_ray", + ) + rga = RayGraphAdapter(result_builder=base.PandasDataFrameResult()) + dr_ray = driver.Builder().with_modules(__main__).with_adapters(rga, tracker_ray).build() + result_ray = dr_ray.execute( + final_vars=[ + "node_5s", + "node_1s_error", + "add_1_to_previous", + ] + ) + print(result_ray) + + except ValueError: + print("UI should display failure.") + finally: + tracker = adapters.HamiltonTracker( + project_id=1, # modify this as needed + username=username, + dag_name="telemetry_without_ray", + ) + dr_without_ray = driver.Builder().with_modules(__main__).with_adapters(tracker).build() + + result_without_ray = dr_without_ray.raw_execute(final_vars=["node_5s", "add_1_to_previous"]) + print(result_without_ray) diff --git a/hamilton/driver.py b/hamilton/driver.py index 419ff8d26..f6fbaed58 100644 --- a/hamilton/driver.py +++ b/hamilton/driver.py @@ -582,32 +582,23 @@ def execute( start_time = time.time() run_successful = True telemetry_error = None - execution_error = None outputs = None _final_vars = self._create_final_vars(final_vars) try: - outputs = self.__raw_execute(_final_vars, overrides, display_graph, inputs=inputs) - if self.adapter.does_method("do_build_result", is_async=False): - # Build the result if we have a result builder - return self.adapter.call_lifecycle_method_sync("do_build_result", outputs=outputs) - # Otherwise just return a dict + outputs = self.__raw_execute( + _final_vars, + overrides, + display_graph, + inputs=inputs, + materialize=False, + ) return outputs except Exception as e: run_successful = False logger.error(SLACK_ERROR_MESSAGE) - execution_error = e telemetry_error = telemetry.sanitize_error(*sys.exc_info()) raise e finally: - if self.adapter.does_hook("post_graph_execute", is_async=False): - self.adapter.call_all_lifecycle_hooks_sync( - "post_graph_execute", - run_id=self.run_id, - graph=self.function_graph, - success=run_successful, - error=execution_error, - results=outputs, - ) duration = time.time() - start_time self.capture_execute_telemetry( telemetry_error, _final_vars, inputs, overrides, run_successful, duration @@ -667,7 +658,7 @@ def capture_execute_telemetry( fail_starting=(2, 0, 0), use_this=None, explanation="This has become a private method and does not guarantee that all the adapters work correctly.", - migration_guide="Don't use this entry point for execution directly. Always go through `.execute()`.", + migration_guide="Don't use this entry point for execution directly. Always go through `.execute()`or `.materialize()`.", ) def raw_execute( self, @@ -677,31 +668,17 @@ def raw_execute( inputs: Dict[str, Any] = None, _fn_graph: graph.FunctionGraph = None, ) -> Dict[str, Any]: - """Don't use this entry point for execution directly. Always go through `.execute()`. + """Don't use this entry point for execution directly. Always go through `.execute()` or `.materialize()`. In case you are using `.raw_execute()` directly, please switch to `.execute()` using a `base.DictResult()`. Note: `base.DictResult()` is the default return of execute if you are using the `driver.Builder()` class to create a `Driver()` object. """ - success = True - error = None - results = None + try: return self.__raw_execute(final_vars, overrides, display_graph, inputs=inputs) except Exception as e: - success = False logger.error(SLACK_ERROR_MESSAGE) - error = e raise e - finally: - if self.adapter.does_hook("post_graph_execute", is_async=False): - self.adapter.call_all_lifecycle_hooks_sync( - "post_graph_execute", - run_id=self.run_id, - graph=self.function_graph, - success=success, - error=error, - results=results, - ) def __raw_execute( self, @@ -710,6 +687,7 @@ def __raw_execute( display_graph: bool = False, inputs: Dict[str, Any] = None, _fn_graph: graph.FunctionGraph = None, + materialize: bool = True, ) -> Dict[str, Any]: """Raw execute function that does the meat of execute. @@ -721,11 +699,11 @@ def __raw_execute( :param inputs: Runtime inputs to the DAG :return: """ - self.function_graph = _fn_graph if _fn_graph is not None else self.graph - self.run_id = str(uuid.uuid4()) - nodes, user_nodes = self.function_graph.get_upstream_nodes(final_vars, inputs, overrides) + function_graph = _fn_graph if _fn_graph is not None else self.graph + run_id = str(uuid.uuid4()) + nodes, user_nodes = function_graph.get_upstream_nodes(final_vars, inputs, overrides) Driver.validate_inputs( - self.function_graph, self.adapter, user_nodes, inputs, nodes + function_graph, self.adapter, user_nodes, inputs, nodes ) # TODO -- validate within the function graph itself if display_graph: # deprecated flow. logger.warning( @@ -734,7 +712,7 @@ def __raw_execute( ) self.visualize_execution(final_vars, "test-output/execute.gv", {"view": True}) if self.has_cycles( - final_vars, self.function_graph + final_vars, function_graph ): # here for backwards compatible driver behavior. raise ValueError("Error: cycles detected in your graph.") all_nodes = nodes | user_nodes @@ -742,23 +720,43 @@ def __raw_execute( if self.adapter.does_hook("pre_graph_execute", is_async=False): self.adapter.call_all_lifecycle_hooks_sync( "pre_graph_execute", - run_id=self.run_id, - graph=self.function_graph, + run_id=run_id, + graph=function_graph, final_vars=final_vars, inputs=inputs, overrides=overrides, ) + success = True + error = None results = None try: results = self.graph_executor.execute( - self.function_graph, + function_graph, final_vars, overrides if overrides is not None else {}, inputs if inputs is not None else {}, - self.run_id, + run_id, ) + if self.adapter.does_method("do_build_result", is_async=False) and not materialize: + results = self.adapter.call_lifecycle_method_sync( + "do_build_result", outputs=results + ) except Exception as e: + success = False + error = e + # With this the correct node display the error + _ = telemetry.sanitize_error(*sys.exc_info()) raise e + finally: + if self.adapter.does_hook("post_graph_execute", is_async=False): + self.adapter.call_all_lifecycle_hooks_sync( + "post_graph_execute", + run_id=run_id, + graph=function_graph, + success=success, + error=error, + results=results, + ) return results @capture_function_usage @@ -1553,7 +1551,6 @@ def materialize( start_time = time.time() run_successful = True error = None - execution_error = None raw_results_output = None final_vars = self._create_final_vars(additional_vars) @@ -1593,6 +1590,7 @@ def materialize( inputs=inputs, overrides=overrides, _fn_graph=function_graph, + materialize=True, ) materialization_output = {key: raw_results[key] for key in materializer_vars} raw_results_output = {key: raw_results[key] for key in final_vars} @@ -1601,19 +1599,9 @@ def materialize( except Exception as e: run_successful = False logger.error(SLACK_ERROR_MESSAGE) - execution_error = e error = telemetry.sanitize_error(*sys.exc_info()) raise e finally: - if self.adapter.does_hook("post_graph_execute", is_async=False): - self.adapter.call_all_lifecycle_hooks_sync( - "post_graph_execute", - run_id=self.run_id, - graph=self.function_graph, - success=run_successful, - error=execution_error, - results=raw_results_output, - ) duration = time.time() - start_time self.capture_execute_telemetry( error, final_vars + materializer_vars, inputs, overrides, run_successful, duration diff --git a/hamilton/execution/graph_functions.py b/hamilton/execution/graph_functions.py index 49cdea2e1..b3379cdde 100644 --- a/hamilton/execution/graph_functions.py +++ b/hamilton/execution/graph_functions.py @@ -205,10 +205,10 @@ def dfs_traverse( execute_lifecycle_for_node_partial = partial( execute_lifecycle_for_node, - node_=node_, - adapter=adapter, - run_id=run_id, - task_id=task_id, + __node_=node_, + __adapter=adapter, + __run_id=run_id, + __task_id=task_id, ) if adapter.does_method("do_remote_execute", is_async=False): @@ -253,11 +253,11 @@ def dfs_traverse( # TODO: better function name def execute_lifecycle_for_node( - node_: node.Node, - adapter: LifecycleAdapterSet, - run_id: str, - task_id: str, - **kwargs: Dict[str, Any], + __node_: node.Node, + __adapter: LifecycleAdapterSet, + __run_id: str, + __task_id: str, + **__kwargs: Dict[str, Any], ): """Helper function to properly execute node lifecycle. @@ -265,11 +265,11 @@ def execute_lifecycle_for_node( For local runtime gets execute directy. Otherwise, serves as a sandwich function that guarantees the pre_node and post_node lifecycle hooks are executed in the remote environment. - :param node_: Node that is being executed - :param adapter: Adapter to use to compute - :param run_id: ID of the run, unique in scope of the driver. - :param task_id: ID of the task, defaults to None if not in a task setting - :param kwargs: Keyword arguments that are being passed into the node + :param __node_: Node that is being executed + :param __adapter: Adapter to use to compute + :param __run_id: ID of the run, unique in scope of the driver. + :param __task_id: ID of the task, defaults to None if not in a task setting + :param ___kwargs: Keyword arguments that are being passed into the node """ error = None @@ -278,28 +278,28 @@ def execute_lifecycle_for_node( pre_node_execute_errored = False try: - if adapter.does_hook("pre_node_execute", is_async=False): + if __adapter.does_hook("pre_node_execute", is_async=False): try: - adapter.call_all_lifecycle_hooks_sync( + __adapter.call_all_lifecycle_hooks_sync( "pre_node_execute", - run_id=run_id, - node_=node_, - kwargs=kwargs, - task_id=task_id, + run_id=__run_id, + node_=__node_, + kwargs=__kwargs, + task_id=__task_id, ) except Exception as e: pre_node_execute_errored = True raise e - if adapter.does_method("do_node_execute", is_async=False): - result = adapter.call_lifecycle_method_sync( + if __adapter.does_method("do_node_execute", is_async=False): + result = __adapter.call_lifecycle_method_sync( "do_node_execute", - run_id=run_id, - node_=node_, - kwargs=kwargs, - task_id=task_id, + run_id=__run_id, + node_=__node_, + kwargs=__kwargs, + task_id=__task_id, ) else: - result = node_(**kwargs) + result = __node_(**__kwargs) return result @@ -307,24 +307,26 @@ def execute_lifecycle_for_node( success = False error = e step = "[pre-node-execute]" if pre_node_execute_errored else "" - message = create_error_message(kwargs, node_, step) + message = create_error_message(__kwargs, __node_, step) logger.exception(message) raise finally: - if not pre_node_execute_errored and adapter.does_hook("post_node_execute", is_async=False): + if not pre_node_execute_errored and __adapter.does_hook( + "post_node_execute", is_async=False + ): try: - adapter.call_all_lifecycle_hooks_sync( + __adapter.call_all_lifecycle_hooks_sync( "post_node_execute", - run_id=run_id, - node_=node_, - kwargs=kwargs, + run_id=__run_id, + node_=__node_, + kwargs=__kwargs, success=success, error=error, result=result, - task_id=task_id, + task_id=__task_id, ) except Exception: - message = create_error_message(kwargs, node_, "[post-node-execute]") + message = create_error_message(__kwargs, __node_, "[post-node-execute]") logger.exception(message) raise diff --git a/tests/lifecycle/test_lifecycle_adapters_end_to_end.py b/tests/lifecycle/test_lifecycle_adapters_end_to_end.py index bd41d29f9..a753ea874 100644 --- a/tests/lifecycle/test_lifecycle_adapters_end_to_end.py +++ b/tests/lifecycle/test_lifecycle_adapters_end_to_end.py @@ -1,5 +1,5 @@ from types import ModuleType -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional import pytest @@ -7,6 +7,7 @@ from hamilton.io.materialization import to from hamilton.lifecycle.base import ( BaseDoNodeExecute, + BaseDoRemoteExecute, BasePostGraphConstruct, BasePostGraphExecute, BasePostNodeExecute, @@ -344,6 +345,78 @@ def post_graph_execute( assert len(calls) == 16 +def test_multi_hook_remote(): + class MultiHook( + BasePreDoAnythingHook, + BasePostGraphConstruct, + BasePreGraphExecute, + BasePreNodeExecute, + BaseDoRemoteExecute, + BasePostNodeExecute, + BasePostGraphExecute, + ExtendToTrackCalls, + ): + def do_remote_execute( + self, + node: node.Node, + execute_lifecycle_for_node: Callable, + **kwargs: Dict[str, Any], + ): + return execute_lifecycle_for_node(**kwargs) + + def pre_do_anything(self): + pass + + def post_graph_construct( + self, graph: "FunctionGraph", modules: List[ModuleType], config: Dict[str, Any] + ): + pass + + def pre_graph_execute( + self, + run_id: str, + graph: "FunctionGraph", + final_vars: List[str], + inputs: Dict[str, Any], + overrides: Dict[str, Any], + ): + pass + + def pre_node_execute( + self, run_id: str, node_: Node, kwargs: Dict[str, Any], task_id: Optional[str] = None + ): + pass + + def post_node_execute( + self, + run_id: str, + node_: node.Node, + kwargs: Dict[str, Any], + success: bool, + error: Optional[Exception], + result: Optional[Any], + task_id: Optional[str] = None, + ): + pass + + def post_graph_execute( + self, + run_id: str, + graph: "FunctionGraph", + success: bool, + error: Optional[Exception], + results: Optional[Dict[str, Any]], + ): + pass + + multi_hook = MultiHook(name="multi_hook") + + dr = _sample_driver(multi_hook) + dr.execute(["d"], inputs={"input": 1}) + calls = multi_hook.calls + assert len(calls) == 16 + + def test_individual_do_validate_input_method(): method_name = "do_validate_input" method = TrackingDoValidateInputMethod(name=method_name, valid=True) diff --git a/z_test_implementation.py b/z_test_implementation.py deleted file mode 100644 index 7ac5131bf..000000000 --- a/z_test_implementation.py +++ /dev/null @@ -1,64 +0,0 @@ -import time - - -def node_5s() -> float: - print("5s executed") - start = time.time() - time.sleep(5) - return time.time() - start - - -def add_1_to_previous(node_5s: float) -> float: - print("1s executed") - start = time.time() - time.sleep(1) - return node_5s + (time.time() - start) - - -def node_1s_error() -> float: - print("1s error executed") - time.sleep(1) - raise ValueError("Does not break telemetry if executed through ray") - - -if __name__ == "__main__": - import __main__ - from hamilton import driver - from hamilton_sdk import adapters - - username = "jf" - - tracker_ray = adapters.HamiltonTracker( - project_id=1, # modify this as needed - username=username, - dag_name="ray_telemetry_bug", - ) - - try: - # ray.init() - # rga = RayGraphAdapter(result_builder=base.PandasDataFrameResult()) - # dr_ray = driver.Builder().with_modules(__main__).with_adapters(rga, tracker_ray).build() - # result_ray = dr_ray.raw_execute( - # final_vars=[ - # "node_5s", - # # "node_1s_error", - # "add_1_to_previous", - # ] - # ) - # print(result_ray) - time.sleep(5) - # ray.shutdown() - - except ValueError: - print("UI displays no problem") - finally: - tracker = adapters.HamiltonTracker( - project_id=1, # modify this as needed - username=username, - dag_name="telemetry_okay", - ) - dr_without_ray = driver.Builder().with_modules(__main__).with_adapters(tracker).build() - - result_without_ray = dr_without_ray.raw_execute( - final_vars=["node_5s", "add_1_to_previous"] - ) # ,"node_5s_error"]) From 3acd95c4e49e8fd0d3c54fffb66b9d96f4037030 Mon Sep 17 00:00:00 2001 From: JFrank Date: Tue, 3 Sep 2024 00:44:37 +0100 Subject: [PATCH 29/33] ruff version comment --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index dbc756987..e41829d23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ dask-distributed = ["dask[distributed]"] datadog = ["ddtrace"] dev = [ "pre-commit", - "ruff==0.5.7", + "ruff==0.5.7", # this should match `.pre-commit-config.yaml` ] diskcache = ["diskcache"] docs = [ From a5565588be0e33bdc3dee907a7bbd6c450d02312 Mon Sep 17 00:00:00 2001 From: JFrank Date: Tue, 3 Sep 2024 20:01:26 +0100 Subject: [PATCH 30/33] Refactored pre- and post-graph-execute hooks outside of raw_execute which now has deprecation warning --- .../ray_Hamilton_UI_tracking/ray_lineage.py | 8 +- hamilton/driver.py | 172 +++++++++++++----- 2 files changed, 130 insertions(+), 50 deletions(-) diff --git a/examples/ray/ray_Hamilton_UI_tracking/ray_lineage.py b/examples/ray/ray_Hamilton_UI_tracking/ray_lineage.py index e261ccc50..89bcd6990 100644 --- a/examples/ray/ray_Hamilton_UI_tracking/ray_lineage.py +++ b/examples/ray/ray_Hamilton_UI_tracking/ray_lineage.py @@ -19,6 +19,8 @@ def node_1s_error() -> float: if __name__ == "__main__": + import ray + import __main__ from hamilton import base, driver from hamilton.plugins.h_ray import RayGraphAdapter @@ -27,6 +29,8 @@ def node_1s_error() -> float: username = "admin" try: + # ray.init() + ray.shutdown() tracker_ray = adapters.HamiltonTracker( project_id=1, # modify this as needed username=username, @@ -37,7 +41,7 @@ def node_1s_error() -> float: result_ray = dr_ray.execute( final_vars=[ "node_5s", - "node_1s_error", + # "node_1s_error", "add_1_to_previous", ] ) @@ -53,5 +57,5 @@ def node_1s_error() -> float: ) dr_without_ray = driver.Builder().with_modules(__main__).with_adapters(tracker).build() - result_without_ray = dr_without_ray.raw_execute(final_vars=["node_5s", "add_1_to_previous"]) + result_without_ray = dr_without_ray.execute(final_vars=["node_5s", "add_1_to_previous"]) print(result_without_ray) diff --git a/hamilton/driver.py b/hamilton/driver.py index f6fbaed58..e76e70afa 100644 --- a/hamilton/driver.py +++ b/hamilton/driver.py @@ -580,29 +580,52 @@ def execute( "Please use visualize_execution()." ) start_time = time.time() + run_id = str(uuid.uuid4()) run_successful = True - telemetry_error = None + error_execution = None + error_telemetry = None outputs = None _final_vars = self._create_final_vars(final_vars) + if self.adapter.does_hook("pre_graph_execute", is_async=False): + self.adapter.call_all_lifecycle_hooks_sync( + "pre_graph_execute", + run_id=run_id, + graph=self.graph, + final_vars=_final_vars, + inputs=inputs, + overrides=overrides, + ) try: outputs = self.__raw_execute( - _final_vars, - overrides, - display_graph, - inputs=inputs, - materialize=False, + _final_vars, overrides, display_graph, inputs=inputs, _run_id=run_id ) - return outputs + if self.adapter.does_method("do_build_result", is_async=False): + # Build the result if we have a result builder + outputs = self.adapter.call_lifecycle_method_sync( + "do_build_result", outputs=outputs + ) + # Otherwise just return a dict except Exception as e: run_successful = False logger.error(SLACK_ERROR_MESSAGE) - telemetry_error = telemetry.sanitize_error(*sys.exc_info()) + error_execution = e + error_telemetry = telemetry.sanitize_error(*sys.exc_info()) raise e finally: + if self.adapter.does_hook("post_graph_execute", is_async=False): + self.adapter.call_all_lifecycle_hooks_sync( + "post_graph_execute", + run_id=run_id, + graph=self.graph, + success=run_successful, + error=error_execution, + results=outputs, + ) duration = time.time() - start_time self.capture_execute_telemetry( - telemetry_error, _final_vars, inputs, overrides, run_successful, duration + error_telemetry, _final_vars, inputs, overrides, run_successful, duration ) + return outputs def _create_final_vars(self, final_vars: List[Union[str, Callable, Variable]]) -> List[str]: """Creates the final variables list - converting functions names as required. @@ -654,7 +677,7 @@ def capture_execute_telemetry( logger.debug(f"Error caught in processing telemetry: \n{e}") @deprecation.deprecated( - warn_starting=(1, 75, 0), + warn_starting=(1, 0, 0), fail_starting=(2, 0, 0), use_this=None, explanation="This has become a private method and does not guarantee that all the adapters work correctly.", @@ -668,30 +691,12 @@ def raw_execute( inputs: Dict[str, Any] = None, _fn_graph: graph.FunctionGraph = None, ) -> Dict[str, Any]: - """Don't use this entry point for execution directly. Always go through `.execute()` or `.materialize()`. + """Raw execute function that does the meat of execute. + + Don't use this entry point for execution directly. Always go through `.execute()` or `.materialize()`. In case you are using `.raw_execute()` directly, please switch to `.execute()` using a `base.DictResult()`. Note: `base.DictResult()` is the default return of execute if you are using the `driver.Builder()` class to create a `Driver()` object. - """ - - try: - return self.__raw_execute(final_vars, overrides, display_graph, inputs=inputs) - except Exception as e: - logger.error(SLACK_ERROR_MESSAGE) - raise e - - def __raw_execute( - self, - final_vars: List[str], - overrides: Dict[str, Any] = None, - display_graph: bool = False, - inputs: Dict[str, Any] = None, - _fn_graph: graph.FunctionGraph = None, - materialize: bool = True, - ) -> Dict[str, Any]: - """Raw execute function that does the meat of execute. - - Private method since the result building and post_graph_execute lifecycle hooks are performed outside and so this returns an incomplete result. :param final_vars: Final variables to compute :param overrides: Overrides to run. @@ -726,9 +731,9 @@ def __raw_execute( inputs=inputs, overrides=overrides, ) - success = True - error = None results = None + error = None + success = False try: results = self.graph_executor.execute( function_graph, @@ -737,15 +742,10 @@ def __raw_execute( inputs if inputs is not None else {}, run_id, ) - if self.adapter.does_method("do_build_result", is_async=False) and not materialize: - results = self.adapter.call_lifecycle_method_sync( - "do_build_result", outputs=results - ) + success = True except Exception as e: - success = False error = e - # With this the correct node display the error - _ = telemetry.sanitize_error(*sys.exc_info()) + success = False raise e finally: if self.adapter.does_hook("post_graph_execute", is_async=False): @@ -759,6 +759,56 @@ def __raw_execute( ) return results + def __raw_execute( + self, + final_vars: List[str], + overrides: Dict[str, Any] = None, + display_graph: bool = False, + inputs: Dict[str, Any] = None, + _fn_graph: graph.FunctionGraph = None, + _run_id: str = None, + ) -> Dict[str, Any]: + """Raw execute function that does the meat of execute. + + Private method since the result building and post_graph_execute lifecycle hooks are performed outside and so this returns an incomplete result. + + :param final_vars: Final variables to compute + :param overrides: Overrides to run. + :param display_graph: DEPRECATED. DO NOT USE. Whether or not to display the graph when running it + :param inputs: Runtime inputs to the DAG + :return: + """ + function_graph = _fn_graph if _fn_graph is not None else self.graph + run_id = _run_id + nodes, user_nodes = function_graph.get_upstream_nodes(final_vars, inputs, overrides) + Driver.validate_inputs( + function_graph, self.adapter, user_nodes, inputs, nodes + ) # TODO -- validate within the function graph itself + if display_graph: # deprecated flow. + logger.warning( + "display_graph=True is deprecated. It will be removed in the 2.0.0 release. " + "Please use visualize_execution()." + ) + self.visualize_execution(final_vars, "test-output/execute.gv", {"view": True}) + if self.has_cycles( + final_vars, function_graph + ): # here for backwards compatible driver behavior. + raise ValueError("Error: cycles detected in your graph.") + all_nodes = nodes | user_nodes + self.graph_executor.validate(list(all_nodes)) + results = None + try: + results = self.graph_executor.execute( + function_graph, + final_vars, + overrides if overrides is not None else {}, + inputs if inputs is not None else {}, + run_id, + ) + return results + except Exception as e: + raise e + @capture_function_usage def list_available_variables( self, *, tag_filter: Dict[str, Union[Optional[str], List[str]]] = None @@ -1550,9 +1600,10 @@ def materialize( additional_vars = [] start_time = time.time() run_successful = True - error = None - raw_results_output = None - + error_execution = None + error_telemetry = None + run_id = str(uuid.uuid4()) + outputs = (None, None) final_vars = self._create_final_vars(additional_vars) # This is so the finally logging statement does not accidentally die materializer_vars = [] @@ -1579,6 +1630,16 @@ def materialize( # Note we will not run the loaders if they're not upstream of the # materializers or additional_vars materializer_vars = [m.id for m in materializer_factories] + if self.adapter.does_hook("pre_graph_execute", is_async=False): + self.adapter.call_all_lifecycle_hooks_sync( + "pre_graph_execute", + run_id=run_id, + graph=function_graph, + final_vars=final_vars + materializer_vars, + inputs=inputs, + overrides=overrides, + ) + nodes, user_nodes = function_graph.get_upstream_nodes( final_vars + materializer_vars, inputs, overrides ) @@ -1590,22 +1651,37 @@ def materialize( inputs=inputs, overrides=overrides, _fn_graph=function_graph, - materialize=True, + _run_id=run_id, ) materialization_output = {key: raw_results[key] for key in materializer_vars} raw_results_output = {key: raw_results[key] for key in final_vars} - - return materialization_output, raw_results_output + outputs = materialization_output, raw_results_output except Exception as e: run_successful = False logger.error(SLACK_ERROR_MESSAGE) - error = telemetry.sanitize_error(*sys.exc_info()) + error_telemetry = telemetry.sanitize_error(*sys.exc_info()) + error_execution = e raise e finally: + if self.adapter.does_hook("post_graph_execute", is_async=False): + self.adapter.call_all_lifecycle_hooks_sync( + "post_graph_execute", + run_id=run_id, + graph=function_graph, + success=run_successful, + error=error_execution, + results=outputs[1], + ) duration = time.time() - start_time self.capture_execute_telemetry( - error, final_vars + materializer_vars, inputs, overrides, run_successful, duration + error_telemetry, + final_vars + materializer_vars, + inputs, + overrides, + run_successful, + duration, ) + return outputs @capture_function_usage def visualize_materialization( From c1b55eeb1618fcef71c76c92c47658366946526f Mon Sep 17 00:00:00 2001 From: Jernej Frank Date: Fri, 6 Sep 2024 23:39:15 +0100 Subject: [PATCH 31/33] added readme, notebook and made script cli interactive --- examples/ray/ray_Hamilton_UI_tracking/README | 29 +++++ .../hamilton_notebook.ipynb | 107 ++++++++++++++++++ .../ray_Hamilton_UI_tracking/ray_lineage.py | 45 +------- .../ray_Hamilton_UI_tracking/requirements.txt | 1 + .../ray_Hamilton_UI_tracking/run_lineage.py | 45 ++++++++ 5 files changed, 183 insertions(+), 44 deletions(-) create mode 100644 examples/ray/ray_Hamilton_UI_tracking/README create mode 100644 examples/ray/ray_Hamilton_UI_tracking/hamilton_notebook.ipynb create mode 100644 examples/ray/ray_Hamilton_UI_tracking/requirements.txt create mode 100644 examples/ray/ray_Hamilton_UI_tracking/run_lineage.py diff --git a/examples/ray/ray_Hamilton_UI_tracking/README b/examples/ray/ray_Hamilton_UI_tracking/README new file mode 100644 index 000000000..baa786687 --- /dev/null +++ b/examples/ray/ray_Hamilton_UI_tracking/README @@ -0,0 +1,29 @@ +# Tracking telemetry in Hamilton UI for Ray clusters + +We show the ability to combine the [RayGraphAdapter](https://hamilton.dagworks.io/en/latest/reference/graph-adapters/RayGraphAdapter/) and [HamiltonTracker](https://hamilton.dagworks.io/en/latest/concepts/ui/) to run a dummy DAG. + +# ray_lineage.py +Has three dummy functions: +- waiting 5s +- waiting 1s +- raising an error + +That represent a basic DAG. + +# run_lineage.py +Is where the driver code lives to create the DAG and exercise it. + +To exercise it: +> Have an open instance of Hamilton UI: https://hamilton.dagworks.io/en/latest/concepts/ui/ + +```bash +python -m run_lineage.py +Usage: python -m run_lineage.py [OPTIONS] COMMAND [ARGS]... + +Options: + --help Show this message and exit. + +Commands: + project_id This command will select the created project in Hamilton UI + username This command will input the correct username to access the selected project_id +``` diff --git a/examples/ray/ray_Hamilton_UI_tracking/hamilton_notebook.ipynb b/examples/ray/ray_Hamilton_UI_tracking/hamilton_notebook.ipynb new file mode 100644 index 000000000..165da38df --- /dev/null +++ b/examples/ray/ray_Hamilton_UI_tracking/hamilton_notebook.ipynb @@ -0,0 +1,107 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Hamilton UI Adapter\n", + "\n", + "Needs a running instance of Hamilton UI: https://hamilton.dagworks.io/en/latest/concepts/ui/" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from hamilton_sdk.adapters import HamiltonTracker\n", + "\n", + "# Inputs required to track into correct project in the UI\n", + "project_id = 2\n", + "username = \"admin\"\n", + "\n", + "tracker_ray = HamiltonTracker(\n", + " project_id=project_id,\n", + " username=username,\n", + " dag_name=\"telemetry_with_ray\",)\n", + "\n", + "tracker_without_ray = HamiltonTracker(\n", + " project_id=project_id,\n", + " username=username,\n", + " dag_name=\"telemetry_without_ray\",\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Ray adapter\n", + "\n", + "https://hamilton.dagworks.io/en/latest/reference/graph-adapters/RayGraphAdapter/" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from hamilton import base\n", + "from hamilton.plugins.h_ray import RayGraphAdapter\n", + "\n", + "rga = RayGraphAdapter(result_builder=base.PandasDataFrameResult())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Importing Hamilton and the DAG modules" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from hamilton import driver\n", + "import ray_lineage" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " dr_ray = driver.Builder().with_modules(ray_lineage).with_adapters(rga, tracker_ray).build()\n", + " result_ray = dr_ray.execute(\n", + " final_vars=[\n", + " \"node_5s\",\n", + " \"node_1s_error\",\n", + " \"add_1_to_previous\",\n", + " ]\n", + " )\n", + " print(result_ray)\n", + "\n", + "except ValueError:\n", + " print(\"UI should display failure.\")\n", + "finally:\n", + " dr_without_ray = driver.Builder().with_modules(ray_lineage).with_adapters(tracker).build()\n", + " result_without_ray = dr_without_ray.execute(final_vars=[\"node_5s\", \"add_1_to_previous\"])\n", + " print(result_without_ray) \n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/ray/ray_Hamilton_UI_tracking/ray_lineage.py b/examples/ray/ray_Hamilton_UI_tracking/ray_lineage.py index 89bcd6990..d9d20c2ca 100644 --- a/examples/ray/ray_Hamilton_UI_tracking/ray_lineage.py +++ b/examples/ray/ray_Hamilton_UI_tracking/ray_lineage.py @@ -13,49 +13,6 @@ def add_1_to_previous(node_5s: float) -> float: return node_5s + (time.time() - start) -def node_1s_error() -> float: +def node_1s_error(node_5s: float) -> float: time.sleep(1) raise ValueError("Does not break telemetry if executed through ray") - - -if __name__ == "__main__": - import ray - - import __main__ - from hamilton import base, driver - from hamilton.plugins.h_ray import RayGraphAdapter - from hamilton_sdk import adapters - - username = "admin" - - try: - # ray.init() - ray.shutdown() - tracker_ray = adapters.HamiltonTracker( - project_id=1, # modify this as needed - username=username, - dag_name="telemetry_with_ray", - ) - rga = RayGraphAdapter(result_builder=base.PandasDataFrameResult()) - dr_ray = driver.Builder().with_modules(__main__).with_adapters(rga, tracker_ray).build() - result_ray = dr_ray.execute( - final_vars=[ - "node_5s", - # "node_1s_error", - "add_1_to_previous", - ] - ) - print(result_ray) - - except ValueError: - print("UI should display failure.") - finally: - tracker = adapters.HamiltonTracker( - project_id=1, # modify this as needed - username=username, - dag_name="telemetry_without_ray", - ) - dr_without_ray = driver.Builder().with_modules(__main__).with_adapters(tracker).build() - - result_without_ray = dr_without_ray.execute(final_vars=["node_5s", "add_1_to_previous"]) - print(result_without_ray) diff --git a/examples/ray/ray_Hamilton_UI_tracking/requirements.txt b/examples/ray/ray_Hamilton_UI_tracking/requirements.txt new file mode 100644 index 000000000..e6916bb92 --- /dev/null +++ b/examples/ray/ray_Hamilton_UI_tracking/requirements.txt @@ -0,0 +1 @@ +sf-hamilton[ray,sdk,ui] diff --git a/examples/ray/ray_Hamilton_UI_tracking/run_lineage.py b/examples/ray/ray_Hamilton_UI_tracking/run_lineage.py new file mode 100644 index 000000000..6d135efd1 --- /dev/null +++ b/examples/ray/ray_Hamilton_UI_tracking/run_lineage.py @@ -0,0 +1,45 @@ +import click +import ray_lineage + +from hamilton import base, driver +from hamilton.plugins.h_ray import RayGraphAdapter +from hamilton_sdk import adapters + + +@click.command() +@click.option("--username", required=True, type=str) +@click.option("--project_id", default=1, type=int) +def run(project_id, username): + try: + tracker_ray = adapters.HamiltonTracker( + project_id=project_id, + username=username, + dag_name="telemetry_with_ray", + ) + rga = RayGraphAdapter(result_builder=base.PandasDataFrameResult()) + dr_ray = driver.Builder().with_modules(ray_lineage).with_adapters(rga, tracker_ray).build() + result_ray = dr_ray.execute( + final_vars=[ + "node_5s", + "node_1s_error", + "add_1_to_previous", + ] + ) + print(result_ray) + + except ValueError: + print("UI should display failure.") + finally: + tracker = adapters.HamiltonTracker( + project_id=project_id, # modify this as needed + username=username, + dag_name="telemetry_without_ray", + ) + dr_without_ray = driver.Builder().with_modules(ray_lineage).with_adapters(tracker).build() + + result_without_ray = dr_without_ray.execute(final_vars=["node_5s", "add_1_to_previous"]) + print(result_without_ray) + + +if __name__ == "__main__": + run() From 089d1de52ad0bd7448b500bfa37575145f91f2ab Mon Sep 17 00:00:00 2001 From: Jernej Frank Date: Sat, 7 Sep 2024 00:08:24 +0100 Subject: [PATCH 32/33] made cluster init optional through inserting config dict --- hamilton/plugins/h_ray.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/hamilton/plugins/h_ray.py b/hamilton/plugins/h_ray.py index e733ec788..932e01760 100644 --- a/hamilton/plugins/h_ray.py +++ b/hamilton/plugins/h_ray.py @@ -99,7 +99,7 @@ def __init__( self, result_builder: base.ResultMixin, ray_init_config: typing.Dict[str, typing.Any] = None, - keep_cluster_open: bool = False, + shutdown_ray_on_completion: bool = False, ): """Constructor @@ -108,7 +108,7 @@ def __init__( :param result_builder: Required. An implementation of base.ResultMixin. :param ray_init_config: allows to connect to an existing cluster or start a new one with custom configuration (https://docs.ray.io/en/latest/ray-core/api/doc/ray.init.html) - :param keep_cluster_open: to access Ray dashboard and logs for the cluster run + :param shutdown_ray_on_completion: by default we leave the cluster open, but we can also shut it down """ self.result_builder = result_builder @@ -117,19 +117,10 @@ def __init__( "Error: ResultMixin object required. Please pass one in for `result_builder`." ) - self.keep_cluster_open = keep_cluster_open - - if ray_init_config: - # Ray breaks if you try to open the cluster twice without this flag - if "ignore_reinit_error" not in ray_init_config: - ray_init_config["ignore_reinit_error"] = True - # If the cluster is already open we don't want to close it with Hamilton - if "address" in ray_init_config: - self.keep_cluster_open = True + self.shutdown_ray_on_completion = shutdown_ray_on_completion + if ray_init_config is not None: ray.init(**ray_init_config) - else: - ray.init(ignore_reinit_error=True) @staticmethod def do_validate_input(node_type: typing.Type, input_value: typing.Any) -> bool: @@ -173,9 +164,9 @@ def do_build_result(self, outputs: typing.Dict[str, typing.Any]) -> typing.Any: return result def post_graph_execute(self, *args, **kwargs): - """When we create a Ray cluster with Hamilton we tear it down after execution, unless manual overwrite.""" + """We have the option to close the cluster down after execution.""" - if not self.keep_cluster_open: + if not self.shutdown_ray_on_completion: # In case we have Hamilton Tracker to have enough time to properly flush time.sleep(5) ray.shutdown() From 1985ef7f524901740eba309ca16b14e5b4b72dfe Mon Sep 17 00:00:00 2001 From: jernejfrank <50105951+jernejfrank@users.noreply.github.com> Date: Sat, 7 Sep 2024 08:01:21 +0100 Subject: [PATCH 33/33] User has option to shutdown ray cluster Co-authored-by: Stefan Krawczyk --- hamilton/plugins/h_ray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hamilton/plugins/h_ray.py b/hamilton/plugins/h_ray.py index 932e01760..66d0a9255 100644 --- a/hamilton/plugins/h_ray.py +++ b/hamilton/plugins/h_ray.py @@ -166,7 +166,7 @@ def do_build_result(self, outputs: typing.Dict[str, typing.Any]) -> typing.Any: def post_graph_execute(self, *args, **kwargs): """We have the option to close the cluster down after execution.""" - if not self.shutdown_ray_on_completion: + if self.shutdown_ray_on_completion: # In case we have Hamilton Tracker to have enough time to properly flush time.sleep(5) ray.shutdown()