From 2431ee9833a5c5c7cb4b41a5d941401032540479 Mon Sep 17 00:00:00 2001 From: Mecoli1219 Date: Tue, 12 Nov 2024 11:37:32 -0800 Subject: [PATCH 01/25] test new nb-rli Signed-off-by: Mecoli1219 --- docs/_ext/notebook_rli.py | 145 ++++++++++++++++++ docs/conf.py | 1 + docs/user_guide/flyte_fundamentals/index.md | 3 + .../jupyter_notebook_interactive_mode.md | 14 ++ 4 files changed, 163 insertions(+) create mode 100644 docs/_ext/notebook_rli.py create mode 100644 docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md diff --git a/docs/_ext/notebook_rli.py b/docs/_ext/notebook_rli.py new file mode 100644 index 0000000000..f936cd80bf --- /dev/null +++ b/docs/_ext/notebook_rli.py @@ -0,0 +1,145 @@ +import requests + +from docutils import nodes +from docutils.parsers.rst import directives +from six import text_type +from typing import Dict, Any, List, Tuple + +from sphinx.locale import __ +from sphinx.util import parselinenos +from sphinx.util.docutils import SphinxDirective +from sphinx.config import Config +from sphinx.util.nodes import set_source_info +from sphinx.directives.code import logger +from sphinx.directives.code import container_wrapper + + +class NotebookRemoteLiteralIncludeReader(object): + + def __init__(self, url: str, options: Dict[str, Any], config: Config) -> None: + self.url = url + self.options = options + + def read_file(self, url: str) -> List[str]: + response = requests.get(url) + json_data = response.json() + + if json_data: + if not response.status_code == requests.codes.ok: + raise ValueError( + "HTTP request returned error code %s" % response.status_code + ) + cell_idx = int(self.options["cell"]) + if "cell" in self.options: + if len(json_data["cells"]) > cell_idx: + if json_data["cells"][cell_idx]["cell_type"] == "code": + lines = json_data["cells"][cell_idx]["source"] + text = "".join(lines) + else: + raise ValueError("Cell is not a code cell") + else: + raise ValueError("Cell exceeds the number of cells in the notebook") + else: + raise ValueError("Cell not specified in options") + + if "tab-width" in self.options: + text = text.expandtabs(self.options["tab-width"]) + + return text.splitlines(True) + else: + raise IOError(__("Include file %r not found or reading it failed") % url) + + def read(self) -> Tuple[str, int]: + lines = self.read_file(self.url) + lines = self.lines_filter(lines) + + return "".join(lines), len(lines) + + def lines_filter(self, lines: List[str]) -> List[str]: + linespec = self.options.get("lines") + if linespec: + linelist = parselinenos(linespec, len(lines)) + if any(i >= len(lines) for i in linelist): + raise ValueError( + "Line number spec is out of range (1 - %s)" % len(lines) + ) + + lines = [lines[n] for n in linelist if n < len(lines)] + if lines == []: + raise ValueError( + __("Line spec %r: no lines pulled from include file %r") + % (linespec, self.url) + ) + + return lines + + +class NotebookRemoteLiteralInclude(SphinxDirective): + """ + Like ``.. include:: :literal:``, but only warns if the include file is + not found, and does not raise errors. Also has several options for + selecting what to include. + """ + + has_content = False + required_arguments = 1 + optional_arguments = 0 + final_argument_whitespace = True + option_spec = { + "tab-width": int, + "language": directives.unchanged_required, + "lines": directives.unchanged_required, + "cell": directives.unchanged_required, + "emphasize-lines": directives.unchanged_required, + "caption": directives.unchanged, + "class": directives.class_option, + "name": directives.unchanged, + } + + def run(self) -> List[nodes.Node]: + document = self.state.document + if not document.settings.file_insertion_enabled: + return [ + document.reporter.warning("File insertion disabled", line=self.lineno) + ] + try: + url = self.arguments[0] + + reader = NotebookRemoteLiteralIncludeReader(url, self.options, self.config) + text, lines = reader.read() + + retnode = nodes.literal_block(text, text, source=url) + set_source_info(self, retnode) + if "language" in self.options: + retnode["language"] = self.options["language"] + retnode["classes"] += self.options.get("class", []) + extra_args = retnode["highlight_args"] = {} + if "emphasize-lines" in self.options: + hl_lines = parselinenos(self.options["emphasize-lines"], lines) + if any(i >= lines for i in hl_lines): + logger.warning( + __("line number spec is out of range(1-%d): %r") + % (lines, self.options["emphasize-lines"]) + ) + extra_args["hl_lines"] = [x + 1 for x in hl_lines if x < lines] + + if "caption" in self.options: + caption = self.options["caption"] or self.arguments[0] + retnode = container_wrapper(self, retnode, caption) + + # retnode will be note_implicit_target that is linked from caption and numref. + # when options['name'] is provided, it should be primary ID. + self.add_name(retnode) + + return [retnode] + except Exception as exc: + return [document.reporter.warning(text_type(exc), line=self.lineno)] + + +def setup(app): + directives.register_directive("nb-rli", NotebookRemoteLiteralInclude) + + return { + "parallel_read_safe": True, + "parallel_write_safe": False, + } diff --git a/docs/conf.py b/docs/conf.py index 316acc60be..fda357ce2e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -79,6 +79,7 @@ # custom extensions "auto_examples", "import_projects", + "notebook_rli", ] source_suffix = { diff --git a/docs/user_guide/flyte_fundamentals/index.md b/docs/user_guide/flyte_fundamentals/index.md index a1a08fd187..7e1d133e77 100644 --- a/docs/user_guide/flyte_fundamentals/index.md +++ b/docs/user_guide/flyte_fundamentals/index.md @@ -22,6 +22,8 @@ use cases. - Develop and deploy workflows to a local Flyte demo cluster. * - {doc}`⏱ Running and scheduling workflows ` - Execute workflows programmatically and schedule them as cron jobs. +* - {doc}`📙 Jupyter notebook interactive mode ` + - Develop and debug Flyte workflows interactively in Jupyter notebooks. * - {doc}`📊 Visualizing task input and output ` - Create rich, customizable static reports for increased visibility into tasks. * - {doc}`🏎 Optimizing tasks ` @@ -45,6 +47,7 @@ cluster, see the {ref}`Deployment Guide `. tasks_workflows_and_launch_plans registering_workflows running_and_scheduling_workflows +jupyter_notebook_interactive_mode visualizing_task_input_and_output optimizing_tasks extending_flyte diff --git a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md new file mode 100644 index 0000000000..cf53f6fd3c --- /dev/null +++ b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md @@ -0,0 +1,14 @@ +(getting_started_jupyter_notebook_interactive_mode)= + +# Hello, World! + +```{nb-rli} https://raw.githubusercontent.com/Mecoli1219/flytesnacks/85966d139a9a3ccdf3323124563960fdd5f5844a/examples/basics/basics/basic_interactive_mode.ipynb +:caption: basics/basic_interactive_mode.ipynb +:cell: 3 +:lines: 1-4 +``` + +```{nb-rli} https://raw.githubusercontent.com/Mecoli1219/flytesnacks/85966d139a9a3ccdf3323124563960fdd5f5844a/examples/basics/basics/basic_interactive_mode.ipynb +:caption: basics/basic_interactive_mode.ipynb +:cell: 6 +``` From 7a1ffdb09862010aad21acce46485c53132f8613 Mon Sep 17 00:00:00 2001 From: Mecoli1219 Date: Tue, 12 Nov 2024 12:11:26 -0800 Subject: [PATCH 02/25] fix error & debug Signed-off-by: Mecoli1219 --- docs/_ext/notebook_rli.py | 12 ++++++------ .../jupyter_notebook_interactive_mode.md | 5 +++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/_ext/notebook_rli.py b/docs/_ext/notebook_rli.py index f936cd80bf..f627d5a96b 100644 --- a/docs/_ext/notebook_rli.py +++ b/docs/_ext/notebook_rli.py @@ -3,15 +3,15 @@ from docutils import nodes from docutils.parsers.rst import directives from six import text_type -from typing import Dict, Any, List, Tuple +from typing import Any, Dict, List, Tuple +from sphinx.application import Sphinx +from sphinx.config import Config +from sphinx.directives.code import container_wrapper, logger from sphinx.locale import __ from sphinx.util import parselinenos from sphinx.util.docutils import SphinxDirective -from sphinx.config import Config from sphinx.util.nodes import set_source_info -from sphinx.directives.code import logger -from sphinx.directives.code import container_wrapper class NotebookRemoteLiteralIncludeReader(object): @@ -136,8 +136,8 @@ def run(self) -> List[nodes.Node]: return [document.reporter.warning(text_type(exc), line=self.lineno)] -def setup(app): - directives.register_directive("nb-rli", NotebookRemoteLiteralInclude) +def setup(app: Sphinx) -> dict: + app.add_directive("nb-rli", NotebookRemoteLiteralInclude) return { "parallel_read_safe": True, diff --git a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md index cf53f6fd3c..20fcaa2994 100644 --- a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md +++ b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md @@ -12,3 +12,8 @@ :caption: basics/basic_interactive_mode.ipynb :cell: 6 ``` + +```{rli} https://raw.githubusercontent.com/Mecoli1219/flytesnacks/85966d139a9a3ccdf3323124563960fdd5f5844a/examples/basics/basics/basic_interactive_mode.ipynb +:caption: basics/basic_interactive_mode.ipynb +:lines: 1-4 +``` From 8c023c426b35f2750f67f17667784c40162b0d20 Mon Sep 17 00:00:00 2001 From: Mecoli1219 Date: Tue, 12 Nov 2024 12:40:53 -0800 Subject: [PATCH 03/25] fix error & debug Signed-off-by: Mecoli1219 --- .../jupyter_notebook_interactive_mode.md | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md index 20fcaa2994..6e91e45901 100644 --- a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md +++ b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md @@ -1,19 +1,35 @@ +--- +kernelspec: + display_name: Python 3 + language: python + name: python3 +--- + (getting_started_jupyter_notebook_interactive_mode)= # Hello, World! -```{nb-rli} https://raw.githubusercontent.com/Mecoli1219/flytesnacks/85966d139a9a3ccdf3323124563960fdd5f5844a/examples/basics/basics/basic_interactive_mode.ipynb +```{eval-rst} +.. tags:: Basic +``` + +```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/69dbe4840031a85d79d9ded25f80397c6834752d/examples/basics/basics/imperative_workflow.py +:caption: basics/imperative_workflow.py +:lines: 1 +``` + +```{rli} https://raw.githubusercontent.com/Mecoli1219/flytesnacks/85966d139a9a3ccdf3323124563960fdd5f5844a/examples/basics/basics/basic_interactive_mode.ipynb :caption: basics/basic_interactive_mode.ipynb -:cell: 3 :lines: 1-4 ``` ```{nb-rli} https://raw.githubusercontent.com/Mecoli1219/flytesnacks/85966d139a9a3ccdf3323124563960fdd5f5844a/examples/basics/basics/basic_interactive_mode.ipynb :caption: basics/basic_interactive_mode.ipynb -:cell: 6 +:cell: 3 +:lines: 1-4 ``` -```{rli} https://raw.githubusercontent.com/Mecoli1219/flytesnacks/85966d139a9a3ccdf3323124563960fdd5f5844a/examples/basics/basics/basic_interactive_mode.ipynb +```{nb-rli} https://raw.githubusercontent.com/Mecoli1219/flytesnacks/85966d139a9a3ccdf3323124563960fdd5f5844a/examples/basics/basics/basic_interactive_mode.ipynb :caption: basics/basic_interactive_mode.ipynb -:lines: 1-4 +:cell: 6 ``` From 562811822b3398896604f8f3141faaa4ce7e3526 Mon Sep 17 00:00:00 2001 From: Mecoli1219 Date: Tue, 12 Nov 2024 13:09:53 -0800 Subject: [PATCH 04/25] fix error & debug Signed-off-by: Mecoli1219 --- .../flyte_fundamentals/jupyter_notebook_interactive_mode.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md index 6e91e45901..82115638c2 100644 --- a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md +++ b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md @@ -23,7 +23,7 @@ kernelspec: :lines: 1-4 ``` -```{nb-rli} https://raw.githubusercontent.com/Mecoli1219/flytesnacks/85966d139a9a3ccdf3323124563960fdd5f5844a/examples/basics/basics/basic_interactive_mode.ipynb + + + From 100f072ea033c244881c2bad73724e2d66cdb7b4 Mon Sep 17 00:00:00 2001 From: Mecoli1219 Date: Tue, 12 Nov 2024 13:54:49 -0800 Subject: [PATCH 05/25] fix error & debug Signed-off-by: Mecoli1219 --- .../flyte_fundamentals/jupyter_notebook_interactive_mode.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md index 82115638c2..f6a82126a9 100644 --- a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md +++ b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md @@ -18,10 +18,10 @@ kernelspec: :lines: 1 ``` -```{rli} https://raw.githubusercontent.com/Mecoli1219/flytesnacks/85966d139a9a3ccdf3323124563960fdd5f5844a/examples/basics/basics/basic_interactive_mode.ipynb + +``` - +:cell: 3 +``` From 74d85b4b275b0b7ed02b6615e02aa8e35a529715 Mon Sep 17 00:00:00 2001 From: Mecoli1219 Date: Tue, 12 Nov 2024 15:00:21 -0800 Subject: [PATCH 07/25] fix error & debug Signed-off-by: Mecoli1219 --- .../flyte_fundamentals/jupyter_notebook_interactive_mode.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md index 3cb00441d5..f1e557ab29 100644 --- a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md +++ b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md @@ -23,7 +23,7 @@ kernelspec: :lines: 1-4 ``` -```{nb-rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/fbe246177d321e0491b652084c1c3a9c0cb0d35c/examples/papermill_plugin/papermill_plugin/nb_simple.ipynb + From 714bd7623327370ba609f5bb229e73d4de60e21f Mon Sep 17 00:00:00 2001 From: Mecoli1219 Date: Tue, 12 Nov 2024 15:42:30 -0800 Subject: [PATCH 08/25] fix error & debug Signed-off-by: Mecoli1219 --- docs/conf.py | 1 + .../flyte_fundamentals/jupyter_notebook_interactive_mode.md | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index fda357ce2e..068796d1e7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -629,6 +629,7 @@ "examples/data_types_and_io", "examples/development_lifecycle", "examples/extending", + "examples/papermill_plugin", "examples/productionizing", "examples/testing" ] diff --git a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md index f1e557ab29..f31619548a 100644 --- a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md +++ b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md @@ -23,13 +23,13 @@ kernelspec: :lines: 1-4 ``` - From 7f5072a7b40d1f5de61e6fd11459a49fbcbc6823 Mon Sep 17 00:00:00 2001 From: Mecoli1219 Date: Tue, 12 Nov 2024 16:12:32 -0800 Subject: [PATCH 09/25] fix error & debug Signed-off-by: Mecoli1219 --- docs/conf.py | 1 - .../flyte_fundamentals/jupyter_notebook_interactive_mode.md | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 068796d1e7..fda357ce2e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -629,7 +629,6 @@ "examples/data_types_and_io", "examples/development_lifecycle", "examples/extending", - "examples/papermill_plugin", "examples/productionizing", "examples/testing" ] diff --git a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md index f31619548a..b27bf9771a 100644 --- a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md +++ b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md @@ -18,7 +18,7 @@ kernelspec: :lines: 1 ``` -```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/fbe246177d321e0491b652084c1c3a9c0cb0d35c/examples/papermill_plugin/papermill_plugin/nb_simple.ipynb +```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/fbe246177d321e0491b652084c1c3a9c0cb0d35c/examples/papermill_plugin/papermill_plugin/simple.py :caption: basics/basic_interactive_mode.ipynb :lines: 1-4 ``` From 215e2ccba104cd20e5b307794289a862911d9a26 Mon Sep 17 00:00:00 2001 From: Mecoli1219 Date: Tue, 12 Nov 2024 16:37:29 -0800 Subject: [PATCH 10/25] fix error & debug Signed-off-by: Mecoli1219 --- .../jupyter_notebook_interactive_mode.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md index b27bf9771a..aeaec494dd 100644 --- a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md +++ b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md @@ -18,11 +18,20 @@ kernelspec: :lines: 1 ``` -```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/fbe246177d321e0491b652084c1c3a9c0cb0d35c/examples/papermill_plugin/papermill_plugin/simple.py -:caption: basics/basic_interactive_mode.ipynb +why? + +```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/69dbe4840031a85d79d9ded25f80397c6834752d/examples/basics/basics/imperative_workflow.py +:caption: basics/imperative_workflow.py :lines: 1-4 ``` +this is not work? + +```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/fbe246177d321e0491b652084c1c3a9c0cb0d35c/examples/papermill_plugin/papermill_plugin/simple.py +:caption: papermill_plugin/simple.py +:lines: 1 +``` + +``` - From 1ed57d2f295e3ffd600512f595bfe8e25c5e37c6 Mon Sep 17 00:00:00 2001 From: Mecoli1219 Date: Thu, 14 Nov 2024 22:49:11 -0800 Subject: [PATCH 13/25] fix error & debug Signed-off-by: Mecoli1219 --- ...book_rli.py => notebook_literalinclude.py} | 71 ++++++++++--------- docs/conf.py | 2 +- .../jupyter_notebook_interactive_mode.md | 17 +++-- 3 files changed, 49 insertions(+), 41 deletions(-) rename docs/_ext/{notebook_rli.py => notebook_literalinclude.py} (67%) diff --git a/docs/_ext/notebook_rli.py b/docs/_ext/notebook_literalinclude.py similarity index 67% rename from docs/_ext/notebook_rli.py rename to docs/_ext/notebook_literalinclude.py index f627d5a96b..a44320f651 100644 --- a/docs/_ext/notebook_rli.py +++ b/docs/_ext/notebook_literalinclude.py @@ -1,9 +1,9 @@ -import requests +import json from docutils import nodes from docutils.parsers.rst import directives from six import text_type -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List, Tuple, TYPE_CHECKING from sphinx.application import Sphinx from sphinx.config import Config @@ -12,45 +12,44 @@ from sphinx.util import parselinenos from sphinx.util.docutils import SphinxDirective from sphinx.util.nodes import set_source_info +from sphinx.util._pathlib import _StrPath +if TYPE_CHECKING: + import os -class NotebookRemoteLiteralIncludeReader(object): - def __init__(self, url: str, options: Dict[str, Any], config: Config) -> None: - self.url = url +class NotebookLiteralIncludeReader(object): + + def __init__( + self, filename: str | os.PathLike[str], options: Dict[str, Any], config: Config + ) -> None: + self.filename = _StrPath(filename) self.options = options - def read_file(self, url: str) -> List[str]: - response = requests.get(url) - json_data = response.json() + def read_file(self, filename: str | os.PathLike[str]) -> List[str]: + filename = _StrPath(filename) + json_data = json.loads(filename.read_text()) - if json_data: - if not response.status_code == requests.codes.ok: - raise ValueError( - "HTTP request returned error code %s" % response.status_code - ) - cell_idx = int(self.options["cell"]) - if "cell" in self.options: - if len(json_data["cells"]) > cell_idx: - if json_data["cells"][cell_idx]["cell_type"] == "code": - lines = json_data["cells"][cell_idx]["source"] - text = "".join(lines) - else: - raise ValueError("Cell is not a code cell") + cell_idx = int(self.options["cell"]) + if "cell" in self.options: + if len(json_data["cells"]) > cell_idx: + if json_data["cells"][cell_idx]["cell_type"] == "code": + lines = json_data["cells"][cell_idx]["source"] + text = "".join(lines) else: - raise ValueError("Cell exceeds the number of cells in the notebook") + raise ValueError("Cell is not a code cell") else: - raise ValueError("Cell not specified in options") + raise ValueError("Cell exceeds the number of cells in the notebook") + else: + raise ValueError("Cell not specified in options") - if "tab-width" in self.options: - text = text.expandtabs(self.options["tab-width"]) + if "tab-width" in self.options: + text = text.expandtabs(self.options["tab-width"]) - return text.splitlines(True) - else: - raise IOError(__("Include file %r not found or reading it failed") % url) + return text.splitlines(True) def read(self) -> Tuple[str, int]: - lines = self.read_file(self.url) + lines = self.read_file(self.filename) lines = self.lines_filter(lines) return "".join(lines), len(lines) @@ -68,13 +67,13 @@ def lines_filter(self, lines: List[str]) -> List[str]: if lines == []: raise ValueError( __("Line spec %r: no lines pulled from include file %r") - % (linespec, self.url) + % (linespec, self.filename) ) return lines -class NotebookRemoteLiteralInclude(SphinxDirective): +class NotebookLiteralInclude(SphinxDirective): """ Like ``.. include:: :literal:``, but only warns if the include file is not found, and does not raise errors. Also has several options for @@ -103,12 +102,14 @@ def run(self) -> List[nodes.Node]: document.reporter.warning("File insertion disabled", line=self.lineno) ] try: - url = self.arguments[0] + location = self.state_machine.get_source_and_line(self.lineno) + rel_filename, filename = self.env.relfn2path(self.arguments[0]) + self.env.note_dependency(rel_filename) - reader = NotebookRemoteLiteralIncludeReader(url, self.options, self.config) + reader = NotebookLiteralIncludeReader(filename, self.options, self.config) text, lines = reader.read() - retnode = nodes.literal_block(text, text, source=url) + retnode = nodes.literal_block(text, text, source=filename) set_source_info(self, retnode) if "language" in self.options: retnode["language"] = self.options["language"] @@ -137,7 +138,7 @@ def run(self) -> List[nodes.Node]: def setup(app: Sphinx) -> dict: - app.add_directive("nb-rli", NotebookRemoteLiteralInclude) + app.add_directive("nb-literalinclude", NotebookLiteralInclude) return { "parallel_read_safe": True, diff --git a/docs/conf.py b/docs/conf.py index 50dec8f375..c936f1d7bd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -79,7 +79,7 @@ # custom extensions "auto_examples", "import_projects", - "notebook_rli", + "notebook_literalinclude", ] source_suffix = { diff --git a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md index 7ad2b0dad0..8080413160 100644 --- a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md +++ b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md @@ -25,15 +25,22 @@ this is not work? :lines: 1 ``` - +``` From c97f6db8c4296754619a8c42b0cc615c917d5dc3 Mon Sep 17 00:00:00 2001 From: Mecoli1219 Date: Fri, 15 Nov 2024 07:40:35 -0800 Subject: [PATCH 14/25] fix buf Signed-off-by: Mecoli1219 --- docs/_ext/notebook_literalinclude.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/_ext/notebook_literalinclude.py b/docs/_ext/notebook_literalinclude.py index a44320f651..bf196dea21 100644 --- a/docs/_ext/notebook_literalinclude.py +++ b/docs/_ext/notebook_literalinclude.py @@ -1,9 +1,10 @@ import json +import os from docutils import nodes from docutils.parsers.rst import directives from six import text_type -from typing import Any, Dict, List, Tuple, TYPE_CHECKING +from typing import Any, Dict, List, Tuple, TYPE_CHECKING, Union from sphinx.application import Sphinx from sphinx.config import Config @@ -14,19 +15,19 @@ from sphinx.util.nodes import set_source_info from sphinx.util._pathlib import _StrPath -if TYPE_CHECKING: - import os - class NotebookLiteralIncludeReader(object): def __init__( - self, filename: str | os.PathLike[str], options: Dict[str, Any], config: Config + self, + filename: Union[str, os.PathLike[str]], + options: Dict[str, Any], + config: Config, ) -> None: self.filename = _StrPath(filename) self.options = options - def read_file(self, filename: str | os.PathLike[str]) -> List[str]: + def read_file(self, filename: Union[str, os.PathLike[str]]) -> List[str]: filename = _StrPath(filename) json_data = json.loads(filename.read_text()) From eb16496b4372fd20fefe747c0a8a146ca163fee2 Mon Sep 17 00:00:00 2001 From: Mecoli1219 Date: Fri, 15 Nov 2024 22:40:03 -0800 Subject: [PATCH 15/25] update post Signed-off-by: Mecoli1219 --- docs/conf.py | 23 ++++++++++ docs/user_guide/flyte_fundamentals/index.md | 3 -- .../jupyter_notebook_interactive_mode.md | 46 ------------------- .../running_and_scheduling_workflows.md | 8 ++++ 4 files changed, 31 insertions(+), 49 deletions(-) delete mode 100644 docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md diff --git a/docs/conf.py b/docs/conf.py index c936f1d7bd..cf49d0e1b7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -643,6 +643,29 @@ ], "local": flytesnacks_local_path is not None, }, + { + "name": "mecoliflytesnacks", + "source": "https://github.com/Mecoli1219/flytesnacks", + "docs_path": "docs", + "dest": "mecoliflytesnacks", + "cmd": [ + ["cd", f"{flytesnacks_path}"], + ["git", "checkout", "origin/jupyter-basic"], + ["cd", "-"], + ["cp", "-R", f"{flytesnacks_path}/examples", "./mecoliexamples"], + [ + # remove un-needed docs files in flytesnacks + "rm", + "-rf", + "flytesnacks/jupyter_execute", + "flytesnacks/auto_examples", + "flytesnacks/_build", + "flytesnacks/_tags", + "flytesnacks/index.md", + ] + ], + "local": False, + }, { "name": "flytekit", "source": flytekit_local_path or "https://github.com/flyteorg/flytekit", diff --git a/docs/user_guide/flyte_fundamentals/index.md b/docs/user_guide/flyte_fundamentals/index.md index 7e1d133e77..a1a08fd187 100644 --- a/docs/user_guide/flyte_fundamentals/index.md +++ b/docs/user_guide/flyte_fundamentals/index.md @@ -22,8 +22,6 @@ use cases. - Develop and deploy workflows to a local Flyte demo cluster. * - {doc}`⏱ Running and scheduling workflows ` - Execute workflows programmatically and schedule them as cron jobs. -* - {doc}`📙 Jupyter notebook interactive mode ` - - Develop and debug Flyte workflows interactively in Jupyter notebooks. * - {doc}`📊 Visualizing task input and output ` - Create rich, customizable static reports for increased visibility into tasks. * - {doc}`🏎 Optimizing tasks ` @@ -47,7 +45,6 @@ cluster, see the {ref}`Deployment Guide `. tasks_workflows_and_launch_plans registering_workflows running_and_scheduling_workflows -jupyter_notebook_interactive_mode visualizing_task_input_and_output optimizing_tasks extending_flyte diff --git a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md deleted file mode 100644 index 8080413160..0000000000 --- a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interactive_mode.md +++ /dev/null @@ -1,46 +0,0 @@ -(getting_started_jupyter_notebook_interactive_mode)= - -# Hello, World! - -```{eval-rst} -.. tags:: Basic -``` - -```{literalinclude} /examples/basics/basics/imperative_workflow.py -:caption: basics/imperative_workflow.py -:lines: 1 -``` - -why? - -```{literalinclude} /examples/basics/basics/imperative_workflow.py -:caption: basics/imperative_workflow.py -:lines: 1-4 -``` - -this is not work? - -```{literalinclude} /examples/papermill_plugin/papermill_plugin/nb_simple.ipynb -:caption: papermill_plugin/nb_simple.ipynb -:lines: 1 -``` - -test - -```{literalinclude} /examples/papermill_plugin/papermill_plugin/nb_simple.ipynb -:caption: papermill_plugin/nb_simple.ipynb -:lines: 1 -``` - -```{nb-literalinclude} /examples/papermill_plugin/papermill_plugin/nb_simple.ipynb -:caption: papermill_plugin/nb_simple.ipynb -:cell: 5 -:lines: 1-2 -``` - -```{nb-literalinclude} /examples/papermill_plugin/papermill_plugin/nb_simple.ipynb -:caption: papermill_plugin/nb_simple.ipynb -:cell: 3 -``` - - diff --git a/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md b/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md index 60bbe98596..b514220ecf 100644 --- a/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md +++ b/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md @@ -52,6 +52,14 @@ remote = FlyteRemote( ) ``` +If you're running this code in a Jupyter notebook, you can also use the +`interactive_mode` to run workflows interactively: + +```{nb-literalinclude} /mecoliexamples/basics/basics/basic_interactive_mode.ipynb +:cell: 3 +:lines: 4-9 +``` + ## Running a workflow You can run workflows using the `FlyteRemote` {py:meth}`~flytekit.remote.remote.FlyteRemote.execute` From 4f02a768391d16c79e64e510748aefaea9f5a677 Mon Sep 17 00:00:00 2001 From: Mecoli1219 Date: Fri, 15 Nov 2024 22:56:23 -0800 Subject: [PATCH 16/25] update clone path Signed-off-by: Mecoli1219 --- docs/conf.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index cf49d0e1b7..161684505a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -649,9 +649,7 @@ "docs_path": "docs", "dest": "mecoliflytesnacks", "cmd": [ - ["cd", f"{flytesnacks_path}"], - ["git", "checkout", "origin/jupyter-basic"], - ["cd", "-"], + ["git", "-C", f"{flytesnacks_path}", "checkout", "origin/jupyter-basic"], ["cp", "-R", f"{flytesnacks_path}/examples", "./mecoliexamples"], [ # remove un-needed docs files in flytesnacks From 9aa1ef50fe5bba4e054bc09784a1d34b0bfe6ef4 Mon Sep 17 00:00:00 2001 From: Mecoli1219 Date: Fri, 15 Nov 2024 23:18:52 -0800 Subject: [PATCH 17/25] update conf.py Signed-off-by: Mecoli1219 --- docs/conf.py | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 161684505a..098e567962 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -625,10 +625,11 @@ import_projects = [ { "name": "flytesnacks", - "source": flytesnacks_local_path or "https://github.com/flyteorg/flytesnacks", + "source": flytesnacks_local_path or "https://github.com/Mecoli1219/flytesnacks", "docs_path": "docs", "dest": "flytesnacks", "cmd": [ + ["git", "-C", f"{flytesnacks_path}", "checkout", "origin/jupyter-basic"], ["cp", "-R", f"{flytesnacks_path}/examples", "./examples"], [ # remove un-needed docs files in flytesnacks @@ -643,27 +644,6 @@ ], "local": flytesnacks_local_path is not None, }, - { - "name": "mecoliflytesnacks", - "source": "https://github.com/Mecoli1219/flytesnacks", - "docs_path": "docs", - "dest": "mecoliflytesnacks", - "cmd": [ - ["git", "-C", f"{flytesnacks_path}", "checkout", "origin/jupyter-basic"], - ["cp", "-R", f"{flytesnacks_path}/examples", "./mecoliexamples"], - [ - # remove un-needed docs files in flytesnacks - "rm", - "-rf", - "flytesnacks/jupyter_execute", - "flytesnacks/auto_examples", - "flytesnacks/_build", - "flytesnacks/_tags", - "flytesnacks/index.md", - ] - ], - "local": False, - }, { "name": "flytekit", "source": flytekit_local_path or "https://github.com/flyteorg/flytekit", From a2f11e9beaae541459b90684992bc6112deb4339 Mon Sep 17 00:00:00 2001 From: Mecoli1219 Date: Fri, 15 Nov 2024 23:57:27 -0800 Subject: [PATCH 18/25] update path Signed-off-by: Mecoli1219 --- .../flyte_fundamentals/running_and_scheduling_workflows.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md b/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md index b514220ecf..acf6a5fe11 100644 --- a/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md +++ b/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md @@ -55,7 +55,8 @@ remote = FlyteRemote( If you're running this code in a Jupyter notebook, you can also use the `interactive_mode` to run workflows interactively: -```{nb-literalinclude} /mecoliexamples/basics/basics/basic_interactive_mode.ipynb +```{nb-literalinclude} /examples/basics/basics/basic_interactive_mode.ipynb +:caption: basics/basic_interactive_mode.ipynb :cell: 3 :lines: 4-9 ``` From 2a420f75d8f16d5cddee068a30991cbd02cccbc9 Mon Sep 17 00:00:00 2001 From: Mecoli1219 Date: Sat, 16 Nov 2024 00:28:06 -0800 Subject: [PATCH 19/25] Update lines Signed-off-by: Mecoli1219 --- .../flyte_fundamentals/running_and_scheduling_workflows.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md b/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md index acf6a5fe11..be79d4ea54 100644 --- a/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md +++ b/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md @@ -58,7 +58,7 @@ If you're running this code in a Jupyter notebook, you can also use the ```{nb-literalinclude} /examples/basics/basics/basic_interactive_mode.ipynb :caption: basics/basic_interactive_mode.ipynb :cell: 3 -:lines: 4-9 +:lines: 5-10 ``` ## Running a workflow From 30722682013d91251b9e3e27bb3705dae827c035 Mon Sep 17 00:00:00 2001 From: Mecoli1219 Date: Tue, 19 Nov 2024 22:48:45 -0800 Subject: [PATCH 20/25] Create a new page Signed-off-by: Mecoli1219 --- docs/_ext/notebook_literalinclude.py | 147 ------------------ docs/conf.py | 1 - docs/user_guide/basics/index.md | 1 + .../running_and_scheduling_workflows.md | 9 -- 4 files changed, 1 insertion(+), 157 deletions(-) delete mode 100644 docs/_ext/notebook_literalinclude.py diff --git a/docs/_ext/notebook_literalinclude.py b/docs/_ext/notebook_literalinclude.py deleted file mode 100644 index bf196dea21..0000000000 --- a/docs/_ext/notebook_literalinclude.py +++ /dev/null @@ -1,147 +0,0 @@ -import json -import os - -from docutils import nodes -from docutils.parsers.rst import directives -from six import text_type -from typing import Any, Dict, List, Tuple, TYPE_CHECKING, Union - -from sphinx.application import Sphinx -from sphinx.config import Config -from sphinx.directives.code import container_wrapper, logger -from sphinx.locale import __ -from sphinx.util import parselinenos -from sphinx.util.docutils import SphinxDirective -from sphinx.util.nodes import set_source_info -from sphinx.util._pathlib import _StrPath - - -class NotebookLiteralIncludeReader(object): - - def __init__( - self, - filename: Union[str, os.PathLike[str]], - options: Dict[str, Any], - config: Config, - ) -> None: - self.filename = _StrPath(filename) - self.options = options - - def read_file(self, filename: Union[str, os.PathLike[str]]) -> List[str]: - filename = _StrPath(filename) - json_data = json.loads(filename.read_text()) - - cell_idx = int(self.options["cell"]) - if "cell" in self.options: - if len(json_data["cells"]) > cell_idx: - if json_data["cells"][cell_idx]["cell_type"] == "code": - lines = json_data["cells"][cell_idx]["source"] - text = "".join(lines) - else: - raise ValueError("Cell is not a code cell") - else: - raise ValueError("Cell exceeds the number of cells in the notebook") - else: - raise ValueError("Cell not specified in options") - - if "tab-width" in self.options: - text = text.expandtabs(self.options["tab-width"]) - - return text.splitlines(True) - - def read(self) -> Tuple[str, int]: - lines = self.read_file(self.filename) - lines = self.lines_filter(lines) - - return "".join(lines), len(lines) - - def lines_filter(self, lines: List[str]) -> List[str]: - linespec = self.options.get("lines") - if linespec: - linelist = parselinenos(linespec, len(lines)) - if any(i >= len(lines) for i in linelist): - raise ValueError( - "Line number spec is out of range (1 - %s)" % len(lines) - ) - - lines = [lines[n] for n in linelist if n < len(lines)] - if lines == []: - raise ValueError( - __("Line spec %r: no lines pulled from include file %r") - % (linespec, self.filename) - ) - - return lines - - -class NotebookLiteralInclude(SphinxDirective): - """ - Like ``.. include:: :literal:``, but only warns if the include file is - not found, and does not raise errors. Also has several options for - selecting what to include. - """ - - has_content = False - required_arguments = 1 - optional_arguments = 0 - final_argument_whitespace = True - option_spec = { - "tab-width": int, - "language": directives.unchanged_required, - "lines": directives.unchanged_required, - "cell": directives.unchanged_required, - "emphasize-lines": directives.unchanged_required, - "caption": directives.unchanged, - "class": directives.class_option, - "name": directives.unchanged, - } - - def run(self) -> List[nodes.Node]: - document = self.state.document - if not document.settings.file_insertion_enabled: - return [ - document.reporter.warning("File insertion disabled", line=self.lineno) - ] - try: - location = self.state_machine.get_source_and_line(self.lineno) - rel_filename, filename = self.env.relfn2path(self.arguments[0]) - self.env.note_dependency(rel_filename) - - reader = NotebookLiteralIncludeReader(filename, self.options, self.config) - text, lines = reader.read() - - retnode = nodes.literal_block(text, text, source=filename) - set_source_info(self, retnode) - if "language" in self.options: - retnode["language"] = self.options["language"] - retnode["classes"] += self.options.get("class", []) - extra_args = retnode["highlight_args"] = {} - if "emphasize-lines" in self.options: - hl_lines = parselinenos(self.options["emphasize-lines"], lines) - if any(i >= lines for i in hl_lines): - logger.warning( - __("line number spec is out of range(1-%d): %r") - % (lines, self.options["emphasize-lines"]) - ) - extra_args["hl_lines"] = [x + 1 for x in hl_lines if x < lines] - - if "caption" in self.options: - caption = self.options["caption"] or self.arguments[0] - retnode = container_wrapper(self, retnode, caption) - - # retnode will be note_implicit_target that is linked from caption and numref. - # when options['name'] is provided, it should be primary ID. - self.add_name(retnode) - - return [retnode] - except Exception as exc: - return [document.reporter.warning(text_type(exc), line=self.lineno)] - - -def setup(app: Sphinx) -> dict: - app.add_directive("nb-literalinclude", NotebookLiteralInclude) - - return { - "parallel_read_safe": True, - "parallel_write_safe": False, - } diff --git a/docs/conf.py b/docs/conf.py index 098e567962..75e6d4ffaf 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -79,7 +79,6 @@ # custom extensions "auto_examples", "import_projects", - "notebook_literalinclude", ] source_suffix = { diff --git a/docs/user_guide/basics/index.md b/docs/user_guide/basics/index.md index bc97b74cc9..9883cddd83 100644 --- a/docs/user_guide/basics/index.md +++ b/docs/user_guide/basics/index.md @@ -22,4 +22,5 @@ imperative_workflows documenting_workflows shell_tasks named_outputs +/examples/basics/basics/basic_interactive_mode ``` diff --git a/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md b/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md index be79d4ea54..60bbe98596 100644 --- a/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md +++ b/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md @@ -52,15 +52,6 @@ remote = FlyteRemote( ) ``` -If you're running this code in a Jupyter notebook, you can also use the -`interactive_mode` to run workflows interactively: - -```{nb-literalinclude} /examples/basics/basics/basic_interactive_mode.ipynb -:caption: basics/basic_interactive_mode.ipynb -:cell: 3 -:lines: 5-10 -``` - ## Running a workflow You can run workflows using the `FlyteRemote` {py:meth}`~flytekit.remote.remote.FlyteRemote.execute` From ea03543d9a8fca0a27fed51274279bf1f9c742aa Mon Sep 17 00:00:00 2001 From: Mecoli1219 Date: Tue, 19 Nov 2024 23:28:14 -0800 Subject: [PATCH 21/25] test direct link & direct ipynb Signed-off-by: Mecoli1219 --- docs/user_guide/basics/index.md | 3 ++- docs/user_guide/basics/test.ipynb | 33 +++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 docs/user_guide/basics/test.ipynb diff --git a/docs/user_guide/basics/index.md b/docs/user_guide/basics/index.md index 9883cddd83..993b27def3 100644 --- a/docs/user_guide/basics/index.md +++ b/docs/user_guide/basics/index.md @@ -21,6 +21,7 @@ launch_plans imperative_workflows documenting_workflows shell_tasks +test +https://raw.githubusercontent.com/Mecoli1219/flytesnacks/85966d139a9a3ccdf3323124563960fdd5f5844a/examples/basics/basics/basic_interactive_mode.ipynb named_outputs -/examples/basics/basics/basic_interactive_mode ``` diff --git a/docs/user_guide/basics/test.ipynb b/docs/user_guide/basics/test.ipynb new file mode 100644 index 0000000000..7a889071cf --- /dev/null +++ b/docs/user_guide/basics/test.ipynb @@ -0,0 +1,33 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Hello World!!!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "@task\n", + "def t1(a: int) -> int:\n", + " return a + 1" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 6bd04c9cacc7e00a1c3e7508eaaf2d01e995b6bc Mon Sep 17 00:00:00 2001 From: Mecoli1219 Date: Wed, 20 Nov 2024 00:52:29 -0800 Subject: [PATCH 22/25] test multiple possible solutions Signed-off-by: Mecoli1219 --- docs/user_guide/basics/index.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/user_guide/basics/index.md b/docs/user_guide/basics/index.md index 993b27def3..0705c53b19 100644 --- a/docs/user_guide/basics/index.md +++ b/docs/user_guide/basics/index.md @@ -1,5 +1,7 @@ # Basics +.. include:: /examples + This section introduces you to the basic building blocks of Flyte using `flytekit`. `flytekit` is a Python SDK for developing Flyte workflows and tasks, and can be used generally, whenever stateful computation is desirable. @@ -20,8 +22,12 @@ workflows launch_plans imperative_workflows documenting_workflows +/examples/basics/basics/basic_interactive_mode.ipynb shell_tasks -test +/examples/basics/basics/basic_interactive_mode https://raw.githubusercontent.com/Mecoli1219/flytesnacks/85966d139a9a3ccdf3323124563960fdd5f5844a/examples/basics/basics/basic_interactive_mode.ipynb +examples/basics/basics/basic_interactive_mode.ipynb +test +examples/basics/basics/basic_interactive_mode named_outputs ``` From 8f86c44f259b03b02042bf90ede7fe07f7971c59 Mon Sep 17 00:00:00 2001 From: Mecoli1219 Date: Fri, 22 Nov 2024 00:49:35 -0800 Subject: [PATCH 23/25] Create a new page Signed-off-by: Mecoli1219 --- docs/conf.py | 3 +- docs/user_guide/basics/index.md | 8 -- docs/user_guide/basics/test.ipynb | 33 ------- docs/user_guide/flyte_fundamentals/index.md | 3 + .../jupyter_notebook_interaction.md | 91 +++++++++++++++++++ .../running_and_scheduling_workflows.md | 5 +- 6 files changed, 99 insertions(+), 44 deletions(-) delete mode 100644 docs/user_guide/basics/test.ipynb create mode 100644 docs/user_guide/flyte_fundamentals/jupyter_notebook_interaction.md diff --git a/docs/conf.py b/docs/conf.py index 75e6d4ffaf..83fca407b7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -624,11 +624,10 @@ import_projects = [ { "name": "flytesnacks", - "source": flytesnacks_local_path or "https://github.com/Mecoli1219/flytesnacks", + "source": flytesnacks_local_path or "https://github.com/flyteorg/flytesnacks", "docs_path": "docs", "dest": "flytesnacks", "cmd": [ - ["git", "-C", f"{flytesnacks_path}", "checkout", "origin/jupyter-basic"], ["cp", "-R", f"{flytesnacks_path}/examples", "./examples"], [ # remove un-needed docs files in flytesnacks diff --git a/docs/user_guide/basics/index.md b/docs/user_guide/basics/index.md index 0705c53b19..bc97b74cc9 100644 --- a/docs/user_guide/basics/index.md +++ b/docs/user_guide/basics/index.md @@ -1,7 +1,5 @@ # Basics -.. include:: /examples - This section introduces you to the basic building blocks of Flyte using `flytekit`. `flytekit` is a Python SDK for developing Flyte workflows and tasks, and can be used generally, whenever stateful computation is desirable. @@ -22,12 +20,6 @@ workflows launch_plans imperative_workflows documenting_workflows -/examples/basics/basics/basic_interactive_mode.ipynb shell_tasks -/examples/basics/basics/basic_interactive_mode -https://raw.githubusercontent.com/Mecoli1219/flytesnacks/85966d139a9a3ccdf3323124563960fdd5f5844a/examples/basics/basics/basic_interactive_mode.ipynb -examples/basics/basics/basic_interactive_mode.ipynb -test -examples/basics/basics/basic_interactive_mode named_outputs ``` diff --git a/docs/user_guide/basics/test.ipynb b/docs/user_guide/basics/test.ipynb deleted file mode 100644 index 7a889071cf..0000000000 --- a/docs/user_guide/basics/test.ipynb +++ /dev/null @@ -1,33 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Hello World!!!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, - "outputs": [], - "source": [ - "@task\n", - "def t1(a: int) -> int:\n", - " return a + 1" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs/user_guide/flyte_fundamentals/index.md b/docs/user_guide/flyte_fundamentals/index.md index a1a08fd187..9ef8a5bd2b 100644 --- a/docs/user_guide/flyte_fundamentals/index.md +++ b/docs/user_guide/flyte_fundamentals/index.md @@ -22,6 +22,8 @@ use cases. - Develop and deploy workflows to a local Flyte demo cluster. * - {doc}`⏱ Running and scheduling workflows ` - Execute workflows programmatically and schedule them as cron jobs. +* - {doc}`📙 Jupyter notebook interaction ` + - Develop and debug Flyte workflows interactively in Jupyter notebooks. * - {doc}`📊 Visualizing task input and output ` - Create rich, customizable static reports for increased visibility into tasks. * - {doc}`🏎 Optimizing tasks ` @@ -45,6 +47,7 @@ cluster, see the {ref}`Deployment Guide `. tasks_workflows_and_launch_plans registering_workflows running_and_scheduling_workflows +jupyter_notebook_interaction visualizing_task_input_and_output optimizing_tasks extending_flyte diff --git a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interaction.md b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interaction.md new file mode 100644 index 0000000000..8436fdf48e --- /dev/null +++ b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interaction.md @@ -0,0 +1,91 @@ +--- +kernelspec: + display_name: Python 3 + language: python + name: python3 +--- + +(getting_started_jupyter_notebook_interaction)= + +# Running and developing workflows in Jupyter notebooks + +Flyte supports the development, running, and debugging of tasks and workflows in an interactive +Jupyter notebook environment, which accelerates the iteration speed when building data- +or machine learning-driven applications. + +```{admonition} Attention +:class: attention + +This feature requires the `flytekit` version `1.14.0` or higher. +``` + +```{admonition} Prerequisites +:class: important + +This guide assumes that you've completed the previous guides for +{ref}`Running and Scheduling Workflows `. +The code snippets in this guide are intended to be run in a Jupyter notebook. +``` + +The code of this guide can be found in the [flytesnacks](https://github.com/flyteorg/flytesnacks/blob/master/examples/basics/basics/basic_interactive_mode.ipynb) + +## Create an interactive `FlyteRemote` object + +In {ref}`Running and Scheduling Workflows `, you learned +how to run registered Flyte workflows from a Python runtime using the +{py:class}`~flytekit.remote.remote.FlyteRemote` client. + +When developing workflows in a Jupyter notebook, `FlyteRemote` provides an +interactive interface to register and run workflows on a Flyte cluster. Let's +create an interactive `FlyteRemote` object: + +```{code-cell} ipython3 +:tags: [remove-output] + +from flytekit.configuration import Config +from flytekit.remote import FlyteRemote + +remote = FlyteRemote( + config=Config.auto(), + default_project="flytesnacks", + default_domain="development", + interactive_mode_enabled=True, +) +``` + +```{admonition} Note +:class: Note + +The `interactive_mode_enabled` flag is automatically set to `True` when running +in a Jupyter notebook environment, enabling interactive registration and execution +of workflows. +``` + +## Running a task or a workflow + +You can run entities (tasks or workflows) using the `FlyteRemote` +{py:meth}`~flytekit.remote.remote.FlyteRemote.execute` method. +During execution, `flytekit` first checks if the entity is registered with the +Flyte backend, and if not, registers it before execution. + +```{code-block} python +execution = remote.execute(my_task, inputs={"name": "Flyte"}) +execution = remote.execute(my_wf, inputs={"name": "Flyte"}) +``` + +You can then fetch the inputs and outputs of the execution by following the steps +in {ref}``. + +## When Does Interactive `FlyteRemote` Re-register an Entity? + +The interactive `FlyteRemote` client re-registers an entity whenever it's +redefined in the notebook, including when you re-execute a cell containing the +entity definition, even if the entity remains unchanged. This behavior facilitates +iterative development and debugging of tasks and workflows in a Jupyter notebook. + +## What's next? + +In this guide, you learned how to develop and run tasks and workflows in a +Jupyter Notebook environment using interactive `FlyteRemote`. + +In the next guide, you'll learn how to visualize tasks using Flyte Decks. diff --git a/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md b/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md index 60bbe98596..4a552a1275 100644 --- a/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md +++ b/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md @@ -184,6 +184,8 @@ execution = remote.execute(flyte_task, inputs={"name": "Kermit"}) You can also launch tasks via `flytectl`, learn more in the {ref}`User Guide ` ``` +(getting_started_run_and_schedule_fetch_execution)= + ## Fetching inputs and outputs of an execution By default, {py:meth}`FlyteRemote.execute ` @@ -342,4 +344,5 @@ In this guide, you learned about how to: - Run tasks, workflows, and launch plans using `FlyteRemote`. - Create a cron schedule to run a launch plan at a specified time interval. -In the next guide, you'll learn how to visualize tasks using Flyte Decks. +In the next guide, you'll learn how to develop and run tasks and workflows in +a Jupyter Notebook environment. From 8018fc2a0b5fd394c414f5b9b55647e135ad1507 Mon Sep 17 00:00:00 2001 From: Mecoli1219 Date: Fri, 22 Nov 2024 01:20:31 -0800 Subject: [PATCH 24/25] Modify anchor Signed-off-by: Mecoli1219 --- .../flyte_fundamentals/running_and_scheduling_workflows.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md b/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md index 4a552a1275..1cc93f06f5 100644 --- a/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md +++ b/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md @@ -185,7 +185,6 @@ You can also launch tasks via `flytectl`, learn more in the {ref}`User Guide ` From 1d710e3d588fa2de04f74e56eb6322e8db891f03 Mon Sep 17 00:00:00 2001 From: Mecoli1219 Date: Fri, 22 Nov 2024 01:44:46 -0800 Subject: [PATCH 25/25] Fix anchor Signed-off-by: Mecoli1219 --- .../flyte_fundamentals/jupyter_notebook_interaction.md | 2 +- .../flyte_fundamentals/running_and_scheduling_workflows.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interaction.md b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interaction.md index 8436fdf48e..7d11828304 100644 --- a/docs/user_guide/flyte_fundamentals/jupyter_notebook_interaction.md +++ b/docs/user_guide/flyte_fundamentals/jupyter_notebook_interaction.md @@ -74,7 +74,7 @@ execution = remote.execute(my_wf, inputs={"name": "Flyte"}) ``` You can then fetch the inputs and outputs of the execution by following the steps -in {ref}``. +in {ref}`getting_started_run_and_schedule_fetch_execution`. ## When Does Interactive `FlyteRemote` Re-register an Entity? diff --git a/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md b/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md index 1cc93f06f5..4a552a1275 100644 --- a/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md +++ b/docs/user_guide/flyte_fundamentals/running_and_scheduling_workflows.md @@ -185,6 +185,7 @@ You can also launch tasks via `flytectl`, learn more in the {ref}`User Guide `