From 49588faa429883a1f399df700b10bc66b505f808 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tun=C3=A7=20Ba=C5=9Far=20K=C3=B6se?= Date: Thu, 5 Sep 2024 15:12:47 +0300 Subject: [PATCH 1/4] Add ability to render Python with PythonCall --- src/QuartoNotebookWorker/Project.toml | 2 + .../ext/QuartoNotebookWorkerPythonCallExt.jl | 17 +++++++ src/server.jl | 45 +++++++++++++++++-- test/examples/integrations/PythonCall.qmd | 20 +++++++++ .../integrations/PythonCall/Project.toml | 2 + test/testsets/integrations/PythonCall.jl | 10 +++++ 6 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 src/QuartoNotebookWorker/ext/QuartoNotebookWorkerPythonCallExt.jl create mode 100644 test/examples/integrations/PythonCall.qmd create mode 100644 test/examples/integrations/PythonCall/Project.toml create mode 100644 test/testsets/integrations/PythonCall.jl diff --git a/src/QuartoNotebookWorker/Project.toml b/src/QuartoNotebookWorker/Project.toml index cf0f4c4..b56d17c 100644 --- a/src/QuartoNotebookWorker/Project.toml +++ b/src/QuartoNotebookWorker/Project.toml @@ -21,6 +21,7 @@ Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" PlotlyBase = "a03496cd-edff-5a9b-9e67-9cda94a718b5" PlotlyJS = "f0f68f2c-4968-5e81-91da-67840de0976a" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" +PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d" RCall = "6f49c342-dc21-5d91-9882-a32aef131414" Revise = "295af30f-e4ad-537b-8983-00126c2a3abe" SymPyCore = "458b697b-88f0-4a86-b56b-78b75cfb3531" @@ -35,6 +36,7 @@ QuartoNotebookWorkerMakieExt = "Makie" QuartoNotebookWorkerPlotlyBaseExt = "PlotlyBase" QuartoNotebookWorkerPlotlyJSExt = "PlotlyJS" QuartoNotebookWorkerPlotsExt = "Plots" +QuartoNotebookWorkerPythonCallExt = "PythonCall" QuartoNotebookWorkerRCallExt = "RCall" QuartoNotebookWorkerReviseExt = "Revise" QuartoNotebookWorkerSymPyCoreExt = "SymPyCore" diff --git a/src/QuartoNotebookWorker/ext/QuartoNotebookWorkerPythonCallExt.jl b/src/QuartoNotebookWorker/ext/QuartoNotebookWorkerPythonCallExt.jl new file mode 100644 index 0000000..2a83b83 --- /dev/null +++ b/src/QuartoNotebookWorker/ext/QuartoNotebookWorkerPythonCallExt.jl @@ -0,0 +1,17 @@ +module QuartoNotebookWorkerPythonCallExt + +import QuartoNotebookWorker +import PythonCall + +function __init__() + if ccall(:jl_generating_output, Cint, ()) == 0 + #RCall_temp_files_ref[] = mktempdir() + #configure() + #QuartoNotebookWorker.add_package_loading_hook!(configure) + #QuartoNotebookWorker.add_package_refresh_hook!(refresh) + #QuartoNotebookWorker.add_post_eval_hook!(display_plots) + #QuartoNotebookWorker.add_post_error_hook!(cleanup_temp_files) + end +end + +end \ No newline at end of file diff --git a/src/server.jl b/src/server.jl index 2f974f3..e70bacf 100644 --- a/src/server.jl +++ b/src/server.jl @@ -318,7 +318,7 @@ function raw_markdown_chunks_from_string(path::String, markdown::String) terminal_line = 1 code_cells = false for (node, enter) in ast - if enter && (is_julia_toplevel(node) || is_r_toplevel(node)) + if enter && (is_julia_toplevel(node) || is_python_toplevel(node) || is_r_toplevel(node)) code_cells = true line = node.sourcepos[1][1] md = join(source_lines[terminal_line:(line-1)], "\n") @@ -341,6 +341,7 @@ function raw_markdown_chunks_from_string(path::String, markdown::String) end language = is_julia_toplevel(node) ? :julia : + is_python_toplevel(node) ? :python : is_r_toplevel(node) ? :r : error("Unhandled code block language") push!( raw_chunks, @@ -775,6 +776,28 @@ function evaluate_raw_cells!( # We also need to hide the real code cell in this case, which contains possible formatting # settings in its YAML front-matter and which can therefore not be omitted entirely. cell_options["echo"] = false + elseif chunk.language === :python + # Same reasoning as :r + push!( + cells, + (; + id = string( + expand_cell ? string(nth, "_", mth) : string(nth), + "_code_prefix", + ), + cell_type = :markdown, + metadata = (;), + source = process_cell_source( + """ + ```python + $(strip_cell_options(chunk.source)) + ``` + """, + Dict(), + ), + ), + ) + cell_options["echo"] = false end source = expand_cell ? remote.code : chunk.source @@ -793,9 +816,9 @@ function evaluate_raw_cells!( end end elseif chunk.type === :markdown - marker = r"{(?:julia|r)} " + marker = r"{(?:julia|r|python)} " source = chunk.source - if contains(chunk.source, r"`{(?:julia|r)} ") + if contains(chunk.source, r"`{(?:julia|r|python)} ") parser = Parser() for (node, enter) in parser(chunk.source) if enter && node.t isa CommonMark.Code @@ -803,6 +826,8 @@ function evaluate_raw_cells!( source_code = replace(node.literal, marker => "") if startswith(node.literal, "{r}") source_code = wrap_with_r_boilerplate(source_code) + elseif startswith(node.literal, "{python}") + source_code = wrap_with_python_boilerplate(source_code) end expr = :(render( $(source_code), @@ -907,6 +932,13 @@ function strip_cell_options(source::AbstractString) join(lines[keep_from:end]) end +function wrap_with_python_boilerplate(code) + """ + @isdefined(PythonCall) && PythonCall isa Module && Base.PkgId(PythonCall).uuid == Base.UUID("6099a3de-0909-46bc-b1f4-468b9a2dfc0d") || error("PythonCall must be imported to execute Python code cells with QuartoNotebookRunner") + @py $code + """ +end + function wrap_with_r_boilerplate(code) """ @isdefined(RCall) && RCall isa Module && Base.PkgId(RCall).uuid == Base.UUID("6f49c342-dc21-5d91-9882-a32aef131414") || error("RCall must be imported to execute R code cells with QuartoNotebookRunner") @@ -921,6 +953,8 @@ function transform_source(chunk) chunk.source elseif chunk.language === :r wrap_with_r_boilerplate(chunk.source) + elseif chunk.language === :python + wrap_with_python_boilerplate(chunk.source) else error("Unhandled code chunk language $(chunk.language)") end @@ -1064,6 +1098,11 @@ is_julia_toplevel(node) = node.t.info == "{julia}" && node.parent.t isa CommonMark.Document +is_python_toplevel(node) = + node.t isa CommonMark.CodeBlock && + node.t.info == "{python}" && + node.parent.t isa CommonMark.Document + is_r_toplevel(node) = node.t isa CommonMark.CodeBlock && node.t.info == "{r}" && diff --git a/test/examples/integrations/PythonCall.qmd b/test/examples/integrations/PythonCall.qmd new file mode 100644 index 0000000..ec1ebf0 --- /dev/null +++ b/test/examples/integrations/PythonCall.qmd @@ -0,0 +1,20 @@ +--- +title: PythonCall integration +julia: + exeflags: ["--project=PythonCall"] +--- + +This should fail since we didn't import PythonCall yet. + +```{python} +import re +``` + +```{julia} +using PythonCall +``` + +```{python} +import re +words = re.findall("[a-zA-Z]+", "PythonCall.jl is very useful!") +``` diff --git a/test/examples/integrations/PythonCall/Project.toml b/test/examples/integrations/PythonCall/Project.toml new file mode 100644 index 0000000..48e0850 --- /dev/null +++ b/test/examples/integrations/PythonCall/Project.toml @@ -0,0 +1,2 @@ +[deps] +PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d" diff --git a/test/testsets/integrations/PythonCall.jl b/test/testsets/integrations/PythonCall.jl new file mode 100644 index 0000000..eef1b8a --- /dev/null +++ b/test/testsets/integrations/PythonCall.jl @@ -0,0 +1,10 @@ +include("../../utilities/prelude.jl") + +test_example(joinpath(@__DIR__, "../../examples/integrations/PythonCall.qmd")) do json + cells = json["cells"] + + @test occursin("PythonCall must be imported", cells[3]["outputs"][1]["traceback"][1]) + + @test cells[8]["data"]["outputs"][1]["text/plain"] == + "Python: ['PythonCall', 'jl', 'is', 'very', 'useful']" +end From 0cc30245cc173a3934bee058b5189a92bda26a67 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Fri, 27 Sep 2024 14:37:40 +0200 Subject: [PATCH 2/4] use `ast` to `exec` piece by piece and only `eval` last if possible --- src/server.jl | 51 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/src/server.jl b/src/server.jl index e70bacf..bc1ad98 100644 --- a/src/server.jl +++ b/src/server.jl @@ -935,7 +935,56 @@ end function wrap_with_python_boilerplate(code) """ @isdefined(PythonCall) && PythonCall isa Module && Base.PkgId(PythonCall).uuid == Base.UUID("6099a3de-0909-46bc-b1f4-468b9a2dfc0d") || error("PythonCall must be imported to execute Python code cells with QuartoNotebookRunner") - @py $code + let + code = \""" + $code + \""" + + ast = PythonCall.pyimport("ast") + tree = ast.parse(code) + + body = tree.body + + result = nothing + if body !== nothing + for (i, node) in enumerate(body) + nodecode = PythonCall.pyconvert(String, ast.unparse(node)) + if i < length(body) + PythonCall.pyexec(nodecode, Main.Notebook) + else + eval_allowed_nodes = ( + ast.Expression, # A wrapper for expressions in eval context + ast.Expr, + ast.BinOp, # Binary operations like 1 + 1 + ast.BoolOp, # Boolean operations like "and", "or" + ast.Call, # Function call like my_func() + ast.Compare, # Comparisons like a > b + ast.Constant, # Constants like numbers, strings (Python 3.8+) + ast.Dict, # Dictionary literals + ast.List, # List literals + ast.Name, # Variable names + ast.Set, # Set literals + ast.Tuple, # Tuple literals + ast.UnaryOp, # Unary operations like -1 + ast.Lambda # Lambda functions + ) + if any(t -> PythonCall.pyisinstance(node, t), eval_allowed_nodes) + result = PythonCall.pyeval(Any, nodecode, Main.Notebook) + else + PythonCall.pyexec(nodecode, Main.Notebook) + if PythonCall.pyisinstance(node, ast.Assign) + for target in node.targets + # TODO: how to know whether it's a single value or a one-element tuple? + # currently throwing away results 2 to n + result = PythonCall.pyeval(Any, ast.unparse(target), Main.Notebook) + end + end + end + end + end + end + result + end """ end From b216b885c87859d56857baa06e04a8e312678396 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Fri, 27 Sep 2024 16:24:38 +0200 Subject: [PATCH 3/4] fix indentation --- src/server.jl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/server.jl b/src/server.jl index bc1ad98..6dd77e9 100644 --- a/src/server.jl +++ b/src/server.jl @@ -936,9 +936,7 @@ function wrap_with_python_boilerplate(code) """ @isdefined(PythonCall) && PythonCall isa Module && Base.PkgId(PythonCall).uuid == Base.UUID("6099a3de-0909-46bc-b1f4-468b9a2dfc0d") || error("PythonCall must be imported to execute Python code cells with QuartoNotebookRunner") let - code = \""" - $code - \""" + code = "$code" ast = PythonCall.pyimport("ast") tree = ast.parse(code) From c884cae74548e11408e99799ceeeec3dec8e514f Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Fri, 27 Sep 2024 16:25:07 +0200 Subject: [PATCH 4/4] format --- .../ext/QuartoNotebookWorkerPythonCallExt.jl | 2 +- src/server.jl | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/QuartoNotebookWorker/ext/QuartoNotebookWorkerPythonCallExt.jl b/src/QuartoNotebookWorker/ext/QuartoNotebookWorkerPythonCallExt.jl index 2a83b83..a9eb7c1 100644 --- a/src/QuartoNotebookWorker/ext/QuartoNotebookWorkerPythonCallExt.jl +++ b/src/QuartoNotebookWorker/ext/QuartoNotebookWorkerPythonCallExt.jl @@ -14,4 +14,4 @@ function __init__() end end -end \ No newline at end of file +end diff --git a/src/server.jl b/src/server.jl index 6dd77e9..ba94fd1 100644 --- a/src/server.jl +++ b/src/server.jl @@ -318,7 +318,8 @@ function raw_markdown_chunks_from_string(path::String, markdown::String) terminal_line = 1 code_cells = false for (node, enter) in ast - if enter && (is_julia_toplevel(node) || is_python_toplevel(node) || is_r_toplevel(node)) + if enter && + (is_julia_toplevel(node) || is_python_toplevel(node) || is_r_toplevel(node)) code_cells = true line = node.sourcepos[1][1] md = join(source_lines[terminal_line:(line-1)], "\n")