From 53d06a17b188b243931d80d2c46ad95ad01cd4b6 Mon Sep 17 00:00:00 2001 From: Jan Pfeifer Date: Wed, 16 Oct 2024 15:13:57 +0200 Subject: [PATCH 01/11] Fixed nbexec: headless chrome started to hang (using 100% cpu and not doing anything) if started with GPU support. Disabled it. --- cmd/nbexec/nbexec.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/cmd/nbexec/nbexec.go b/cmd/nbexec/nbexec.go index a1b86d3..5eba2d1 100644 --- a/cmd/nbexec/nbexec.go +++ b/cmd/nbexec/nbexec.go @@ -103,7 +103,7 @@ func main() { jupyterStop.Trigger() if e != nil { // Wait for jupyter notebook to get killed, before re-throwing the panic. - klog.Errorf("%v", e) + klog.Errorf("Panic: %+v", e) jupyterDone.Wait() panic(e) } @@ -271,14 +271,24 @@ func executeNotebook(url string, inputBoxes []string) { // Use system's Google Chrome is available, for sandboxing: var controlURL string chromePath, err := exec.LookPath("google-chrome") + var l *launcher.Launcher if err == nil { - controlURL = launcher.New().Bin(chromePath).MustLaunch() + klog.V(1).Infof("Using system's Google Chrome") + l = launcher.New().Bin(chromePath) } else { klog.Warningf("Using rod downloaded chromium, with --no-sandbox") - controlURL = launcher.New().NoSandbox(true).MustLaunch() + l = launcher.New().NoSandbox(true) } - page := rod.New().ControlURL(controlURL).MustConnect().MustPage(url) - klog.V(1).Infof("Waiting for opening of page %q", url) + controlURL = l. + Set("disable-gpu", "true"). + Set("disable-software-rasterizer", "true"). + Logger(os.Stderr). + MustLaunch() + klog.V(1).Infof("Using controlURL=%q", controlURL) + browser := rod.New().ControlURL(controlURL).MustConnect() + klog.V(1).Info("Connected to browser.") + page := browser.MustPage(url) + klog.V(1).Infof("Connected to page in browser, waiting for opening of page %q", url) page.MustWaitStable() if *flagConsoleLog { From 1efadd7c77dd49c1878c8b7d8d22838706759fe0 Mon Sep 17 00:00:00 2001 From: Jan Pfeifer Date: Wed, 16 Oct 2024 15:15:10 +0200 Subject: [PATCH 02/11] Added `%capture [-a] ` special command. --- docs/CHANGELOG.md | 1 + examples/tests/capture.ipynb | 213 ++++++++++++++++++++++++++++++ go.mod | 4 +- go.sum | 4 + gonbui/dom/script.go | 2 +- gonbui/gonbui.go | 33 +++-- internal/goexec/execcode.go | 36 ++++- internal/goexec/goexec.go | 6 + internal/jpyexec/jpyexec.go | 15 +++ internal/jpyexec/namedpipes.go | 12 ++ internal/nbtests/nbtests_test.go | 50 ++++++- internal/specialcmd/help.md | 8 +- internal/specialcmd/specialcmd.go | 53 +++++++- 13 files changed, 408 insertions(+), 29 deletions(-) create mode 100644 examples/tests/capture.ipynb diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 2cd598a..fd1c971 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -8,6 +8,7 @@ * Added support for `autostart.sh` that if present in the mounted container `/notebooks` directory, and if root owned and set as executable. * Updated Dockerfile to latest version to JupyterLab -- now the base docker is served `quay.io/jupyter/base-notebook` +* Added `%capture` to capture the output of a cell (#142) ## v0.10.5, Added SendAsDownload diff --git a/examples/tests/capture.ipynb b/examples/tests/capture.ipynb new file mode 100644 index 0000000..8b66199 --- /dev/null +++ b/examples/tests/capture.ipynb @@ -0,0 +1,213 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "fe105ef6-5264-4cb4-b467-60d9b0a337fd", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Capturing output to \"/tmp/gonb_nbtests_writefile_1126554417//pingpong.txt\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Ping\n" + ] + } + ], + "source": [ + "%capture ${TEST_DIR}/pingpong.txt\n", + "%%\n", + "fmt.Println(\"Ping\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "ee9acf4f-2488-4f92-bfae-6998d8351175", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Ping\n" + ] + } + ], + "source": [ + "!cat ${TEST_DIR}/pingpong.txt" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "004e9622-e3a4-4780-816a-e5d1c77055b4", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Capturing output to \"/tmp/gonb_nbtests_writefile_1126554417//pingpong.txt\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Pong\n" + ] + } + ], + "source": [ + "%capture -a ${TEST_DIR}/pingpong.txt\n", + "%%\n", + "fmt.Println(\"Pong\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "aee500e4-5728-4224-9746-f61d58909851", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Ping\n", + "Pong\n" + ] + } + ], + "source": [ + "!cat ${TEST_DIR}/pingpong.txt" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "c8214b58-5946-4db5-a329-9e2f2c2fc215", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Capturing output to \"/tmp/gonb_nbtests_writefile_1126554417//pingpong.txt\"\n" + ] + }, + { + "data": { + "text/html": [ + "Ping\n", + "Pong\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "%capture ${TEST_DIR}/pingpong.txt\n", + "import \"github.com/janpfeifer/gonb/gonbui\"\n", + "%%\n", + "gonbui.DisplayHTML(\"Ping\\nPong\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "f67d51c5-dbcb-4e96-aba5-380bc55dbf53", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Ping\n", + "Pong\n" + ] + } + ], + "source": [ + "!cat ${TEST_DIR}/pingpong.txt" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "9b88a712-deae-4d48-80a1-c4795db31c22", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Capturing output to \"/tmp/gonb_nbtests_writefile_1126554417//pingpong.txt\"\n" + ] + }, + { + "data": { + "text/markdown": [ + "# Ping\n", + "# Pong\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "%capture ${TEST_DIR}/pingpong.txt\n", + "import \"github.com/janpfeifer/gonb/gonbui\"\n", + "%%\n", + "gonbui.DisplayMarkdown(\"# Ping\\n# Pong\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "e884811a-e7c9-425b-aa0b-394b1628ccd8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "# Ping\n", + "# Pong\n" + ] + } + ], + "source": [ + "!cat ${TEST_DIR}/pingpong.txt" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Go (gonb)", + "language": "go", + "name": "gonb" + }, + "language_info": { + "codemirror_mode": "", + "file_extension": ".go", + "mimetype": "", + "name": "go", + "nbconvert_exporter": "", + "pygments_lexer": "", + "version": "go1.23.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/go.mod b/go.mod index 7ef3a2f..2e80103 100644 --- a/go.mod +++ b/go.mod @@ -16,8 +16,8 @@ require ( github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.1 go.lsp.dev/jsonrpc2 v0.10.0 - golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa - golang.org/x/mod v0.20.0 + golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c + golang.org/x/mod v0.21.0 k8s.io/klog/v2 v2.130.1 ) diff --git a/go.sum b/go.sum index c6ab314..fa0185f 100644 --- a/go.sum +++ b/go.sum @@ -243,6 +243,8 @@ golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRj golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI= golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= +golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -253,6 +255,8 @@ golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/gonbui/dom/script.go b/gonbui/dom/script.go index 3c5626f..3b914c2 100644 --- a/gonbui/dom/script.go +++ b/gonbui/dom/script.go @@ -176,7 +176,7 @@ func loadScriptOrRequireJSModuleAndRunImpl(moduleName, src string, attributes ma if transient { TransientJavascript(js) } else { - gonbui.DisplayHtmlf("", "UTF-8", js) + gonbui.DisplayHTMLF("", "UTF-8", js) } return nil } diff --git a/gonbui/gonbui.go b/gonbui/gonbui.go index 93fb02d..b8fb9f4 100644 --- a/gonbui/gonbui.go +++ b/gonbui/gonbui.go @@ -314,8 +314,13 @@ func UniqueID() string { return UniqueId() } -// DisplayHtml will display the given HTML in the notebook, as the output of the cell being executed. +// DisplayHtml is an alias to DisplayHTML. func DisplayHtml(html string) { + DisplayHTML(html) +} + +// DisplayHTML will display the given HTML in the notebook, as the output of the cell being executed. +func DisplayHTML(html string) { if !IsNotebook { return } @@ -324,14 +329,9 @@ func DisplayHtml(html string) { }) } -// DisplayHTML is an alias to DisplayHtml. -func DisplayHTML(html string) { - DisplayHtml(html) -} - -// DisplayHtmlf is similar to DisplayHtml, but it takes a format string and its args which +// DisplayHTMLF is similar to DisplayHTML, but it takes a format string and its args which // are passed to fmt.Sprintf. -func DisplayHtmlf(htmlFormat string, args ...any) { +func DisplayHTMLF(htmlFormat string, args ...any) { if !IsNotebook { return } @@ -339,6 +339,11 @@ func DisplayHtmlf(htmlFormat string, args ...any) { DisplayHtml(html) } +// DisplayHtmlf is an alias to DisplayHTMLF. +func DisplayHtmlf(htmlFormat string, args ...any) { + DisplayHTMLF(htmlFormat, args...) +} + // DisplayMarkdown will display the given markdown content in the notebook, as the output of // the cell being executed. // This also renders math formulas using latex, use `$x^2$` for formulas inlined in text, or @@ -353,7 +358,7 @@ func DisplayMarkdown(markdown string) { }) } -// UpdateHtml displays the given HTML in the notebook on an output block with the given `id`: +// UpdateHTML displays the given HTML in the notebook on an output block with the given `id`: // the block identified by 'id' is created automatically the first time this function is // called, and simply updated thereafter. // @@ -372,7 +377,7 @@ func DisplayMarkdown(markdown string) { // Notice that the value of `counterDisplayId` is not a DOM element id -- unfortunately. // If you want a `
` that you can manipulate with the [dom] package, create an empty `
` // with another unique id (see [gonbui.UniqueID]) and use that instead. -func UpdateHtml(id, html string) { +func UpdateHTML(id, html string) { if !IsNotebook { return } @@ -382,10 +387,10 @@ func UpdateHtml(id, html string) { }) } -// UpdateHTML is an alias for UpdateHtml. -// Deprecated: use UpdateHtml instead, it's the same. -func UpdateHTML(id, html string) { - UpdateHtml(id, html) +// UpdateHtml is an alias for UpdateHTML. +// Deprecated: use UpdateHTML instead, it's the same. +func UpdateHtml(id, html string) { + UpdateHTML(id, html) } // UpdateMarkdown updates the contents of the output identified by id: diff --git a/internal/goexec/execcode.go b/internal/goexec/execcode.go index c582472..1c4ac84 100644 --- a/internal/goexec/execcode.go +++ b/internal/goexec/execcode.go @@ -58,9 +58,9 @@ func (s *State) serializeExecuteCell() { for { select { case params := <-s.cellExecChan: - // Received new execution request. - params.done.Trigger(s.executeCellImpl( - params.msg, params.cellId, params.lines, params.skipLines)) + // New execution request: execute it, and report back error in the params.done latch. + err := s.executeCellImpl(params.msg, params.cellId, params.lines, params.skipLines) + params.done.Trigger(err) case <-stopC: // Kernel stopped, exit. @@ -74,9 +74,11 @@ func (s *State) serializeExecuteCell() { // It is not reentrant, and calls to it should be serialized. // ExecuteCell serializes the calls to this method. func (s *State) executeCellImpl(msg kernel.Message, cellId int, lines []string, skipLines Set[int]) error { + // Makes sure at exit state is reset of any "one-shot" state. + defer s.PostExecuteCell() + klog.V(1).Infof("ExecuteCell: %q", lines) - defer s.PostExecuteCell() klog.V(2).Infof("ExecuteCell(): CellIsTest=%v, CellIsWasm=%v", s.CellIsTest, s.CellIsWasm) if s.CellIsTest && s.CellIsWasm { return errors.Errorf("Cannot execute test in a %%wasm cell. Please, choose either `%%wasm` or `%%test`.") @@ -138,6 +140,13 @@ func (s *State) PostExecuteCell() { s.CellHasBenchmarks = false s.CellIsWasm = false s.WasmDivId = "" + if s.CaptureFile != nil { + err := s.CaptureFile.Close() + if err != nil { + klog.Errorf("goexec.PostExecuteCell(): failed to close capture file: %+v", err) + } + s.CaptureFile = nil + } } // BinaryPath is the path to the generated binary file. @@ -178,6 +187,12 @@ func (s *State) AlternativeDefinitionsPath() string { return path.Join(s.TempDir, "other.go") } +// Execute cell code already prepared to `${GONB_TMP_DIR}/main.go`. +// +// If errors in execution happen, fileToCellIdAndLine helps to map the `main.go` line numbers to cell id and line, +// so errors can be annotated. +// +// If s.CellIsWasm is true, it passes through State.ExecuteWasm. func (s *State) Execute(msg kernel.Message, fileToCellIdAndLine []CellIdAndLine) error { if s.CellIsWasm { return s.ExecuteWasm(msg) @@ -186,10 +201,21 @@ func (s *State) Execute(msg kernel.Message, fileToCellIdAndLine []CellIdAndLine) if len(args) == 0 && s.CellIsTest { args = s.DefaultCellTestArgs() } + + // Create stdout and stderr pipes that write to Jupyter stdout/stderr streams. + stdout := kernel.NewJupyterStreamWriter(msg, kernel.StreamStdout) + stderrWithAnnotator := newJupyterStackTraceMapperWriter(msg, "stderr", s.CodePath(), fileToCellIdAndLine) + if s.CaptureFile != nil { + stdout = io.MultiWriter(stdout, s.CaptureFile) + stderrWithAnnotator = io.MultiWriter(stderrWithAnnotator, s.CaptureFile) + } + err := jpyexec.New(msg, s.BinaryPath(), args...). UseNamedPipes(s.Comms). ExecutionCount(msg.Kernel().ExecCounter). - WithStderr(newJupyterStackTraceMapperWriter(msg, "stderr", s.CodePath(), fileToCellIdAndLine)). + WithStdout(stdout). + WithStderr(stderrWithAnnotator). + CaptureDisplayDataOutput(s.CaptureFile). Exec() if err != nil { klog.Infof("goexec.Execute(): failed to run the compiled cell: %+v", msg) diff --git a/internal/goexec/goexec.go b/internal/goexec/goexec.go index a5dda94..1242c5e 100644 --- a/internal/goexec/goexec.go +++ b/internal/goexec/goexec.go @@ -12,6 +12,7 @@ import ( "github.com/janpfeifer/gonb/internal/goexec/goplsclient" "github.com/janpfeifer/gonb/internal/kernel" "github.com/pkg/errors" + "io" "k8s.io/klog/v2" "os" "os/exec" @@ -96,6 +97,11 @@ type State struct { // Comms represents the communication with the front-end. Comms *comms.State + + // CaptureFile is the file where to write any cell output. It is closed and set to nil at the end of the cell + // executions. + // If nil, no output is to be captured. + CaptureFile io.WriteCloser } // Declarations is a collection of declarations that we carry over from one cell to another. diff --git a/internal/jpyexec/jpyexec.go b/internal/jpyexec/jpyexec.go index 2b98755..a1c743c 100644 --- a/internal/jpyexec/jpyexec.go +++ b/internal/jpyexec/jpyexec.go @@ -56,6 +56,12 @@ type Executor struct { // Currently, it is assumed that it will be used by the CommsHandler. PipeWriterFifo chan *protocol.CommValue + // captureDisplayDataOutput is a writer to where all data to be displayed send through the named pipe is + // copied. + // + // Notice the contents are written raw, without the mime-type. + captureDisplayDataOutput io.Writer + isDone bool doneChan chan struct{} muDone sync.Mutex @@ -156,6 +162,15 @@ func (exec *Executor) WithStaticInput(stdinContent []byte) *Executor { return exec } +// CaptureDisplayDataOutput configures the Executor to capture the output of the program +// and send it as protocol.DisplayMessage messages, in the named pipe. +// +// The captured data is sent to the given `io.Writer` raw without any processing or attached mime-type. +func (exec *Executor) CaptureDisplayDataOutput(writer io.Writer) *Executor { + exec.captureDisplayDataOutput = writer + return exec +} + // WaitToKill is the to wait after an interrupt signal, before killing the process. var WaitToKill = 5 * time.Second diff --git a/internal/jpyexec/namedpipes.go b/internal/jpyexec/namedpipes.go index f95fdc6..e3204a7 100644 --- a/internal/jpyexec/namedpipes.go +++ b/internal/jpyexec/namedpipes.go @@ -281,7 +281,19 @@ func (exec *Executor) dispatchDisplayData(data *protocol.DisplayData) { } for mimeType, content := range data.Data { msgData.Data[string(mimeType)] = content + + // Capture display data output, if requested. + if exec.captureDisplayDataOutput != nil { + str, ok := content.(string) + if ok { + _, err := exec.captureDisplayDataOutput.Write([]byte(str)) + if err != nil { + klog.Errorf("failed to capture display data output: %v", err) + } + } + } } + if klog.V(1).Enabled() { kernel.LogDisplayData(msgData.Data) } diff --git a/internal/nbtests/nbtests_test.go b/internal/nbtests/nbtests_test.go index 4735aa8..6f0342f 100644 --- a/internal/nbtests/nbtests_test.go +++ b/internal/nbtests/nbtests_test.go @@ -146,7 +146,7 @@ func executeNotebookWithInputBoxes(t *testing.T, notebook string, inputBoxValues notebookRelPath := path.Join("examples", "tests", notebook+".ipynb") args := []string{"-n=" + notebookRelPath, "-jupyter_dir=" + rootDir, "-logtostderr"} if *flagLogExec { - args = append(args, "-jupyter_log", "-console_log", "-vmodule=main=1,nbexec=1") + args = append(args, "-jupyter_log", "-console_log", "-vmodule=main=2,nbexec=2") } if len(inputBoxValues) > 0 { // Check there are no commas in the values. @@ -712,3 +712,51 @@ func TestVarTuple(t *testing.T) { require.NoError(t, os.Remove(f.Name())) clearNotebook(t, "vartuple") } + +func TestCapture(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration (nbconvert) test for short tests.") + return + } + klog.Infof("GOCOVERDIR=%s", os.Getenv("GOCOVERDIR")) + + // Create directory where to write the file, and set TEST_DIR env variable. + testDir, err := os.MkdirTemp("", "gonb_nbtests_writefile_") + require.NoError(t, err) + require.NoError(t, os.Setenv("TEST_DIR", testDir+"/")) + klog.Infof("TEST_DIR=%s/", testDir) + + notebook := "capture" + f := executeNotebook(t, notebook) + err = Check(f, + Sequence( + Match(OutputLine(2), + Separator, + "Ping", + Separator), + + Match(OutputLine(4), + Separator, + "Ping", + "Pong", + Separator), + + Match(OutputLine(6), + Separator, + "Ping", + "Pong", + Separator), + + Match(OutputLine(8), + Separator, + "# Ping", + "# Pong", + Separator), + ), + *flagPrintNotebook) + + require.NoError(t, err) + require.NoError(t, f.Close()) + require.NoError(t, os.Remove(f.Name())) + clearNotebook(t, notebook) +} diff --git a/internal/specialcmd/help.md b/internal/specialcmd/help.md index 4eab1b0..77b1a3a 100644 --- a/internal/specialcmd/help.md +++ b/internal/specialcmd/help.md @@ -66,6 +66,10 @@ This way each cell can create its own `init_...()` and have it called at every c you to enter one last value after the shell script executes. - `%with_password`: will prompt for a password passed to the next shell command. Do this is if your next shell command requires a password. +- `%capture [-a] ` will make a copy of all **cell execution output** to the given file. By default + it overwrites the file contents each time the cell is executed. Use `-a` instead to append to the file. + It works only for the current cell. See also `%%writefile` to write files with a specific content. + It doesn't work with `%wasm` cells. **Notes**: @@ -236,7 +240,9 @@ it will append the cell contents to the file. This can be handy if for instance the notebook needs to write a configuration file, or simply to dump the code inside the cell into some file. -File path passes through a tilde (`~`) expansion to the user's home directory, as well as environment variable substitution (e.g.: `${HOME}` or `$MY_DIR/a/b`). +File path passes through a tilde (`~`) expansion to the user's home directory, as well as environment variable substitution (e.g.: `${HOME}` or `$MY_DIR/a/b`). + +See also `%capture` to instead write the output of the execution of the cell to a file. ### `%%script`, `%%bash` and `%%sh` diff --git a/internal/specialcmd/specialcmd.go b/internal/specialcmd/specialcmd.go index 32adf80..5f7df81 100644 --- a/internal/specialcmd/specialcmd.go +++ b/internal/specialcmd/specialcmd.go @@ -227,7 +227,7 @@ func execSpecialConfig(msg kernel.Message, goExec *goexec.State, cmdStr string, klog.Errorf("Failed publishing contents: %+v", err) } - // Automatic `go get` control: + // Automatic `go get` control: case "autoget": goExec.AutoGet = true case "noautoget": @@ -239,7 +239,7 @@ func execSpecialConfig(msg kernel.Message, goExec *goexec.State, cmdStr string, klog.Errorf("Failed publishing help contents: %+v", err) } - // Definitions management. + // Definitions management. case "reset": if len(parts) == 1 { resetDefinitions(msg, goExec) @@ -254,7 +254,7 @@ func execSpecialConfig(msg kernel.Message, goExec *goexec.State, cmdStr string, case "rm", "remove": removeDefinitions(msg, goExec, parts[1:]) - // Input handling. + // Input handling. case "with_inputs": allowInput := content["allow_stdin"].(bool) if !allowInput && (status.withInputs || status.withPassword) { @@ -268,16 +268,59 @@ func execSpecialConfig(msg kernel.Message, goExec *goexec.State, cmdStr string, } status.withPassword = true - // Files that need tracking for `gopls` (for auto-complete and contextual help). + // Files that need tracking for `gopls` (for auto-complete and contextual help). case "track": + if len(parts) != 2 { + return errors.New("%track takes one argument, the name Go file to tack") + } execTrack(msg, goExec, parts[1:]) case "untrack": + if len(parts) != 2 { + return errors.New("%untrack takes one argument, the name Go file to tack") + } execUntrack(msg, goExec, parts[1:]) - // Fix issues with `go work`. + // Fix issues with `go work`. case "goworkfix": return goExec.GoWorkFix(msg) + // Capture output of cell. + case "capture": + args := parts[1:] + var appendToFile bool + if len(args) > 1 && args[0] == "-a" { + appendToFile = true + args = args[1:] + } + if len(args) != 1 { + return errors.New("%capture takes one argument, the name of the file where to save the captured output") + } + filePath := args[0] + filePath = ReplaceTildeInDir(filePath) + filePath = ReplaceEnvVars(filePath) + err := kernel.PublishWriteStream(msg, kernel.StreamStderr, fmt.Sprintf("Capturing output to %q\n", filePath)) + if err != nil { + klog.Errorf("Failed to write to kernel stdout: %+v", err) + } + var f *os.File + if appendToFile { + f, err = os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + err = errors.Wrapf(err, "failed to append to \"%%capture\" file %q", filePath) + klog.Errorf("Error: %+v", err) + return err + } + } else { + f, err = os.Create(filePath) + if err != nil { + err = errors.Wrapf(err, "failed to create \"%%capture\" file %q", filePath) + klog.Errorf("Error: %+v", err) + return err + } + } + // Notice, file will be closed in goExec.PostExecuteCell(), where all "one-shot" state is cleaned up. + goExec.CaptureFile = f + default: if CellSpecialCommands.Has("%" + parts[0]) { // Cell special commands should always come first, and if they are parsed here (as opposed to being processed by specialCells) From 40a207761546b235f27d0d203dd4efceb5189e40 Mon Sep 17 00:00:00 2001 From: Jan Pfeifer Date: Wed, 16 Oct 2024 15:44:30 +0200 Subject: [PATCH 03/11] Fixed `%track` to list files tracked. --- internal/specialcmd/specialcmd.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/internal/specialcmd/specialcmd.go b/internal/specialcmd/specialcmd.go index 5f7df81..9019056 100644 --- a/internal/specialcmd/specialcmd.go +++ b/internal/specialcmd/specialcmd.go @@ -270,9 +270,6 @@ func execSpecialConfig(msg kernel.Message, goExec *goexec.State, cmdStr string, // Files that need tracking for `gopls` (for auto-complete and contextual help). case "track": - if len(parts) != 2 { - return errors.New("%track takes one argument, the name Go file to tack") - } execTrack(msg, goExec, parts[1:]) case "untrack": if len(parts) != 2 { From a992fd105c40edb9667b61603f96a4e733558382 Mon Sep 17 00:00:00 2001 From: Jan Pfeifer Date: Wed, 16 Oct 2024 15:47:29 +0200 Subject: [PATCH 04/11] Updated CHANGELOG. --- docs/CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index fd1c971..7cd8839 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -8,7 +8,8 @@ * Added support for `autostart.sh` that if present in the mounted container `/notebooks` directory, and if root owned and set as executable. * Updated Dockerfile to latest version to JupyterLab -- now the base docker is served `quay.io/jupyter/base-notebook` -* Added `%capture` to capture the output of a cell (#142) +* Added `%capture [-a] ` to capture the output of a cell (#142) +* Fixed `nbexec`: added `--disable-gpu` and `--disable-software-rasterizer` when executing "headless" chrome for tests. ## v0.10.5, Added SendAsDownload From cb3f489923a70192f5e74fbe26c16a48c6c355ef Mon Sep 17 00:00:00 2001 From: Jan Pfeifer Date: Wed, 16 Oct 2024 15:54:53 +0200 Subject: [PATCH 05/11] Updated coverage. --- docs/coverage.txt | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/docs/coverage.txt b/docs/coverage.txt index 348853f..c5a5e8c 100644 --- a/docs/coverage.txt +++ b/docs/coverage.txt @@ -28,7 +28,7 @@ github.com/janpfeifer/gonb/cache/cache.go CacheWith 91.7% github.com/janpfeifer/gonb/cache/cache.go Cache 0.0% github.com/janpfeifer/gonb/cmd/nbexec/nbexec.go main 74.3% github.com/janpfeifer/gonb/cmd/nbexec/nbexec.go startJupyterNotebook 73.2% -github.com/janpfeifer/gonb/cmd/nbexec/nbexec.go executeNotebook 78.6% +github.com/janpfeifer/gonb/cmd/nbexec/nbexec.go executeNotebook 82.4% github.com/janpfeifer/gonb/cmd/nbexec/nbexec.go checkForInputBoxes 95.7% github.com/janpfeifer/gonb/common/common.go Panicf 0.0% github.com/janpfeifer/gonb/common/common.go Pause 0.0% @@ -68,12 +68,13 @@ github.com/janpfeifer/gonb/gonbui/gonbui.go pollReaderPipe 66.7% github.com/janpfeifer/gonb/gonbui/gonbui.go Sync 92.9% github.com/janpfeifer/gonb/gonbui/gonbui.go UniqueId 100.0% github.com/janpfeifer/gonb/gonbui/gonbui.go UniqueID 100.0% -github.com/janpfeifer/gonb/gonbui/gonbui.go DisplayHtml 66.7% -github.com/janpfeifer/gonb/gonbui/gonbui.go DisplayHTML 100.0% +github.com/janpfeifer/gonb/gonbui/gonbui.go DisplayHtml 100.0% +github.com/janpfeifer/gonb/gonbui/gonbui.go DisplayHTML 66.7% +github.com/janpfeifer/gonb/gonbui/gonbui.go DisplayHTMLF 0.0% github.com/janpfeifer/gonb/gonbui/gonbui.go DisplayHtmlf 0.0% github.com/janpfeifer/gonb/gonbui/gonbui.go DisplayMarkdown 66.7% -github.com/janpfeifer/gonb/gonbui/gonbui.go UpdateHtml 66.7% -github.com/janpfeifer/gonb/gonbui/gonbui.go UpdateHTML 0.0% +github.com/janpfeifer/gonb/gonbui/gonbui.go UpdateHTML 66.7% +github.com/janpfeifer/gonb/gonbui/gonbui.go UpdateHtml 100.0% github.com/janpfeifer/gonb/gonbui/gonbui.go UpdateMarkdown 0.0% github.com/janpfeifer/gonb/gonbui/gonbui.go DisplayPng 0.0% github.com/janpfeifer/gonb/gonbui/gonbui.go DisplayPNG 0.0% @@ -198,12 +199,12 @@ github.com/janpfeifer/gonb/internal/goexec/errorpublish.go JupyterErrorSplit github.com/janpfeifer/gonb/internal/goexec/execcode.go *State.ExecuteCell 100.0% github.com/janpfeifer/gonb/internal/goexec/execcode.go *State.serializeExecuteCell 100.0% github.com/janpfeifer/gonb/internal/goexec/execcode.go *State.executeCellImpl 68.0% -github.com/janpfeifer/gonb/internal/goexec/execcode.go *State.PostExecuteCell 88.9% +github.com/janpfeifer/gonb/internal/goexec/execcode.go *State.PostExecuteCell 85.7% github.com/janpfeifer/gonb/internal/goexec/execcode.go *State.BinaryPath 100.0% github.com/janpfeifer/gonb/internal/goexec/execcode.go *State.CodePath 100.0% github.com/janpfeifer/gonb/internal/goexec/execcode.go *State.RemoveCode 83.3% github.com/janpfeifer/gonb/internal/goexec/execcode.go *State.AlternativeDefinitionsPath 0.0% -github.com/janpfeifer/gonb/internal/goexec/execcode.go *State.Execute 77.8% +github.com/janpfeifer/gonb/internal/goexec/execcode.go *State.Execute 85.7% github.com/janpfeifer/gonb/internal/goexec/execcode.go *State.Compile 70.0% github.com/janpfeifer/gonb/internal/goexec/execcode.go *State.GoImports 70.6% github.com/janpfeifer/gonb/internal/goexec/execcode.go newJupyterStackTraceMapperWriter 75.0% @@ -321,10 +322,11 @@ github.com/janpfeifer/gonb/internal/jpyexec/jpyexec.go *Executor.UseNamedPipes github.com/janpfeifer/gonb/internal/jpyexec/jpyexec.go *Executor.ExecutionCount 100.0% github.com/janpfeifer/gonb/internal/jpyexec/jpyexec.go *Executor.InDir 100.0% github.com/janpfeifer/gonb/internal/jpyexec/jpyexec.go *Executor.WithStderr 100.0% -github.com/janpfeifer/gonb/internal/jpyexec/jpyexec.go *Executor.WithStdout 0.0% +github.com/janpfeifer/gonb/internal/jpyexec/jpyexec.go *Executor.WithStdout 100.0% github.com/janpfeifer/gonb/internal/jpyexec/jpyexec.go *Executor.WithInputs 100.0% github.com/janpfeifer/gonb/internal/jpyexec/jpyexec.go *Executor.WithPassword 100.0% github.com/janpfeifer/gonb/internal/jpyexec/jpyexec.go *Executor.WithStaticInput 100.0% +github.com/janpfeifer/gonb/internal/jpyexec/jpyexec.go *Executor.CaptureDisplayDataOutput 100.0% github.com/janpfeifer/gonb/internal/jpyexec/jpyexec.go *Executor.Exec 68.8% github.com/janpfeifer/gonb/internal/jpyexec/jpyexec.go *Executor.done 92.3% github.com/janpfeifer/gonb/internal/jpyexec/jpyexec.go *Executor.handleJupyterInput 91.3% @@ -335,7 +337,7 @@ github.com/janpfeifer/gonb/internal/jpyexec/namedpipes.go *Executor.createTmpF github.com/janpfeifer/gonb/internal/jpyexec/namedpipes.go *Executor.openPipeReader 89.3% github.com/janpfeifer/gonb/internal/jpyexec/namedpipes.go *Executor.pollNamedPipeReader 73.1% github.com/janpfeifer/gonb/internal/jpyexec/namedpipes.go *Executor.reportCellError 0.0% -github.com/janpfeifer/gonb/internal/jpyexec/namedpipes.go *Executor.dispatchDisplayData 78.6% +github.com/janpfeifer/gonb/internal/jpyexec/namedpipes.go *Executor.dispatchDisplayData 80.0% github.com/janpfeifer/gonb/internal/jpyexec/namedpipes.go *Executor.dispatchInputRequest 83.3% github.com/janpfeifer/gonb/internal/jpyexec/namedpipes.go *Executor.openPipeWriter 86.7% github.com/janpfeifer/gonb/internal/jpyexec/namedpipes.go *Executor.pollPipeWriterFifo 66.7% @@ -409,11 +411,11 @@ github.com/janpfeifer/gonb/internal/specialcmd/definitions.go removeDefinitionI github.com/janpfeifer/gonb/internal/specialcmd/definitions.go removeDefinitions 91.7% github.com/janpfeifer/gonb/internal/specialcmd/specialcmd.go Parse 84.6% github.com/janpfeifer/gonb/internal/specialcmd/specialcmd.go joinLine 87.5% -github.com/janpfeifer/gonb/internal/specialcmd/specialcmd.go execSpecialConfig 55.3% +github.com/janpfeifer/gonb/internal/specialcmd/specialcmd.go execSpecialConfig 58.2% github.com/janpfeifer/gonb/internal/specialcmd/specialcmd.go execShell 100.0% github.com/janpfeifer/gonb/internal/specialcmd/specialcmd.go splitCmd 97.0% github.com/janpfeifer/gonb/internal/specialcmd/track.go execTrack 27.3% github.com/janpfeifer/gonb/internal/specialcmd/track.go execUntrack 54.5% github.com/janpfeifer/gonb/internal/specialcmd/track.go showTrackedList 91.7% github.com/janpfeifer/gonb/internal/websocket/websocket.go Javascript 83.3% -total (statements) 64.4% +total (statements) 64.6% From 1c8ba695a026ac392a3ef83b2b8066cf536d81f8 Mon Sep 17 00:00:00 2001 From: Jan Pfeifer Date: Wed, 16 Oct 2024 15:58:49 +0200 Subject: [PATCH 06/11] Updated tutorial.ipynb with the help entry for the new `%capture` special command. --- examples/tutorial.ipynb | 93 ++++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 43 deletions(-) diff --git a/examples/tutorial.ipynb b/examples/tutorial.ipynb index 867f7ff..7eb4f3f 100644 --- a/examples/tutorial.ipynb +++ b/examples/tutorial.ipynb @@ -208,8 +208,8 @@ "text": [ "\t...VeryExpensive() call...\n", "\t...VeryExpensive() call...\n", - "NonCachedValue=976\n", - " CachedValue=16\n" + "NonCachedValue=665\n", + " CachedValue=452\n" ] } ], @@ -293,7 +293,7 @@ "name": "stdout", "output_type": "stream", "text": [ - " 100% |████████████████████████████████████████| (25 steps/s) [3s:0s]:0s]\n", + " 100% |████████████████████████████████████████| (25 steps/s) [3s:0s]0s]\n", "Done\n" ] } @@ -413,7 +413,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "lastRenderTime=1.684685\n" + "lastRenderTime=1.132695\n" ] }, { @@ -638,7 +638,7 @@ { "data": { "text/html": [ - "
12345678910X1.02.04.08.016.032.064.0128.0256.0512.0YA diagram of sorts 📊 📈
" + "
12345678910X1.02.04.08.016.032.064.0128.0256.0512.0YA diagram of sorts 📊 📈
" ] }, "metadata": {}, @@ -723,7 +723,7 @@ { "data": { "text/html": [ - "Animated Sine" + "Animated Sine" ] }, "metadata": {}, @@ -1013,7 +1013,7 @@ { "data": { "text/html": [ - "
" + "
" ] }, "metadata": {}, @@ -1031,15 +1031,16 @@ "\t\tmodule = window.Plotly;\n", "\t}\n", "\tlet data = JSON.parse('{\"data\":[{\"type\":\"bar\",\"x\":[1,2,3],\"y\":[1,2,3]}],\"layout\":{\"title\":{\"text\":\"A Figure Specified By Go Struct\"}}}');\n", - "\tmodule.newPlot('07e85c7d', data);\n", + "\tmodule.newPlot('d30bf355', data);\n", "\n", "\t}\n", "\t\n", " if (typeof requirejs === \"function\") {\n", " // Use RequireJS to load module.\n", + "\t\tlet srcWithoutExtension = src.substring(0, src.lastIndexOf(\".js\"));\n", " requirejs.config({\n", " paths: {\n", - " 'plotly': 'https://cdn.plot.ly/plotly-2.29.1.min'\n", + " 'plotly': srcWithoutExtension\n", " }\n", " });\n", " require(['plotly'], function(plotly) {\n", @@ -1353,7 +1354,7 @@ { "data": { "text/html": [ - "
" + "
" ] }, "metadata": {}, @@ -1383,7 +1384,7 @@ { "data": { "text/html": [ - "  5.50 " + "  5.50 " ] }, "metadata": {}, @@ -1392,7 +1393,7 @@ { "data": { "text/html": [ - "Animated Sine" + "Animated Sine" ] }, "metadata": {}, @@ -1525,13 +1526,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "go version go1.22.1 linux/amd64\n", + "go version go1.23.2 linux/amd64\n", "/home/janpf/Projects/gonb/examples\n", - "total 320\n", - "-rwxr-xr-x 1 janpf janpf 87160 Dec 15 08:39 google_colab_demo.ipynb\n", - "drwxr-xr-x 3 janpf janpf 4096 Apr 7 10:34 tests\n", - "-rw-r--r-- 1 janpf janpf 218236 Mar 25 15:26 tutorial.ipynb\n", - "-rw-r--r-- 1 janpf janpf 10090 Dec 15 08:39 wasm_demo.ipynb\n" + "total 344\n", + "-rw-rw-r-- 1 janpf janpf 108240 Sep 23 22:37 google_colab_demo.ipynb\n", + "drwxr-xr-x 3 janpf janpf 4096 Oct 16 15:55 tests\n", + "-rw-r--r-- 1 janpf janpf 222917 Jul 10 09:19 tutorial.ipynb\n", + "-rw-r--r-- 1 janpf janpf 10090 Dec 15 2023 wasm_demo.ipynb\n" ] } ], @@ -1563,13 +1564,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "/tmp/gonb_b75e8f56\n", - "total 13720\n", - "-rw-r--r-- 1 janpf janpf 1379 Apr 7 10:40 go.mod\n", - "-rwxr-xr-x 1 janpf janpf 14017798 Apr 7 10:40 gonb_b75e8f56\n", - "srwxr-xr-x 1 janpf janpf 0 Apr 7 10:39 gopls_socket\n", - "-rw-r--r-- 1 janpf janpf 15907 Apr 7 10:40 go.sum\n", - "-rw-r--r-- 1 janpf janpf 6128 Apr 7 10:40 main.go\n" + "/tmp/gonb_936a10c9\n", + "total 14132\n", + "-rw-r--r-- 1 janpf janpf 1379 Oct 16 15:57 go.mod\n", + "-rwxr-xr-x 1 janpf janpf 14445280 Oct 16 15:58 gonb_936a10c9\n", + "srwxr-xr-x 1 janpf janpf 0 Oct 16 15:57 gopls_socket\n", + "-rw-r--r-- 1 janpf janpf 9849 Oct 16 15:57 go.sum\n", + "-rw-r--r-- 1 janpf janpf 6128 Oct 16 15:58 main.go\n" ] } ], @@ -1687,10 +1688,10 @@ "--- PASS: TestIncr (0.00s)\n", "goos: linux\n", "goarch: amd64\n", - "pkg: gonb_b75e8f56\n", + "pkg: gonb_936a10c9\n", "cpu: 12th Gen Intel(R) Core(TM) i9-12900K\n", "BenchmarkIncr\n", - "BenchmarkIncr-24 \t1000000000\t 0.1362 ns/op\n", + "BenchmarkIncr-24 \t1000000000\t 0.1152 ns/op\n", "PASS\n" ] } @@ -1764,19 +1765,19 @@ "name": "stdout", "output_type": "stream", "text": [ - "module gonb_b75e8f56\n", + "module gonb_936a10c9\n", "\n", - "go 1.22.1\n", + "go 1.23.2\n", "\n", "require (\n", - "\tgithub.com/MetalBlueberry/go-plotly v0.4.0\n", + "\tgithub.com/MetalBlueberry/go-plotly v0.5.0\n", "\tgithub.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b\n", "\tgithub.com/benc-uk/gofract v0.0.0-20230120162050-a6f644f92fd6\n", "\tgithub.com/erkkah/margaid v0.1.1-0.20230128143048-d60b2efd2f5a\n", - "\tgithub.com/janpfeifer/gonb v0.10.0\n", - "\tgithub.com/schollz/progressbar/v3 v3.14.2\n", - "\tgithub.com/stretchr/testify v1.8.1\n", - "\tgolang.org/x/exp v0.0.0-20240404231335-c0f41cb1a7a0\n", + "\tgithub.com/janpfeifer/gonb v0.10.5\n", + "\tgithub.com/schollz/progressbar/v3 v3.16.1\n", + "\tgithub.com/stretchr/testify v1.9.0\n", + "\tgolang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c\n", "\tgonum.org/v1/plot v0.14.0\n", ")\n", "\n", @@ -1786,7 +1787,7 @@ "\tgithub.com/davecgh/go-spew v1.1.1 // indirect\n", "\tgithub.com/go-fonts/liberation v0.3.1 // indirect\n", "\tgithub.com/go-latex/latex v0.0.0-20230307184459-12ec69307ad9 // indirect\n", - "\tgithub.com/go-logr/logr v1.4.1 // indirect\n", + "\tgithub.com/go-logr/logr v1.4.2 // indirect\n", "\tgithub.com/go-pdf/fpdf v0.8.0 // indirect\n", "\tgithub.com/gofrs/uuid v4.4.0+incompatible // indirect\n", "\tgithub.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect\n", @@ -1796,12 +1797,12 @@ "\tgithub.com/pmezard/go-difflib v1.0.0 // indirect\n", "\tgithub.com/rivo/uniseg v0.4.7 // indirect\n", "\tgolang.org/x/image v0.11.0 // indirect\n", - "\tgolang.org/x/sys v0.17.0 // indirect\n", - "\tgolang.org/x/term v0.17.0 // indirect\n", - "\tgolang.org/x/text v0.14.0 // indirect\n", - "\tgopkg.in/yaml.v2 v2.4.0 // indirect\n", + "\tgolang.org/x/sys v0.25.0 // indirect\n", + "\tgolang.org/x/term v0.24.0 // indirect\n", + "\tgolang.org/x/text v0.17.0 // indirect\n", + "\tgopkg.in/yaml.v2 v2.3.0 // indirect\n", "\tgopkg.in/yaml.v3 v3.0.1 // indirect\n", - "\tk8s.io/klog/v2 v2.120.1 // indirect\n", + "\tk8s.io/klog/v2 v2.130.1 // indirect\n", ")\n", "\n", "replace github.com/janpfeifer/gonb => /home/janpf/Projects/gonb\n" @@ -1953,7 +1954,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 33, "id": "ba9172b8-ede5-44cd-baa1-a58a1d668a98", "metadata": { "tags": [] @@ -2030,6 +2031,10 @@ " you to enter one last value after the shell script executes.\n", "- `%with_password`: will prompt for a password passed to the next shell command.\n", " Do this is if your next shell command requires a password.\n", + "- `%capture [-a] ` will make a copy of all **cell execution output** to the given file. By default\n", + " it overwrites the file contents each time the cell is executed. Use `-a` instead to append to the file.\n", + " It works only for the current cell. See also `%%writefile` to write files with a specific content.\n", + " It doesn't work with `%wasm` cells.\n", "\n", "**Notes**: \n", "\n", @@ -2200,7 +2205,9 @@ "This can be handy if for instance the notebook needs to write a configuration file, or simply to dump the code inside\n", "the cell into some file.\n", "\n", - "File path passes through a tilde (`~`) expansion to the user's home directory, as well as environment variable substitution (e.g.: `${HOME}` or `$MY_DIR/a/b`). \n", + "File path passes through a tilde (`~`) expansion to the user's home directory, as well as environment variable substitution (e.g.: `${HOME}` or `$MY_DIR/a/b`).\n", + "\n", + "See also `%capture` to instead write the output of the execution of the cell to a file.\n", "\n", "### `%%script`, `%%bash` and `%%sh`\n", "\n", @@ -2251,7 +2258,7 @@ "name": "go", "nbconvert_exporter": "", "pygments_lexer": "", - "version": "go1.22.3" + "version": "go1.23.2" } }, "nbformat": 4, From ac41c493d11ea4f41d35114bfcb64fbbc17b6c24 Mon Sep 17 00:00:00 2001 From: Jan Pfeifer Date: Wed, 16 Oct 2024 17:38:53 +0200 Subject: [PATCH 07/11] Removed `%capture` log line. --- internal/specialcmd/specialcmd.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/internal/specialcmd/specialcmd.go b/internal/specialcmd/specialcmd.go index 9019056..0b114cd 100644 --- a/internal/specialcmd/specialcmd.go +++ b/internal/specialcmd/specialcmd.go @@ -295,10 +295,7 @@ func execSpecialConfig(msg kernel.Message, goExec *goexec.State, cmdStr string, filePath := args[0] filePath = ReplaceTildeInDir(filePath) filePath = ReplaceEnvVars(filePath) - err := kernel.PublishWriteStream(msg, kernel.StreamStderr, fmt.Sprintf("Capturing output to %q\n", filePath)) - if err != nil { - klog.Errorf("Failed to write to kernel stdout: %+v", err) - } + var err error var f *os.File if appendToFile { f, err = os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) From 2985a91eb7bdb46a7081b15cc65f2d6a077eb4d1 Mon Sep 17 00:00:00 2001 From: Jan Pfeifer Date: Wed, 16 Oct 2024 17:43:26 +0200 Subject: [PATCH 08/11] Fixed the "//" in the `%capture` test. --- examples/tests/capture.ipynb | 28 ---------------------------- internal/nbtests/nbtests_test.go | 4 ++-- 2 files changed, 2 insertions(+), 30 deletions(-) diff --git a/examples/tests/capture.ipynb b/examples/tests/capture.ipynb index 8b66199..01d9423 100644 --- a/examples/tests/capture.ipynb +++ b/examples/tests/capture.ipynb @@ -6,13 +6,6 @@ "id": "fe105ef6-5264-4cb4-b467-60d9b0a337fd", "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Capturing output to \"/tmp/gonb_nbtests_writefile_1126554417//pingpong.txt\"\n" - ] - }, { "name": "stdout", "output_type": "stream", @@ -51,13 +44,6 @@ "id": "004e9622-e3a4-4780-816a-e5d1c77055b4", "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Capturing output to \"/tmp/gonb_nbtests_writefile_1126554417//pingpong.txt\"\n" - ] - }, { "name": "stdout", "output_type": "stream", @@ -97,13 +83,6 @@ "id": "c8214b58-5946-4db5-a329-9e2f2c2fc215", "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Capturing output to \"/tmp/gonb_nbtests_writefile_1126554417//pingpong.txt\"\n" - ] - }, { "data": { "text/html": [ @@ -147,13 +126,6 @@ "id": "9b88a712-deae-4d48-80a1-c4795db31c22", "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Capturing output to \"/tmp/gonb_nbtests_writefile_1126554417//pingpong.txt\"\n" - ] - }, { "data": { "text/markdown": [ diff --git a/internal/nbtests/nbtests_test.go b/internal/nbtests/nbtests_test.go index 6f0342f..1b8e202 100644 --- a/internal/nbtests/nbtests_test.go +++ b/internal/nbtests/nbtests_test.go @@ -723,8 +723,8 @@ func TestCapture(t *testing.T) { // Create directory where to write the file, and set TEST_DIR env variable. testDir, err := os.MkdirTemp("", "gonb_nbtests_writefile_") require.NoError(t, err) - require.NoError(t, os.Setenv("TEST_DIR", testDir+"/")) - klog.Infof("TEST_DIR=%s/", testDir) + require.NoError(t, os.Setenv("TEST_DIR", testDir)) + klog.Infof("TEST_DIR=%q", testDir) notebook := "capture" f := executeNotebook(t, notebook) From 630ec6c78d858a6c7fa294304d7eac954d192596 Mon Sep 17 00:00:00 2001 From: Jan Pfeifer Date: Wed, 16 Oct 2024 17:45:19 +0200 Subject: [PATCH 09/11] Added `tzdata` deb package to docker. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0550477..bffa27b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ FROM ${BASE_IMAGE}:${BASE_TAG} USER root RUN apt update --yes RUN apt install --yes --no-install-recommends \ - sudo wget git openssh-client rsync curl + sudo tzdata wget git openssh-client rsync curl # Give NB_USER sudo power for "/usr/bin/apt-get install/update" or "/usr/bin/apt install/update". USER root From 348fb7671cf3891b5504332810a646fc4c420218 Mon Sep 17 00:00:00 2001 From: Jan Pfeifer Date: Wed, 16 Oct 2024 17:50:21 +0200 Subject: [PATCH 10/11] Updated CHANGELOG, bumped version. --- docs/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 7cd8839..4b09621 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,6 @@ # GoNB Changelog -## Next +## v0.10.6, 2024/10/16, Improved Docker, added `%capture` * Feature request #138 * Added openssh-client, rsync and curl, to allow users to install other dependencies. From ccbb91fee7518cb5efca96c14c85e43be7efb492 Mon Sep 17 00:00:00 2001 From: Jan Pfeifer Date: Wed, 16 Oct 2024 17:50:52 +0200 Subject: [PATCH 11/11] Bumped release version. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bffa27b..7d546cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,7 +61,7 @@ RUN wget --quiet --output-document=- "https://go.dev/dl/go${GO_VERSION}.linux-am && go version # Install GoNB (https://github.com/janpfeifer/gonb) in the user account -ARG GONB_VERSION="v0.10.5" +ARG GONB_VERSION="v0.10.6" USER $NB_USER WORKDIR ${HOME} RUN export GOPROXY=direct && \