From 4d75aa913e9c7ad3135bfd1ba9a58239decde972 Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 1 Jul 2021 12:52:46 +0200 Subject: [PATCH] add example notebooks to docs with nbsphinx --- docs/requirements.txt | 3 + docs/source/conf.py | 6 + .../examples/Data Publication API.ipynb | 40 +-- docs/source/examples/Futures.ipynb | 64 +--- docs/source/examples/Index.ipynb | 329 ++++++++++++++---- .../Monitoring an MPI Simulation - 1.ipynb | 22 +- .../Monitoring an MPI Simulation - 2.ipynb | 36 +- .../source/examples/Monte Carlo Options.ipynb | 41 +-- .../examples/Parallel Decorator and map.ipynb | 16 +- docs/source/examples/Parallel Magics.ipynb | 132 +++---- docs/source/examples/Using Dill.ipynb | 88 ++--- .../Using MPI with IPython Parallel.ipynb | 12 +- docs/source/examples/dask.ipynb | 60 +--- docs/source/examples/joblib.ipynb | 12 +- docs/source/examples/rmt/rmt.ipynb | 28 +- docs/source/index.rst | 13 +- 16 files changed, 422 insertions(+), 480 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 212c2c566..7efc0c516 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,7 @@ +matplotlib myst-parser +nbsphinx +numpy pydata-sphinx-theme sphinx sphinx-copybutton diff --git a/docs/source/conf.py b/docs/source/conf.py index 56813b7a5..55a023b80 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -50,6 +50,7 @@ 'sphinx.ext.intersphinx', 'sphinx.ext.napoleon', 'myst_parser', + 'nbsphinx', 'IPython.sphinxext.ipython_console_highlighting', ] @@ -357,3 +358,8 @@ 'distributed': ('https://distributed.readthedocs.io/en/stable/', None), 'jupyterclient': ('https://jupyter-client.readthedocs.io/en/stable/', None), } + + +# nbsphinx config +# https://nbsphinx.readthedocs.io/en/latest/usage.html +nbsphinx_execute = 'auto' diff --git a/docs/source/examples/Data Publication API.ipynb b/docs/source/examples/Data Publication API.ipynb index 02daa7598..2a04cb71a 100644 --- a/docs/source/examples/Data Publication API.ipynb +++ b/docs/source/examples/Data Publication API.ipynb @@ -31,9 +31,7 @@ { "cell_type": "code", "execution_count": 1, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%matplotlib inline" @@ -42,9 +40,7 @@ { "cell_type": "code", "execution_count": 2, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "import ipyparallel as ipp" @@ -53,9 +49,7 @@ { "cell_type": "code", "execution_count": 3, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -92,9 +86,7 @@ { "cell_type": "code", "execution_count": 4, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "def publish_it():\n", @@ -112,9 +104,7 @@ { "cell_type": "code", "execution_count": 5, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "ar = dv.apply_async(publish_it)" @@ -130,9 +120,7 @@ { "cell_type": "code", "execution_count": 6, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -173,9 +161,7 @@ { "cell_type": "code", "execution_count": 7, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "def simulation_loop():\n", @@ -197,9 +183,7 @@ { "cell_type": "code", "execution_count": 8, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "ar = dv.apply_async(simulation_loop)" @@ -215,9 +199,7 @@ { "cell_type": "code", "execution_count": 9, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt" @@ -226,9 +208,7 @@ { "cell_type": "code", "execution_count": 10, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { diff --git a/docs/source/examples/Futures.ipynb b/docs/source/examples/Futures.ipynb index eab70749f..7135ee8ec 100644 --- a/docs/source/examples/Futures.ipynb +++ b/docs/source/examples/Futures.ipynb @@ -15,9 +15,7 @@ { "cell_type": "code", "execution_count": 1, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -47,9 +45,7 @@ { "cell_type": "code", "execution_count": 2, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%%px --local\n", @@ -83,9 +79,7 @@ { "cell_type": "code", "execution_count": 4, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -106,9 +100,7 @@ { "cell_type": "code", "execution_count": 5, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -135,9 +127,7 @@ { "cell_type": "code", "execution_count": 6, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -166,9 +156,7 @@ { "cell_type": "code", "execution_count": 7, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -188,9 +176,7 @@ { "cell_type": "code", "execution_count": 8, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -227,9 +213,7 @@ { "cell_type": "code", "execution_count": 9, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -300,9 +284,7 @@ { "cell_type": "code", "execution_count": 10, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -323,9 +305,7 @@ { "cell_type": "code", "execution_count": 11, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -361,9 +341,7 @@ { "cell_type": "code", "execution_count": 12, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -397,9 +375,7 @@ { "cell_type": "code", "execution_count": 20, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor\n", @@ -448,9 +424,7 @@ { "cell_type": "code", "execution_count": 22, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -500,9 +474,7 @@ { "cell_type": "code", "execution_count": 24, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -532,9 +504,7 @@ { "cell_type": "code", "execution_count": 25, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -561,9 +531,7 @@ { "cell_type": "code", "execution_count": 26, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", diff --git a/docs/source/examples/Index.ipynb b/docs/source/examples/Index.ipynb index bd7369db4..f011503a0 100644 --- a/docs/source/examples/Index.ipynb +++ b/docs/source/examples/Index.ipynb @@ -2,65 +2,120 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, - "source": [ - "" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Back to the main [Index](../Index.ipynb)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Parallel Computing" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, + "metadata": { + "tags": [] + }, "source": [ - "IPython includes an architecture and library for interactive parallel computing. The enables Python functions, along with their arguments, to be run in parallel a multicore CPU, cluster or cloud using a simple Python API." + "# examples\n", + "\n", + "Here you will find example notebooks and scripts for working with IPython Parallel." ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "tags": [ + "nbsphinx-tocree" + ] + }, "source": [ - "## Tutorials" + "## Tutorials\n", + "\n", + "* [Creating Clusters from Python](Cluster%20API.ipynb)\n", + "* [Data Publication API](Data%20Publication%20API.ipynb)" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "tags": [ + "nbsphinx-tocree" + ] + }, "source": [ - "* [Data Publication API](Data Publication API.ipynb)\n", - "* [Using Futures with IPython Parallel](Futures.ipynb) " + "## Examples\n", + "\n", + "* [Parallel Magics](Parallel%20Magics.ipynb)\n", + "* [Monitoring an MPI Simulation - 1](Monitoring%20an%20MPI%20Simulation%20-%201.ipynb)\n", + "* [Monitoring an MPI Simulation - 2](Monitoring%20an%20MPI%20Simulation%20-%202.ipynb)\n", + "* [Parallel decorator and map](Parallel%20Decorator%20and%20map.ipynb)\n", + "* [Using MPI with IPython Parallel](Using%20MPI%20with%20IPython%20Parallel.ipynb)\n", + "* [Monte Carlo Options](Monte%20Carlo%20Options.ipynb)\n", + "* [rmt](rmt/rmt.ipynb)\n" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "tags": [ + "nbsphinx-tocree" + ] + }, "source": [ - "## Examples" + "## Integrating IPython Parallel with other tools\n", + "\n", + "* [Using Futures with IPython Parallel](Futures.ipynb)\n", + "* [joblib](joblib.ipynb)\n", + "* [dask](dask.ipynb)\n", + "* [dill](Using%20Dill.ipynb)" ] }, { - "cell_type": "markdown", - "metadata": {}, + "cell_type": "code", + "execution_count": 1, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "./
\n", + "  Data Publication API.ipynb
\n", + "  Monitoring an MPI Simulation - 2.ipynb
\n", + "  Untitled.ipynb
\n", + "  Monte Carlo Options.ipynb
\n", + "  Using MPI with IPython Parallel.ipynb
\n", + "  Parallel Magics.ipynb
\n", + "  Using Dill.ipynb
\n", + "  Cluster API.ipynb
\n", + "  Monitoring an MPI Simulation - 1.ipynb
\n", + "  joblib.ipynb
\n", + "  Futures.ipynb
\n", + "  Parallel Decorator and map.ipynb
\n", + "  dask.ipynb
\n", + "  Index.ipynb
" + ], + "text/plain": [ + "./\n", + " Data Publication API.ipynb\n", + " Monitoring an MPI Simulation - 2.ipynb\n", + " Untitled.ipynb\n", + " Monte Carlo Options.ipynb\n", + " Using MPI with IPython Parallel.ipynb\n", + " Parallel Magics.ipynb\n", + " Using Dill.ipynb\n", + " Cluster API.ipynb\n", + " Monitoring an MPI Simulation - 1.ipynb\n", + " joblib.ipynb\n", + " Futures.ipynb\n", + " Parallel Decorator and map.ipynb\n", + " dask.ipynb\n", + " Index.ipynb" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "* [Monitoring an MPI Simulation - 1](Monitoring an MPI Simulation - 1.ipynb)\n", - "* [Monitoring an MPI Simulation - 2](Monitoring an MPI Simulation - 2.ipynb)\n", - "* [Parallel Decorator and map](Parallel Decorator and map.ipynb)\n", - "* [Parallel Magics](Parallel Magics.ipynb)\n", - "* [Using Dill](Using Dill.ipynb)\n", - "* [Using MPI with IPython Parallel](Using MPI with IPython Parallel.ipynb)\n", - "* [Monte Carlo Options](Monte Carlo Options.ipynb)" + "import glob\n", + "import os\n", + "\n", + "from IPython.display import FileLink, FileLinks, display\n", + "\n", + "FileLinks(\".\", included_suffixes=[\".ipynb\"], recursive=False)" ] }, { @@ -79,23 +134,52 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": { - "collapsed": false + "tags": [] }, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "customresults.py fetchparse.py nwmerge.py throughput.py\r\n", - "dagdeps.py iopubwatcher.py phistogram.py\r\n", - "dependencies.py itermapresult.py task_profiler.py\r\n" - ] + "data": { + "text/html": [ + "./
\n", + "  dagdeps.py
\n", + "  customresults.py
\n", + "  throughput.py
\n", + "  itermapresult.py
\n", + "  task_profiler.py
\n", + "  fetchparse.py
\n", + "  iopubwatcher.py
\n", + "  nwmerge.py
\n", + "  phistogram.py
\n", + "  dependencies.py
" + ], + "text/plain": [ + "./\n", + " dagdeps.py\n", + " customresults.py\n", + " throughput.py\n", + " itermapresult.py\n", + " task_profiler.py\n", + " fetchparse.py\n", + " iopubwatcher.py\n", + " nwmerge.py\n", + " phistogram.py\n", + " dependencies.py" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "ls *.py" + "import glob\n", + "import os\n", + "\n", + "from IPython.display import FileLink, FileLinks, display\n", + "\n", + "FileLinks(\".\", included_suffixes=[\".py\"], recursive=False)" ] }, { @@ -107,41 +191,124 @@ }, { "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": false - }, + "execution_count": 3, + "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "daVinci Word Count/:\r\n", - "pwordfreq.py wordfreq.py\r\n", - "\r\n", - "interengine/:\r\n", - "bintree.py \u001b[31mbintree_script.py\u001b[m\u001b[m* communicator.py interengine.py\r\n", - "\r\n", - "pi/:\r\n", - "parallelpi.py pidigits.py\r\n", - "\r\n", - "rmt/:\r\n", - "rmt.ipy rmt.ipynb rmtkernel.py\r\n", - "\r\n", - "wave2D/:\r\n", - "\u001b[31mRectPartitioner.py\u001b[m\u001b[m* \u001b[31mparallelwave-mpi.py\u001b[m\u001b[m* \u001b[31mwavesolver.py\u001b[m\u001b[m*\r\n", - "communicator.py \u001b[31mparallelwave.py\u001b[m\u001b[m*\r\n" - ] + "data": { + "text/html": [ + "wave2D/
\n", + "  wavesolver.py
\n", + "  RectPartitioner.py
\n", + "  parallelwave-mpi.py
\n", + "  communicator.py
\n", + "  parallelwave.py
" + ], + "text/plain": [ + "wave2D/\n", + " wavesolver.py\n", + " RectPartitioner.py\n", + " parallelwave-mpi.py\n", + " communicator.py\n", + " parallelwave.py" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "pi/
\n", + "  pidigits.py
\n", + "  parallelpi.py
" + ], + "text/plain": [ + "pi/\n", + " pidigits.py\n", + " parallelpi.py" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "daVinci Word Count/
\n", + "  wordfreq.py
\n", + "  pwordfreq.py
" + ], + "text/plain": [ + "daVinci Word Count/\n", + " wordfreq.py\n", + " pwordfreq.py" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "interengine/
\n", + "  bintree_script.py
\n", + "  bintree.py
\n", + "  communicator.py
\n", + "  interengine.py
" + ], + "text/plain": [ + "interengine/\n", + " bintree_script.py\n", + " bintree.py\n", + " communicator.py\n", + " interengine.py" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "broadcast/
\n", + "  test.py
" + ], + "text/plain": [ + "broadcast/\n", + " test.py" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "rmt/
\n", + "  rmtkernel.py
\n", + "  rmt.ipynb
" + ], + "text/plain": [ + "rmt/\n", + " rmtkernel.py\n", + " rmt.ipynb" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ - "ls */" + "for path in os.listdir(\".\"):\n", + " if os.path.isdir(path) and not path.startswith(\".\"):\n", + " display(FileLinks(path, included_suffixes=['.py', '.ipynb']))" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -155,9 +322,19 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.1" + "version": "3.8.8" + }, + "nbsphinx": { + "execute": "always" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "state": {}, + "version_major": 2, + "version_minor": 0 + } } }, "nbformat": 4, - "nbformat_minor": 0 + "nbformat_minor": 4 } diff --git a/docs/source/examples/Monitoring an MPI Simulation - 1.ipynb b/docs/source/examples/Monitoring an MPI Simulation - 1.ipynb index ab6a52d06..35717ab80 100644 --- a/docs/source/examples/Monitoring an MPI Simulation - 1.ipynb +++ b/docs/source/examples/Monitoring an MPI Simulation - 1.ipynb @@ -15,7 +15,6 @@ "cell_type": "code", "execution_count": 1, "metadata": { - "collapsed": false, "slideshow": { "slide_start": false } @@ -38,9 +37,7 @@ { "cell_type": "code", "execution_count": 2, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -74,7 +71,6 @@ "cell_type": "code", "execution_count": 3, "metadata": { - "collapsed": false, "slideshow": { "slide_start": false } @@ -120,7 +116,6 @@ "cell_type": "code", "execution_count": 4, "metadata": { - "collapsed": false, "slideshow": { "slide_start": false } @@ -165,7 +160,6 @@ "cell_type": "code", "execution_count": 5, "metadata": { - "collapsed": false, "slideshow": { "slide_start": false } @@ -238,7 +232,6 @@ "cell_type": "code", "execution_count": 6, "metadata": { - "collapsed": false, "slideshow": { "slide_start": false } @@ -297,7 +290,6 @@ "cell_type": "code", "execution_count": 7, "metadata": { - "collapsed": false, "slideshow": { "slide_start": false } @@ -325,7 +317,6 @@ "cell_type": "code", "execution_count": 8, "metadata": { - "collapsed": false, "slideshow": { "slide_start": false } @@ -393,7 +384,6 @@ "cell_type": "code", "execution_count": 9, "metadata": { - "collapsed": false, "slideshow": { "slide_start": false } @@ -416,7 +406,6 @@ "cell_type": "code", "execution_count": 10, "metadata": { - "collapsed": false, "slideshow": { "slide_start": false } @@ -465,7 +454,6 @@ "cell_type": "code", "execution_count": 11, "metadata": { - "collapsed": false, "slideshow": { "slide_start": false } @@ -478,9 +466,7 @@ { "cell_type": "code", "execution_count": 12, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -517,9 +503,7 @@ { "cell_type": "code", "execution_count": 13, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%%px --target 0\n", diff --git a/docs/source/examples/Monitoring an MPI Simulation - 2.ipynb b/docs/source/examples/Monitoring an MPI Simulation - 2.ipynb index 9deb2a94e..cab96416f 100644 --- a/docs/source/examples/Monitoring an MPI Simulation - 2.ipynb +++ b/docs/source/examples/Monitoring an MPI Simulation - 2.ipynb @@ -35,9 +35,7 @@ { "cell_type": "code", "execution_count": 1, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "import ipyparallel as ipp\n", @@ -56,9 +54,7 @@ { "cell_type": "code", "execution_count": 2, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%matplotlib inline\n", @@ -78,9 +74,7 @@ { "cell_type": "code", "execution_count": 3, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -119,9 +113,7 @@ { "cell_type": "code", "execution_count": 4, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "ranks = view['rank']\n", @@ -153,9 +145,7 @@ { "cell_type": "code", "execution_count": 5, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%%px\n", @@ -222,9 +212,7 @@ { "cell_type": "code", "execution_count": 6, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "from IPython.display import display, clear_output\n", @@ -280,9 +268,7 @@ { "cell_type": "code", "execution_count": 7, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "def monitor_simulation(ar, refresh=5.0, plots_in_place=True):\n", @@ -350,9 +336,7 @@ { "cell_type": "code", "execution_count": 8, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# Create the local client that controls our IPython cluster with MPI support\n", @@ -368,9 +352,7 @@ { "cell_type": "code", "execution_count": 9, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { diff --git a/docs/source/examples/Monte Carlo Options.ipynb b/docs/source/examples/Monte Carlo Options.ipynb index bfd228c26..5135bbcb6 100644 --- a/docs/source/examples/Monte Carlo Options.ipynb +++ b/docs/source/examples/Monte Carlo Options.ipynb @@ -24,9 +24,7 @@ { "cell_type": "code", "execution_count": 1, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%matplotlib inline\n", @@ -88,9 +86,7 @@ { "cell_type": "code", "execution_count": 5, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -124,9 +120,7 @@ { "cell_type": "code", "execution_count": 6, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "def price_option(S=100.0, K=100.0, sigma=0.25, r=0.05, days=260, paths=10000):\n", @@ -184,9 +178,7 @@ { "cell_type": "code", "execution_count": 7, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -270,9 +262,7 @@ { "cell_type": "code", "execution_count": 10, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "async_results = []" @@ -282,7 +272,6 @@ "cell_type": "code", "execution_count": 11, "metadata": { - "collapsed": false, "scrolled": true }, "outputs": [ @@ -310,9 +299,7 @@ { "cell_type": "code", "execution_count": 12, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -389,9 +376,7 @@ { "cell_type": "code", "execution_count": 15, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -439,9 +424,7 @@ { "cell_type": "code", "execution_count": 16, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -489,9 +472,7 @@ { "cell_type": "code", "execution_count": 17, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -539,9 +520,7 @@ { "cell_type": "code", "execution_count": 18, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { diff --git a/docs/source/examples/Parallel Decorator and map.ipynb b/docs/source/examples/Parallel Decorator and map.ipynb index eac66a620..d69c55e5c 100644 --- a/docs/source/examples/Parallel Decorator and map.ipynb +++ b/docs/source/examples/Parallel Decorator and map.ipynb @@ -22,9 +22,7 @@ { "cell_type": "code", "execution_count": 2, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "rc = ipp.Client()\n", @@ -34,9 +32,7 @@ { "cell_type": "code", "execution_count": 3, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -54,9 +50,7 @@ { "cell_type": "code", "execution_count": 4, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -77,9 +71,7 @@ { "cell_type": "code", "execution_count": 5, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", diff --git a/docs/source/examples/Parallel Magics.ipynb b/docs/source/examples/Parallel Magics.ipynb index d471e4f08..75154009c 100644 --- a/docs/source/examples/Parallel Magics.ipynb +++ b/docs/source/examples/Parallel Magics.ipynb @@ -20,13 +20,14 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "import ipyparallel as ipp\n", - "rc = ipp.Client()\n", + "cluster = ipp.Cluster()\n", + "cluster.start_cluster_sync(n=4)\n", + "rc = cluster.connect_client_sync()\n", + "rc.wait_for_engines(4)\n", "dv = rc[:]\n", "rc.ids" ] @@ -49,9 +50,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%px a=5" @@ -60,9 +59,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%px print(a)" @@ -71,9 +68,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%px a" @@ -82,9 +77,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "with dv.sync_imports():\n", @@ -94,9 +87,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%px from __future__ import print_function\n", @@ -113,9 +104,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%pxconfig --noblock" @@ -124,13 +113,11 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%px import time\n", - "%px time.sleep(5)\n", + "%px time.sleep(1)\n", "%px time.time()" ] }, @@ -145,9 +132,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%pxresult" @@ -163,9 +148,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%pxconfig --block\n", @@ -173,11 +156,11 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "`%%px` can be used to lower the priority of the engines to improve system performance under heavy CPU load." - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -187,15 +170,13 @@ "source": [ "%%px\n", "import psutil\n", - "psutil.Process().nice(20 if psutil.LINUX else psutil.IDLE_PRIORITY_CLASS)" + "psutil.Process().nice(20 if psutil.POSIX else psutil.IDLE_PRIORITY_CLASS)" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%%px\n", @@ -215,9 +196,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "dv.scatter('id', dv.targets, flatten=True)\n", @@ -227,9 +206,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%%px --noblock\n", @@ -243,9 +220,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%pxresult" @@ -269,9 +244,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%%px --group-outputs=engine\n", @@ -296,9 +269,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%pxresult --group-outputs=order" @@ -322,7 +293,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": false + "tags": [] }, "outputs": [], "source": [ @@ -364,9 +335,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "e0 = rc[-1]\n", @@ -377,9 +346,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%px0 generate_output()" @@ -388,9 +355,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%px generate_output()" @@ -406,9 +371,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%pxresult --group-outputs order" @@ -417,9 +380,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%pxresult --group-outputs engine" @@ -444,13 +405,15 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": false + "tags": [ + "raises-exception" + ] }, "outputs": [], "source": [ "%%px\n", "from numpy.random import random\n", - "A = random((100,100,'invalid shape'))" + "A = random((100, 100, 'invalid shape'))" ] }, { @@ -470,16 +433,14 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%%px\n", "%%timeit\n", "from numpy.random import random\n", "from numpy.linalg import norm\n", - "A = random((100,100))\n", + "A = random((100, 100))\n", "norm(A, 2)" ] }, @@ -494,7 +455,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "As of IPython 1.0, you can instruct `%%px` to also execute the cell locally.\n", + "You can instruct `%%px` to also execute the cell locally.\n", "This is useful for interactive definitions,\n", "or if you want to load a data source everywhere,\n", "not just on the engines." @@ -503,9 +464,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "%%px --local\n", @@ -517,7 +476,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -531,9 +490,16 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.1" + "version": "3.8.8" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "state": {}, + "version_major": 2, + "version_minor": 0 + } } }, "nbformat": 4, - "nbformat_minor": 0 + "nbformat_minor": 4 } diff --git a/docs/source/examples/Using Dill.ipynb b/docs/source/examples/Using Dill.ipynb index b1d35b14b..2ea66cc92 100644 --- a/docs/source/examples/Using Dill.ipynb +++ b/docs/source/examples/Using Dill.ipynb @@ -32,9 +32,7 @@ { "cell_type": "code", "execution_count": 1, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "def make_closure(a):\n", @@ -52,9 +50,7 @@ { "cell_type": "code", "execution_count": 2, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "!rm -f /tmp/dilltest\n", @@ -64,9 +60,7 @@ { "cell_type": "code", "execution_count": 3, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -86,9 +80,7 @@ { "cell_type": "code", "execution_count": 4, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -105,9 +97,7 @@ { "cell_type": "code", "execution_count": 5, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "import pickle" @@ -123,9 +113,7 @@ { "cell_type": "code", "execution_count": 6, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "ename": "AttributeError", @@ -153,9 +141,7 @@ { "cell_type": "code", "execution_count": 7, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "import dill" @@ -164,9 +150,7 @@ { "cell_type": "code", "execution_count": 8, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -207,9 +191,7 @@ { "cell_type": "code", "execution_count": 9, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "import ipyparallel as parallel\n", @@ -227,9 +209,7 @@ { "cell_type": "code", "execution_count": 10, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "ename": "TypeError", @@ -266,9 +246,7 @@ { "cell_type": "code", "execution_count": 11, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -315,9 +293,7 @@ { "cell_type": "code", "execution_count": 12, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -337,9 +313,7 @@ { "cell_type": "code", "execution_count": 13, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -368,9 +342,7 @@ { "cell_type": "code", "execution_count": 14, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -391,9 +363,7 @@ { "cell_type": "code", "execution_count": 15, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -421,9 +391,7 @@ { "cell_type": "code", "execution_count": 16, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "def outer(a):\n", @@ -447,9 +415,7 @@ { "cell_type": "code", "execution_count": 17, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -476,9 +442,7 @@ { "cell_type": "code", "execution_count": 18, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -516,9 +480,7 @@ { "cell_type": "code", "execution_count": 19, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -548,9 +510,7 @@ { "cell_type": "code", "execution_count": 20, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "import testdill" @@ -559,9 +519,7 @@ { "cell_type": "code", "execution_count": 21, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -581,9 +539,7 @@ { "cell_type": "code", "execution_count": 22, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { diff --git a/docs/source/examples/Using MPI with IPython Parallel.ipynb b/docs/source/examples/Using MPI with IPython Parallel.ipynb index 3183af644..22e785d37 100644 --- a/docs/source/examples/Using MPI with IPython Parallel.ipynb +++ b/docs/source/examples/Using MPI with IPython Parallel.ipynb @@ -68,9 +68,7 @@ { "cell_type": "code", "execution_count": 3, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -98,9 +96,7 @@ { "cell_type": "code", "execution_count": 4, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -129,9 +125,7 @@ { "cell_type": "code", "execution_count": 5, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { diff --git a/docs/source/examples/dask.ipynb b/docs/source/examples/dask.ipynb index 4113fba33..892e7a9b7 100644 --- a/docs/source/examples/dask.ipynb +++ b/docs/source/examples/dask.ipynb @@ -14,9 +14,7 @@ { "cell_type": "code", "execution_count": 1, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -45,9 +43,7 @@ { "cell_type": "code", "execution_count": 2, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -85,9 +81,7 @@ { "cell_type": "code", "execution_count": 3, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "from distributed import progress\n", @@ -107,9 +101,7 @@ { "cell_type": "code", "execution_count": 4, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -138,9 +130,7 @@ { "cell_type": "code", "execution_count": 5, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -177,9 +167,7 @@ { "cell_type": "code", "execution_count": 6, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -212,9 +200,7 @@ { "cell_type": "code", "execution_count": 7, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -242,9 +228,7 @@ { "cell_type": "code", "execution_count": 8, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -272,9 +256,7 @@ { "cell_type": "code", "execution_count": 9, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stderr", @@ -311,9 +293,7 @@ { "cell_type": "code", "execution_count": 10, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stderr", @@ -340,9 +320,7 @@ { "cell_type": "code", "execution_count": 11, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stderr", @@ -379,9 +357,7 @@ { "cell_type": "code", "execution_count": 12, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "view = rc[one_engine_per_host]\n", @@ -399,9 +375,7 @@ { "cell_type": "code", "execution_count": 13, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stderr", @@ -453,9 +427,7 @@ { "cell_type": "code", "execution_count": 14, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -486,9 +458,7 @@ { "cell_type": "code", "execution_count": 15, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { diff --git a/docs/source/examples/joblib.ipynb b/docs/source/examples/joblib.ipynb index c2fabb714..f08d63172 100644 --- a/docs/source/examples/joblib.ipynb +++ b/docs/source/examples/joblib.ipynb @@ -34,9 +34,7 @@ { "cell_type": "code", "execution_count": 2, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -114,9 +112,7 @@ { "cell_type": "code", "execution_count": 4, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -144,9 +140,7 @@ { "cell_type": "code", "execution_count": 5, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { diff --git a/docs/source/examples/rmt/rmt.ipynb b/docs/source/examples/rmt/rmt.ipynb index f38771d69..f04062e5f 100644 --- a/docs/source/examples/rmt/rmt.ipynb +++ b/docs/source/examples/rmt/rmt.ipynb @@ -19,9 +19,7 @@ { "cell_type": "code", "execution_count": 1, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "from rmtkernel import ensemble_diffs, normalize_diffs, GOE\n", @@ -76,9 +74,7 @@ { "cell_type": "code", "execution_count": 4, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "s, rhos = generate_wigner_data()" @@ -87,9 +83,7 @@ { "cell_type": "code", "execution_count": 17, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -160,9 +154,7 @@ { "cell_type": "code", "execution_count": 8, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -179,9 +171,7 @@ { "cell_type": "code", "execution_count": 9, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "serial_diffs = serial_diffs(serial_nmats, serial_matsize)" @@ -198,9 +188,7 @@ { "cell_type": "code", "execution_count": 10, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -289,9 +277,7 @@ { "cell_type": "code", "execution_count": 14, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", diff --git a/docs/source/index.rst b/docs/source/index.rst index 6b189391a..50d8673b9 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -21,11 +21,11 @@ or:: As of IPython Parallel 6.2, this will additionally install and enable the IPython Clusters tab in the Jupyter Notebook dashboard. -You can also enable/install the clusters tab yourself after the fact with: +You can also enable/install the clusters tab yourself after the fact with:: ipcluster nbextension enable -Or the individual Jupyter commands: +Or the individual Jupyter commands:: jupyter serverextension install [--sys-prefix] --py ipyparallel jupyter nbextension install [--sys-prefix] --py ipyparallel @@ -41,13 +41,18 @@ Contents tutorial/index reference/index +.. toctree:: + :maxdepth: 2 + + examples/Index + .. toctree:: :maxdepth: 1 changelog -ipyparallel API -=============== +IPython Parallel API +==================== .. toctree:: :maxdepth: 2