Migration to latest jupyter version

README.md

@@ -43,9 +43,14 @@ Due to this, sampling profilers typically end up profiling the least interesting
 
 ## Building <a name="building"></a>
 
-To build xpedite, you will need [cmake](http://cmake.org/) 3.4 or later, [GNU gcc](https://gcc.gnu.org/) 5.2 or later, and a version 2.5 or later Linux kernel.
-
-With cmake and g++ installed, to use xpedite, clone this repository and run the following from xpedite source dir:
+To build xpedite, you will need a linux machine (kernel 2.5 or later) running on intel hardware with the following packages.
+  1. [cmake](http://cmake.org/) (3.4 or later)
+  2. [GNU gcc](https://gcc.gnu.org/) (5.2 or later)
+  3. [python 3](https://www.python.org/downloads/)
+  4. [pybind11](https://github.com/pybind/pybind11)
+  5. [python 3 venv](https://docs.python.org/3/library/venv.html)
+
+With the above installed, to use xpedite, clone this repository and run the following from xpedite source dir:
 
 ```
 $ ./build.sh      # builds xpedite c++ library

include/xpedite/probes/Probe.H

@@ -14,6 +14,7 @@
 #include <xpedite/util/Util.H>
 #include <xpedite/probes/CallSite.H>
 #include <xpedite/probes/RecorderCtl.H>
+#include <cstdint>
 
 namespace xpedite { namespace probes {
 

include/xpedite/probes/RecorderCtl.H

@@ -11,6 +11,7 @@
 #pragma once
 #include <xpedite/probes/CallSite.H>
 #include <xpedite/probes/Recorders.H>
+#include <cstdint>
 
 using XpediteRecorder = void (*)(const void*, uint64_t);
 using XpediteDataProbeRecorder = void (*)(const void*, uint64_t, __uint128_t);

include/xpedite/probes/Recorders.H

@@ -17,6 +17,7 @@
 #pragma once
 #include <xpedite/platform/Builtins.H>
 #include <array>
+#include <cstdint>
 
 extern "C" {
 

scripts/lib/setup.py

@@ -63,7 +63,9 @@
         'cement',
         'termcolor',
         'py-cpuinfo',
-        'jupyter',
+        'nbclassic',
         'six',
+        'setuptools'
+        'ipynbname',
       ],
       zip_safe=False)

scripts/lib/xpedite/jupyter/commands.py

@@ -14,7 +14,7 @@
 """
 
 import logging
-from IPython.core.display      import display, HTML
+from IPython.display           import display, HTML
 from xpedite.jupyter.context   import context
 from xpedite.report.markup     import ERROR_TEXT
 
@@ -241,7 +241,7 @@ def __repr__(self):
       ) if self.profile.current.isEventsEnabled() else ''
       threshold = 1000
       if len(self.profile.current) > threshold:
-        LOGGER.warn('too many transaction - showing only %d out of %d', threshold, len(self.profile.current))
+        LOGGER.warning('too many transaction - showing only %d out of %d', threshold, len(self.profile.current))
       strRepr += str(ReportBuilder().buildTimelineTable(
         self.profile.current, self.profile.probes, ResultOrder.Chronological, threshold, uid
       ))

scripts/lib/xpedite/jupyter/data/config/custom/custom.css

@@ -6,12 +6,12 @@
 **
 *******************************************************************************************/
 
-@import 'flot.css';
-@import 'notebook.css';
-@import 'sunburst.css';
-@import 'bipartite.css';
-@import 'xpedite.css';
-@import 'darkTheme.css';
+@import '/static/flot.css';
+@import '/static/notebook.css';
+@import '/static/sunburst.css';
+@import '/static/bipartite.css';
+@import '/static/xpedite.css';
+@import '/static/darkTheme.css';
 
 body.notebook_app, body.notebook_app.command_mode {
   font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;

scripts/lib/xpedite/jupyter/data/config/jupyter_server_config.py

@@ -0,0 +1,18 @@
+# Configuration file for jupyter-notebook.
+
+# Add serverextension to sys.path for jupyter to load tornadoExtension module on startup
+import os
+import sys
+
+currDir = os.path.dirname(__file__)
+packagePath = os.path.join(currDir, '../config/serverextensions')
+sys.path.insert(0, packagePath)
+c = get_config()
+c.ServerApp.jpserver_extensions = {
+    'tornadoExtension' : True,
+}
+
+staticPath = os.path.join(currDir, '../config/custom')
+c.ServerApp.extra_static_paths = [os.path.join(currDir, '../js'), staticPath]
+c.ServerApp.allow_origin = '*'  #allow all origins
+c.ServerApp.ip = '0.0.0.0'      # listen on all IPs

scripts/lib/xpedite/jupyter/data/config/serverextensions/tornadoExtension.py → scripts/lib/xpedite/jupyter/data/config/serverextensions/tornadoExtension/__init__.py

@@ -5,8 +5,10 @@
 Author: Dhruv Shekhawat, Morgan Stanley
 """
 
-from notebook.utils          import url_path_join
-from notebook.base.handlers  import IPythonHandler
+#from notebook.utils          import url_path_join
+import jupyter_server
+from jupyter_server.base.handlers import JupyterHandler
+import tornado
 from tornado                 import template
 import tornado.web
 import json
@@ -15,13 +17,14 @@
 import os
 import sys
 
-class HtmlReportHandler(tornado.web.RequestHandler):
+class HtmlReportHandler(JupyterHandler):
   """Class to serve html reports through links with
   query params as notebook path and (cellId, reportId)
   as indices to read metadata from notebook
   """
+  @tornado.web.authenticated
   def get(self):
-    xpeditePath = os.path.normpath(os.path.join(__file__, '../../../../../..'))
+    xpeditePath = os.path.normpath(os.path.join(__file__, '../../../../../../..'))
     sys.path.append(xpeditePath)
 
     from xpedite.jupyter.xpediteData import XpediteDataReader
@@ -57,11 +60,16 @@ def get_init_cell(jsonReport):
     if(('isInit' in metadata) and (metadata['isInit'] == '0xFFFFFFFFA5A55A5DUL')):
       return cell
 
-def load_jupyter_server_extension(nb_server_app):
-  """Method called first to load the
-  server extension on jupyter startup
-  """
-  web_app = nb_server_app.web_app
-  host_pattern = '.*$'
-  route_pattern = url_path_join(web_app.settings['base_url'], '/xpedite')
-  web_app.add_handlers(host_pattern, [(route_pattern, HtmlReportHandler)])
+def _load_jupyter_server_extension(serverapp: jupyter_server.serverapp.ServerApp):
+    """
+    This function is called when the extension is loaded.
+    """
+    handlers = [('/xpedite', HtmlReportHandler)]
+    serverapp.web_app.add_handlers(".*$", handlers)
+
+def _jupyter_server_extension_points():
+    """
+    Returns a list of dictionaries with metadata describing
+    where to find the `_load_jupyter_server_extension` function.
+    """
+    return [{"module": "tornadoExtension"}]

scripts/lib/xpedite/jupyter/driver.py

@@ -202,15 +202,17 @@ def launchJupyter(homeDir):
   LOGGER.info('')
   pyPath = os.path.dirname(binPath('python')) + os.pathsep + os.environ['PATH']
   initPath = os.path.dirname(__file__)
+  runtimePath = tempfile.mkdtemp(prefix=SHELL_PREFIX, dir='/tmp')
   jupyterEnv = os.environ
   jupyterEnv[Context.xpediteHomeKey] = os.path.abspath(homeDir)
   jupyterEnv['JUPYTER_PATH'] = os.path.join(initPath, 'data/extensions/')
   jupyterEnv['JUPYTER_CONFIG_DIR'] = os.path.join(initPath, 'data/config/')
+  jupyterEnv['JUPYTER_RUNTIME_DIR'] = runtimePath
+  jupyterEnv['HOME'] = runtimePath
   jupyterEnv['XPEDITE_PATH'] = os.path.abspath(os.path.join(initPath, '../../'))
   jupyterEnv['PATH'] = pyPath
-  jupyterEnv['HOME'] = tempfile.mkdtemp(prefix=SHELL_PREFIX, dir='/tmp')
   jupyterBinary = binPath('jupyter')
-  os.execle(jupyterBinary, 'Xpedite', 'notebook', '--no-browser', '--notebook-dir='+homeDir, jupyterEnv)
+  os.execle(jupyterBinary, 'Xpedite', 'nbclassic', '--no-browser', '--notebook-dir='+homeDir, jupyterEnv)
 
 def validatePath(homeDir, reportName):
   """Validates the path to store xpedite notebook and data files"""

scripts/lib/xpedite/jupyter/templates/initCell.fmt

@@ -2,13 +2,13 @@ import os, sys
 from IPython.display import display, HTML
 sys.path.append(os.environ['XPEDITE_PATH'])
 import xpedite
+import ipynbname
 from xpedite.jupyter.commands import routes, txns, plot, stat, filter, diff
 from xpedite.analytics.timelineTree import buildTimelineTree
 from xpedite.jupyter.templates.initCell import INTRO_FRMT
 from xpedite.jupyter.context import Context, context
 
-thismodule = sys.modules[__name__]
-notebookPath = getattr(thismodule, Context.notebookPathKey)
+notebookPath = ipynbname.path()
 result = 'Failed to resolve jupyter notebook path'
 if notebookPath:
   context.initialize(notebookPath)

scripts/lib/xpedite/requirements.txt

@@ -5,8 +5,10 @@ netifaces>=0.10.4
 numpy>=1.15.2
 pygments>=2.0.2
 rpyc>=4.0.1
-cement==2.8.2
+cement>=3.0.10
 termcolor>=1.1.0
 py-cpuinfo>=0.1.2
-jupyter>=1.0.0
 six>=1.12.0
+nbclassic>=1.0.0
+setuptools
+ipynbname

test/runTest.sh

@@ -134,7 +134,6 @@ function runPytests() {
     SCENARIO_TYPES="--scenarioTypes=Regular,Benchmark,PMC"
   fi
 
-  set -x
   PYTEST_ARGS="${COV} ${TEST_NAME} ${TEST_PATTERN} -v ${APP_HOST} ${PYTEST_ARGS} ${TRANSACTION_COUNT} ${THREAD_COUNT}"
   PYTEST_ARGS="${PYTEST_ARGS} ${WORKSPACE} ${RUN_DIR_ARG} ${APPS} ${SCENARIO_TYPES} ${RECORD_PMC}"
   if ! PYTHONPATH=${XPEDITE_DIR}:${PYTHONPATH} FORCE_COLOR=true pytest ${PYTEST_ARGS}; then