API evolution and performance improvements

codecov.yml

@@ -1,5 +1,5 @@
 comment:
-  after_n_builds: 11
+  after_n_builds: 9
 coverage:
   status:
     project:

conftest.py

@@ -2,13 +2,38 @@
 Global test configuration. Use this file to define fixtures to use
 in both doctests and regular tests.
 """
+import os
+import importlib
+from contextlib import contextmanager
+
 from libertem.executor.inline import InlineJobExecutor
 from libertem.executor.pipelined import PipelinedExecutor
+
 import pytest
+import numpy as np
 
 from libertem.viz.base import Dummy2DPlot
 
+from libertem_live.detectors.dectris.sim import DectrisSim
+from libertem_live.detectors.merlin.sim import CameraSim
 from libertem_live import api as ltl
+# A bit of gymnastics to import the test utilities since this
+# conftest.py file is shared between the doctests and unit tests
+# and this file is outside the package
+basedir = os.path.dirname(__file__)
+location = os.path.join(basedir, "tests/utils.py")
+spec = importlib.util.spec_from_file_location("utils", location)
+utils = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(utils)
+
+
+DECTRIS_TESTDATA_PATH = os.path.join(
+    utils.get_testdata_path(),
+    'dectris', 'zmqdump.dat.128x128-id34-exte-bslz4'
+)
+HAVE_DECTRIS_TESTDATA = os.path.exists(DECTRIS_TESTDATA_PATH)
+MIB_TESTDATA_PATH = os.path.join(utils.get_testdata_path(), 'default.mib')
+HAVE_MIB_TESTDATA = os.path.exists(MIB_TESTDATA_PATH)
 
 
 @pytest.fixture
@@ -37,10 +62,71 @@ def ctx_pipelined():
 
 @pytest.fixture()
 def default_aq(ltl_ctx):
-    return ltl_ctx.prepare_acquisition('memory', trigger=None, datashape=[16, 16, 32, 32])
+    conn = ltl_ctx.make_connection('memory').open(
+        data=None, extra_kwargs={'datashape': [16, 16, 32, 32]}
+    )
+    return ltl_ctx.make_acquisition(conn=conn)
 
 
 def pytest_collectstart(collector):
     # nbval: ignore some output types
     if collector.fspath and collector.fspath.ext == '.ipynb':
         collector.skip_compare += 'text/html', 'application/javascript', 'stderr',
+
+
+@pytest.fixture(autouse=True, scope='session')
+def add_np(doctest_namespace):
+    doctest_namespace['np'] = np
+
+
+@pytest.fixture(autouse=True, scope='session')
+def add_helpers(doctest_namespace, ctx_pipelined):
+    from libertem.udf.sum import SumUDF
+    doctest_namespace['ctx'] = ctx_pipelined
+    doctest_namespace['SumUDF'] = SumUDF
+
+
+@pytest.fixture(autouse=True, scope='session')
+def add_sims(doctest_namespace):
+    if not HAVE_DECTRIS_TESTDATA or not HAVE_MIB_TESTDATA:
+        # FIXME: add some kind of proxy object that calls
+        # pytest.skip on access? is this possible somehow?
+        yield
+        return
+    path = DECTRIS_TESTDATA_PATH
+
+    sim = contextmanager(utils.run_camera_sim)
+
+    with sim(
+        cls=DectrisSim,
+        path=path,
+        port=0,
+        zmqport=0,
+    ) as dectris_runner, sim(
+        cls=CameraSim,
+        host='127.0.0.1',
+        data_port=0,
+        control_port=0,
+        trigger_port=0,
+        path=MIB_TESTDATA_PATH,
+        nav_shape=(32, 32),
+        wait_trigger=False,
+        initial_params={
+            'IMAGEX': "256",
+            'IMAGEY': "256",
+            'COUNTERDEPTH': '12',
+        },
+    ) as merlin_runner:
+        dectris_api_port, dectris_data_port = dectris_runner.port, dectris_runner.zmqport
+        merlin_api_port, merlin_data_port = (
+            merlin_runner.control_t.sockname[1],
+            merlin_runner.server_t.sockname[1]
+        )
+
+        doctest_namespace['DCU_API_PORT'] = dectris_api_port
+        doctest_namespace['DCU_DATA_PORT'] = dectris_data_port
+
+        doctest_namespace['MERLIN_API_PORT'] = merlin_api_port
+        doctest_namespace['MERLIN_DATA_PORT'] = merlin_data_port
+
+        yield

docs/source/changelog/features/api-evolution.rst

@@ -0,0 +1,14 @@
+[Feature] Update user-facing APIs
+=================================
+
+* Most operations are now reachable just by importing the :class:`libertem_live.api.LiveContext`,
+  no need to manually import detector-specific classes etc.
+* Introduce :class:`libertem_live.hooks.Hooks` as a replacement
+  for the trigger function that is open for future enhancements.
+* Generic interface :meth:`libertem_live.api.LiveContext.make_acquisition` for
+  creating acquisition objects; the detector specific parameters are now mostly
+  specified in :meth:`libertem_live.api.LiveContext.make_connection`. Connecting
+  and creating an acquisition object is now a two-step process.
+* Support for a passive acquisition workflow using
+  :meth:`libertem_live.detectors.base.connection.DetectorConnection.wait_for_acquisition`
+  (:pr:`74`).

docs/source/changelog/features/dectris.rst

@@ -1,3 +1,5 @@
 [Feature] Support for DECTRIS detectors
 =======================================
 * Initial support for DECTRIS detectors, including QUADRO and ARINA (:pr:`51`).
+* Updated to use shared memory as data transport to worker processes
+  for efficiency gains (:pr:`74`).

docs/source/changelog/features/shape-auto.rst

@@ -0,0 +1,13 @@
+[Feature] Automatically determine :code:`nav_shape` if possible
+===============================================================
+
+* This is only possible in passive mode, as we have to know
+  the :code:`nav_shape` in active mode.
+* If the automatic mode doesn't work for you, it's still possible to
+  manually specify the shape, or to override :meth:`libertem_live.hooks.Hooks.on_determine_nav_shape`
+  which takes precedence over the automatic detection.
+* It's now possible to have placeholders in the :code:`nav_shape`,
+  which will be filled with the remainder of the shape. For example,
+  It's possible to specify :code:`(-1, -1)` and get a 2D shape, or
+  something like :code:`(128, -1)` to fill the last dimension automatically,
+  based on the number of frames in the acquisition (:pr:`74`).

docs/source/conf.py

@@ -208,7 +208,7 @@
 todo_include_todos = True
 
 intersphinx_mapping = {
-    'numpy': ('http://docs.scipy.org/doc/numpy/', None),
+    'numpy': ('https://numpy.org/doc/stable/', None),
     'python': ('https://docs.python.org/3.10', None),
     'libertem': ('https://libertem.github.io/LiberTEM/', None),
 }
@@ -237,4 +237,9 @@ def get_testdata_path():
     'dectris', 'zmqdump.dat.128x128-id34-exte-bslz4'
 )
 HAVE_DECTRIS_TESTDATA = os.path.exists(DECTRIS_TESTDATA_PATH)
+
+MERLIN_TESTDATA_PATH = os.path.join(
+    get_testdata_path(), 'default.mib',
+)
+HAVE_MERLIN_TESTDATA = os.path.exists(MERLIN_TESTDATA_PATH)
 '''

docs/source/dectris-acquisition-example.nblink

@@ -0,0 +1,3 @@
+{
+    "path": "../../examples/dectris-acquisition.ipynb"
+}

docs/source/detectors.rst

@@ -1,149 +1,18 @@
 Supported Detectors
 ===================
 
-.. _`dectris detectors`:
+These are the different detectors and detector APIs supported in LiberTEM-live. Note
+that level of support can vary between these, but the basic functionality
+of receiving data and running UDFs on it works for each of these.
 
-DECTRIS EIGER2-based
---------------------
+Support for additional detectors is planned, please talk to us if you are
+missing your detector in this list!
 
-.. versionadded:: 0.2
+In addition, there is also a :ref:`memory "detector" <memory detector>` available,
+which is mostly useful for testing, and is not optimized for production use.
 
-LiberTEM-live has support for all DECTRIS detectors that support
-the `SIMPLON API <https://media.dectris.com/210607-DECTRIS-SIMPLON-API-Manual_EIGER2-chip-based_detectros.pdf>`_,
-including QUADRO and ARINA.
+.. toctree::
+    :maxdepth: 2
+    :glob:
 
-.. note::
-    Python 3.7+ is required to use the DECTRIS-related features of
-    LiberTEM-live, and we recommend to use Python 3.10. Currently, Windows and
-    Linux on x86_64 are supported.
-
-Installation
-............
-
-There are a few extra dependencies needed for DECTRIS support. You can easily
-install them using:
-
-.. code-block:: shell
-
-    (libertem) $ python -m pip install "libertem-live[dectris]"
-
-Currently, at least LiberTEM 0.10 is required. Using LiberTEM master is
-recommended until a number of stability fixes for the pipelined executor are
-released in LiberTEM.
-
-Usage example
-.............
-
-.. testsetup::
-    :skipif: not HAVE_DECTRIS_TESTDATA
-
-    from libertem_live.detectors.dectris.sim import DectrisSim
-    server = DectrisSim(path=DECTRIS_TESTDATA_PATH, port=0, zmqport=0, verbose=False)
-    server.start()
-    server.wait_for_listen()
-    DCU_API_PORT = server.port
-    DCU_DATA_PORT = server.zmqport
-
-.. testoutput::
-    :hide:
-
-    RustedReplay listening on tcp://127.0.0.1:...
-    Waiting for arm
-
-.. testcode::
-    :skipif: not HAVE_DECTRIS_TESTDATA
-
-    from libertem.executor.pipelined import PipelinedExecutor
-    from libertem.viz.bqp import BQLive2DPlot
-    from libertem_live.api import LiveContext
-    from libertem.udf.sum import SumUDF
-
-    executor = PipelinedExecutor(spec=PipelinedExecutor.make_spec(cpus=range(10), cudas=[]))
-    ctx = LiveContext(executor=executor, plot_class=BQLive2DPlot)
-
-    def trigger(aq):
-        """
-        You can trigger the scan here, if you have a microscope control API
-        """
-        # you can imagine something like:
-        # trigger_scan(shape=tuple(aq.shape.nav), dwelltime=67e-6)
-        pass
-
-    # prepare for acquisition, setting up scan parameters etc.
-    aq = ctx.prepare_acquisition(
-        'dectris',
-        api_host="127.0.0.1",
-        api_port=DCU_API_PORT,
-        data_host="127.0.0.1",
-        data_port=DCU_DATA_PORT,
-        nav_shape=(128, 128),
-        trigger_mode="exte",
-        trigger=trigger,
-        frames_per_partition=512,
-    )
-
-    # run one or more UDFs on the live data stream:
-    # (this can be run multiple times on the same `aq` object)
-    ctx.run_udf(dataset=aq, udf=SumUDF(), plots=True)
-
-.. testoutput::
-    :hide:
-
-    ...
-
-.. testcleanup::
-    :skipif: not HAVE_DECTRIS_TESTDATA
-
-    ctx.close()
-    server.stop()
-    server.maybe_raise()
-
-See the :ref:`DECTRIS reference section <dectris reference>` for a description of
-the acquisition parameters.
-
-.. _`merlin detector`:
-
-Quantum Detectors Merlin
-------------------------
-
-No extra dependencies are needed for using Merlin detectors.
-
-Supported are currently 1 bit, 6 bit and 12 bit :code:`COUNTERDEPTH` for both the "binary"
-(:code:`FILEFORMAT 0`) and the "raw binary" format (:code:`FILEFORMAT 2`).
-
-For testing, an acquisition with soft trigger (:code:`TRIGGERSTART 5`) is
-recommended since internal trigger (:code:`TRIGGERSTART 0`) may cause issues
-with finding the beginning of the data stream. For a real STEM acquisition a
-hardware trigger setup that matches the given instrument is required. See the
-MerlinEM User Manual from Quantum Detectors for details!
-
-A simple simulator for testing live acquisition without the actual hardware is
-included in LiberTEM-live. It replays an MIB dataset and accepts the following
-parameters:
-
-.. code-block:: shell
-
-    (libertem) $ libertem-live-mib-sim --help
-    Usage: libertem-live-mib-sim [OPTIONS] PATH
-
-    Options:
-    --nav-shape <INTEGER INTEGER>...
-    --continuous
-    --cached [NONE|MEM|MEMFD]
-    --host TEXT                     Address to listen on (data, control, and
-                                    trigger sockets)
-    --data-port INTEGER
-    --control-port INTEGER
-    --wait-trigger                  Wait for a SOFTTRIGGER command on the
-                                    control port, or a trigger signal on the
-                                    trigger socket
-    --garbage                       Send garbage before trigger. Implies --wait-
-                                    trigger
-    --max-runs INTEGER
-    --help                          Show this message and exit.
-
-A suitable MIB dataset for testing can be downloaded at
-https://zenodo.org/record/5113449.
-
-See the :ref:`Merlin reference section <merlin reference>` for a description of
-the acquisition parameters.
+    detectors/*