Merge pull request #692 from opesci/no-mpi

devito without mpi

.travis.yml

@@ -19,7 +19,7 @@ matrix:
           packages:
             - gcc-4.9
             - g++-4.9
-      env: DEVITO_ARCH=gcc-4.9 DEVITO_OPENMP=0 INSTALL_TYPE=pip_setup RUN_EXAMPLES=False
+      env: DEVITO_ARCH=gcc-4.9 DEVITO_OPENMP=0 INSTALL_TYPE=pip_setup RUN_EXAMPLES=False MPI_INSTALL=1
     - os: linux
       python: "3.6"
       addons:
@@ -29,7 +29,7 @@ matrix:
           packages:
             - gcc-5
             - g++-5
-      env: DEVITO_ARCH=gcc-5 DEVITO_OPENMP=0 RUN_EXAMPLES=True INSTALL_TYPE=conda
+      env: DEVITO_ARCH=gcc-5 DEVITO_OPENMP=0 RUN_EXAMPLES=False INSTALL_TYPE=conda MPI_INSTALL=0
     - os: linux
       python: "3.6"
       addons:
@@ -39,7 +39,7 @@ matrix:
           packages:
             - gcc-4.9
             - g++-4.9
-      env: DEVITO_ARCH=gcc-4.9 DEVITO_OPENMP=1 OMP_NUM_THREADS=2 RUN_EXAMPLES=True INSTALL_TYPE=conda
+      env: DEVITO_ARCH=gcc-4.9 DEVITO_OPENMP=1 OMP_NUM_THREADS=2 RUN_EXAMPLES=True INSTALL_TYPE=conda MPI_INSTALL=1
     - os: linux
       python: "3.6"
       addons:
@@ -49,7 +49,7 @@ matrix:
           packages:
             - gcc-7
             - g++-7
-      env: DEVITO_ARCH=gcc-7 DEVITO_OPENMP=0 DEVITO_BACKEND=yask YC_CXX=g++-7 INSTALL_TYPE=conda RUN_EXAMPLES=False
+      env: DEVITO_ARCH=gcc-7 DEVITO_OPENMP=0 DEVITO_BACKEND=yask YC_CXX=g++-7 INSTALL_TYPE=conda RUN_EXAMPLES=False MPI_INSTALL=1
     - os: linux
       python: "3.6"
       addons:
@@ -59,7 +59,7 @@ matrix:
           packages:
             - gcc-7
             - g++-7
-      env: DEVITO_ARCH=gcc-7 DEVITO_OPENMP=0 DEVITO_BACKEND=ops YC_CXX=g++-7 INSTALL_TYPE=conda RUN_EXAMPLES=False
+      env: DEVITO_ARCH=gcc-7 DEVITO_OPENMP=0 DEVITO_BACKEND=ops YC_CXX=g++-7 INSTALL_TYPE=conda RUN_EXAMPLES=False MPI_INSTALL=0
   allow_failures:
     - os: linux
       python: "2.7"
@@ -87,7 +87,9 @@ before_install:
   - conda update -q conda
   # Useful for debugging any issues with conda
   - conda info -a
-  - sudo apt-get install -y -q mpich libmpich-dev
+  - if [[ "$MPI_INSTALL" == '1' ]]; then
+      sudo apt-get install -y -q mpich libmpich-dev;
+    fi
 
 install:
   # Install devito with conda
@@ -97,8 +99,13 @@ install:
       pip install -e .;
       conda list;
     fi
+  - if [[ "$MPI_INSTALL" == '1' ]]; then
+      pip install -r requirements-optional.txt;
+    fi
+
   # Install devito with pip
   - if [[ $INSTALL_TYPE == 'pip_setup' ]]; then python setup.py install; fi
+  - if [[ $MPI_INSTALL == '1' ]]; then pip install -e .[extras]; fi
 
 before_script:
   - echo -e "Host github.com\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config

Jenkinsfile

@@ -19,6 +19,7 @@ pipeline {
                      environment {
                          HOME="${WORKSPACE}"
                          DEVITO_OPENMP=0
+                         INSTALL_MPI=1
                          PYTHONPATH="${WORKSPACE}/lib/python3.6/site-packages/"
                      }
                      steps {
@@ -27,13 +28,14 @@ pipeline {
                          runPipTests()
                      }
                 }
-                stage('Build and test gcc-4.9 OpenMP container') {
+                stage('Build and test gcc-4.9 OpenMP container no MPI') {
                      agent { dockerfile { label 'azure-linux-8core'
                                           filename 'Dockerfile.jenkins'
                                           additionalBuildArgs "--build-arg gccvers=4.9" } }
                      environment {
                          HOME="${WORKSPACE}"
                          DEVITO_OPENMP=1
+                         INSTALL_MPI=0
                          OMP_NUM_THREADS=2
                      }
                      steps {
@@ -52,10 +54,12 @@ pipeline {
                      environment {
                          HOME="${WORKSPACE}"
                          DEVITO_OPENMP=0
+                         INSTALL_MPI=1
                      }
                      steps {
                          cleanWorkspace()
                          condaInstallDevito()
+                         condaInstallMPI()
                          runCondaTests()
                          runExamples()
                          runCodecov()
@@ -70,11 +74,13 @@ pipeline {
                          HOME="${WORKSPACE}"
                          DEVITO_BACKEND="yask"
                          DEVITO_OPENMP="0"
+                         INSTALL_MPI="1"
                          YC_CXX="g++-7"
                      }
                      steps {
                          cleanWorkspace()
                          condaInstallDevito()
+                         condaInstallMPI()
                          installYask()
                          runCondaTests()
                          runCodecov()
@@ -104,10 +110,12 @@ pipeline {
                      environment {
                          HOME="${WORKSPACE}"
                          DEVITO_OPENMP=0
+                         INSTALL_MPI=1
                      }
                      steps {
                          cleanWorkspace()
                          condaInstallDevito()
+                         condaInstallMPI()
                          runCondaTests()
                          runExamples()
                          runCodecov()
@@ -130,6 +138,10 @@ def condaInstallDevito () {
     sh 'source activate devito ; flake8 --exclude .conda,.git --builtins=ArgumentError .'
 }
 
+def condaInstallMPI () {
+    sh 'source activate devito ; pip install -r requirements-optional.txt'
+}
+
 def pipInstallDevito () {
     sh "mkdir -p ${WORKSPACE}/lib/python3.6/site-packages/"
     sh "python setup.py install --prefix=${WORKSPACE}"

devito/__init__.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 import os
 from subprocess import PIPE, Popen
 from collections import namedtuple

devito/mpi/distributed.py

@@ -19,9 +19,26 @@
 # Do not prematurely initialize MPI
 # This allows launching a Devito program from within another Python program
 # that has *already* initialized MPI
-import mpi4py
-mpi4py.rc(initialize=False, finalize=False)
-from mpi4py import MPI  # noqa
+try:
+    import mpi4py
+    mpi4py.rc(initialize=False, finalize=False)
+    from mpi4py import MPI  # noqa
+except ImportError:
+    # Dummy fallback in case mpi4py/MPI aren't available
+    class MPI(object):
+        COMM_NULL = None
+
+        @classmethod
+        def Is_initialized(cls):
+            return False
+
+        def _sizeof(comm):
+            return None
+
+        @property
+        def Comm(self):
+            return None
+
 
 __all__ = ['Distributor', 'SparseDistributor', 'MPI']
 

environment.yml

@@ -8,7 +8,6 @@ dependencies:
   - sympy==1.2
   - cgen
   - scipy
-  - mpi4py
   - matplotlib
   - pytest
   - flake8>=2.1.0

requirements-optional.txt

@@ -0,0 +1,2 @@
+mpi4py
+matplotlib

requirements.txt

@@ -1,8 +1,6 @@
 numpy==1.14
 sympy>=1.2
 scipy
-mpi4py
-matplotlib
 pytest-runner
 flake8>=2.1.0
 jedi

setup.py

@@ -5,6 +5,9 @@
 with open('requirements.txt') as f:
     required = f.read().splitlines()
 
+with open('requirements-optional.txt') as f:
+    optionals = f.read().splitlines()
+
 reqs = []
 links = []
 for ir in required:
@@ -14,6 +17,15 @@
     else:
         reqs += [ir]
 
+opt_reqs = []
+opt_links = []
+for ir in optionals:
+    if ir[0:3] == 'git':
+        opt_links += [ir + '#egg=' + ir.split('/')[-1] + '-0']
+        opt_reqs += [ir.split('/')[-1]]
+    else:
+        opt_reqs += [ir]
+
 setup(name='devito',
       version=versioneer.get_version(),
       cmdclass=versioneer.get_cmdclass(),
@@ -25,11 +37,12 @@
       equations defined in SymPy to create and execute highly
       optimised Finite Difference kernels on multiple computer
       platforms.""",
-      url='http://www.opesci.org/devito',
+      url='http://www.devitoproject.org',
       author="Imperial College London",
       author_email='[email protected]',
       license='MIT',
       packages=find_packages(exclude=['docs', 'tests', 'examples']),
       install_requires=reqs,
+      extras_require={'extras': opt_reqs},
       dependency_links=links,
       test_suite='tests')

tests/conftest.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 import pytest
 
 import os
@@ -17,13 +15,37 @@
 from devito.ir.iet import Iteration
 from devito.tools import as_tuple
 
-
-def skipif_backend(backends):
-    conditions = []
-    for b in backends:
-        conditions.append(b == configuration['backend'])
-    return pytest.mark.skipif(any(conditions),
-                              reason="{} testing is currently restricted".format(b))
+try:
+    from mpi4py import MPI  # noqa
+except ImportError:
+    MPI = None
+
+
+def skipif(items):
+    items = as_tuple(items)
+    # Sanity check
+    accepted = set(configuration._accepted['backend'])
+    accepted.update({'no%s' % i for i in configuration._accepted['backend']})
+    accepted.update({'nompi'})
+    unknown = sorted(set(items) - accepted)
+    if unknown:
+        raise ValueError("Illegal skipif argument(s) `%s`" % unknown)
+    for i in items:
+        # Skip if no MPI
+        if i == 'nompi':
+            if MPI is None:
+                return pytest.mark.skipif(True, reason="mpi4py/MPI not installed")
+            continue
+        # Skip if an unsupported backend
+        if i == configuration['backend']:
+            return pytest.mark.skipif(True, reason="`%s` backend unsupported" % i)
+        try:
+            _, noi = i.split('no')
+            if noi != configuration['backend']:
+                return pytest.mark.skipif(True, reason="`%s` backend unsupported" % i)
+        except ValueError:
+            pass
+    return pytest.mark.skipif(False, reason="")
 
 
 # Testing dimensions for space and time
@@ -279,19 +301,18 @@ def wrapper(*args, **kwargs):
     return dec
 
 
-# Support to run MPI tests
-# This is partly extracted from:
-# `https://github.com/firedrakeproject/firedrake/blob/master/tests/conftest.py`
-
-mpi_exec = 'mpiexec'
-mpi_distro = sniff_mpi_distro(mpi_exec)
-
-
 def parallel(item):
     """Run a test in parallel.
 
     :parameter item: The test item to run.
     """
+    # Support to run MPI tests
+    # This is partly extracted from:
+    # `https://github.com/firedrakeproject/firedrake/blob/master/tests/conftest.py`
+
+    mpi_exec = 'mpiexec'
+    mpi_distro = sniff_mpi_distro(mpi_exec)
+
     marker = item.get_closest_marker("parallel")
     nprocs = as_tuple(marker.kwargs.get("nprocs", 2))
     for i in nprocs:

tests/test_adjoint.py

@@ -1,15 +1,14 @@
 import numpy as np
 import pytest
 from numpy import linalg
-from conftest import unit_box, points
-from devito import clear_cache, Operator, configuration
+
+from conftest import unit_box, points, skipif
+from devito import clear_cache, Operator
 from devito.logger import info
 from examples.seismic import demo_model, Receiver
 from examples.seismic.acoustic import acoustic_setup
 
-pytestmark = pytest.mark.skipif(configuration['backend'] == 'yask' or
-                                configuration['backend'] == 'ops',
-                                reason="testing is currently restricted")
+pytestmark = skipif(['yask', 'ops'])
 
 presets = {
     'constant': {'preset': 'constant-isotropic'},

tests/test_autotuner.py

@@ -3,14 +3,14 @@
 
 import pytest
 import numpy as np
+from unittest.mock import patch
+
+from conftest import skipif
 from devito import (Grid, Function, TimeFunction, Eq, Operator, configuration,
                     switchconfig)
 from devito.types import LEFT
-from unittest.mock import patch
 
-pytestmark = pytest.mark.skipif(configuration['backend'] == 'yask' or
-                                configuration['backend'] == 'ops',
-                                reason="testing is currently restricted")
+pytestmark = skipif(['yask', 'ops'])
 
 
 # To enforce cross-compilation for a 4-core architecture
@@ -210,6 +210,7 @@ def test_discarding_runs():
     assert op._state['autotuning'][1]['tuned']['nthreads'] == 1
 
 
+@skipif('nompi')
 @pytest.mark.parallel(nprocs=2)
 def test_at_w_mpi():
     """Make sure autotuning works in presence of MPI. MPI ranks work

tests/test_checkpointing.py

@@ -1,16 +1,16 @@
-from examples.checkpointing.checkpoint import DevitoCheckpoint, CheckpointOperator
-from examples.seismic.acoustic.acoustic_example import acoustic_setup
-from examples.seismic import Receiver
+from functools import reduce
+
+import pytest
 from pyrevolve import Revolver
 import numpy as np
-import pytest
-from functools import reduce
 
-from devito import Grid, TimeFunction, Operator, Function, Eq, configuration, switchconfig
+from conftest import skipif
+from devito import Grid, TimeFunction, Operator, Function, Eq, switchconfig
+from examples.checkpointing.checkpoint import DevitoCheckpoint, CheckpointOperator
+from examples.seismic.acoustic.acoustic_example import acoustic_setup
+from examples.seismic import Receiver
 
-pytestmark = pytest.mark.skipif(configuration['backend'] == 'yask' or
-                                configuration['backend'] == 'ops',
-                                reason="testing is currently restricted")
+pytestmark = skipif(['yask', 'ops'])
 
 
 @switchconfig(log_level='WARNING')