diff --git a/.github/workflows/test_suite_ubuntu.yml b/.github/workflows/test_suite_ubuntu.yml index aab34ede..7d71ffaf 100644 --- a/.github/workflows/test_suite_ubuntu.yml +++ b/.github/workflows/test_suite_ubuntu.yml @@ -66,7 +66,7 @@ jobs: - name: Install an MPI distribution run: | sudo apt update - sudo apt install mpich + sudo apt install openmpi-bin openmpi-common libopenmpi-dev - name: Install pFUnit run: | diff --git a/examples/7_MPI/CMakeLists.txt b/examples/7_MPI/CMakeLists.txt new file mode 100644 index 00000000..4179be86 --- /dev/null +++ b/examples/7_MPI/CMakeLists.txt @@ -0,0 +1,66 @@ +cmake_minimum_required(VERSION 3.15...3.31) +# policy CMP0076 - target_sources source files are relative to file where +# target_sources is run +cmake_policy(SET CMP0076 NEW) + +set(PROJECT_NAME MPIExample) + +project(${PROJECT_NAME} LANGUAGES Fortran) + +# Build in Debug mode if not specified +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE + Debug + CACHE STRING "" FORCE) +endif() + +find_package(FTorch) +find_package(MPI REQUIRED) +message(STATUS "Building with Fortran PyTorch coupling") + +# Fortran example +add_executable(mpi_infer_fortran mpi_infer_fortran.f90) +target_link_libraries(mpi_infer_fortran PRIVATE FTorch::ftorch) +target_link_libraries(mpi_infer_fortran PRIVATE MPI::MPI_Fortran) + +# Integration testing +if(CMAKE_BUILD_TESTS) + include(CTest) + + # 1. Check the PyTorch model runs and its outputs meet expectations + add_test(NAME simplenet COMMAND ${Python_EXECUTABLE} + ${PROJECT_SOURCE_DIR}/simplenet.py) + + # 2. Check the model is saved to file in the expected location with the + # pt2ts.py script + add_test( + NAME pt2ts + COMMAND ${Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/pt2ts.py --filepath + ${PROJECT_BINARY_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) + + # 3. Check the model can be loaded from file and run with MPI in Python and + # that its outputs meet expectations + add_test( + NAME mpi_infer_python + COMMAND + ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 2 ${Python_EXECUTABLE} + ${PROJECT_SOURCE_DIR}/mpi_infer_python.py --filepath ${PROJECT_BINARY_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) + set_tests_properties( + mpi_infer_python PROPERTIES PASS_REGULAR_EXPRESSION + "MPI Python example ran successfully") + + # 4. Check the model can be loaded from file and run with MPI in Fortran and + # that its outputs meet expectations + add_test( + NAME mpi_infer_fortran + COMMAND + ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 2 ./mpi_infer_fortran + ${PROJECT_BINARY_DIR}/saved_simplenet_model_cpu.pt + # Command line argument: model file + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) + set_tests_properties( + mpi_infer_fortran PROPERTIES PASS_REGULAR_EXPRESSION + "MPI Fortran example ran successfully") +endif() diff --git a/examples/7_MPI/README.md b/examples/7_MPI/README.md new file mode 100644 index 00000000..e2dd48e3 --- /dev/null +++ b/examples/7_MPI/README.md @@ -0,0 +1,117 @@ +# Example 7 - MPI + +This example revisits the SimpleNet example and demonstrates how to run it using +MPI parallelism. + + +## Description + +The Python file `simplenet.py` is copied from the earlier example. Recall that +it defines a very simple PyTorch network that takes an input of length 5 and +applies a single `Linear` layer to multiply it by 2. + +The same `pt2ts.py` tool is used to save the simple network to TorchScript. + +A series of files `mpi_infer_` then bind from other languages to run the +TorchScript model in inference mode. + +## Dependencies + +To run this example requires: + +- CMake +- An MPI installation. +- mpif90 +- FTorch (installed as described in main package) +- Python 3 + +## Running + +To run this example install FTorch as described in the main documentation. Then +from this directory create a virtual environment and install the necessary +Python modules: +``` +python3 -m venv venv +source venv/bin/activate +pip install -r requirements.txt +``` + +You can check the network is set up correctly by running `simplenet.py`: +``` +python3 simplenet.py +``` +As before, this defines the network and runs it with an input tensor +[0.0, 1.0, 2.0, 3.0, 4.0] to produce the result: +``` +tensor([[0, 2, 4, 6, 8]]) +``` + +To save the `SimpleNet`` model to TorchScript run the modified version of the +`pt2ts.py` tool: +``` +python3 pt2ts.py +``` +which will generate `saved_simplenet_model_cpu.pt` - the TorchScript instance +of the network. + +You can check that everything is working by running the `mpi_infer_python.py` +script. It's set up with MPI such that a different GPU device is associated +with each MPI rank. You should substitute `` with the number of GPUs you +wish to run with: +``` +mpiexec -np python3 multigpu_infer_python.py +``` +This reads the model in from the TorchScript file and runs it with an different +input tensor on each GPU device: [0.0, 1.0, 2.0, 3.0, 4.0], plus the device +index in each entry. Running with `NP=2`, the result should be (some +permutation of): +``` +rank 0: result: +tensor([[0., 2., 4., 6., 8.]]) +rank 1: result: +tensor([[ 2., 4., 6., 8., 10.]]) +``` + +At this point we no longer require Python, so can deactivate the virtual +environment: +``` +deactivate +``` + +To call the saved `SimpleNet` model from Fortran we need to compile the +`mpi_infer_fortran.f90` file. This can be done using the included +`CMakeLists.txt` as follows, noting that we need to use an MPI-enabled Fortran +compiler: +``` +mkdir build +cd build +cmake .. -DCMAKE_PREFIX_PATH= -DCMAKE_BUILD_TYPE=Release +cmake --build . +``` + +(Note that the Fortran compiler can be chosen explicitly with the +`-DCMAKE_Fortran_COMPILER` flag, and should match the compiler that was used to +locally build FTorch.) + +To run the compiled code calling the saved `SimpleNet` TorchScript from Fortran, +run the executable with an argument of the saved model file. Again, specify the +number of MPI processes according to the desired number of GPUs: +``` +mpiexec -np ./mpi_infer_fortran ../saved_simplenet_model_cpu.pt +``` + +This runs the model with the same inputs as described above and should produce (some +permutation of) the output: +``` +input on rank 0: [ 0.0, 1.0, 2.0, 3.0, 4.0] +input on rank 1: [ 1.0, 2.0, 3.0, 4.0, 5.0] +output on rank 0: [ 0.0, 2.0, 4.0, 6.0, 8.0] +output on rank 1: [ 2.0, 4.0, 6.0, 8.0, 10.0] +``` + +Alternatively, we can use `make`, instead of CMake, copying the Makefile over from the +first example: +``` +cp ../1_SimpleNet/Makefile . +``` +See the instructions in that example directory for further details. diff --git a/examples/7_MPI/mpi_infer_fortran.f90 b/examples/7_MPI/mpi_infer_fortran.f90 new file mode 100644 index 00000000..2bf40298 --- /dev/null +++ b/examples/7_MPI/mpi_infer_fortran.f90 @@ -0,0 +1,124 @@ +program inference + + ! Import precision info from iso + use, intrinsic :: iso_fortran_env, only : sp => real32 + + ! Import our library for interfacing with PyTorch + use ftorch, only : torch_model, torch_tensor, torch_kCPU, torch_delete, & + torch_tensor_from_array, torch_model_load, torch_model_forward + + ! Import our tools module for testing utils + use ftorch_test_utils, only : assert_allclose + + ! Import MPI + use mpi, only : mpi_comm_rank, mpi_comm_size, mpi_comm_world, mpi_finalize, mpi_float, & + mpi_gather, mpi_init + + implicit none + + ! Set working precision for reals + integer, parameter :: wp = sp + + integer :: num_args, ix + character(len=128), dimension(:), allocatable :: args + + ! Set up Fortran data structures + real(wp), dimension(5), target :: in_data + real(wp), dimension(5), target :: out_data + real(wp), dimension(5), target :: expected + integer, parameter :: tensor_layout(1) = [1] + + ! Set up Torch data structures + ! The net, a vector of input tensors (in this case we only have one), and the output tensor + type(torch_model) :: model + type(torch_tensor), dimension(1) :: in_tensors + type(torch_tensor), dimension(1) :: out_tensors + + ! Flag for testing + logical :: test_pass + + ! MPI configuration + integer :: rank, size, ierr, i + + ! Variables for testing + real(wp), allocatable, dimension(:,:) :: recvbuf + real(wp), dimension(5) :: result_chk + integer :: rank_chk + + call mpi_init(ierr) + call mpi_comm_rank(mpi_comm_world, rank, ierr) + call mpi_comm_size(mpi_comm_world, size, ierr) + + ! Check MPI was configured correctly + if (size == 1) then + write(*,*) "MPI communicator size is 1, indicating that it is not configured correctly" + write(*,*) "(assuming you specified more than one rank)" + call clean_up() + stop 999 + end if + + ! Get TorchScript model file as a command line argument + num_args = command_argument_count() + allocate(args(num_args)) + do ix = 1, num_args + call get_command_argument(ix,args(ix)) + end do + + ! Initialise data and print the values used on each MPI rank + in_data = [(rank + i, i = 0, 4)] + write(unit=6, fmt="('input on rank ',i1,': ')", advance="no") rank + write(unit=6, fmt=100) in_data(:) + 100 format('[',4(f5.1,','),f5.1,']') + + ! Create Torch input/output tensors from the above arrays + call torch_tensor_from_array(in_tensors(1), in_data, tensor_layout, torch_kCPU) + call torch_tensor_from_array(out_tensors(1), out_data, tensor_layout, torch_kCPU) + + ! Load ML model + call torch_model_load(model, args(1), torch_kCPU) + + ! Run inference on each MPI rank + call torch_model_forward(model, in_tensors, out_tensors) + + ! Print the values computed on each MPI rank + write(unit=6, fmt="('output on rank ',i1,': ')", advance="no") rank + write(unit=6, fmt=100) out_data(:) + + ! Gather the outputs onto rank 0 + allocate(recvbuf(5,size)) + call mpi_gather(out_data, 5, mpi_float, recvbuf, 5, mpi_float, 0, mpi_comm_world, ierr) + + ! Check that the correct values were attained + if (rank == 0) then + + ! Check output tensor matches expected value + do rank_chk = 0, size-1 + expected = [(2 * (rank_chk + i), i = 0, 4)] + result_chk(:) = recvbuf(:,rank_chk+1) + test_pass = assert_allclose(result_chk, expected, test_name="MPI") + if (.not. test_pass) then + write(unit=6, fmt="('rank ',i1,' result: ')") rank_chk + write(unit=6, fmt=100) result_chk(:) + write(unit=6, fmt="('does not match expected value')") + write(unit=6, fmt=100) expected(:) + call clean_up() + stop 999 + end if + end do + + write (*,*) "MPI Fortran example ran successfully" + end if + + call clean_up() + + contains + + subroutine clean_up() + call torch_delete(model) + call torch_delete(in_tensors) + call torch_delete(out_tensors) + call mpi_finalize(ierr) + deallocate(recvbuf) + end subroutine clean_up + +end program inference diff --git a/examples/7_MPI/mpi_infer_python.py b/examples/7_MPI/mpi_infer_python.py new file mode 100644 index 00000000..dccca102 --- /dev/null +++ b/examples/7_MPI/mpi_infer_python.py @@ -0,0 +1,103 @@ +"""Load saved SimpleNet to TorchScript and run inference example.""" + +import os +import sys + +import torch +from mpi4py import MPI + + +def deploy(saved_model: str, device: str, batch_size: int = 1) -> torch.Tensor: + """ + Load TorchScript SimpleNet and run inference with example Tensor. + + Parameters + ---------- + saved_model : str + location of SimpleNet model saved to Torchscript + device : str + Torch device to run model on, 'cpu' or 'cuda' + batch_size : int + batch size to run (default 1) + + Returns + ------- + output : torch.Tensor + result of running inference on model with example Tensor input + """ + input_tensor = torch.tensor([0.0, 1.0, 2.0, 3.0, 4.0]).repeat(batch_size, 1) + + # Add the rank (device index) to each tensor to make them differ + input_tensor += MPI.COMM_WORLD.rank + + if device == "cpu": + # Load saved TorchScript model + model = torch.jit.load(saved_model) + # Inference + output = model.forward(input_tensor) + + elif device == "cuda": + # All previously saved modules, no matter their device, are first + # loaded onto CPU, and then are moved to the devices they were saved + # from, so we don't need to manually transfer the model to the GPU + model = torch.jit.load(saved_model) + input_tensor_gpu = input_tensor.to(torch.device("cuda")) + output_gpu = model.forward(input_tensor_gpu) + output = output_gpu.to(torch.device("cpu")) + + else: + device_error = f"Device '{device}' not recognised." + raise ValueError(device_error) + + return output + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--filepath", + help="Path to the file containing the PyTorch model", + type=str, + default=os.path.dirname(__file__), + ) + parsed_args = parser.parse_args() + filepath = parsed_args.filepath + saved_model_file = os.path.join(filepath, "saved_simplenet_model_cpu.pt") + + comm = MPI.COMM_WORLD + rank = comm.rank + device_to_run = "cpu" + if comm.size == 1: + size_error = ( + "MPI communicator size is 1, indicating that it is not configured correctly" + " (assuming you specified more than one rank)" + ) + raise ValueError(size_error) + + batch_size_to_run = 1 + + # Run inference on each rank + with torch.no_grad(): + result = deploy(saved_model_file, device_to_run, batch_size_to_run) + print(f"rank {rank}: result:\n{result}") + + # Gather the outputs onto rank 0 + recvbuf = torch.empty([comm.size, 5], dtype=torch.float32) if rank == 0 else None + comm.Gather(result, recvbuf, root=0) + + # Check that the correct values were attained + if rank == 0: + for rank_chk, result_chk in enumerate(recvbuf): + expected = torch.Tensor([2 * (i + rank_chk) for i in range(5)]) + if not torch.allclose(expected, result_chk): + result_error = ( + f"rank {rank_chk}: result:\n{result_chk}\n" + f"does not match expected value:\n{expected}" + ) + raise ValueError(result_error) + + print("MPI Python example ran successfully") diff --git a/examples/7_MPI/pt2ts.py b/examples/7_MPI/pt2ts.py new file mode 100644 index 00000000..58755dde --- /dev/null +++ b/examples/7_MPI/pt2ts.py @@ -0,0 +1,197 @@ +"""Load a PyTorch model and convert it to TorchScript.""" + +import os +import sys +from typing import Optional + +# FPTLIB-TODO +# Add a module import with your model here: +# This example assumes the model architecture is in an adjacent module `my_ml_model.py` +import simplenet +import torch + + +def script_to_torchscript( + model: torch.nn.Module, filename: Optional[str] = "scripted_model.pt" +) -> None: + """ + Save PyTorch model to TorchScript using scripting. + + Parameters + ---------- + model : torch.NN.Module + a PyTorch model + filename : str + name of file to save to + """ + print("Saving model using scripting...", end="") + scripted_model = torch.jit.script(model) + # print(scripted_model.code) + scripted_model.save(filename) + print("done.") + + +def trace_to_torchscript( + model: torch.nn.Module, + dummy_input: torch.Tensor, + filename: Optional[str] = "traced_model.pt", +) -> None: + """ + Save PyTorch model to TorchScript using tracing. + + Parameters + ---------- + model : torch.NN.Module + a PyTorch model + dummy_input : torch.Tensor + appropriate size Tensor to act as input to model + filename : str + name of file to save to + """ + print("Saving model using tracing...", end="") + traced_model = torch.jit.trace(model, dummy_input) + frozen_model = torch.jit.freeze(traced_model) + ## print(frozen_model.graph) + ## print(frozen_model.code) + frozen_model.save(filename) + print("done.") + + +def load_torchscript(filename: Optional[str] = "saved_model.pt") -> torch.nn.Module: + """ + Load a TorchScript from file. + + Parameters + ---------- + filename : str + name of file containing TorchScript model + """ + model = torch.jit.load(filename) + + return model + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--device_type", + help="Device type to run the inference on", + type=str, + choices=["cpu", "cuda", "xpu", "mps"], + default="cpu", + ) + parser.add_argument( + "--filepath", + help="Path to the file containing the PyTorch model", + type=str, + default=os.path.dirname(__file__), + ) + parsed_args = parser.parse_args() + device_type = parsed_args.device_type + filepath = parsed_args.filepath + + # ===================================================== + # Load model and prepare for saving + # ===================================================== + + # FPTLIB-TODO + # Load a pre-trained PyTorch model + # Insert code here to load your model as `trained_model`. + # This example assumes my_ml_model has a method `initialize` to load + # architecture, weights, and place in inference mode + trained_model = simplenet.SimpleNet() + + # Switch off specific layers/parts of the model that behave + # differently during training and inference. + # This may have been done by the user already, so just make sure here. + trained_model.eval() + + # ===================================================== + # Prepare dummy input and check model runs + # ===================================================== + + # FPTLIB-TODO + # Generate a dummy input Tensor `dummy_input` to the model of appropriate size. + # This example assumes one input of size (5) + trained_model_dummy_input = torch.ones(5) + + # Transfer the model and inputs to GPU device, if appropriate + if device_type != "cpu": + device = torch.device(device_type) + trained_model = trained_model.to(device) + trained_model.eval() + trained_model_dummy_input = trained_model_dummy_input.to(device) + + # FPTLIB-TODO + # Run model for dummy inputs + # If something isn't working This will generate an error + trained_model_dummy_outputs = trained_model( + trained_model_dummy_input, + ) + + # ===================================================== + # Save model + # ===================================================== + + # FPTLIB-TODO + # Set the name of the file you want to save the torchscript model to: + saved_ts_filename = f"saved_simplenet_model_{device_type}.pt" + # A filepath may also be provided. To do this, pass the filepath as an argument to + # this script when it is run from the command line, i.e. `./pt2ts.py path/to/model`. + + # FPTLIB-TODO + # Save the PyTorch model using either scripting (recommended if possible) or tracing + # ----------- + # Scripting + # ----------- + script_to_torchscript(trained_model, filename=saved_ts_filename) + + # ----------- + # Tracing + # ----------- + # trace_to_torchscript( + # trained_model, trained_model_dummy_input, filename=saved_ts_filename + # ) + + print(f"Saved model to TorchScript in '{saved_ts_filename}'.") + + # ===================================================== + # Check model saved OK + # ===================================================== + + # Load torchscript and run model as a test, scaling inputs as above + trained_model_dummy_input = 2.0 * trained_model_dummy_input + trained_model_testing_outputs = trained_model( + trained_model_dummy_input, + ) + ts_model = load_torchscript(filename=saved_ts_filename) + ts_model_outputs = ts_model( + trained_model_dummy_input, + ) + + if not isinstance(ts_model_outputs, tuple): + ts_model_outputs = (ts_model_outputs,) + if not isinstance(trained_model_testing_outputs, tuple): + trained_model_testing_outputs = (trained_model_testing_outputs,) + for ts_output, output in zip(ts_model_outputs, trained_model_testing_outputs): + if torch.all(ts_output.eq(output)): + print("Saved TorchScript model working as expected in a basic test.") + print("Users should perform further validation as appropriate.") + else: + model_error = ( + "Saved Torchscript model is not performing as expected.\n" + "Consider using scripting if you used tracing, or investigate further." + ) + raise RuntimeError(model_error) + + # Check that the model file is created + if not os.path.exists(os.path.join(filepath, saved_ts_filename)): + torchscript_file_error = ( + f"Saved TorchScript file {os.path.join(filepath, saved_ts_filename)} " + "cannot be found." + ) + raise FileNotFoundError(torchscript_file_error) diff --git a/examples/7_MPI/requirements.txt b/examples/7_MPI/requirements.txt new file mode 100644 index 00000000..604f78d3 --- /dev/null +++ b/examples/7_MPI/requirements.txt @@ -0,0 +1,2 @@ +torch +mpi4py diff --git a/examples/7_MPI/simplenet.py b/examples/7_MPI/simplenet.py new file mode 100644 index 00000000..66e89952 --- /dev/null +++ b/examples/7_MPI/simplenet.py @@ -0,0 +1,58 @@ +"""Module defining a simple PyTorch 'Net' for coupling to Fortran.""" + +import torch +from torch import nn + + +class SimpleNet(nn.Module): + """PyTorch module multiplying an input vector by 2.""" + + def __init__( + self, + ) -> None: + """ + Initialize the SimpleNet model. + + Consists of a single Linear layer with weights predefined to + multiply the input by 2. + """ + super().__init__() + self._fwd_seq = nn.Sequential( + nn.Linear(5, 5, bias=False), + ) + with torch.no_grad(): + self._fwd_seq[0].weight = nn.Parameter(2.0 * torch.eye(5)) + + def forward(self, batch: torch.Tensor) -> torch.Tensor: + """ + Pass ``batch`` through the model. + + Parameters + ---------- + batch : torch.Tensor + A mini-batch of input vectors of length 5. + + Returns + ------- + torch.Tensor + batch scaled by 2. + + """ + return self._fwd_seq(batch) + + +if __name__ == "__main__": + model = SimpleNet() + model.eval() + + input_tensor = torch.Tensor([0.0, 1.0, 2.0, 3.0, 4.0]) + with torch.no_grad(): + output_tensor = model(input_tensor) + + print(output_tensor) + if not torch.allclose(output_tensor, 2 * input_tensor): + result_error = ( + f"result:\n{output_tensor}\ndoes not match expected value:\n" + f"{2 * input_tensor}" + ) + raise ValueError(result_error) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index c69812f8..9dcbb52d 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -10,4 +10,8 @@ if(CMAKE_BUILD_TESTS) # practice, as opposed to functionality. add_subdirectory(5_Looping) add_subdirectory(6_Autograd) + # NOTE: We do not currently support running with MPI on Windows + if(UNIX) + add_subdirectory(7_MPI) + endif() endif() diff --git a/pages/examples.md b/pages/examples.md index 3c55b1ad..aba4f664 100644 --- a/pages/examples.md +++ b/pages/examples.md @@ -185,3 +185,8 @@ is currently under development. Eventually, it will demonstrate how to perform automatic differentiation in FTorch by leveraging PyTorch's Autograd module. Currently, it just demonstrates how to use `torch_tensor_to_array` and compute mathematical expressions involving Torch tensors. + +#### 7) MPI +[This worked example](https://github.com/Cambridge-ICCS/FTorch/tree/main/examples/7_MPI) +demonstrates how to run the SimpleNet example in the context of MPI parallelism, +running the net with different input arrays on each MPI rank. diff --git a/pages/hpc.md b/pages/hpc.md index 9e5b40f3..eeb70adf 100644 --- a/pages/hpc.md +++ b/pages/hpc.md @@ -152,3 +152,19 @@ or similar. This process should also add FTorch to the `LD_LIBRARY_PATH` and `CMAKE_PREFIX_PATH` rather than requiring the user to specify them manually as suggested elsewhere in this documentation. + +## Parallelism + +If you are investigating running FTorch on HPC then you are probably interested +in improving computational efficiency via parallelism. + +#### MPI + +For a worked example of running with MPI, see the +[associated example](https://github.com/Cambridge-ICCS/FTorch/tree/main/examples/7_MPI). + +#### GPU + +For information on running on GPU architectures, see the +[GPU user guide page](pages/gpu.html) and/or the +[MultiGPU example](https://github.com/Cambridge-ICCS/FTorch/tree/main/examples/3_MultiGPU). diff --git a/run_test_suite.sh b/run_test_suite.sh index 755c419a..e8528023 100755 --- a/run_test_suite.sh +++ b/run_test_suite.sh @@ -83,9 +83,9 @@ fi # Run integration tests if [ "${RUN_INTEGRATION}" = true ]; then if [ -e "${BUILD_DIR}/examples/3_MultiGPU" ]; then - EXAMPLES="1_SimpleNet 2_ResNet18 3_MultiGPU 4_MultiIO 6_Autograd" + EXAMPLES="1_SimpleNet 2_ResNet18 3_MultiGPU 4_MultiIO 6_Autograd 7_MPI" else - EXAMPLES="1_SimpleNet 2_ResNet18 4_MultiIO 6_Autograd" + EXAMPLES="1_SimpleNet 2_ResNet18 4_MultiIO 6_Autograd 7_MPI" fi export PIP_REQUIRE_VIRTUALENV=true for EXAMPLE in ${EXAMPLES}; do