From 832b66cd1f889ee52342cb7b0fed4b274df79d28 Mon Sep 17 00:00:00 2001 From: Fabio Luporini Date: Wed, 14 Jun 2017 14:20:32 +0200 Subject: [PATCH 1/2] Remove unusued, broken memmap functionality --- devito/memmap_manager.py | 97 ---------------------------------------- 1 file changed, 97 deletions(-) delete mode 100644 devito/memmap_manager.py diff --git a/devito/memmap_manager.py b/devito/memmap_manager.py deleted file mode 100644 index 84370f881f..0000000000 --- a/devito/memmap_manager.py +++ /dev/null @@ -1,97 +0,0 @@ -import atexit -import os -import sys -from signal import SIGABRT, SIGINT, SIGSEGV, SIGTERM, signal -from tempfile import gettempdir - -from devito.logger import warning - - -class MemmapManager(): - """Class for managing all memmap related settings""" - # used to enable memmap as default - _use_memmap = False - # determine whether created files are deleted by default - _delete_file = True - # flag for registering exit func - _registered = False - # default directory to store memmap file - _default_disk_path = os.path.join(gettempdir(), "devito_disk") - # contains str name of all memmap file created - _created_data = {} - # unique id - _id = 0 - # exit code used for normal exiting - _default_exit_code = 0 - - @staticmethod - def set_memmap(memmap): - """Call this method to set default value of memmap""" - MemmapManager._use_memmap = memmap - - @staticmethod - def set_delete_file(delete): - """Call this method to set default flag contolling whether to delete - crated files. - """ - MemmapManager._delete_file = delete - - @staticmethod - def set_default_disk_path(default_disk_path): - """Call this method to change the default disk path for memmap""" - MemmapManager._default_disk_path = default_disk_path - - @staticmethod - def setup(data_self, *args, **kwargs): - """This method is used to setup memmap parameters for data classes. - - :param name: Name of data - :param memmap: Boolean indicates whether memmap is used. Optional - :param disk_path: String indicates directory to create memmap file. Optional - :param delete_file: Boolean indicates whether to delete created file. Optional - - Note: If memmap, disk_path or delete_file are not provided, the default values - are used. - """ - data_self.memmap = kwargs.get('memmap', MemmapManager._use_memmap) - - if data_self.memmap: - disk_path = kwargs.get('disk_path', MemmapManager._default_disk_path) - - if not os.path.exists(disk_path): - os.makedirs(disk_path) - - data_self.f = "%s/data_%s_%s" % (disk_path, kwargs.get('name'), - str(MemmapManager._id)) - MemmapManager._id += 1 - data_self.delete_file = kwargs.get('delete_file', MemmapManager._delete_file) - MemmapManager._created_data[data_self.f] = data_self.delete_file - - if not MemmapManager._registered: - MemmapManager._register_remove_memmap_file_signal() - MemmapManager._registered = True - - @staticmethod - def _remove_memmap_file(): - """This method is used to clean up memmap file""" - for f in MemmapManager._created_data: - if MemmapManager._created_data[f]: - try: - os.remove(f) - except OSError: - warning("error removing %s it may be already removed, skipping", f) - else: - warning("file %s has been left", f) - - @staticmethod - def _remove_memmap_file_on_signal(*args): - """This method is used to clean memmap file on signal, internal method""" - sys.exit(MemmapManager._default_exit_code) - - @staticmethod - def _register_remove_memmap_file_signal(): - """This method is used to register clean up method for chosen signals""" - atexit.register(MemmapManager._remove_memmap_file) - - for sig in (SIGABRT, SIGINT, SIGSEGV, SIGTERM): - signal(sig, MemmapManager._remove_memmap_file_on_signal) From 0a10314f2690968c042246ed848c32f9a9c60f7f Mon Sep 17 00:00:00 2001 From: Fabio Luporini Date: Wed, 14 Jun 2017 14:42:33 +0200 Subject: [PATCH 2/2] Fix data allocation and cleanup potential leaks and avoid extra array zeroing sweeps --- devito/interfaces.py | 47 ++++++++++++-------------------------------- devito/memory.py | 1 - 2 files changed, 13 insertions(+), 35 deletions(-) diff --git a/devito/interfaces.py b/devito/interfaces.py index aee9df1b19..7454b247e0 100644 --- a/devito/interfaces.py +++ b/devito/interfaces.py @@ -9,7 +9,6 @@ first_derivative, left, right, second_derivative) from devito.logger import debug, error, warning -from devito.memmap_manager import MemmapManager from devito.memory import CMemory, first_touch __all__ = ['DenseData', 'TimeData', 'Forward', 'Backward'] @@ -270,13 +269,11 @@ def __init__(self, *args, **kwargs): self.indices = self._indices(**kwargs) self.dtype = kwargs.get('dtype', np.float32) self.space_order = kwargs.get('space_order', 1) - initializer = kwargs.get('initializer', None) - if initializer is not None: - assert(callable(initializer)) - self.initializer = initializer + self.initializer = kwargs.get('initializer', None) + if self.initializer is not None: + assert(callable(self.initializer)) self.numa = kwargs.get('numa', False) - self._data = kwargs.get('_data', None) - MemmapManager.setup(self, *args, **kwargs) + self._data_object = None @classmethod def _indices(cls, **kwargs): @@ -304,21 +301,13 @@ def _indices(cls, **kwargs): return dimensions def _allocate_memory(self): - """Function to allocate memmory in terms of numpy ndarrays. - - Note: memmap is a subclass of ndarray. - """ - if self.memmap: - self._data = np.memmap(filename=self.f, dtype=self.dtype, mode='w+', - shape=self.shape, order='C') + """Allocate memory in terms of numpy ndarrays.""" + debug("Allocating memory for %s (%s)" % (self.name, str(self.shape))) + self._data_object = CMemory(self.shape, dtype=self.dtype) + if self.numa: + first_touch(self) else: - debug("Allocating memory for %s (%s)" % (self.name, str(self.shape))) - self._data_object = CMemory(self.shape, dtype=self.dtype) - self._data = self._data_object.ndpointer - if self.numa: - first_touch(self) - else: - self.data.fill(0) + self.data.fill(0) @property def data(self): @@ -326,16 +315,15 @@ def data(self): :returns: The ndarray containing the data """ - if self._data is None: + if self._data_object is None: self._allocate_memory() - return self._data + return self._data_object.ndpointer def initialize(self): """Apply the data initilisation function, if it is not None.""" if self.initializer is not None: self.initializer(self.data) - # Ignore if no initializer exists - assume no initialisation necessary @property def dx(self): @@ -521,7 +509,6 @@ class TimeData(DenseData): def __init__(self, *args, **kwargs): if not self._cached(): super(TimeData, self).__init__(*args, **kwargs) - self._full_data = self._data.view() if self._data else None time_dim = kwargs.get('time_dim', None) self.time_order = kwargs.get('time_order', 1) self.save = kwargs.get('save', False) @@ -543,9 +530,7 @@ def __init__(self, *args, **kwargs): def initialize(self): if self.initializer is not None: - if self._full_data is None: - self._allocate_memory() - self.initializer(self._full_data) + self.initializer(self.data) @classmethod def _indices(cls, **kwargs): @@ -562,12 +547,6 @@ def _indices(cls, **kwargs): _indices = DenseData._indices(**kwargs) return tuple([tidx] + list(_indices)) - def _allocate_memory(self): - """function to allocate memmory in terms of numpy ndarrays.""" - super(TimeData, self)._allocate_memory() - - self._full_data = self._data.view() - @property def dim(self): """Returns the spatial dimension of the data object""" diff --git a/devito/memory.py b/devito/memory.py index 2a37532ac8..15727c6eb3 100644 --- a/devito/memory.py +++ b/devito/memory.py @@ -16,7 +16,6 @@ class CMemory(object): def __init__(self, shape, dtype=np.float32, alignment=None): self.ndpointer, self.data_pointer = malloc_aligned(shape, alignment, dtype) - self.fill(0) def __del__(self): free(self.data_pointer)