Merge pull request #153 from JdeRobot/python-shared-memory

First implementation of wires using Python 3.8s shared memory

Author: Suhas-G

backend/staticfiles/synthesis/console.py

@@ -0,0 +1,9 @@
+class InvalidOutputNameException(Exception):
+    """Raised when Output name has not been declared in ports"""
+
+
+class InvalidInputNameException(Exception):
+    """Raised when Input name has not been declared in ports"""
+
+class InvalidParameterNameException(Exception):
+    """Raised when Parameter name has not been declared in ports"""

backend/staticfiles/synthesis/lib/block.py

@@ -0,0 +1,74 @@
+from multiprocessing import shared_memory
+
+import numpy as np
+from lib.exceptions import InvalidInputNameException
+from lib.utils import create_ndbuffer
+
+
+def create_readonly_wire(name):
+    try:
+        shm = shared_memory.SharedMemory(name, create=False)
+    except FileNotFoundError:
+        shm = None
+    return shm
+
+
+class Inputs:
+    def __init__(self, input_data) -> None:
+        self.inputs = input_data
+
+    def _read_npy_matrix(self, name, dtype):
+        if self.inputs[name].get("created", False):
+            dim = create_ndbuffer((1,), np.int64, self.inputs[name]["dim"].buf)[:][0]
+            shape = create_ndbuffer((dim,), np.int64, self.inputs[name]["shape"].buf)
+            data = create_ndbuffer(shape, dtype, self.inputs[name]["data"].buf)
+        else:
+            wire_name = self.inputs[name]["wire"]
+            data_wire = create_readonly_wire(wire_name)
+            shape_wire = create_readonly_wire(wire_name + "_shape")
+            dim_wire = create_readonly_wire(wire_name + "_dim")
+            if data_wire is None or shape_wire is None or dim_wire is None:
+                return None
+
+            self.inputs[name]["dim"] = dim_wire
+            dim = create_ndbuffer((1,), np.int64, dim_wire.buf)[:][0]
+            self.inputs[name]["shape"] = shape_wire
+            shape = create_ndbuffer((dim,), np.int64, shape_wire.buf)
+            self.inputs[name]["data"] = data_wire
+            data = create_ndbuffer(shape, dtype, data_wire.buf)
+            self.inputs[name]["created"] = True
+
+        return data
+
+    def read_image(self, name):
+        if self.inputs.get(name) is None:
+            raise InvalidInputNameException(f"{name} is not declared in inputs")
+
+        data = self._read_npy_matrix(name, np.uint8)
+        return data
+
+    def read_number(self, name):
+        if self.inputs.get(name) is None:
+            raise InvalidInputNameException(f"{name} is not declared in inputs")
+
+        number = None
+        if self.inputs[name].get("created", False):
+            number = self.inputs[name]["data"][:][0]
+        else:
+            wire_name = self.inputs[name]["wire"]
+            data_wire = create_readonly_wire(wire_name)
+            if data_wire is not None:
+                self.inputs[name]["data"] = create_ndbuffer(
+                    (1,), np.float64, data_wire.buf
+                )
+                number = self.inputs[name]["data"][:][0]
+                self.inputs[name]["created"] = True
+
+        return number
+
+    def read_array(self, name):
+        if self.inputs.get(name) is None:
+            raise InvalidInputNameException(f"{name} is not declared in inputs")
+
+        data = self._read_npy_matrix(name, np.float64)
+        return data

backend/staticfiles/synthesis/lib/exceptions.py

@@ -0,0 +1,77 @@
+from multiprocessing import shared_memory
+
+import numpy as np
+from lib.exceptions import InvalidOutputNameException
+from lib.utils import create_ndbuffer
+
+
+class Outputs:
+    def __init__(self, output_data) -> None:
+        self.outputs = output_data
+        self.shms = []
+
+    def _create_wire(self, name, size):
+        shm = shared_memory.SharedMemory(name=name, create=True, size=size)
+        self.shms.append(shm)
+        return shm
+
+    def _share_npy_matrix(self, name, matrix, shape):
+        dim = np.array([len(matrix.shape)], dtype=np.int64)
+        if self.outputs[name].get("created", False):
+            self.outputs[name]["shape"][:] = shape[:]
+            self.outputs[name]["data"][:] = matrix[:]
+        else:
+            wire_name = self.outputs[name]["wire"]
+            data_wire = self._create_wire(wire_name, matrix.nbytes)
+            shape_wire = self._create_wire(wire_name + "_shape", shape.nbytes)
+            dim_wire = self._create_wire(wire_name + "_dim", dim.nbytes)
+            self.outputs[name]["dim"] = create_ndbuffer((1,), np.int64, dim_wire.buf)
+            self.outputs[name]["dim"][:] = dim[:]
+            self.outputs[name]["shape"] = create_ndbuffer(
+                shape.shape, shape.dtype, shape_wire.buf
+            )
+            self.outputs[name]["shape"][:] = shape[:]
+            self.outputs[name]["data"] = create_ndbuffer(
+                shape, matrix.dtype, data_wire.buf
+            )
+            self.outputs[name]["data"][:] = matrix[:]
+            self.outputs[name]["created"] = True
+
+    def share_image(self, name, image):
+        if self.outputs.get(name) is None:
+            raise InvalidOutputNameException(f"{name} is not declared in outputs")
+
+        image = np.array(image, dtype=np.uint8)
+        if len(image.shape) != 2 and len(image.shape) != 3:
+            raise ValueError("Image must be 2D or 3D")
+
+        shape = (
+            image.shape
+            if len(image.shape) == 3
+            else (image.shape[0], image.shape[1], 1)
+        )
+        shape = np.array(shape, dtype=np.int64)
+        self._share_npy_matrix(name, image, shape)
+
+    def share_number(self, name, number):
+        if self.outputs.get(name) is None:
+            raise InvalidOutputNameException(f"{name} is not declared in outputs")
+
+        if self.outputs[name].get("created", False):
+            self.outputs[name]["data"][:] = number
+        else:
+            wire_name = self.outputs[name]["wire"]
+            data_wire = self._create_wire(
+                wire_name, np.array([1], dtype=np.float64).nbytes
+            )
+            self.outputs[name]["data"] = create_ndbuffer(
+                (1,), np.float64, data_wire.buf
+            )
+            self.outputs[name]["data"][:] = number
+            self.outputs[name]["created"] = True
+
+    def share_array(self, name, array):
+        if self.outputs.get(name) is None:
+            raise InvalidOutputNameException(f"{name} is not declared in outputs")
+        array = np.array(array, dtype=np.float64)
+        self._share_npy_matrix(name, array, np.array(array.shape, dtype=np.int64))

backend/staticfiles/synthesis/lib/inputs.py

@@ -0,0 +1,24 @@
+from lib.exceptions import InvalidParameterNameException
+
+
+class Parameters:
+    def __init__(self, parameter_data) -> None:
+        self.parameters = parameter_data
+
+    def read_number(self, name):
+        if self.parameters.get(name) is None:
+            raise InvalidParameterNameException(f"{name} is not declared in parameters")
+        
+        return float(self.parameters[name])
+
+    def read_string(self, name):
+        if self.parameters.get(name) is None:
+            raise InvalidParameterNameException(f"{name} is not declared in parameters")
+        
+        return str(self.parameters[name])
+
+    def read_bool(self, name):
+        if self.parameters.get(name) is None:
+            raise InvalidParameterNameException(f"{name} is not declared in parameters")
+
+        return bool(self.parameters[name])