support different buffer name and field name

pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "PyCXpress"
-version = "0.0.6"
+version = "0.0.7"
 description = "PyCXpress is a high-performance hybrid framework that seamlessly integrates Python and C++ to harness the flexibility of Python and the speed of C++ for efficient and expressive computation, particularly in the realm of deep learning and numerical computing."
 readme = "README.md"
 authors = ["chaoqing <[email protected]>"]

src/PyCXpress/core.py

@@ -79,6 +79,7 @@ def __new__(
         fields: Dict[str, TensorMeta],
         type: ModelAnnotationType,
         mode: ModelRuntimeType,
+        raw: bool = True,
     ):
         if type == ModelAnnotationType.Input:
             generate_property = mcs.generate_input_property
@@ -89,7 +90,7 @@ def __new__(
 
         for field_name, field_meta in fields.items():
             field_meta.setdefault(field_name)
-            attrs[field_name] = generate_property(field_meta)
+            attrs[field_name] = generate_property(field_meta, raw)
 
         get_buffer_shape, set_buffer_value, init_func = mcs.general_funcs(
             name, [field_meta.name for field_meta in fields.values()]
@@ -106,25 +107,27 @@ def __new__(
     @staticmethod
     def general_funcs(name: str, field_names: List[str]):
         def get_buffer_shape(self, name: str) -> Tuple[int]:
-            shape: Tuple[int] = getattr(self.__buffer_data__, name).shape
+            shape: Tuple[int] = self.__buffer_data__[name].shape
             return shape
 
         def set_buffer_value(self, name: str, value: np.ndarray) -> None:
-            buffer = getattr(self.__buffer_data__, name)
-            buffer.data = value
+            self.__buffer_data__[name].data = value
 
         def init_func(self):
-            _BufferData_ = namedtuple("_BufferData_", field_names)
-            self.__buffer_data__ = _BufferData_(
-                *tuple(TensorWithShape() for _ in field_names)
-            )
+            self.__buffer_data__ = {field: TensorWithShape() for field in field_names}
 
         return get_buffer_shape, set_buffer_value, init_func
 
     @staticmethod
-    def generate_input_property(field: TensorMeta):
+    def generate_input_property(field: TensorMeta, raw: bool):
         def get_func(self):
-            return getattr(self.__buffer_data__, field.name).data
+            data = self.__buffer_data__[field.name].data
+            if raw:
+                return data
+            else:
+                import tensorflow as tf
+
+                return tf.Variable(data, name=field.name)
 
         def set_func(*_):
             raise AssertionError("Not supported for input tensor")
@@ -135,16 +138,18 @@ def del_func(_):
         return property(fget=get_func, fset=set_func, fdel=del_func, doc=field.doc)
 
     @staticmethod
-    def generate_output_property(field: TensorMeta):
+    def generate_output_property(field: TensorMeta, raw: bool):
         def get_func(self):
             logger.warning(f"Only read the data field {field.name} in debugging mode")
-            buffer = getattr(self.__buffer_data__, field.name)
+            buffer = self.__buffer_data__[field.name]
             return buffer.data[: np.prod(buffer.shape)].reshape(buffer.shape)
 
         def set_func(self, data):
-            buffer = getattr(self.__buffer_data__, field.name)
+            buffer = self.__buffer_data__[field.name]
             buffer.shape = data.shape
-            buffer.data[: np.prod(data.shape)] = data.flatten()
+            len = np.prod(data.shape)
+            assert len <= buffer.data.size
+            buffer.data[:len] = (data if raw else data.numpy()).flatten()
 
         def del_func(_):
             raise AssertionError("Not supported for output tensor")

src/PyCXpress/example/Makefile

@@ -1,7 +1,7 @@
 # Compiler
 CC = c++
 PYTHONPATH=../../
-LD_PRELOAD:=$(shell find_libpython)
+LD_PRELOAD:=$(shell find_libpython):/opt/conda/envs/py38/lib/libstdc++.so.6.0.29
 
 # Compiler flags
 CFLAGS = -g -Wall -std=c++17 -fPIC

src/PyCXpress/example/main.cpp

@@ -17,7 +17,7 @@ void show_test(pcx::Model &model) {
     std::vector<uint8_t> shape   = {3, 4};
     void                *pBuffer = nullptr;
     size_t               nBytes  = 0;
-    std::tie(pBuffer, nBytes) = model.set_buffer("data_to_be_reshaped", {12});
+    std::tie(pBuffer, nBytes)    = model.set_buffer("input/data", {12});
     assert(data.size() * sizeof(double) == nBytes);
 
     memcpy(pBuffer, data.data(), nBytes);

src/PyCXpress/example/model.py

@@ -30,6 +30,7 @@ def show(a: np.ndarray):
 
 InputFields = dict(
     data_to_be_reshaped=TensorMeta(
+        name="input/data",
         dtype=np.float_,
         shape=(100,),
     ),
@@ -45,6 +46,7 @@ class InputDataSet(
     fields=InputFields,
     type=ModelAnnotationType.Input,
     mode=ModelRuntimeType.EagerExecution,
+    raw=False,
 ):
     pass
 
@@ -62,6 +64,7 @@ class OutputDataSet(
     fields=OutputFields,
     type=ModelAnnotationType.Output,
     mode=ModelRuntimeType.EagerExecution,
+    raw=False,
 ):
     pass
 
@@ -84,11 +87,22 @@ def run(self):
         self.model(self.input, self.output)
 
     @staticmethod
-    def model(input: InputDataSet, output: OutputDataSet):
+    def model(input: InputDataSet, output: OutputDataSet, use_tensorflow: bool = True):
         with nullcontext():
             # print(input.data_to_be_reshaped)
             # print(input.new_2d_shape)
-            output.output_a = input.data_to_be_reshaped.reshape(input.new_2d_shape).T
+            if use_tensorflow:
+                import tensorflow as tf
+
+                output.output_a = tf.transpose(
+                    tf.reshape(
+                        input.data_to_be_reshaped, tf.cast(input.new_2d_shape, tf.int32)
+                    )
+                )
+            else:
+                output.output_a = input.data_to_be_reshaped.reshape(
+                    input.new_2d_shape
+                ).T
             # print(output.output_a)
 
 
@@ -98,8 +112,7 @@ def main():
     input_data, output_data, spec = model.initialize()
     print(spec)
 
-    pycxpress_debugger()
-    input_data.set_buffer_value("data_to_be_reshaped", np.arange(12, dtype=np.float_))
+    input_data.set_buffer_value("input/data", np.arange(12, dtype=np.float_))
     print(input_data.data_to_be_reshaped)
     input_data.set_buffer_value("new_2d_shape", np.array([3, 4]).astype(np.uint8))
     print(input_data.new_2d_shape)
@@ -109,6 +122,8 @@ def main():
     print(output_data.output_a)
     print(output_data.get_buffer_shape("output_a"))
 
+    # pycxpress_debugger()
+
 
 if __name__ == "__main__":
     main()

src/PyCXpress/include/PyCXpress/core.hpp

@@ -162,15 +162,15 @@ class PYCXPRESS_EXPORT Model {
 
     BufferPtr set_buffer(const std::string         &name,
                          const std::vector<size_t> &shape) {
-        auto &buf  = m_buffers[name];
+        auto &buf  = m_buffers.at(name);
         auto  pBuf = buf.set(shape);
         m_input.attr("set_buffer_value")(name, buf.array());
         return pBuf;
     }
 
     std::pair<BufferPtr, std::vector<size_t>> get_buffer(
         const std::string &name) {
-        auto &buf   = m_buffers[name];
+        auto &buf   = m_buffers.at(name);
         auto &array = buf.array();
 
         auto                iter = m_output_buffer_sizes.find(name);