diff --git a/assets/images/coverage.svg b/assets/images/coverage.svg
index 5cc1bb5..cb3cdc0 100644
--- a/assets/images/coverage.svg
+++ b/assets/images/coverage.svg
@@ -15,7 +15,7 @@
coverage
coverage
- 42%
- 42%
+ 41%
+ 41%
diff --git a/pyproject.toml b/pyproject.toml
index 3614082..0246a09 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -5,7 +5,7 @@ build-backend = "poetry.core.masonry.api"
[tool.poetry]
name = "PyCXpress"
-version = "0.0.6"
+version = "0.0.7"
description = "PyCXpress is a high-performance hybrid framework that seamlessly integrates Python and C++ to harness the flexibility of Python and the speed of C++ for efficient and expressive computation, particularly in the realm of deep learning and numerical computing."
readme = "README.md"
authors = ["chaoqing "]
diff --git a/src/PyCXpress/core.py b/src/PyCXpress/core.py
index 7ae79c2..01af457 100644
--- a/src/PyCXpress/core.py
+++ b/src/PyCXpress/core.py
@@ -79,6 +79,7 @@ def __new__(
fields: Dict[str, TensorMeta],
type: ModelAnnotationType,
mode: ModelRuntimeType,
+ raw: bool = True,
):
if type == ModelAnnotationType.Input:
generate_property = mcs.generate_input_property
@@ -89,7 +90,7 @@ def __new__(
for field_name, field_meta in fields.items():
field_meta.setdefault(field_name)
- attrs[field_name] = generate_property(field_meta)
+ attrs[field_name] = generate_property(field_meta, raw)
get_buffer_shape, set_buffer_value, init_func = mcs.general_funcs(
name, [field_meta.name for field_meta in fields.values()]
@@ -106,25 +107,27 @@ def __new__(
@staticmethod
def general_funcs(name: str, field_names: List[str]):
def get_buffer_shape(self, name: str) -> Tuple[int]:
- shape: Tuple[int] = getattr(self.__buffer_data__, name).shape
+ shape: Tuple[int] = self.__buffer_data__[name].shape
return shape
def set_buffer_value(self, name: str, value: np.ndarray) -> None:
- buffer = getattr(self.__buffer_data__, name)
- buffer.data = value
+ self.__buffer_data__[name].data = value
def init_func(self):
- _BufferData_ = namedtuple("_BufferData_", field_names)
- self.__buffer_data__ = _BufferData_(
- *tuple(TensorWithShape() for _ in field_names)
- )
+ self.__buffer_data__ = {field: TensorWithShape() for field in field_names}
return get_buffer_shape, set_buffer_value, init_func
@staticmethod
- def generate_input_property(field: TensorMeta):
+ def generate_input_property(field: TensorMeta, raw: bool):
def get_func(self):
- return getattr(self.__buffer_data__, field.name).data
+ data = self.__buffer_data__[field.name].data
+ if raw:
+ return data
+ else:
+ import tensorflow as tf
+
+ return tf.Variable(data, name=field.name)
def set_func(*_):
raise AssertionError("Not supported for input tensor")
@@ -135,16 +138,18 @@ def del_func(_):
return property(fget=get_func, fset=set_func, fdel=del_func, doc=field.doc)
@staticmethod
- def generate_output_property(field: TensorMeta):
+ def generate_output_property(field: TensorMeta, raw: bool):
def get_func(self):
logger.warning(f"Only read the data field {field.name} in debugging mode")
- buffer = getattr(self.__buffer_data__, field.name)
+ buffer = self.__buffer_data__[field.name]
return buffer.data[: np.prod(buffer.shape)].reshape(buffer.shape)
def set_func(self, data):
- buffer = getattr(self.__buffer_data__, field.name)
+ buffer = self.__buffer_data__[field.name]
buffer.shape = data.shape
- buffer.data[: np.prod(data.shape)] = data.flatten()
+ len = np.prod(data.shape)
+ assert len <= buffer.data.size
+ buffer.data[:len] = (data if raw else data.numpy()).flatten()
def del_func(_):
raise AssertionError("Not supported for output tensor")
diff --git a/src/PyCXpress/example/Makefile b/src/PyCXpress/example/Makefile
index 06aa78b..feaa76d 100644
--- a/src/PyCXpress/example/Makefile
+++ b/src/PyCXpress/example/Makefile
@@ -1,7 +1,7 @@
# Compiler
CC = c++
PYTHONPATH=../../
-LD_PRELOAD:=$(shell find_libpython)
+LD_PRELOAD:=$(shell find_libpython):/opt/conda/envs/py38/lib/libstdc++.so.6.0.29
# Compiler flags
CFLAGS = -g -Wall -std=c++17 -fPIC
diff --git a/src/PyCXpress/example/main.cpp b/src/PyCXpress/example/main.cpp
index d8c3255..1208548 100644
--- a/src/PyCXpress/example/main.cpp
+++ b/src/PyCXpress/example/main.cpp
@@ -17,7 +17,7 @@ void show_test(pcx::Model &model) {
std::vector shape = {3, 4};
void *pBuffer = nullptr;
size_t nBytes = 0;
- std::tie(pBuffer, nBytes) = model.set_buffer("data_to_be_reshaped", {12});
+ std::tie(pBuffer, nBytes) = model.set_buffer("input/data", {12});
assert(data.size() * sizeof(double) == nBytes);
memcpy(pBuffer, data.data(), nBytes);
diff --git a/src/PyCXpress/example/model.py b/src/PyCXpress/example/model.py
index 51da35d..4e0652a 100644
--- a/src/PyCXpress/example/model.py
+++ b/src/PyCXpress/example/model.py
@@ -30,6 +30,7 @@ def show(a: np.ndarray):
InputFields = dict(
data_to_be_reshaped=TensorMeta(
+ name="input/data",
dtype=np.float_,
shape=(100,),
),
@@ -45,6 +46,7 @@ class InputDataSet(
fields=InputFields,
type=ModelAnnotationType.Input,
mode=ModelRuntimeType.EagerExecution,
+ raw=False,
):
pass
@@ -62,6 +64,7 @@ class OutputDataSet(
fields=OutputFields,
type=ModelAnnotationType.Output,
mode=ModelRuntimeType.EagerExecution,
+ raw=False,
):
pass
@@ -84,11 +87,22 @@ def run(self):
self.model(self.input, self.output)
@staticmethod
- def model(input: InputDataSet, output: OutputDataSet):
+ def model(input: InputDataSet, output: OutputDataSet, use_tensorflow: bool = True):
with nullcontext():
# print(input.data_to_be_reshaped)
# print(input.new_2d_shape)
- output.output_a = input.data_to_be_reshaped.reshape(input.new_2d_shape).T
+ if use_tensorflow:
+ import tensorflow as tf
+
+ output.output_a = tf.transpose(
+ tf.reshape(
+ input.data_to_be_reshaped, tf.cast(input.new_2d_shape, tf.int32)
+ )
+ )
+ else:
+ output.output_a = input.data_to_be_reshaped.reshape(
+ input.new_2d_shape
+ ).T
# print(output.output_a)
@@ -98,8 +112,7 @@ def main():
input_data, output_data, spec = model.initialize()
print(spec)
- pycxpress_debugger()
- input_data.set_buffer_value("data_to_be_reshaped", np.arange(12, dtype=np.float_))
+ input_data.set_buffer_value("input/data", np.arange(12, dtype=np.float_))
print(input_data.data_to_be_reshaped)
input_data.set_buffer_value("new_2d_shape", np.array([3, 4]).astype(np.uint8))
print(input_data.new_2d_shape)
@@ -109,6 +122,8 @@ def main():
print(output_data.output_a)
print(output_data.get_buffer_shape("output_a"))
+ # pycxpress_debugger()
+
if __name__ == "__main__":
main()
diff --git a/src/PyCXpress/include/PyCXpress/core.hpp b/src/PyCXpress/include/PyCXpress/core.hpp
index e2272b0..7dc9fd2 100644
--- a/src/PyCXpress/include/PyCXpress/core.hpp
+++ b/src/PyCXpress/include/PyCXpress/core.hpp
@@ -162,7 +162,7 @@ class PYCXPRESS_EXPORT Model {
BufferPtr set_buffer(const std::string &name,
const std::vector &shape) {
- auto &buf = m_buffers[name];
+ auto &buf = m_buffers.at(name);
auto pBuf = buf.set(shape);
m_input.attr("set_buffer_value")(name, buf.array());
return pBuf;
@@ -170,7 +170,7 @@ class PYCXPRESS_EXPORT Model {
std::pair> get_buffer(
const std::string &name) {
- auto &buf = m_buffers[name];
+ auto &buf = m_buffers.at(name);
auto &array = buf.array();
auto iter = m_output_buffer_sizes.find(name);