add tensorflow as extra

.vscode/launch.json

@@ -26,9 +26,17 @@
             "stopAtEntry": false,
             "cwd": "${workspaceFolder}/src/PyCXpress/example/",
             "environment": [
+                {
+                    "name": "PYTHONPATH",
+                    "value": "${workspaceFolder}/src"
+                },
+                {
+                    "name": "LD_PRELOAD",
+                    "value": "/usr/lib/aarch64-linux-gnu/libpython3.10.so.1.0"
+                },
                 {
                     "name": "PATH",
-                    "value": "/opt/conda/envs/py38/bin"
+                    "value": "${workspaceFolder}/.venv/bin/"
                 },
                 {
                     "name": "PYCXPRESS_DEBUGGER_TYPE",
@@ -55,6 +63,9 @@
             "type": "debugpy",
             "request": "launch",
             "program": "${file}",
+            "cwd": "${workspaceFolder}",
+            "python": "${workspaceFolder}/.venv/bin/python",
+            "justMyCode": false,
             "console": "integratedTerminal"
         }
     ]

.vscode/settings.json

@@ -16,6 +16,8 @@
         "limits": "cpp",
         "sstream": "cpp",
         "streambuf": "cpp",
-        "complex": "cpp"
-    }
-}
+        "complex": "cpp",
+        "stdexcept": "cpp"
+    },
+    "debugpy.showPythonInlineValues": true
+}

Makefile

@@ -18,10 +18,16 @@ define message
 @echo $(1)
 endef
 
+#* Quick commands
+.PHONY: example
+example: 
+	env -C $(REPO_DIR)/src/PyCXpress/example poetry run make run
+
 #* Poetry
 .PHONY: poetry-download
 poetry-download:
 	curl -sSL https://install.python-poetry.org | $(PYTHON) -
+	~/.local/share/pypoetry/venv/bin/pip install poetry-plugin-export
 
 .PHONY: poetry-remove
 poetry-remove:
@@ -30,8 +36,10 @@ poetry-remove:
 #* Installation
 .PHONY: install
 install:
-	poetry lock -n && poetry export --without-hashes > requirements.txt
-	poetry install -n
+	poetry lock --no-update -n 
+	poetry export --without-hashes > requirements.txt
+	poetry export -E tensorflow --without-hashes > requirements.tensorflow.txt
+	env POETRY_VIRTUALENVS_IN_PROJECT=true poetry install -n --extras tensorflow
 	-poetry run mypy --install-types --non-interactive ./
 
 .PHONY: install-conda-deps install-conda-deps-manually

pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "PyCXpress"
-version = "0.0.8"
+version = "0.0.9"
 description = "PyCXpress is a high-performance hybrid framework that seamlessly integrates Python and C++ to harness the flexibility of Python and the speed of C++ for efficient and expressive computation, particularly in the realm of deep learning and numerical computing."
 readme = "README.md"
 authors = ["chaoqing <[email protected]>"]
@@ -43,9 +43,14 @@ numpy = [
     {version = "<1.22", python = "~3.8"},
     {version = "^1.22", python = ">=3.9"}
 ]
+tensorflow = [
+    {version = "^2.10", python = "^3.9", source="jp61", optional = true}, # markers="platform_release == '5.15.148-tegra'",
+]
 
+[tool.poetry.extras]
+tensorflow = [ "tensorflow" ]
 
-[tool.poetry.dev-dependencies]
+[tool.poetry.group.dev.dependencies]
 bandit = "^1.7.1"
 black = {version = "^24.4", allow-prereleases = true}
 darglint = "^1.8.1"
@@ -63,12 +68,16 @@ coverage = "^7.4.4"
 coverage-badge = "^1.1.0"
 pytest-html = "^3.1.1"
 pytest-cov = "^5.0.0"
-
-[tool.poetry.group.dev.dependencies]
 debugpy = "^1.8.1"
 find-libpython = "^0.4.0"
 pydevd-pycharm = "^241.15989.155"
 
+
+[[tool.poetry.source]]
+name = "jp61"
+url = "https://developer.download.nvidia.com/compute/redist/jp/v61"
+priority = "supplemental"
+
 [tool.black]
 # https://github.com/psf/black
 target-version = ["py38"]

requirements.tensorflow.txt

@@ -0,0 +1,44 @@
+--extra-index-url https://developer.download.nvidia.com/compute/redist/jp/v61
+
+absl-py==2.1.0 ; python_version >= "3.9" and python_version < "4.0"
+astunparse==1.6.3 ; python_version >= "3.9" and python_version < "4.0"
+certifi==2024.8.30 ; python_version >= "3.9" and python_version < "4.0"
+charset-normalizer==3.4.0 ; python_version >= "3.9" and python_version < "4.0"
+flatbuffers==24.3.25 ; python_version >= "3.9" and python_version < "4.0"
+gast==0.6.0 ; python_version >= "3.9" and python_version < "4.0"
+google-pasta==0.2.0 ; python_version >= "3.9" and python_version < "4.0"
+grpcio==1.68.1 ; python_version >= "3.9" and python_version < "4.0"
+h5py==3.12.1 ; python_version >= "3.9" and python_version < "4.0"
+idna==3.10 ; python_version >= "3.9" and python_version < "4.0"
+importlib-metadata==8.5.0 ; python_version >= "3.9" and python_version < "3.10"
+keras==3.7.0 ; python_version >= "3.9" and python_version < "4.0"
+libclang==18.1.1 ; python_version >= "3.9" and python_version < "4.0"
+markdown-it-py==3.0.0 ; python_version >= "3.9" and python_version < "4.0"
+markdown==3.7 ; python_version >= "3.9" and python_version < "4.0"
+markupsafe==2.1.5 ; python_version >= "3.9" and python_version < "4.0"
+mdurl==0.1.2 ; python_version >= "3.9" and python_version < "4.0"
+ml-dtypes==0.3.2 ; python_version >= "3.9" and python_version < "4.0"
+namex==0.0.8 ; python_version >= "3.9" and python_version < "4.0"
+numpy==1.21.6 ; python_version >= "3.8" and python_version < "3.9"
+numpy==1.26.4 ; python_version >= "3.9" and python_version < "4.0"
+opt-einsum==3.4.0 ; python_version >= "3.9" and python_version < "4.0"
+optree==0.13.1 ; python_version >= "3.9" and python_version < "4.0"
+packaging==24.2 ; python_version >= "3.9" and python_version < "4.0"
+protobuf==4.25.5 ; python_version >= "3.9" and python_version < "4.0"
+pybind11==2.13.6 ; python_version >= "3.8" and python_version < "4.0"
+pygments==2.18.0 ; python_version >= "3.9" and python_version < "4.0"
+requests==2.32.3 ; python_version >= "3.9" and python_version < "4.0"
+rich==13.9.4 ; python_version >= "3.9" and python_version < "4.0"
+setuptools==75.3.0 ; python_version >= "3.9" and python_version < "4.0"
+six==1.16.0 ; python_version >= "3.9" and python_version < "4.0"
+tensorboard-data-server==0.7.2 ; python_version >= "3.9" and python_version < "4.0"
+tensorboard==2.16.2 ; python_version >= "3.9" and python_version < "4.0"
+tensorflow-io-gcs-filesystem==0.37.1 ; python_version >= "3.9" and python_version < "3.12"
+tensorflow==2.16.1+nv24.08 ; python_version >= "3.9" and python_version < "4.0"
+termcolor==2.5.0 ; python_version >= "3.9" and python_version < "4.0"
+typing-extensions==4.12.2 ; python_version >= "3.9" and python_version < "4.0"
+urllib3==2.2.3 ; python_version >= "3.9" and python_version < "4.0"
+werkzeug==3.1.3 ; python_version >= "3.9" and python_version < "4.0"
+wheel==0.45.1 ; python_version >= "3.9" and python_version < "4.0"
+wrapt==1.17.0 ; python_version >= "3.9" and python_version < "4.0"
+zipp==3.20.2 ; python_version >= "3.9" and python_version < "3.10"

requirements.txt

@@ -1,3 +1,3 @@
 numpy==1.21.6 ; python_version >= "3.8" and python_version < "3.9"
 numpy==1.26.4 ; python_version >= "3.9" and python_version < "4.0"
-pybind11==2.12.0 ; python_version >= "3.8" and python_version < "4.0"
+pybind11==2.13.6 ; python_version >= "3.8" and python_version < "4.0"

src/PyCXpress/example/Makefile

@@ -1,12 +1,13 @@
 # Compiler
+SHELL_PREFIX=poetry run
 CC = c++
 PYTHONPATH=../../
-LD_PRELOAD:=$(shell find_libpython):/opt/conda/envs/py38/lib/libstdc++.so.6.0.29
+LD_PRELOAD:=$(shell $(SHELL_PREFIX) find_libpython):
 
 # Compiler flags
 CFLAGS = -g -Wall -std=c++17 -fPIC
-CFLAGS += $(shell python3-config --cflags --ldflags --embed)
-CFLAGS += $(shell PYTHONPATH=$(PYTHONPATH) python3 -m PyCXpress --includes)
+CFLAGS += $(shell $(SHELL_PREFIX) python3-config --cflags --ldflags --embed)
+CFLAGS += $(shell PYTHONPATH=$(PYTHONPATH) $(SHELL_PREFIX) python3 -m PyCXpress --includes)
 
 # The build target executable
 TARGET = example.out

src/PyCXpress/example/model.py

@@ -6,10 +6,17 @@
 
 logging.basicConfig(level=logging.DEBUG)
 
-import os
+import sys
 from contextlib import nullcontext
 
 import numpy as np
+try:
+    import tensorflow as tf
+    logging.info(f"tensorflow found: {tf.version.VERSION}")
+except ImportError:
+    tf = None
+    logging.warn(f"tensorflow not found inside {sys.path}")
+
 
 from PyCXpress import (
     ModelAnnotationCreator,
@@ -47,7 +54,7 @@ class InputDataSet(
     fields=InputFields,
     type=ModelAnnotationType.Input,
     mode=ModelRuntimeType.EagerExecution,
-    raw=False,
+    raw=tf is None,
 ):
     pass
 
@@ -65,7 +72,7 @@ class OutputDataSet(
     fields=OutputFields,
     type=ModelAnnotationType.Output,
     mode=ModelRuntimeType.EagerExecution,
-    raw=False,
+    raw=tf is None,
 ):
     pass
 
@@ -87,16 +94,15 @@ def initialize(self):
 
     def run(self):
         print("current status: ", getenv("PYCXPRESS_STATUS", ""))
-        self.model(self.input, self.output)
+        self.model(self.input, self.output, use_tensorflow=tf is not None)
 
     @staticmethod
     def model(input: InputDataSet, output: OutputDataSet, use_tensorflow: bool = True):
         with nullcontext():
             # print(input.data_to_be_reshaped)
             # print(input.new_2d_shape)
             if use_tensorflow:
-                import tensorflow as tf
-
+                assert tf is not None
                 output.output_a = tf.transpose(
                     tf.reshape(
                         input.data_to_be_reshaped, tf.cast(input.new_2d_shape, tf.int32)