Merge pull request #96 from singnet/imp/issue-79-handletrie-wrapper

[#79] HandleTrie wrapper and benchmark

experiments/handletrie_benchmark/CMakeListsCXX.txt

@@ -0,0 +1,23 @@
+cmake_minimum_required(VERSION 3.5.1)
+project(HandleTrie)
+
+SET(CMAKE_CXX_FLAGS  "-O2")
+
+find_package(OpenSSL REQUIRED)
+
+set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
+file(GLOB_RECURSE sources ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cc)
+file(GLOB_RECURSE includes ${CMAKE_CURRENT_SOURCE_DIR}/*.h)
+# set(exclude_files
+#         ${CMAKE_CURRENT_SOURCE_DIR}/cpython.cc
+# )
+# list(REMOVE_ITEM sources ${exclude_files})
+
+message(${sources})
+
+
+add_executable(HandleTrie ${sources} ${includes})
+
+target_link_libraries(HandleTrie 
+        PUBLIC OpenSSL::Crypto
+                )

experiments/handletrie_benchmark/CMakeListsNanoBind.txt

@@ -0,0 +1,26 @@
+cmake_minimum_required(VERSION 3.12)
+project(handletrie_nanobind)
+
+find_package(OpenSSL REQUIRED)
+
+file(GLOB_RECURSE sources ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cc)
+file(GLOB_RECURSE includes ${CMAKE_CURRENT_SOURCE_DIR}/*.h)
+file(GLOB_RECURSE cpp ${CMAKE_CURRENT_SOURCE_DIR}/_nanobind.cpp)
+# find_package(Python3 REQUIRED)
+# include_directories(${Python3_INCLUDE_DIRS})
+find_package(Python COMPONENTS Interpreter Development REQUIRED)
+# set(nanobind_DIR ${CMAKE_CURRENT_SOURCE_DIR}/venv/lib/python3.10/site-packages/nanobind/cmake)
+
+execute_process(
+  COMMAND "${Python_EXECUTABLE}" -m nanobind --cmake_dir
+  OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE nanobind_ROOT)
+find_package(nanobind CONFIG REQUIRED)
+# find_package(nanobind REQUIRED)
+
+
+# Add the pybind11 module
+nanobind_add_module(handletrie_nanobind MODULE ${cpp} ${sources} ${includes})
+
+target_include_directories(handletrie_nanobind PRIVATE ${OpenSSL_INCLUDE_DIR})
+
+target_link_libraries(handletrie_nanobind PRIVATE OpenSSL::Crypto )

experiments/handletrie_benchmark/CMakeListsPyBind.txt

@@ -0,0 +1,22 @@
+cmake_minimum_required(VERSION 3.12)
+project(handletrie_pybind)
+
+find_package(OpenSSL REQUIRED)
+
+file(GLOB_RECURSE sources ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cc)
+file(GLOB_RECURSE includes ${CMAKE_CURRENT_SOURCE_DIR}/*.h)
+file(GLOB_RECURSE cpp ${CMAKE_CURRENT_SOURCE_DIR}/_pybind.cpp)
+# Find Python and pybind11
+find_package(Python3 COMPONENTS Interpreter Development REQUIRED)
+set(pybind11_DIR ${CMAKE_CURRENT_SOURCE_DIR}/venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11)
+find_package(pybind11 REQUIRED)
+
+
+# Add the pybind11 module
+pybind11_add_module(handletrie_pybind ${cpp} ${sources} ${includes})
+
+target_include_directories(handletrie_pybind PRIVATE ${OpenSSL_INCLUDE_DIR})
+
+target_link_libraries(handletrie_pybind PRIVATE
+                OpenSSL::Crypto
+                )

experiments/handletrie_benchmark/Makefile

@@ -0,0 +1,23 @@
+build-nanobind: clean
+	@rm -f CMakeLists.txt cxx/handletrie_nanobind*.so
+	@ln -s CMakeListsNanoBind.txt CMakeLists.txt
+	@cd build && cmake .. && make && cp *.so ../cxx/
+
+build-pybind: clean
+	@rm -f CMakeLists.txt cxx/handletrie_pybind*.so
+	@ln -s CMakeListsPyBind.txt CMakeLists.txt 
+	@cd build && cmake .. && make && cp *.so ../cxx/
+
+
+build-cpython: clean
+	@rm -f cxx/handletrie_cpython*.so
+	@python setup.py build_ext --inplace && mv *.so cxx/
+
+build: clean
+	@rm -f CMakeLists.txt
+	@ln -s CMakeListsCXX.txt CMakeLists.txt
+	@cd build && cmake .. && make
+
+clean:
+	@rm -rf build/*
+	@rm -rf cpython/*

experiments/handletrie_benchmark/README.md

@@ -0,0 +1,73 @@
+# HandleTrie C++ and Python benchmarks
+
+
+### Time average in seconds
+
+|  Impl |  1,000n  | 100,000n  | 1,000,000n  | 10,000,000n |60,000,000* |
+|---|---|---|---|---|---|
+|C++ Handle Trie      |0.00129|0.15719|1.86882|23.91808|149.15487*|
+|C++ unordered map std|0.00054|0.15927|2.12357|22.52662|-|
+|C++ map std          |0.00151|0.32405|5.21409|71.53995|561.42147*|
+|Python's Dict        |0.00037|0.06631|0.89053|11.75517|79.204370*|
+|CPython Handle Trie  |0.00130|0.38828|5.00333|63.05321|510.58422*| 
+|NanoBind Handle Trie |0.00281|0.52542|6.52422|84.37761|651.15217*| 
+|PyBind Handle Trie   |0.01159|1.41426|15.5277|168.3574|1159,28535*|
+
+> \* The 60,000,000 column values may not be accurate, as the test fill up all the available memory on the test machine, leading to inaccurate results.
+
+> **Note:** Although both the C++ and Python implementations account for overhead, the operations being measured are not exactly the same. To minimize this discrepancy, the time spent on random number generation for populating keys has been estimated and excluded. This discrepancy could affect performance comparisons. For the complete runtime, please check the "runs" folder.
+
+
+
+### Memory average in GB
+
+|  Impl |  1,000n  | 100,000n  | 1,000,000n  | 10,000,000n |60,000,000n |
+|---|---|---|---|---|---|
+|C++ Handle Trie      |0.01428|0.01597|0.23844|3.01968| - | 
+|C++ map std          |0.01587|0.01600|0.47400|3.19017| - |
+|Python's Dict        |0.00448|0.02477|0.26700|3.10800|20.59636|
+|CPython Handle Trie  |0.00448|0.06898|0.59080|5.45343|33.04674|
+|NanoBind Handle Trie |0.00448|0.07680|0.61002|5.70955|34.56202|
+|PyBind Handle Trie   |0.00967|0.07858|0.59319|5.56300|35.54603|
+
+
+### Setup Python
+
+```
+python3 -m venv venv
+source venv/bin/activate
+pip install pybind11
+pip install nanobind
+```
+
+### Build C++
+
+```
+make build
+```
+
+### Build CPython
+```
+make build-cpython
+```
+
+
+### Build PyBind
+
+```
+make build-pybind
+```
+
+
+### Build NanoBind
+
+```
+make build-nanobind
+```
+
+
+### Run
+
+```
+python main.py
+```