feat(compiler): add concrete-sys rust bindings

.github/workflows/concrete_compiler_publish_docker_images.yml

@@ -37,6 +37,7 @@ jobs:
           backend: aws
           profile: cpu-test
 
+  # Not sure what to do about this one...
   hpx-image:
     needs: [setup-instance]
     runs-on: ${{ needs.setup-instance.outputs.runner-name }}

.github/workflows/concrete_compiler_test_cpu_distributed.yml

@@ -66,8 +66,8 @@ jobs:
         run: |
           cd compilers/concrete-compiler/compiler
           rm -rf /shared/build
-          make HPX_DIR=/shared/hpx install-hpx-from-source
-          make HPX_DIR=/shared/hpx BUILD_DIR=/shared/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON BINDINGS_PYTHON_ENABLED=OFF build-end-to-end-tests
+          cargo install cxxbridge-cmd
+          make BUILD_DIR=/shared/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON BINDINGS_PYTHON_ENABLED=OFF build-end-to-end-tests
 
       - name: Run end-to-end distributed test
         run: |

.github/workflows/concrete_compiler_test_macos_cpu.yml

@@ -60,6 +60,7 @@ jobs:
           cd compilers/concrete-compiler/compiler
           echo "Debug: ccache statistics (prior to the build):"
           ccache -s
+          cargo install cxxbridge-cmd
           make Python3_EXECUTABLE="${PYTHON_EXEC}" all
           echo "Debug: ccache statistics (after the build):"
           ccache -s

compilers/concrete-compiler/compiler/CMakeLists.txt

@@ -11,6 +11,10 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 # Needed on linux with clang 15 and on MacOS because cxx emits dollars in the optimizer C++ API
 add_definitions("-Wno-dollar-in-identifier-extension")
 add_definitions("-Wno-c++98-compat-extra-semi")
+add_definitions("-Wno-strict-aliasing")
+add_definitions("-Wno-date-time")
+add_definitions("-Wno-suggest-override")
+add_definitions("-Wno-ctad-maybe-unsupported")
 add_definitions("-Wall ")
 add_definitions("-Werror ")
 add_definitions("-Wfatal-errors")
@@ -85,35 +89,7 @@ install(EXPORT concrete-protocol DESTINATION "./")
 # -------------------------------------------------------------------------------
 
 set(CONCRETE_OPTIMIZER_DIR "${PROJECT_SOURCE_DIR}/../../concrete-optimizer")
-if(CMAKE_BUILD_TYPE STREQUAL "Debug")
-  set(CONCRETE_OPTIMIZER_BUILD_DIR "${CONCRETE_OPTIMIZER_DIR}/target/debug")
-  set(CONCRETE_OPTIMIZER_PROFILE "dev")
-else()
-  set(CONCRETE_OPTIMIZER_BUILD_DIR "${CONCRETE_OPTIMIZER_DIR}/target/release")
-  set(CONCRETE_OPTIMIZER_PROFILE "release")
-endif()
 set(CONCRETE_OPTIMIZER_INCLUDE_DIR "${CONCRETE_OPTIMIZER_DIR}/concrete-optimizer-cpp/src/cpp")
-set(CONCRETE_OPTIMIZER_STATIC_LIB "${CONCRETE_OPTIMIZER_BUILD_DIR}/libconcrete_optimizer_cpp.a")
-
-ExternalProject_Add(
-  concrete_optimizer_rust
-  DOWNLOAD_COMMAND ""
-  CONFIGURE_COMMAND "" OUTPUT "${CONCRETE_OPTIMIZER_STATIC_LIB}"
-  BUILD_ALWAYS true
-  BUILD_COMMAND cargo build -p concrete-optimizer-cpp --profile ${CONCRETE_OPTIMIZER_PROFILE}
-  BINARY_DIR "${CONCRETE_OPTIMIZER_DIR}"
-  INSTALL_COMMAND cp ${CONCRETE_OPTIMIZER_STATIC_LIB} ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}
-  LOG_BUILD ON
-  LOG_OUTPUT_ON_FAILURE ON)
-
-add_library(concrete_optimizer STATIC ${CONCRETE_OPTIMIZER_DIR}/concrete-optimizer-cpp/src/cpp/concrete-optimizer.cpp)
-
-target_link_libraries(concrete_optimizer PRIVATE pthread m dl "${CONCRETE_OPTIMIZER_STATIC_LIB}")
-install(TARGETS concrete_optimizer EXPORT concrete_optimizer)
-install(EXPORT concrete_optimizer DESTINATION "./")
-
-add_dependencies(concrete_optimizer concrete_optimizer_rust)
-# TODO - Remove the global include directory
 include_directories(${CONCRETE_OPTIMIZER_INCLUDE_DIR})
 
 # -------------------------------------------------------------------------------
@@ -127,48 +103,44 @@ set(CONCRETE_BACKENDS_DIR "${PROJECT_SOURCE_DIR}/../../../backends")
 # -------------------------------------------------------------------------------
 
 set(CONCRETE_CPU_DIR "${CONCRETE_BACKENDS_DIR}/concrete-cpu/implementation")
-set(CONCRETE_CPU_RELEASE_DIR "${CONCRETE_CPU_DIR}/target/release")
 set(CONCRETE_CPU_INCLUDE_DIR "${CONCRETE_CPU_DIR}/include")
-set(CONCRETE_CPU_STATIC_LIB "${CONCRETE_CPU_RELEASE_DIR}/libconcrete_cpu.a")
-
-ExternalProject_Add(
-  concrete_cpu_rust
-  DOWNLOAD_COMMAND ""
-  CONFIGURE_COMMAND "" OUTPUT "${CONCRETE_CPU_STATIC_LIB}"
-  BUILD_ALWAYS true
-  BUILD_COMMAND cargo +nightly-2024-09-30 build --release --features=nightly
-  BINARY_DIR "${CONCRETE_CPU_DIR}"
-  INSTALL_COMMAND ""
-  LOG_BUILD ON
-  LOG_OUTPUT_ON_FAILURE ON)
-
-add_library(concrete_cpu STATIC IMPORTED)
-set_target_properties(concrete_cpu PROPERTIES IMPORTED_LOCATION "${CONCRETE_CPU_STATIC_LIB}")
-add_dependencies(concrete_cpu concrete_cpu_rust)
 
 # -------------------------------------------------------------------------------
 # Concrete CPU Noise Model Configuration
 # -------------------------------------------------------------------------------
 
 set(CONCRETE_CPU_NOISE_MODEL_DIR "${CONCRETE_BACKENDS_DIR}/concrete-cpu/noise-model")
-set(CONCRETE_CPU_NOISE_MODEL_RELEASE_DIR "${CONCRETE_CPU_NOISE_MODEL_DIR}/target/release")
 set(CONCRETE_CPU_NOISE_MODEL_INCLUDE_DIR "${CONCRETE_CPU_NOISE_MODEL_DIR}/include")
-set(CONCRETE_CPU_NOISE_MODEL_STATIC_LIB "${CONCRETE_CPU_NOISE_MODEL_RELEASE_DIR}/libconcrete_cpu_noise_model.a")
+
+# -------------------------------------------------------------------------------
+# RUST DEPS BUNDLE
+# -------------------------------------------------------------------------------
+
+set(RUST_DEPS_BUNDLE_DIR "${PROJECT_SOURCE_DIR}/../../../tools/rust_deps_bundle")
+if(CMAKE_BUILD_TYPE STREQUAL "Debug")
+  set(RUST_DEPS_BUNDLE_BUILD_DIR "${RUST_DEPS_BUNDLE_DIR}/target/debug")
+  set(RUST_DEPS_BUNDLE_TARGET "")
+else()
+  set(RUST_DEPS_BUNDLE_BUILD_DIR "${RUST_DEPS_BUNDLE_DIR}/target/release")
+  set(RUST_DEPS_BUNDLE_TARGET "--release")
+endif()
+set(RUST_DEPS_BUNDLE_LIB "${RUST_DEPS_BUNDLE_BUILD_DIR}/librust_deps_bundle.a")
 
 ExternalProject_Add(
-  concrete_cpu_noise_model_rust
+  rust_deps_bundle_build
   DOWNLOAD_COMMAND ""
-  CONFIGURE_COMMAND "" OUTPUT "${CONCRETE_CPU_NOISE_MODEL_STATIC_LIB}"
+  CONFIGURE_COMMAND "" OUTPUT "${RUST_DEPS_BUNDLE_LIB}"
   BUILD_ALWAYS true
-  BUILD_COMMAND cargo build --release
-  BINARY_DIR "${CONCRETE_CPU_NOISE_MODEL_DIR}"
-  INSTALL_COMMAND ""
+  BUILD_COMMAND cargo +nightly-2024-09-30 build ${RUST_DEPS_BUNDLE_TARGET}
+  BINARY_DIR "${RUST_DEPS_BUNDLE_DIR}"
+  INSTALL_COMMAND cp ${RUST_DEPS_BUNDLE_LIB} ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}
   LOG_BUILD ON
   LOG_OUTPUT_ON_FAILURE ON)
 
-add_library(concrete_cpu_noise_model STATIC IMPORTED)
-set_target_properties(concrete_cpu_noise_model PROPERTIES IMPORTED_LOCATION "${CONCRETE_CPU_NOISE_MODEL_STATIC_LIB}")
-add_dependencies(concrete_cpu_noise_model concrete_cpu_noise_model_rust)
+add_library(rust_deps_bundle STATIC IMPORTED)
+set_target_properties(rust_deps_bundle PROPERTIES IMPORTED_LOCATION "${RUST_DEPS_BUNDLE_LIB}")
+add_dependencies(rust_deps_bundle rust_deps_bundle_build)
+target_link_libraries(rust_deps_bundle INTERFACE dl)
 
 # --------------------------------------------------------------------------------
 # Concrete Cuda Configuration
@@ -216,8 +188,9 @@ option(CONCRETELANG_TIMING_ENABLED "Enables execution timing." ON)
 if(CONCRETELANG_DATAFLOW_EXECUTION_ENABLED)
   message(STATUS "ConcreteLang dataflow execution enabled.")
 
-  find_package(HPX REQUIRED CONFIG)
-  list(APPEND CMAKE_MODULE_PATH "${HPX_CMAKE_DIR}")
+  include(FetchHpxLibrary)
+  fetch_hpx_library()
+
   add_compile_options(-DCONCRETELANG_DATAFLOW_EXECUTION_ENABLED
                       -DHPX_DEFAULT_CONFIG_FILE="${PROJECT_SOURCE_DIR}/hpx.ini")
 

compilers/concrete-compiler/compiler/Makefile

@@ -27,13 +27,6 @@ KEYSETCACHEDEV=/tmp/KeySetCache
 KEYSETCACHECI ?= ../KeySetCache
 KEYSETCACHENAME ?= KeySetCacheV4
 
-HPX_VERSION?=1.9.1
-HPX_URL=https://github.com/STEllAR-GROUP/hpx/archive/refs/tags/v$(HPX_VERSION).tar.gz
-HPX_DIR?=$(shell pwd)
-HPX_TARBALL=hpx-$(HPX_VERSION).tar.gz
-HPX_LOCAL_DIR=$(HPX_DIR)/hpx-$(HPX_VERSION)
-HPX_INSTALL_DIR?=$(HPX_LOCAL_DIR)/build
-
 ML_BENCH_SUBSET_ID=
 
 # Find OS
@@ -120,26 +113,9 @@ ifeq ($(BUILD_TYPE),Debug)
 	LIBOMP_LINK_TO_LIBSTDCXX_OPT=-DLIBOMP_USE_STDCPPLIB=ON
 endif
 
-all: concretecompiler python-bindings build-tests build-benchmarks doc
-
-# HPX #####################################################
-
-install-hpx-from-source: $(HPX_LOCAL_DIR)
-	mkdir -p $(HPX_LOCAL_DIR)/build
-	cd $(HPX_LOCAL_DIR)/build && cmake \
-		-DHPX_WITH_MAX_CPU_COUNT="" \
-		-DHPX_WITH_FETCH_ASIO=on \
-		-DHPX_FILESYSTEM_WITH_BOOST_FILESYSTEM_COMPATIBILITY=ON \
-		-DHPX_WITH_MALLOC=system \
-		$(CXX_COMPILER_OPTION) $(CC_COMPILER_OPTION) ..
-	cd $(HPX_LOCAL_DIR)/build && make -j2
-
-$(HPX_TARBALL):
-	mkdir -p $(HPX_DIR)
-	cd $(HPX_DIR) && curl -L $(HPX_URL) -o $(HPX_TARBALL)
+all: concretecompiler runtime python-bindings build-tests build-benchmarks doc concrete-sys
 
-$(HPX_LOCAL_DIR): $(HPX_TARBALL)
-	cd $(HPX_DIR) && tar xzvf $(HPX_TARBALL)
+# COMPILER #####################################################
 
 $(BUILD_DIR)/configured.stamp:
 	mkdir -p $(BUILD_DIR)
@@ -154,12 +130,11 @@ $(BUILD_DIR)/configured.stamp:
 	-DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
 	-DLLVM_ENABLE_ASSERTIONS=ON \
 	-DLLVM_ENABLE_RTTI=ON \
-	-DLLVM_LINK_LLVM_DYLIB=on \
+	-DLLVM_LINK_LLVM_DYLIB=ON \
 	-DMLIR_ENABLE_BINDINGS_PYTHON=$(BINDINGS_PYTHON_ENABLED) \
 	-DCONCRETELANG_BINDINGS_PYTHON_ENABLED=$(BINDINGS_PYTHON_ENABLED) \
 	-DCONCRETELANG_DATAFLOW_EXECUTION_ENABLED=$(DATAFLOW_EXECUTION_ENABLED) \
 	-DCONCRETELANG_TIMING_ENABLED=$(TIMING_ENABLED) \
-	-DHPX_DIR=${HPX_INSTALL_DIR}/lib/cmake/HPX \
 	-DLLVM_EXTERNAL_PROJECTS=concretelang \
 	-DLLVM_EXTERNAL_CONCRETELANG_SOURCE_DIR=. \
 	-DPython3_EXECUTABLE=${Python3_EXECUTABLE} \
@@ -184,7 +159,10 @@ update-dialect-doc: doc
 concretecompiler: build-initialized
 	cmake --build $(BUILD_DIR) --target concretecompiler
 
-python-bindings: build-initialized
+runtime: build-initialized
+	cmake --build $(BUILD_DIR) --target ConcretelangRuntime
+
+python-bindings: build-initialized runtime
 	cmake --build $(BUILD_DIR) --target ConcretelangMLIRPythonModules
 	cmake --build $(BUILD_DIR) --target ConcretelangPythonModules
 	PYTHONPATH=${PYTHONPATH}:$(BUILD_DIR)/tools/concretelang/python_packages/concretelang_core LD_PRELOAD=$(BUILD_DIR)/lib/libConcretelangRuntime.so $(STUBGEN) -m mlir._mlir_libs._concretelang._compiler --include-docstrings -o $(BUILD_DIR)/tools/concretelang/python_packages/concretelang_core
@@ -195,7 +173,8 @@ clientlib: build-initialized
 serverlib: build-initialized
 	cmake --build $(BUILD_DIR) --target ConcretelangServerLib
 
-
+concrete-sys: build-initialized
+	cmake --build $(BUILD_DIR) --target ConcreteSys
 
 GITHUB_URL=https://api.github.com/repos/zama-ai/concrete-compiler-internal
 GITHUB_URL_LIST_ARTIFACTS="${GITHUB_URL}/actions/artifacts?name=${KEYSETCACHENAME}&per_page=1"
@@ -233,7 +212,7 @@ endif
 
 build-tests: build-unit-tests build-end-to-end-tests
 
-run-tests: run-check-tests run-unit-tests run-end-to-end-tests run-random-end-to-end-tests-for-each-options run-python-tests
+run-tests: run-check-tests run-unit-tests run-end-to-end-tests run-random-end-to-end-tests-for-each-options run-python-tests run-rust-tests
 
 ## check-tests
 
@@ -250,9 +229,15 @@ run-unit-tests: build-unit-tests
 
 ## python-tests
 
-run-python-tests: python-bindings concretecompiler
+run-python-tests: python-bindings concretecompiler runtime
 	PYTHONPATH=${PYTHONPATH}:$(BUILD_DIR)/tools/concretelang/python_packages/concretelang_core LD_PRELOAD=$(BUILD_DIR)/lib/libConcretelangRuntime.so pytest -vs -m $(PYTHON_TESTS_MARKER) tests/python
 
+## rust-tests
+
+run-rust-tests: concrete-sys
+	$(eval _abs_build_dir = $(realpath $(BUILD_DIR)))
+	cd lib/Bindings/Rust && COMPILER_BUILD_DIRECTORY=$(_abs_build_dir) cargo test
+
 test-compiler-file-output: concretecompiler
 	pytest -vs tests/test_compiler_file_output
 

compilers/concrete-compiler/compiler/README.md

@@ -21,16 +21,11 @@ The Compiler depends on some Rust libraries, including the optimizer and the con
 
 **Note:** some crates use Rust nightly currently, and it might be required to install both stable and nightly distributions.
 
-### Prerequisite: Building HPX and enable dataflow parallelism (optional)
+### Prerequisite: Enable dataflow parallelism (optional)
 
-In order to implement the dataflow parallelism and the distribution of the computation we use the [HPX Standard Library](https://hpx-docs.stellar-group.org/). You can else use your own HPX installation by set the `HPX_INSTALL_DIR` environment variable or you can install HPX on the default path of our build system thanks the following command:
+In order to implement the dataflow parallelism and the distribution of the computation we use the [HPX Standard Library](https://hpx-docs.stellar-group.org/). The build script takes care of installing HPX, but it may rely on a local installation of BOOST.
 
-```sh
-make install-hpx-from-source
-```
-
-This may fail on some systems when dependencies are missing. Some recent packages required are Cmake, HWLOC and BOOST. For full details see [HPX Quickstart guide](https://hpx-docs.stellar-group.org/tags/1.7.1/html/quickstart.html).
-Once you have a proper installation of HPX to enable the dataflow parallelism set the `DATAFLOW_EXECUTION_ENABLED=ON`.
+Once you have a proper installation of BOOST you can enable the dataflow parallelism set the `DATAFLOW_EXECUTION_ENABLED=ON`.
 
 ### Prerequisite: Fetch git submodules
 

compilers/concrete-compiler/compiler/cmake/modules/FetchHpxLibrary.cmake

@@ -0,0 +1,28 @@
+include(FetchContent)
+
+function(fetch_hpx_library)
+    FetchContent_Declare(
+        HPX
+        GIT_REPOSITORY https://github.com/STEllAR-GROUP/hpx.git
+        GIT_TAG stable # release 1.10 + fixes for compilation with fopenmp with gcc and clang :,)
+        GIT_SHALLOW    TRUE
+        GIT_PROGRESS   TRUE
+    )
+    set(HPX_WITH_FETCH_ASIO ON CACHE BOOL INTERNAL)
+    set(HPX_WITH_FETCH_HWLOC ON CACHE BOOL INTERNAL)
+    set(HPX_WITH_FETCH_BOOST OFF CACHE BOOL INTERNAL)
+    set(HPX_WITH_MALLOC system CACHE STRING INTERNAL)
+    set(HPX_WITH_EXAMPLES OFF CACHE BOOL INTERNAL)
+    set(HPX_WITH_TESTS OFF CACHE BOOL INTERNAL)
+    set(HPX_WITH_STATIC_LINKING ON CACHE BOOL INTERNAL)
+    set(HPX_WITH_PKGCONFIG OFF CACHE BOOL INTERNAL)
+    set(HPX_WITH_MAX_CPU_COUNT "" CACHE STRING INTERNAL)
+    set(HPX_WITH_CXX_STANDARD 17 CACHE STRING INTERNAL)
+    set(HPX_WITH_COMPILER_WARNINGS OFF CACHE BOOL INTERNAL)
+    unset(CMAKE_CXX_STANDARD CACHE)
+    unset(CMAKE_CXX_STANDARD)
+    remove_definitions("-Wall ")
+    remove_definitions("-Werror ")
+    remove_definitions("-Wfatal-errors")
+    FetchContent_MakeAvailable(HPX)
+endfunction()

compilers/concrete-compiler/compiler/include/concretelang/Support/CompilerEngine.h

@@ -6,8 +6,6 @@
 #ifndef CONCRETELANG_SUPPORT_COMPILER_ENGINE_H
 #define CONCRETELANG_SUPPORT_COMPILER_ENGINE_H
 
-#include "capnp/message.h"
-#include "concrete-protocol.capnp.h"
 #include "concretelang/Common/Protocol.h"
 #include "concretelang/Conversion/Utils/GlobalFHEContext.h"
 #include "concretelang/Support/Encodings.h"
@@ -315,9 +313,6 @@ enum class Target {
 
 class CompilerEngine {
 public:
-
-
-
   CompilerEngine(std::shared_ptr<CompilationContext> compilationContext)
       : overrideMaxEintPrecision(), overrideMaxMANP(), compilerOptions(),
         generateProgramInfo(true),

compilers/concrete-compiler/compiler/lib/Bindings/CMakeLists.txt

@@ -1,3 +1,5 @@
 if(CONCRETELANG_BINDINGS_PYTHON_ENABLED)
   add_subdirectory(Python)
 endif()
+
+add_subdirectory(Rust)

compilers/concrete-compiler/compiler/lib/Bindings/Python/CMakeLists.txt

@@ -22,6 +22,9 @@ declare_mlir_python_extension(
   PRIVATE_LINK_LIBS
   ConcretelangSupport
   ConcretelangCommon
+  ConcretelangRuntime
+  ConcretelangClientLib
+  ConcretelangServerLib
   EMBED_CAPI_LINK_LIBS
   MLIRCAPIRegisterEverything)
 

compilers/concrete-compiler/compiler/lib/Bindings/Python/CompilerAPIModule.cpp

@@ -124,9 +124,8 @@ std::string roundTrip(const char *module) {
   std::string backingString;
   llvm::raw_string_ostream os(backingString);
 
-  llvm::Expected<mlir::concretelang::CompilationResult>
-      retOrErr = ce.compile(
-          module, mlir::concretelang::Target::ROUND_TRIP);
+  llvm::Expected<mlir::concretelang::CompilationResult> retOrErr =
+      ce.compile(module, mlir::concretelang::Target::ROUND_TRIP);
   if (!retOrErr) {
     os << "MLIR parsing failed: " << llvm::toString(retOrErr.takeError());
     throw std::runtime_error(os.str());

compilers/concrete-compiler/compiler/lib/Bindings/Rust/.gitignore

@@ -0,0 +1,3 @@
+target
+Cargo.lock
+.cargo