gRPC upgrade version

This commit upgrades gRPC from 1.37.x to the latest release version 1.48.0. Over the past ten versions of gRPC multiple optimizations and bug fixes have been introduced. Please see https://github.com/grpc/grpc/releases for the detailed breakdown of updates from 1.38.0 to 1.48.0. This upgrade would also allow for batching of updates natively via setting the buffer hint flag for improved performance. This change updates client, TRS and UTT CMakeLists files to account for the upgrade.
vmware · Sep 21, 2022 · db98910 · db98910
1 parent ce36ed4
commit db98910
Show file tree

Hide file tree

Showing 6 changed files with 217 additions and 82 deletions.
diff --git a/client/clientservice/CMakeLists.txt b/client/clientservice/CMakeLists.txt
@@ -1,4 +1,9 @@
+cmake_minimum_required(VERSION 3.5)
+set(CMAKE_CXX_STANDARD 17)
+
 find_package(GRPC REQUIRED)
+find_package(absl CONFIG REQUIRED)
+find_package(Protobuf CONFIG REQUIRED)
 find_package(Boost ${MIN_BOOST_VERSION} COMPONENTS program_options REQUIRED)
 find_package(jaegertracing REQUIRED)
 
@@ -13,6 +18,7 @@ target_link_libraries(clientservice-lib PUBLIC
   concordclient
   gRPC::grpc++
   gRPC::grpc++_reflection
+  protobuf::libprotobuf
   logging
   yaml-cpp
   secret_retriever

diff --git a/client/proto/CMakeLists.txt b/client/proto/CMakeLists.txt
@@ -1,21 +1,102 @@
+cmake_minimum_required(VERSION 3.5)
+set(CMAKE_CXX_STANDARD 17)
+
+project(concord)
+
 find_package(Protobuf REQUIRED)
+find_package(absl CONFIG REQUIRED)
 find_package(GRPC REQUIRED)
 
-include_directories(${GRPC_INCLUDE_DIR})
-
-protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ${CMAKE_CURRENT_BINARY_DIR}
-  request/v1/request.proto
-  event/v1/event.proto
-  state_snapshot/v1/state_snapshot.proto
-  ../concordclient/proto/concord_client_request/v1/concord_client_request.proto
-)
-grpc_generate_cpp(GRPC_SRCS GRPC_HDRS ${CMAKE_CURRENT_BINARY_DIR}
-  request/v1/request.proto
-  event/v1/event.proto
-  state_snapshot/v1/state_snapshot.proto
-  ../concordclient/proto/concord_client_request/v1/concord_client_request.proto
-)
-
-add_library(clientservice-proto STATIC ${PROTO_SRCS} ${GRPC_SRCS})
-target_link_libraries(clientservice-proto PRIVATE protobuf::libprotobuf gRPC::grpc++)
+# Proto file
+get_filename_component(rs_proto "request/v1/request.proto" ABSOLUTE)
+get_filename_component(es_proto "event/v1/event.proto" ABSOLUTE)
+get_filename_component(ss_proto "state_snapshot/v1/state_snapshot.proto" ABSOLUTE)
+get_filename_component(ccr_proto "../concordclient/proto/concord_client_request/v1/concord_client_request.proto" ABSOLUTE)
+get_filename_component(rs_proto_path "${rs_proto}" PATH)
+get_filename_component(es_proto_path "${es_proto}" PATH)
+get_filename_component(ss_proto_path "${ss_proto}" PATH)
+get_filename_component(ccr_proto_path "${ccr_proto}" PATH)
+
+get_target_property(grpc_cpp_plugin_location gRPC::grpc_cpp_plugin LOCATION)
+
+# Generated sources
+set(rs_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/request.pb.cc")
+set(rs_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/request.pb.h")
+set(rs_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/request.grpc.pb.cc")
+set(rs_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/request.grpc.pb.h")
+add_custom_command(
+      OUTPUT "${rs_proto_srcs}" "${rs_proto_hdrs}" "${rs_grpc_srcs}" "${rs_grpc_hdrs}"
+      COMMAND protoc
+      ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}"
+        --cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
+        -I "${rs_proto_path}"
+        --plugin=protoc-gen-grpc="${grpc_cpp_plugin_location}"
+        "${rs_proto}"
+      DEPENDS "${rs_proto}")
+
+set(es_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/event.pb.cc")
+set(es_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/event.pb.h")
+set(es_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/event.grpc.pb.cc")
+set(es_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/event.grpc.pb.h")
+add_custom_command(
+      OUTPUT "${es_proto_srcs}" "${es_proto_hdrs}" "${es_grpc_srcs}" "${es_grpc_hdrs}"
+      COMMAND protoc
+      ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}"
+        --cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
+        -I "${es_proto_path}"
+        --plugin=protoc-gen-grpc="${grpc_cpp_plugin_location}"
+        "${es_proto}"
+      DEPENDS "${es_proto}")
+
+set(ss_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/state_snapshot.pb.cc")
+set(ss_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/state_snapshot.pb.h")
+set(ss_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/state_snapshot.grpc.pb.cc")
+set(ss_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/state_snapshot.grpc.pb.h")
+add_custom_command(
+      OUTPUT "${ss_proto_srcs}" "${ss_proto_hdrs}" "${ss_grpc_srcs}" "${ss_grpc_hdrs}"
+      COMMAND protoc
+      ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}"
+        --cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
+        -I "${ss_proto_path}"
+        --plugin=protoc-gen-grpc="${grpc_cpp_plugin_location}"
+        "${ss_proto}"
+      DEPENDS "${ss_proto}")
+
+set(ccr_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/concord_client_request.pb.cc")
+set(ccr_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/concord_client_request.pb.h")
+set(ccr_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/concord_client_request.grpc.pb.cc")
+set(ccr_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/concord_client_request.grpc.pb.h")
+add_custom_command(
+      OUTPUT "${ccr_proto_srcs}" "${ccr_proto_hdrs}" "${ccr_grpc_srcs}" "${ccr_grpc_hdrs}"
+      COMMAND protoc
+      ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}"
+        --cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
+        -I "${ccr_proto_path}"
+        --plugin=protoc-gen-grpc="${grpc_cpp_plugin_location}"
+        "${ccr_proto}"
+      DEPENDS "${ccr_proto}")
+
+
+# Include generated *.pb.h files
+include_directories("${CMAKE_CURRENT_BINARY_DIR}")
+
+# clientservice-proto
+add_library(clientservice-proto STATIC
+  ${rs_grpc_srcs}
+  ${rs_grpc_hdrs}
+  ${rs_proto_srcs}
+  ${rs_proto_hdrs}
+  ${es_grpc_srcs}
+  ${es_grpc_hdrs}
+  ${es_proto_srcs}
+  ${es_proto_hdrs}
+  ${ss_grpc_srcs}
+  ${ss_grpc_hdrs}
+  ${ss_proto_srcs}
+  ${ss_proto_hdrs}
+  ${ccr_grpc_srcs}
+  ${ccr_grpc_hdrs}
+  ${ccr_proto_srcs}
+  ${ccr_proto_hdrs})
+target_link_libraries(clientservice-proto PRIVATE protobuf::libprotobuf gRPC::grpc++ absl::synchronization)
 target_include_directories(clientservice-proto PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
diff --git a/install_deps.sh b/install_deps.sh
@@ -76,9 +76,9 @@ install_third_party_libraries() {
         pytest \
         pycryptodome \
         ecdsa \
-        protobuf==3.15.8 \
-        grpcio==1.37.1 \
-        grpcio-tools==1.37.1
+        protobuf==3.21.5 \
+        grpcio==1.48.0 \
+        grpcio-tools==1.48.0
 }
 
 
@@ -300,36 +300,20 @@ install_openssl() {
 # https://github.com/grpc/grpc/blob/master/test/distrib/cpp/run_distrib_test_cmake.sh
 install_grpc() {
     cd ${HOME}
-    git clone -b v1.37.1 --depth 1 --recurse-submodules https://github.com/grpc/grpc && \
-        cd grpc && \
-        mkdir -p ${HOME}/grpc/third_party/abseil-cpp/cmake/build && \
-        cd ${HOME}/grpc/third_party/abseil-cpp/cmake/build && \
-        cmake -DCMAKE_BUILD_TYPE=Release \
-            -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE \
-            -DCMAKE_INSTALL_PREFIX=/usr/local \
-            ../.. && \
-        make -j$(nproc) install && \
-        mkdir -p ${HOME}/grpc/third_party/protobuf/cmake/build && \
-        cd ${HOME}/grpc/third_party/protobuf/cmake/build && \
-        cmake -DBUILD_SHARED_LIBS=ON \
-            -DCMAKE_BUILD_TYPE=Release \
-            -DCMAKE_INSTALL_PREFIX=/usr/local \
-            .. && \
-        make -j$(nproc) install && \
-        mkdir -p ${HOME}/grpc/cmake/build && \
-        cd ${HOME}/grpc/cmake/build && \
-        cmake -DgRPC_INSTALL=ON \
-            -DgRPC_ABSL_PROVIDER=package \
-            -DgRPC_PROTOBUF_PROVIDER=package \
-            -DgRPC_SSL_PROVIDER=package \
-            -DBUILD_SHARED_LIBS=ON \
-            -DCMAKE_BUILD_TYPE=Release \
-            -DCMAKE_INSTALL_PREFIX=/usr/local \
-            ../.. && \
-        make -j$(nproc) install &&
-        cd ${HOME} && \
-        rm -r ${HOME}/grpc
-
+    git clone -b v1.48.x --depth 1 --recurse-submodules https://github.com/grpc/grpc && \
+    mkdir -p ${HOME}/grpc/cmake/build && \
+    cd ${HOME}/grpc/cmake/build && \
+    cmake -DCMAKE_CXX_STANDARD=17 \
+          -DCMAKE_BUILD_TYPE=Release \
+          -DBUILD_SHARED_LIBS=ON \
+          -DgRPC_INSTALL=ON \
+          -DgRPC_BUILD_TESTS=OFF \
+          -DgRPC_SSL_PROVIDER=package \
+          -DCMAKE_INSTALL_PREFIX=/opt/grpc \
+          ../.. && \
+    make -j$(nproc) install && \
+    cd ${HOME} && \
+    rm -r ${HOME}/grpc
 }
 
 install_prometheus() {

diff --git a/kvbc/proto/CMakeLists.txt b/kvbc/proto/CMakeLists.txt
@@ -1,10 +1,27 @@
-find_package(Protobuf REQUIRED)
+cmake_minimum_required(VERSION 3.5)
+set(CMAKE_CXX_STANDARD 17)
+
+find_package(absl CONFIG REQUIRED)
+find_package(GRPC REQUIRED)
+find_package(Protobuf CONFIG REQUIRED)
+
+# Proto file
+get_filename_component(concord_kvbc_proto "concord_kvbc.proto" ABSOLUTE)
+get_filename_component(concord_kvbc_proto_path "${concord_kvbc_proto}" PATH)
+
+set(ck_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/concord_kvbc.pb.cc")
+set(ck_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/concord_kvbc.pb.h")
+
+add_custom_command(
+      OUTPUT "${ck_proto_srcs}" "${ck_proto_hdrs}"
+      COMMAND protoc
+      ARGS --proto_path="${concord_kvbc_proto_path}"
+        --cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
+        "${concord_kvbc_proto}"
+      DEPENDS "${concord_kvbc_proto}")
 
-protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ${CMAKE_CURRENT_BINARY_DIR}
-  concord_kvbc.proto
-)
 message(STATUS "Concord KVBC protobuf generated - see " ${CMAKE_CURRENT_BINARY_DIR})
 
-add_library(concord-kvbc-proto STATIC ${PROTO_SRCS})
+add_library(concord-kvbc-proto STATIC ${ck_proto_srcs})
 target_link_libraries(concord-kvbc-proto protobuf::libprotobuf)
 target_include_directories(concord-kvbc-proto PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
diff --git a/thin-replica-server/proto/CMakeLists.txt b/thin-replica-server/proto/CMakeLists.txt
@@ -1,30 +1,58 @@
-find_package(Protobuf REQUIRED)
+cmake_minimum_required(VERSION 3.5)
+set(CMAKE_CXX_STANDARD 17)
+
 find_package(GRPC REQUIRED)
+find_package(absl CONFIG REQUIRED)
+find_package(Protobuf CONFIG REQUIRED)
 
 include_directories(${GRPC_INCLUDE_DIR})
 
-protobuf_generate_cpp(THIN_REPLICA_PROTO_SRCS THIN_REPLICA_PROTO_HDRS ${CMAKE_CURRENT_BINARY_DIR}
-  thin_replica.proto
-)
-grpc_generate_cpp(THIN_REPLICA_GRPC_SRCS THIN_REPLICA_GRPC_HDRS ${CMAKE_CURRENT_BINARY_DIR}
-  thin_replica.proto
-)
+get_filename_component(tr_proto "thin_replica.proto" ABSOLUTE)
+get_filename_component(tr_proto_path "${tr_proto}" PATH)
+
+get_target_property(grpc_cpp_plugin_location gRPC::grpc_cpp_plugin LOCATION)
+
+set(tr_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/thin_replica.pb.cc")
+set(tr_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/thin_replica.pb.h")
+set(tr_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/thin_replica.grpc.pb.cc")
+set(tr_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/thin_replica.grpc.pb.h")
+add_custom_command(
+      OUTPUT "${tr_proto_srcs}" "${tr_proto_hdrs}" "${tr_grpc_srcs}" "${tr_grpc_hdrs}"
+      COMMAND protoc
+      ARGS --grpc_out=generate_mock_code=true:"${CMAKE_CURRENT_BINARY_DIR}"
+        --cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
+        -I "${tr_proto_path}"
+        --plugin=protoc-gen-grpc="${grpc_cpp_plugin_location}"
+        "${tr_proto}"
+      DEPENDS "${tr_proto}")
+
 message(STATUS "Thin replica gRPC/protobuf generated - see " ${CMAKE_CURRENT_BINARY_DIR})
 
-protobuf_generate_cpp(REPLICA_STATE_SNAPSHOT_PROTO_SRCS REPLICA_STATE_SNAPSHOT_PROTO_HDRS ${CMAKE_CURRENT_BINARY_DIR}
-  replica_state_snapshot.proto
-)
-grpc_generate_cpp(REPLICA_STATE_SNAPSHOT_GRPC_SRCS REPLICA_STATE_SNAPSHOT_GRPC_HDRS ${CMAKE_CURRENT_BINARY_DIR}
-  replica_state_snapshot.proto
-)
+get_filename_component(ss_proto "replica_state_snapshot.proto" ABSOLUTE)
+get_filename_component(ss_proto_path "${ss_proto}" PATH)
+
+set(ss_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/replica_state_snapshot.pb.cc")
+set(ss_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/replica_state_snapshot.pb.h")
+set(ss_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/replica_state_snapshot.grpc.pb.cc")
+set(ss_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/replica_state_snapshot.grpc.pb.h")
+add_custom_command(
+      OUTPUT "${ss_proto_srcs}" "${ss_proto_hdrs}" "${ss_grpc_srcs}" "${ss_grpc_hdrs}"
+      COMMAND protoc
+      ARGS --grpc_out=generate_mock_code=true:"${CMAKE_CURRENT_BINARY_DIR}"
+        --cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
+        -I "${ss_proto_path}"
+        --plugin=protoc-gen-grpc="${grpc_cpp_plugin_location}"
+        "${ss_proto}"
+      DEPENDS "${ss_proto}")
+
 message(STATUS "State snapshot gRPC/protobuf generated - see " ${CMAKE_CURRENT_BINARY_DIR})
 
-add_library(thin-replica-proto STATIC ${THIN_REPLICA_PROTO_SRCS} ${THIN_REPLICA_PROTO_HDRS} 
-            ${THIN_REPLICA_GRPC_SRCS} ${THIN_REPLICA_GRPC_HDRS})
-target_link_libraries(thin-replica-proto protobuf::libprotobuf gRPC::grpc++)
+add_library(thin-replica-proto STATIC ${tr_proto_srcs} ${tr_proto_hdrs} 
+            ${tr_grpc_srcs} ${tr_grpc_hdrs})
+target_link_libraries(thin-replica-proto protobuf::libprotobuf gRPC::grpc++ gRPC::grpc++_reflection)
 target_include_directories(thin-replica-proto PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
 
-add_library(replica-state-snapshot-proto STATIC ${REPLICA_STATE_SNAPSHOT_PROTO_SRCS} ${REPLICA_STATE_SNAPSHOT_PROTO_HDRS} 
-            ${REPLICA_STATE_SNAPSHOT_GRPC_SRCS} ${REPLICA_STATE_SNAPSHOT_GRPC_HDRS})
-target_link_libraries(replica-state-snapshot-proto protobuf::libprotobuf gRPC::grpc++)
+add_library(replica-state-snapshot-proto STATIC ${ss_proto_srcs} ${ss_proto_hdrs} 
+            ${ss_grpc_srcs} ${ss_grpc_hdrs})
+target_link_libraries(replica-state-snapshot-proto protobuf::libprotobuf gRPC::grpc++ gRPC::grpc++_reflection)
 target_include_directories(replica-state-snapshot-proto PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
diff --git a/utt/wallet-cli/proto/CMakeLists.txt b/utt/wallet-cli/proto/CMakeLists.txt
@@ -1,15 +1,34 @@
+cmake_minimum_required(VERSION 3.5)
+set(CMAKE_CXX_STANDARD 17)
+
+project(concord)
+
 find_package(Protobuf REQUIRED)
 find_package(GRPC REQUIRED)
 
 include_directories(${GRPC_INCLUDE_DIR})
 
-protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ${CMAKE_CURRENT_BINARY_DIR}
-  api/v1/api.proto
-)
-grpc_generate_cpp(GRPC_SRCS GRPC_HDRS ${CMAKE_CURRENT_BINARY_DIR}
-  api/v1/api.proto
-)
+# Proto file
+get_filename_component(utt_wallet_api_proto "api/v1/api.proto" ABSOLUTE)
+get_filename_component(utt_wallet_api_proto_path "${utt_wallet_api_proto}" PATH)
+
+get_target_property(grpc_cpp_plugin_location gRPC::grpc_cpp_plugin LOCATION)
+
+set(utt_wallet_api_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/api.pb.cc")
+set(utt_wallet_api_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/api.pb.h")
+set(utt_wallet_api_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/api.grpc.pb.cc")
+set(utt_wallet_api_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/api.grpc.pb.h")
+
+add_custom_command(
+      OUTPUT "${utt_wallet_api_proto_srcs}" "${utt_wallet_api_proto_hdrs}" "${utt_wallet_api_grpc_srcs}" "${utt_wallet_api_grpc_hdrs}"
+      COMMAND protoc
+      ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}"
+        --cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
+        -I "${utt_wallet_api_proto_path}"
+        --plugin=protoc-gen-grpc="${grpc_cpp_plugin_location}"
+        "${utt_wallet_api_proto}"
+      DEPENDS "${utt_wallet_api_proto}")
 
-add_library(utt-wallet-api-proto STATIC ${PROTO_SRCS} ${GRPC_SRCS})
+add_library(utt-wallet-api-proto STATIC ${utt_wallet_api_proto_srcs} ${utt_wallet_api_grpc_srcs})
 target_link_libraries(utt-wallet-api-proto PRIVATE protobuf::libprotobuf gRPC::grpc++)
-target_include_directories(utt-wallet-api-proto PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
+target_include_directories(utt-wallet-api-proto PUBLIC ${CMAKE_CURRENT_BINARY_DIR})