From dfbb9fb6f65f8e1782372951e30de078ca61cbaa Mon Sep 17 00:00:00 2001 From: Alex Upshaw Date: Mon, 21 Jun 2021 16:37:34 +0000 Subject: [PATCH] Define CMF messages for SKVBC This commit defines CMF messages intended for use with SKVBC (i.e. simpleKVBC, implemented in tests/simpleKVBC) for communication between the SKVBC clients and replicas. The intention is to eventually move SKVBC clients and replicas (including those implemented in tests/simpleKVBC/TesterClient and tests/simpleKVBC/TesterReplica and also Apollo's usage of SKVBC implemented in tests/apollo/util/skvbc.py) to use these messages defined in CMF rather than their current ad-hoc message format rooted in manually packing C++ struct and object data. We hope the switch to CMF in this area will improve the code cleanliness, readability, and long-term maintainability. If this commit is accepted, it will be followed up at a later time with changes to actually switch the SKVBC clients and replicas (as outlined above) to use the messages newly defined in this commit. --- Makefile | 2 + messages/compiler/cpp/cpp_visitor.py | 4 +- messages/compiler/grammar.ebnf | 2 +- messages/compiler/python/py_visitor.py | 6 +++ messages/example.cmf | 7 +++ tests/apollo/util/CMakeLists.txt | 3 +- tests/simpleKVBC/CMakeLists.txt | 3 +- tests/simpleKVBC/cmf/CMakeLists.txt | 4 ++ tests/simpleKVBC/cmf/skvbc_messages.cmf | 62 +++++++++++++++++++++++++ 9 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 tests/simpleKVBC/cmf/CMakeLists.txt create mode 100644 tests/simpleKVBC/cmf/skvbc_messages.cmf diff --git a/Makefile b/Makefile index 1718e3d0c3..1c6e2703ec 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ CONCORD_BFT_KVBC_CMF_PATHS?=${CONCORD_BFT_TARGET_SOURCE_PATH}/build/kvbc/cmf CONCORD_BFT_RECONFIGURATION_CMF_PATHS?=${CONCORD_BFT_TARGET_SOURCE_PATH}/build/reconfiguration/cmf CONCORD_BFT_BFTENGINE_CMF_PATHS?=${CONCORD_BFT_TARGET_SOURCE_PATH}/build/bftengine/cmf CONCORD_BFT_CCRON_CMF_PATHS?=${CONCORD_BFT_TARGET_SOURCE_PATH}/build/ccron/cmf +CONCORD_BFT_SKVBC_CMF_PATHS?=${CONCORD_BFT_TARGET_SOURCE_PATH}/build/tests/simpleKVBC/cmf CONCORD_BFT_CLIENT_PROTO_PATH?=${CONCORD_BFT_TARGET_SOURCE_PATH}/build/client/clientservice/proto CONCORD_BFT_THIN_REPLICA_PROTO_PATH?=${CONCORD_BFT_TARGET_SOURCE_PATH}/build/thin-replica-server/proto CONCORD_BFT_KVBC_PROTO_PATH?=${CONCORD_BFT_TARGET_SOURCE_PATH}/build/kvbc/proto @@ -174,6 +175,7 @@ tidy-check: gen_cmake ## Run clang-tidy make -C ${CONCORD_BFT_RECONFIGURATION_CMF_PATHS} &> /dev/null && \ make -C ${CONCORD_BFT_BFTENGINE_CMF_PATHS} &> /dev/null && \ make -C ${CONCORD_BFT_CCRON_CMF_PATHS} &> /dev/null && \ + make -C ${CONCORD_BFT_SKVBC_CMF_PATHS} &> /dev/null && \ make -C ${CONCORD_BFT_CLIENT_PROTO_PATH} &> /dev/null && \ make -C ${CONCORD_BFT_THIN_REPLICA_PROTO_PATH} &> /dev/null && \ make -C ${CONCORD_BFT_KVBC_PROTO_PATH} &> /dev/null && \ diff --git a/messages/compiler/cpp/cpp_visitor.py b/messages/compiler/cpp/cpp_visitor.py index 95c02e712a..4d82b1ab42 100644 --- a/messages/compiler/cpp/cpp_visitor.py +++ b/messages/compiler/cpp/cpp_visitor.py @@ -127,7 +127,9 @@ def equalop_str_declaration(msg_name): def equalop_str(msg_name, fields): """ Create an 'operator==' function for the current message struct """ - comparison = " && ".join([f"l.{f} == r.{f}" for f in fields]) + comparison = "true" + if fields: + comparison = " && ".join([f"l.{f} == r.{f}" for f in fields]) return equalop_str_fn.format(msg_name=msg_name) + f""" {{ return {comparison}; }}""" diff --git a/messages/compiler/grammar.ebnf b/messages/compiler/grammar.ebnf index 0e41684356..c7baed1242 100644 --- a/messages/compiler/grammar.ebnf +++ b/messages/compiler/grammar.ebnf @@ -5,7 +5,7 @@ start = {toplevel}+ $ ; toplevel = msgs:{msg}+ | enums:{enum_def}+ ; -msg = 'Msg' name:msgname id:msgid '{' fields:{field}+ '}' ; +msg = 'Msg' name:msgname id:msgid '{' fields:{field}* '}' ; enum_def = 'Enum' name:enumname '{' tags:','.{name}+ '}' ; msgname = name:name; diff --git a/messages/compiler/python/py_visitor.py b/messages/compiler/python/py_visitor.py index 33f176cf4b..f34d3c9dad 100644 --- a/messages/compiler/python/py_visitor.py +++ b/messages/compiler/python/py_visitor.py @@ -76,6 +76,9 @@ def __init__(self): # The current field name being processed self.field_name = '' + # Whether the current message has any fields + self.has_any_fields = False + # Messages are represented as Classes in python self.msg_class = '' @@ -115,6 +118,8 @@ def msg_start(self, name, id): self.eq = eq_start() def msg_end(self): + if (not self.has_any_fields): + self.msg_class += ' pass\n' self.serialize += ' return serializer.buf' self.deserialize += ' return obj, deserializer.pos\n' self.eq += ' return True\n' @@ -127,6 +132,7 @@ def create_enum(self, name, tags): def field_start(self, name, type): self.field_name = name + self.has_any_fields = True self.msg_class += f' self.{name} = None\n' def field_end(self): diff --git a/messages/example.cmf b/messages/example.cmf index 35563b534e..9196a56c92 100644 --- a/messages/example.cmf +++ b/messages/example.cmf @@ -85,6 +85,13 @@ Msg ComposedOneof 18 { list oneof {Transaction NewViewElement} x } +# Note empty messages are supported. An example of a use for an empty message +# would be a request that has no parameters that vary between requests, such as +# a request for the latest entire block committed to a blockchain, or a request +# for the health status of a particular service. +Msg EmptyRequest 19 { +} + ###### # Uncomment each message to induce various parse errors ###### diff --git a/tests/apollo/util/CMakeLists.txt b/tests/apollo/util/CMakeLists.txt index 20e777a428..ac4c5bb92e 100644 --- a/tests/apollo/util/CMakeLists.txt +++ b/tests/apollo/util/CMakeLists.txt @@ -1 +1,2 @@ -cmf_generate_python(concord_msgs ../../../reconfiguration/cmf/concord.cmf) \ No newline at end of file +cmf_generate_python(concord_msgs ../../../reconfiguration/cmf/concord.cmf) +cmf_generate_python(skvbc_messages ../../simpleKVBC/cmf/skvbc_messages.cmf) diff --git a/tests/simpleKVBC/CMakeLists.txt b/tests/simpleKVBC/CMakeLists.txt index 5123a9a53d..1611de37a5 100644 --- a/tests/simpleKVBC/CMakeLists.txt +++ b/tests/simpleKVBC/CMakeLists.txt @@ -1,5 +1,6 @@ +add_subdirectory(cmf) add_subdirectory(TesterClient) add_subdirectory(TesterReplica) add_custom_target(copy_blockchain_scripts2 ALL COMMENT "Copying scripts abcd") add_custom_command(TARGET copy_blockchain_scripts2 - COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/scripts ${CMAKE_CURRENT_BINARY_DIR}/scripts) \ No newline at end of file + COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/scripts ${CMAKE_CURRENT_BINARY_DIR}/scripts) diff --git a/tests/simpleKVBC/cmf/CMakeLists.txt b/tests/simpleKVBC/cmf/CMakeLists.txt new file mode 100644 index 0000000000..5e258c8024 --- /dev/null +++ b/tests/simpleKVBC/cmf/CMakeLists.txt @@ -0,0 +1,4 @@ +cmf_generate_cpp(header cpp skvbc::messages skvbc_messages.cmf) +add_library(skvbc_messages_cmf ${cpp}) +set_target_properties(skvbc_messages_cmf PROPERTIES LINKER_LANGUAGE CXX) +target_include_directories(skvbc_messages_cmf PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/tests/simpleKVBC/cmf/skvbc_messages.cmf b/tests/simpleKVBC/cmf/skvbc_messages.cmf new file mode 100644 index 0000000000..cdaac97332 --- /dev/null +++ b/tests/simpleKVBC/cmf/skvbc_messages.cmf @@ -0,0 +1,62 @@ +# Definition of client-server interface for the SimpleKVBC test application. +# +# Note SimpleKVBC effectively implements a versioned key-value store supporting +# only keys and values that are byte strings. + +Msg SKVBCReadRequest 3 { + + # Note an SKVBCReadRequest with read version 0 indicates that the latest + # version should be read. + uint64 read_version + + list bytes keys +} + +Msg SKVBCWriteRequest 4 { + uint64 read_version + + # if set to true, the SimpleKVBC replica will waste some time when it works + # on pre-processing this request, which may be useful for simulating certain + # workloads in testing. + bool long_exec + + list bytes readset + list kvpair bytes bytes writeset +} + +Msg SKVBCGetLastBlockRequest 5 { +} + +Msg SKVBCGetBlockDataRequest 6 { + uint64 block_id +} + +Msg SKVBCReadReply 7 { + list kvpair bytes bytes reads +} + +Msg SKVBCWriteReply 8 { + bool success + uint64 latest_block +} + +Msg SKVBCGetLastBlockReply 9 { + uint64 latest_block +} + +Msg SKVBCRequest 1 { + oneof { + SKVBCReadRequest + SKVBCWriteRequest + SKVBCGetLastBlockRequest + SKVBCGetBlockDataRequest + } request +} + +Msg SKVBCReply 2 { + oneof { + SKVBCReadReply + SKVBCWriteReply + SKVBCGetLastBlockReply + } reply +}