Define CMF messages for SKVBC

This commit defines CMF messages intended for use with SKVBC (i.e.
simpleKVBC, implemented in tests/simpleKVBC) for communication between
the SKVBC clients and replicas.

The intention is to eventually move SKVBC clients and replicas
(including those implemented in tests/simpleKVBC/TesterClient and
tests/simpleKVBC/TesterReplica and also Apollo's usage of SKVBC
implemented in tests/apollo/util/skvbc.py) to use these messages
defined in CMF rather than their current ad-hoc message format rooted
in manually packing C++ struct and object data. We hope the switch to
CMF in this area will improve the code cleanliness, readability, and
long-term maintainability.

If this commit is accepted, it will be followed up at a later time with
changes to actually switch the SKVBC clients and replicas (as outlined
above) to use the messages newly defined in this commit.

Makefile

@@ -10,6 +10,7 @@ CONCORD_BFT_KVBC_CMF_PATHS?=${CONCORD_BFT_TARGET_SOURCE_PATH}/build/kvbc/cmf
 CONCORD_BFT_RECONFIGURATION_CMF_PATHS?=${CONCORD_BFT_TARGET_SOURCE_PATH}/build/reconfiguration/cmf
 CONCORD_BFT_BFTENGINE_CMF_PATHS?=${CONCORD_BFT_TARGET_SOURCE_PATH}/build/bftengine/cmf
 CONCORD_BFT_CCRON_CMF_PATHS?=${CONCORD_BFT_TARGET_SOURCE_PATH}/build/ccron/cmf
+CONCORD_BFT_SKVBC_CMF_PATHS?=${CONCORD_BFT_TARGET_SOURCE_PATH}/build/tests/simpleKVBC/cmf
 CONCORD_BFT_CLIENT_PROTO_PATH?=${CONCORD_BFT_TARGET_SOURCE_PATH}/build/client/clientservice/proto
 CONCORD_BFT_THIN_REPLICA_PROTO_PATH?=${CONCORD_BFT_TARGET_SOURCE_PATH}/build/thin-replica-server/proto
 CONCORD_BFT_KVBC_PROTO_PATH?=${CONCORD_BFT_TARGET_SOURCE_PATH}/build/kvbc/proto
@@ -174,6 +175,7 @@ tidy-check: gen_cmake ## Run clang-tidy
 		make -C ${CONCORD_BFT_RECONFIGURATION_CMF_PATHS} &> /dev/null && \
 		make -C ${CONCORD_BFT_BFTENGINE_CMF_PATHS} &> /dev/null && \
 		make -C ${CONCORD_BFT_CCRON_CMF_PATHS} &> /dev/null && \
+		make -C ${CONCORD_BFT_SKVBC_CMF_PATHS} &> /dev/null && \
 		make -C ${CONCORD_BFT_CLIENT_PROTO_PATH} &> /dev/null && \
 		make -C ${CONCORD_BFT_THIN_REPLICA_PROTO_PATH} &> /dev/null && \
 		make -C ${CONCORD_BFT_KVBC_PROTO_PATH} &> /dev/null && \

messages/compiler/cpp/cpp_visitor.py

@@ -127,7 +127,9 @@ def equalop_str_declaration(msg_name):
 
 def equalop_str(msg_name, fields):
     """ Create an 'operator==' function for the current message struct """
-    comparison = " && ".join([f"l.{f} == r.{f}" for f in fields])
+    comparison = "true"
+    if fields:
+        comparison = " && ".join([f"l.{f} == r.{f}" for f in fields])
     return equalop_str_fn.format(msg_name=msg_name) + f""" {{
   return {comparison};
 }}"""

messages/compiler/grammar.ebnf

@@ -5,7 +5,7 @@ start = {toplevel}+ $ ;
 
 toplevel = msgs:{msg}+ | enums:{enum_def}+ ;
 
-msg = 'Msg' name:msgname id:msgid '{' fields:{field}+ '}' ;
+msg = 'Msg' name:msgname id:msgid '{' fields:{field}* '}' ;
 enum_def = 'Enum' name:enumname '{' tags:','.{name}+ '}' ;
 
 msgname = name:name;

messages/compiler/python/py_visitor.py

@@ -76,6 +76,9 @@ def __init__(self):
         # The current field name being processed
         self.field_name = ''
 
+        # Whether the current message has any fields
+        self.has_any_fields = False
+
         # Messages are represented as Classes in python
         self.msg_class = ''
 
@@ -115,6 +118,8 @@ def msg_start(self, name, id):
         self.eq = eq_start()
 
     def msg_end(self):
+        if (not self.has_any_fields):
+            self.msg_class += '         pass\n'
         self.serialize += '        return serializer.buf'
         self.deserialize += '        return obj, deserializer.pos\n'
         self.eq += '        return True\n'
@@ -127,6 +132,7 @@ def create_enum(self, name, tags):
 
     def field_start(self, name, type):
         self.field_name = name
+        self.has_any_fields = True
         self.msg_class += f'         self.{name} = None\n'
 
     def field_end(self):

messages/example.cmf

@@ -85,6 +85,13 @@ Msg ComposedOneof 18 {
     list oneof {Transaction NewViewElement} x
 }
 
+# Note empty messages are supported. An example of a use for an empty message
+# would be a request that has no parameters that vary between requests, such as
+# a request for the latest entire block committed to a blockchain, or a request
+# for the health status of a particular service.
+Msg EmptyRequest 19 {
+}
+
 ######
 # Uncomment each message to induce various parse errors
 ######

tests/apollo/util/CMakeLists.txt

@@ -1 +1,2 @@
-cmf_generate_python(concord_msgs ../../../reconfiguration/cmf/concord.cmf)
+cmf_generate_python(concord_msgs ../../../reconfiguration/cmf/concord.cmf)
+cmf_generate_python(skvbc_messages ../../simpleKVBC/cmf/skvbc_messages.cmf)

tests/simpleKVBC/CMakeLists.txt

@@ -1,5 +1,6 @@
+add_subdirectory(cmf)
 add_subdirectory(TesterClient)
 add_subdirectory(TesterReplica)
 add_custom_target(copy_blockchain_scripts2 ALL COMMENT "Copying scripts abcd")
 add_custom_command(TARGET copy_blockchain_scripts2
-        COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/scripts ${CMAKE_CURRENT_BINARY_DIR}/scripts)
+        COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/scripts ${CMAKE_CURRENT_BINARY_DIR}/scripts)

tests/simpleKVBC/cmf/CMakeLists.txt

@@ -0,0 +1,4 @@
+cmf_generate_cpp(header cpp skvbc::messages skvbc_messages.cmf)
+add_library(skvbc_messages_cmf ${cpp})
+set_target_properties(skvbc_messages_cmf PROPERTIES LINKER_LANGUAGE CXX)
+target_include_directories(skvbc_messages_cmf PUBLIC ${CMAKE_CURRENT_BINARY_DIR})

tests/simpleKVBC/cmf/skvbc_messages.cmf

@@ -0,0 +1,62 @@
+# Definition of client-server interface for the SimpleKVBC test application.
+#
+# Note SimpleKVBC effectively implements a versioned key-value store supporting
+# only keys and values that are byte strings.
+
+Msg SKVBCReadRequest 3 {
+
+    # Note an SKVBCReadRequest with read version 0 indicates that the latest
+    # version should be read.
+    uint64 read_version
+
+    list bytes keys
+}
+
+Msg SKVBCWriteRequest 4 {
+    uint64 read_version
+
+    # if set to true, the SimpleKVBC replica will waste some time when it works
+    # on pre-processing this request, which may be useful for simulating certain
+    # workloads in testing.
+    bool long_exec
+
+    list bytes readset
+    list kvpair bytes bytes writeset
+}
+
+Msg SKVBCGetLastBlockRequest 5 {
+}
+
+Msg SKVBCGetBlockDataRequest 6 {
+    uint64 block_id
+}
+
+Msg SKVBCReadReply 7 {
+    list kvpair bytes bytes reads
+}
+
+Msg SKVBCWriteReply 8 {
+    bool success
+    uint64 latest_block
+}
+
+Msg SKVBCGetLastBlockReply 9 {
+    uint64 latest_block
+}
+
+Msg SKVBCRequest 1 {
+    oneof {
+        SKVBCReadRequest
+        SKVBCWriteRequest
+        SKVBCGetLastBlockRequest
+        SKVBCGetBlockDataRequest
+    } request
+}
+
+Msg SKVBCReply 2 {
+    oneof {
+        SKVBCReadReply
+        SKVBCWriteReply
+        SKVBCGetLastBlockReply
+    } reply
+}