Significant update to build and install process for ccf ledger

build/cmake/ProjectVariables.cmake

@@ -55,3 +55,25 @@ ELSE()
   ADD_COMPILE_OPTIONS(-Wno-deprecated)
   ADD_COMPILE_OPTIONS(-Wno-deprecated-declarations)
 ENDIF()
+
+IF (NOT DEFINED ENV{PDO_INSTALL_ROOT})
+  MESSAGE(FATAL_ERROR "PDO_INSTALL_ROOT not defined")
+ENDIF()
+SET(PDO_INSTALL_ROOT $ENV{PDO_INSTALL_ROOT})
+
+IF (NOT DEFINED ENV{PDO_SOURCE_ROOT})
+  MESSAGE(FATAL_ERROR "PDO_SOURCE_ROOT not defined")
+ENDIF()
+SET(PDO_SOURCE_ROOT $ENV{PDO_SOURCE_ROOT})
+
+# Get the current version using the get_version
+# utility; note that this will provide 0.0.0 as
+# the version if something goes wrong (like running
+# without any annotated version tags)
+EXECUTE_PROCESS(
+  COMMAND ./get_version
+  WORKING_DIRECTORY ${PDO_SOURCE_ROOT}/bin
+  OUTPUT_VARIABLE PDO_VERSION
+  ERROR_QUIET
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+)

build/cmake/Python.cmake

@@ -0,0 +1,53 @@
+# Copyright 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# assumes that project variables have been
+# set including PDO_INSTALL_ROOT and PDO_CONTRACT_VERSION
+
+IF (NOT DEFINED PDO_INSTALL_ROOT)
+  MESSAGE(FATAL_ERROR "PDO_INSTALL_ROOT not defined")
+ENDIF()
+
+IF (NOT DEFINED PDO_VERSION)
+  MESSAGE(FATAL_ERROR "PDO_VERSION not defined")
+ENDIF()
+
+SET(PIP "${PDO_INSTALL_ROOT}/bin/pip3" CACHE STRING "Pip executable in virtual environment")
+SET(PYTHON "${PDO_INSTALL_ROOT}/bin/python3" CACHE STRING "Python executable in virtual environment")
+SET(RESOURCE_INSTALLER "${PDO_INSTALL_ROOT}/bin/pdo-install-plugin-resources" CACHE STRING "PDO resource installer")
+SET(WHEEL_PATH "${CMAKE_BINARY_DIR}/dist" CACHE STRING "Path where python wheels will be placed")
+
+FUNCTION(BUILD_WHEEL package)
+  SET(SOURCE ${CMAKE_CURRENT_SOURCE_DIR})
+  SET(WHEEL_FILE "${WHEEL_PATH}/${package}-${PDO_VERSION}-py3-none-any.whl")
+  FILE(STRINGS "${SOURCE}/MANIFEST" MANIFEST)
+
+  # adding the build and egg-info directories to the output means that
+  # they will be cleaned up with the global clean target
+  ADD_CUSTOM_COMMAND(
+    OUTPUT ${WHEEL_FILE} ${SOURCE}/${package}.egg-info
+    COMMAND ${PYTHON}
+    ARGS -m build --wheel --outdir ${WHEEL_PATH}
+    WORKING_DIRECTORY ${SOURCE}
+    DEPENDS ${MANIFEST})
+
+  ADD_CUSTOM_TARGET(${package}-package ALL DEPENDS ${WHEEL_FILE})
+
+  STRING(JOIN "\n" INSTALL_COMMAND
+    "MESSAGE(\"INSTALL ${package}\")"
+    "EXECUTE_PROCESS(COMMAND ${PIP} uninstall --yes ${WHEEL_FILE})"
+    "EXECUTE_PROCESS(COMMAND ${PIP} install ${WHEEL_FILE})" )
+
+  INSTALL(CODE ${INSTALL_COMMAND})
+ENDFUNCTION()

docker/tools/build_client.sh

@@ -22,5 +22,5 @@ export PDO_HOSTNAME=localhost
 export PDO_LEDGER_URL=https://127.0.0.1:6600
 
 try make -C ${PDO_SOURCE_ROOT}/build environment
-try make -C ${PDO_SOURCE_ROOT}/build template
 try make -C ${PDO_SOURCE_ROOT}/build build-client
+try make -C ${PDO_SOURCE_ROOT}/ledgers/ccf install-python

docker/tools/build_services.sh

@@ -27,6 +27,6 @@ check_pdo_build_env
 yell Build and install services into ${PDO_INSTALL_ROOT}
 # -----------------------------------------------------------------
 try make -C ${PDO_SOURCE_ROOT}/build environment
-try make -C ${PDO_SOURCE_ROOT}/build template
 try make -C ${PDO_SOURCE_ROOT}/build system-keys
 try make -C ${PDO_SOURCE_ROOT}/build verified-build
+try make -C ${PDO_SOURCE_ROOT}/ledgers/ccf install-python

ledgers/ccf/.gitignore

@@ -1 +1 @@
-build
+build*

ledgers/ccf/CMakeLists.txt

@@ -14,16 +14,15 @@
 
 CMAKE_MINIMUM_REQUIRED(VERSION 3.16)
 
-INCLUDE(${CCF_DIR}/cmake/preproject.cmake)
-PROJECT(pdoenc LANGUAGES C CXX)
-INCLUDE(${CCF_DIR}/cmake/ccf_app.cmake)
+IF (NOT DEFINED ENV{PDO_SOURCE_ROOT})
+  MESSAGE(FATAL_ERROR "PDO_SOURCE_ROOT not defined")
+ENDIF()
+SET(PDO_SOURCE_ROOT $ENV{PDO_SOURCE_ROOT})
 
-FIND_PACKAGE(ccf_virtual REQUIRED)
+LIST(APPEND CMAKE_MODULE_PATH "${PDO_SOURCE_ROOT}/build/cmake")
+INCLUDE(ProjectVariables)
 
-# Add the PDO transaction processor target
-add_ccf_app(pdoenc
-  SRCS transaction_processor/pdo_tp.cpp transaction_processor/verify_signatures.cpp
-  INCLUDE_DIRS ${CCF_DIR}/include/ccf ${CCF_DIR}/include/3rdparty ${CCF_DIR}/include
-)
+PROJECT(pdo_ccf_ledger)
 
-INSTALL(TARGETS pdoenc.virtual DESTINATION lib)
+INCLUDE(Python)
+BUILD_WHEEL(pdo_ccf_ledger)

ledgers/ccf/MANIFEST

@@ -0,0 +1,12 @@
+MANIFEST.in
+setup.py
+pdo/__init__.py
+pdo/ledgers/__init__.py
+pdo/ledgers/ccf/__init__.py
+pdo/ledgers/ccf/common.py
+pdo/ledgers/ccf/scripts/__init__.py
+pdo/ledgers/ccf/scripts/fetch_ledger_authority.py
+pdo/ledgers/ccf/scripts/ping_test.py
+pdo/ledgers/ccf/scripts/register_enclave_attestation_verification_policy.py
+pdo/ledgers/ccf/scripts/generate_ledger_authority.py
+pdo/ledgers/ccf/scripts/configure_ccf_network.py

ledgers/ccf/Makefile

@@ -35,7 +35,14 @@ CCF_LEDGER_DIR ?= $(PDO_INSTALL_ROOT)/opt/pdo/ccf
 NINJA ?= ninja $(NINJA_OPTIONS)
 
 SCRIPTDIR ?= $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
-BLDDIR := $(SCRIPTDIR)/build
+
+# Build the PDO TP in the transaction processor subdirectory
+PDO_TP_SRCDIR := $(SCRIPTDIR)/transaction_processor
+PDO_TP_BLDDIR := $(SCRIPTDIR)/transaction_processor/build
+
+# Build the Python wheel in this directory
+PYTHON_SRCDIR := $(SCRIPTDIR)
+PYTHON_BLDDIR := $(SCRIPTDIR)/build
 
 all: environment install
 
@@ -76,22 +83,28 @@ $(CCF_LEDGER_DIR)/lib/python3.8 : pdo-environment
 # -----------------------------------------------------------------
 build : build-pdo-tp
 
-build-pdo-tp : $(BLDDIR)
-	cmake --build $(BLDDIR)
+build-pdo-tp : $(PDO_TP_BLDDIR)
+	cmake --build $(PDO_TP_BLDDIR)
 
-$(BLDDIR) :
-	cmake -S . -B $(BLDDIR) -GNinja \
+$(PDO_TP_BLDDIR) :
+	cmake -S $(PDO_TP_SRCDIR) -B $(PDO_TP_BLDDIR) -GNinja \
 		-DCCF_DIR=$(CCF_BASE) \
 		-DCOMPILE_TARGET=$(CCF_COMPILE_TARGET) \
 		-DCMAKE_INSTALL_PREFIX=$(CCF_LEDGER_DIR)
 
+build-python : $(PYTHON_BLDDIR)
+	cmake --build $(PYTHON_BLDDIR)
+
+$(PYTHON_BLDDIR) :
+	cmake -S $(PYTHON_SRCDIR) -B $(PYTHON_BLDDIR)
+
 # -----------------------------------------------------------------
 # clean up
 # -----------------------------------------------------------------
 clean : clean-build clean-ledger
 
 clean-build:
-	rm -rf $(BLDDIR)
+	rm -rf $(PDO_TP_BLDDIR) $(PYTHON_BLDDIR)
 
 clean-ledger :
 	rm -rf $(CCF_LEDGER_DIR)
@@ -109,15 +122,14 @@ clean-ledger :
 PDO_BASH_SCRIPTS = $(wildcard scripts/*.sh)
 PDO_PYTHON_SCRIPTS = $(wildcard scripts/*.py)
 
-install : install-pdo-tp install-pdo-scripts
+install : install-pdo-tp install-python
 
-# cd $(BLDDIR) && $(NINJA) install
 install-pdo-tp : ledger-environment build-pdo-tp
-	cmake --install $(BLDDIR)
+	cmake --install $(PDO_TP_BLDDIR)
 	cp $(PDO_BASH_SCRIPTS) $(CCF_LEDGER_DIR)/bin
 
-install-pdo-scripts : pdo-environment
-	cp $(PDO_PYTHON_SCRIPTS) $(PDO_INSTALL_ROOT)/bin
+install-python : pdo-environment build-python
+	cmake --install $(PYTHON_BLDDIR)
 
 # -----------------------------------------------------------------
 .PHONY : all

ledgers/ccf/pdo/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2018 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__import__('pkg_resources').declare_namespace('pdo')

ledgers/ccf/pdo/ledgers/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

ledgers/ccf/pdo/ledgers/ccf/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

ledgers/ccf/pdo/ledgers/ccf/common.py

@@ -0,0 +1,144 @@
+# Copyright 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import sys
+from urllib.parse import urlparse
+
+from ccf.clients import Identity
+from ccf.clients import CCFClient
+
+from loguru import logger as LOG
+
+# -----------------------------------------------------------------
+# parse options and initialize the common variables
+# -----------------------------------------------------------------
+def parse_common_arguments(args, description, member_keys_required = False) :
+
+    parser = argparse.ArgumentParser(description=description)
+
+    parser.add_argument(
+        '--logfile',
+        help='Name of the log file, __screen__ for standard output',
+        default='__screen__',
+        type=str)
+
+    parser.add_argument(
+        '--loglevel',
+        help='Logging level',
+        default='WARNING',
+        type=str)
+
+    parser.add_argument(
+        '--url',
+        help='URL for the ledger',
+        default = os.environ.get("PDO_LEDGER_URL"),
+        type=str)
+
+    parser.add_argument(
+        '--interface',
+        help='Host interface where CCF is listening',
+        type=str)
+
+    parser.add_argument(
+        '--port',
+        help='Port where CCF is listening',
+        type=int,
+        default=6600)
+
+    parser.add_argument(
+        '--key-dir',
+        help='Directory where certificate files are located, defaults to PDO_LEDGER_KEY_ROOT',
+        default=os.environ.get("PDO_LEDGER_KEY_ROOT"),
+        type=str)
+
+    parser.add_argument(
+        '--cert',
+        help='Name of the network certificate file',
+        type=str,
+        default='networkcert.pem')
+
+    parser.add_argument(
+        '--member',
+        help="Name of the network membership certificate",
+        default = "memberccf",
+        type=str)
+
+    (options, unprocessed_args) = parser.parse_known_args(args)
+
+    # set up the logging
+    LOG.remove()
+    if options.logfile == '__screen__' :
+        LOG.add(sys.stderr, level=options.loglevel)
+    else :
+        LOG.add(options.logfile)
+
+    # precedence is given to ledger interface through the interface/port parameters; the fall back
+    # is to use the ledger url parameter
+    if options.interface :
+        pass
+    elif options.url :
+        (options.interface, options.port) = urlparse(options.url).netloc.split(':')
+    else :
+        LOG.error('no ledger interface specified')
+        sys.exit(-1)
+
+    # the key directory must be specified either through the PDO_LEDGER_KEY_ROOT
+    # environment variable or the key-dir parameter
+    if not options.key_dir or not os.path.exists(options.key_dir) :
+        LOG.error('unable to locate key dir')
+        sys.exit(-1)
+
+    network_cert = os.path.join(options.key_dir, options.cert)
+    if not os.path.exists(network_cert) :
+        LOG.error('network certificate ({}) does not exist'.format(network_cert))
+        sys.exit(-1)
+
+    # now create the client
+    if member_keys_required :
+        member_cert = os.path.join(options.key_dir, "{}_cert.pem".format(options.member))
+        if not os.path.exists(member_cert) :
+            LOG.error('member certificate ({}) does not exist'.format(member_cert))
+            sys.exit(-1)
+
+        member_key = os.path.join(options.key_dir, "{}_privk.pem".format(options.member))
+        if not os.path.exists(member_key) :
+            LOG.error('member key ({}) does not exist'.format(member_key))
+            sys.exit(-1)
+
+        try :
+            client = CCFClient(
+                options.interface,
+                options.port,
+                network_cert,
+                session_auth=Identity(member_key, member_cert, "member"),
+                signing_auth=Identity(member_key, member_cert, "member"),
+            )
+        except Exception as e:
+            LOG.error('failed to connect to CCF service : {}'.format(str(e)))
+            sys.exit(-1)
+
+    else :
+        try :
+            client = CCFClient(
+                options.interface,
+                options.port,
+                network_cert)
+        except Exception as e:
+            LOG.error('failed to connect to CCF service : {}'.format(str(e)))
+            sys.exit(-1)
+
+    # and return the client plus any operation-specific arguments that have not been processed
+    return (options, unprocessed_args, client)