Prepare for using azure cleanroom in tests

.devcontainer/devcontainer.json

@@ -1,17 +1,17 @@
 {
   "name": "KMS Dev",
-  "build": {
-    "dockerfile": "Dockerfile.devcontainer",
-    "context": ".."
-  },
+  "image": "mcr.microsoft.com/devcontainers/universal:2",
   "features": {
-    "ghcr.io/devcontainers/features/docker-in-docker:2": {
-      "version": "latest",
-      "enableNonRootDocker": "true",
-      "moby": "true"
-    },
+    "ghcr.io/devcontainers/features/docker-in-docker:2": {},
     "ghcr.io/devcontainers/features/common-utils:2": {},
-    "ghcr.io/devcontainers/features/node:1": {}
+    "ghcr.io/devcontainers/features/node:1": {},
+    "ghcr.io/devcontainers/features/powershell:1": {
+      "modules": "powershell-yaml"
+    },
+    "ghcr.io/devcontainers/features/azure-cli:1": {
+			"version": "2.61.0",
+      "extensions": "confcom,managedccfs"
+    }
   },
   "customizations": {
     "vscode": {

.github/workflows/endpoint-test.yml

@@ -22,7 +22,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        env: [local]
+        env: [ccf/sandbox-local]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -42,9 +42,4 @@ jobs:
       - name: Run System Tests
         env:
             TEST_ENVIRONMENT: ${{ matrix.env }}
-        run: |
-          if [[ "$TEST_ENVIRONMENT" == "cloud" ]]; then
-            pytest -sv -n auto test/system-test/test_${{ inputs.endpoint }}.py
-          else
-            pytest -sv test/system-test/test_${{ inputs.endpoint }}.py
-          fi
+        run: pytest -sv test/system-test/test_${{ inputs.endpoint }}.py

Makefile

@@ -43,37 +43,32 @@ setup: ## Setup proposals and generate an initial key
 
 stop-host:  ## 🏃 Stop the host
 	@echo -e "\e[34m$@\e[0m" || true
-	sudo lsof -t -i :8000 | xargs -r sudo kill -9
+	source ./scripts/ccf/sandbox-local/down.sh
 
 stop-idp:  ## 🏃 Stop the idp
 	@echo -e "\e[34m$@\e[0m" || true
-	sudo lsof -t -i :3000 | xargs -r sudo kill -9
+	source ./scripts/jwt-issuer/down.sh
 
 stop-all: stop-host stop-idp # Stop all services
 	@echo -e "\e[34m$@\e[0m" || true
 
 # idp commands to issue JWT
 start-idp:  ## 🏃 Start the idp for testing jwt
 	@echo -e "\e[34m$@\e[0m" || true
-	mkdir -p ${KMS_WORKSPACE}
-	cd test/utils/jwt && KMS_WORKSPACE=${KMS_WORKSPACE} nohup npm run start > nohup.out 2>&1 &
-	./scripts/wait_idp_ready.sh
+	source ./scripts/jwt-issuer/up.sh
 
 # Start hosting the application using `sandbox.sh` and enable custom JWT authentication
 start-host: stop-host  ## 🏃 Start the CCF network using Sandbox.sh
 	@echo -e "\e[34m$@\e[0m" || true
-	$(CCFSB)/sandbox.sh --js-app-bundle ./dist/ --initial-member-count ${MEMBER_COUNT} --initial-user-count 1 --constitution ./governance/constitution/kms_actions.js  -v --http2
+	MEMBER_COUNT=${MEMBER_COUNT} source ./scripts/ccf/sandbox-local/up.sh && \
+	source ./scripts/kms/js-app-set.sh && \
+	source ./scripts/kms/constitution-set.sh ./governance/constitution/kms_actions.js
 
-start-host-idp: stop-host stop-idp start-idp build ## 🏃 Start the CCF network && idp using Sandbox.sh
+start-host-idp: stop-host stop-idp start-idp start-host ## 🏃 Start the CCF network && idp using Sandbox.sh
 	@echo -e "\e[34m$@\e[0m" || true
 	@echo "Executing: $(COMMAND)"
-	if [ "$(RUN_BACK)" = "true" ]; then \
-		 env -i PATH=${PATH} KMS_WORKSPACE=${KMS_WORKSPACE} $(CCFSB)/sandbox.sh --js-app-bundle ./dist/ --initial-member-count  ${MEMBER_COUNT} --initial-user-count 1 --constitution ./governance/constitution/kms_actions.js --jwt-issuer ${KMS_WORKSPACE}/proposals/set_jwt_issuer_test_sandbox.json  -v --http2 \
-		 	${CCF_SANDBOX_EXTRA_ARGS} & \
-	else \
-		 env -i PATH=${PATH} KMS_WORKSPACE=${KMS_WORKSPACE} $(CCFSB)/sandbox.sh --js-app-bundle ./dist/ --initial-member-count  ${MEMBER_COUNT} --initial-user-count 1 --constitution ./governance/constitution/kms_actions.js --jwt-issuer ${KMS_WORKSPACE}/proposals/set_jwt_issuer_test_sandbox.json  -v --http2 \
-		 	${CCF_SANDBOX_EXTRA_ARGS};  \
-	fi
+	MEMBER_COUNT=${MEMBER_COUNT} source ./scripts/ccf/sandbox-local/up.sh > /dev/null 2>&1 && \
+	source ./scripts/kms/jwt-issuer-trust.sh
 
 demo: stop-all start-host-idp ## 🎬 Demo the KMS Application in the Sandbox
 	@echo -e "\e[34m$@\e[0m" || true
@@ -168,36 +163,36 @@ jwt-issuer-up:
 	@WORKSPACE=${KMS_WORKSPACE} \
 	DEPLOYMENT_ENV=${DEPLOYMENT_ENV} \
 	IMAGE_TAG=${IMAGE_TAG} \
-		./scripts/jwt-issuer-up.sh
+		./scripts/jwt-issuer/up.sh
 
 jwt-issuer-down:
 	@DEPLOYMENT_ENV=${DEPLOYMENT_ENV} \
-		./scripts/jwt-issuer-down.sh
+		./scripts/jwt-issuer/down.sh
 
 jwt-issuer-trust:
 	@WORKSPACE=${KMS_WORKSPACE} \
 	KMS_URL=${KMS_URL} \
 	DEPLOYMENT_ENV=${DEPLOYMENT_ENV} \
-		./scripts/jwt-issuer-trust.sh
+		./scripts/kms/jwt-issuer-trust.sh
 
 # Manage KMS -------------------------------------------------------------------
 
 js-app-set:
 	@WORKSPACE=${KMS_WORKSPACE} \
 	KMS_URL=${KMS_URL} \
-		./scripts/js-app-set.sh
+		./scripts/kms/js-app-set.sh
 
 constitution-set:
 	@WORKSPACE=${KMS_WORKSPACE} \
 	KMS_URL=${KMS_URL} \
 	CONSTITUTION_PATH=./governance/constitution/kms_actions.js \
-		./scripts/constitution-set.sh
+		./scripts/kms/constitution-set.sh
 
 release-policy-set:
 	@WORKSPACE=${KMS_WORKSPACE} \
 	KMS_URL=${KMS_URL} \
 	RELEASE_POLICY_PROPOSAL=$(release-policy-proposal) \
-		./scripts/release-policy-set.sh
+		./scripts/kms/release-policy-set.sh
 
 test-system:
 	@pytest -s test/system-test/$(filter-out $@,$(MAKECMDGOALS))

scripts/ccf/propose.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+ccf-propose() {
+    set -e
+
+    echo "Proposing: $1"
+    echo "  to $KMS_URL"
+    echo "    cert: $KMS_SERVICE_CERT_PATH"
+    echo "  as $KMS_MEMBER_CERT_PATH"
+    ccf_cose_sign1 \
+        --content $1 \
+        --signing-cert ${KMS_MEMBER_CERT_PATH} \
+        --signing-key ${KMS_MEMBER_PRIVK_PATH} \
+        --ccf-gov-msg-type proposal \
+        --ccf-gov-msg-created_at $(date -Is) \
+            | curl $KMS_URL/gov/proposals -k -H "Content-Type: application/cose" \
+            --data-binary @- \
+            -s \
+            --cacert $KMS_SERVICE_CERT_PATH -w '\n' \
+                | jq
+
+    set +e
+}
+
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    ccf-propose "$@"
+fi

scripts/ccf/sandbox-local/down.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+ccf-sandbox-local-down() {
+    set -e
+
+    REPO_ROOT="$(realpath "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/../../..")"
+    docker compose -f $REPO_ROOT/services/docker-compose.yml down ccf-sandbox --remove-orphans
+
+    unset KMS_URL
+    unset KMS_SERVICE_CERT_PATH
+    unset KMS_MEMBER_CERT_PATH
+    unset KMS_MEMBER_PRIVK_PATH
+
+    set +e
+}
+
+ccf-sandbox-local-down

scripts/ccf/sandbox-local/up.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+ccf-sandbox-local-up() {
+    set -e
+
+    REPO_ROOT="$(realpath "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/../../..")"
+
+    export WORKSPACE="$(realpath ${WORKSPACE:-$REPO_ROOT/workspace})"
+    mkdir -p $WORKSPACE
+    docker compose -f $REPO_ROOT/services/docker-compose.yml build ccf-sandbox > /dev/null 2>&1
+    docker compose -f $REPO_ROOT/services/docker-compose.yml up ccf-sandbox --wait
+    sudo chown $USER:$USER -R $WORKSPACE
+
+    export KMS_URL="https://127.0.0.1:8000"
+    export KMS_SERVICE_CERT_PATH="$WORKSPACE/sandbox_common/service_cert.pem"
+    export KMS_MEMBER_CERT_PATH="$WORKSPACE/sandbox_common/member0_cert.pem"
+    export KMS_MEMBER_PRIVK_PATH="$WORKSPACE/sandbox_common/member0_privk.pem"
+
+    set +e
+}
+
+ccf-sandbox-local-up
+
+jq -n '{
+    WORKSPACE: env.WORKSPACE,
+    KMS_URL: env.KMS_URL,
+    KMS_SERVICE_CERT_PATH: env.KMS_SERVICE_CERT_PATH,
+    KMS_MEMBER_CERT_PATH: env.KMS_MEMBER_CERT_PATH,
+    KMS_MEMBER_PRIVK_PATH: env.KMS_MEMBER_PRIVK_PATH
+}'

scripts/jwt-issuer/down.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+set -e
+
+docker compose -f services/docker-compose.yml down jwt-issuer --remove-orphans

scripts/jwt-issuer/up.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+set -e
+
+REPO_ROOT="$(realpath "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/../..")"
+export WORKSPACE=$(realpath ${WORKSPACE:-$REPO_ROOT/workspace})
+
+docker compose -f services/docker-compose.yml up jwt-issuer --wait
+
+sudo chown $USER:$USER -R $WORKSPACE

scripts/kms/constitution-set.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+constitution-set() {
+  set -e
+
+  REPO_ROOT="$(realpath "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/../..")"
+  CONSTITUTION_PATH=$1
+
+  # Get the current constitution
+  curl -k $KMS_URL/gov/service/constitution?api-version=2024-07-01 > $WORKSPACE/proposals/constitution.js
+
+  # Append the consitution given
+  cat "$CONSTITUTION_PATH" >> $WORKSPACE/proposals/constitution.js
+
+  # Construct the proposal
+  jq --arg constitution "$(tr -s ' ' < "$WORKSPACE/proposals/constitution.js")" \
+    '.actions[0].args.constitution = $constitution' \
+      $REPO_ROOT/governance/proposals/set_constitution.json > $WORKSPACE/proposals/set_constitution.json
+
+  # Submit the proposal
+  source $REPO_ROOT/scripts/ccf/propose.sh
+  ccf-propose $WORKSPACE/proposals/set_constitution.json
+
+  set +e
+}
+
+constitution-set "$@"

scripts/kms/endpoints/heartbeat.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+heartbeat() {
+    curl $KMS_URL/app/heartbeat \
+        --cacert $KMS_SERVICE_CERT_PATH \
+        -w '\n%{http_code}\n'
+}
+
+heartbeat