build: add helper scripts and instructions

README.md

@@ -1,3 +1,19 @@
+# Notes for Constellation
+
+The e2e tests are shipped as a docker container that executes multiple payloads.
+Each payload is a S3 client program that sends a list of requests to a configurable S3 server.
+
+Checkout `build.sh`, `upload.sh` and `run.sh` to see how the image is built, uploaded and run.
+
+To locally run e2e tests you will have to:
+- deploy s3proxy using the instructions in `../deploy/README.md`.
+- Run `build.sh && run.sh`
+
+You don't need to upload a new image manually as the CI will automatically build and upload mint images.
+
+The CI uses the `deploy.sh` script to run the test image as a Kubernetes job inside the cluster.
+This is not done for local development so you don't have to upload the image for every change.
+
 # Mint [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/mint.svg?maxAge=604800)](https://hub.docker.com/r/minio/mint/)
 
 Mint is a testing framework for Minio object server, available as a podman image. It runs correctness, benchmarking and stress tests. Following are the SDKs/tools used in correctness tests.

build.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+script_path=$(realpath $0)
+
+bazel build //bazel/settings:tag
+pseudo_version=$(cat ../../../bazel-bin/bazel/settings/_tag.tags.txt)
+
+docker build -t mint . -f Dockerfile
+if [[ "$?" -ne 0 ]]; then
+  echo "Failed to build docker image"
+  exit 1
+fi
+
+tag=ghcr.io/edgelesssys/constellation/mint:"$pseudo_version"
+docker tag mint:latest "$tag"
+
+if [[ "$?" -eq 0 ]]; then
+  echo "Successfully built docker image: $tag"
+fi

deploy.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+function terminate_mint() {
+  kubectl logs job/mint-deploy
+  kubectl delete job mint-deploy
+}
+
+
+if [[ ! "$1" =~ ^ghcr.io/edgelesssys/constellation/mint:v.*$ ]]; then
+  echo "Error: invalid tag, expected input to match pattern '^ghcr.io\/edgelesssys\/constellation\/mint:v*$'"
+  exit 1
+fi
+
+if [[ -z "$KUBECONFIG" ]]; then
+  echo "Error: KUBECONFIG environment variable not set"
+  exit 1
+fi
+
+if [[ -z "$ACCESS_KEY" ]]; then
+  echo "Error: ACCESS_KEY environment variable not set"
+  exit 1
+fi
+
+if [[ -z "$SECRET_KEY" ]]; then
+  echo "Error: SECRET_KEY environment variable not set"
+  exit 1
+fi
+
+# Wait for the s3proxy service to be created. kubectl wait can not wait for resources to be created.
+start_time=$(date +%s)
+timeout=300
+while true; do
+    if [[ -n "$(kubectl get svc -l app=s3proxy -o jsonpath='{.items[*]}')" ]]; then
+        echo "Service with label app=s3proxy found"
+        service_ip=$(kubectl get svc s3proxy-service -o=jsonpath='{.spec.clusterIP}')
+        break
+    else
+        current_time=$(date +%s)
+        elapsed_time=$((current_time - start_time))
+        if [[ $elapsed_time -ge $timeout ]]; then
+            echo "Timeout waiting for service with label app=s3proxy"
+            exit 1
+        else
+            echo "Waiting for service with label app=s3proxy"
+            sleep 5
+        fi
+    fi
+done
+
+kubectl delete job mint-deploy --ignore-not-found=true
+
+cat << EOF | kubectl apply -f -
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: mint-deploy
+spec:
+  template:
+    metadata:
+      name: mint-deploy
+    spec:
+      restartPolicy: Never
+      hostAliases:
+      - ip: "$service_ip"
+        hostnames:
+        - "s3.eu-west-1.amazonaws.com"
+      containers:
+      - name: mint
+        image: "$1"
+        args:
+        - "aws-sdk-go"
+        - "versioning"
+        volumeMounts:
+        - name: ca-cert
+          mountPath: /etc/ssl/certs/kube-ca.crt
+          subPath: kube-ca.crt
+        env:
+        - name: SERVER_REGION
+          value: eu-west-1
+        - name: SERVER_ENDPOINT
+          value: s3.eu-west-1.amazonaws.com:443
+        - name: ENABLE_HTTPS
+          value: "1"
+        - name: AWS_CA_BUNDLE
+          value: /etc/ssl/certs/kube-ca.crt
+        - name: ACCESS_KEY
+          value: "$ACCESS_KEY"
+        - name: SECRET_KEY
+          value: "$SECRET_KEY"
+      volumes:
+      - name: ca-cert
+        secret:
+            secretName: s3proxy-tls
+            items:
+            - key: ca.crt
+              path: kube-ca.crt
+EOF
+
+# Remove job before this script finishes.
+trap "terminate_mint" EXIT
+
+# Tests have to complete within 10 minutes, otherwise they have failed.
+if kubectl wait --for=condition=complete job/mint-deploy --timeout=600s; then
+  echo "Mint tests completed successfully"
+  exit 0
+else
+  echo "Mint tests failed"
+  exit 1
+fi

run.sh

@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+
+# Validate inputs and environment variables.
+if [[ ! "$1" =~ ^ghcr.io/edgelesssys/constellation/mint:v.*$ ]]; then
+  echo "Error: invalid tag, expected input to match pattern '^ghcr.io\/edgelesssys\/constellation\/mint:v*$'"
+  exit 1
+fi
+mint_image=$1
+
+if [[ -z "$KUBECONFIG" ]]; then
+  echo "Error: KUBECONFIG environment variable not set"
+  exit 1
+fi
+
+if [[ -z "$AWS_ACCESS_KEY_ID" ]]; then
+  echo "Error: AWS_ACCESS_KEY_ID environment variable not set"
+  exit 1
+fi
+
+if [[ -z "$AWS_SECRET_ACCESS_KEY" ]]; then
+  echo "Error: AWS_SECRET_ACCESS_KEY environment variable not set"
+  exit 1
+fi
+
+# Wait for the s3proxy pod to be created. kubectl wait can not wait for resources to be created.
+start_time=$(date +%s)
+timeout=300
+while true; do
+    if [[ -n "$(kubectl get svc -l app=s3proxy -o jsonpath='{.items[*]}')" ]]; then
+        echo "Service with label app=s3proxy found"
+        break
+    else
+        current_time=$(date +%s)
+        elapsed_time=$((current_time - start_time))
+        if [[ $elapsed_time -ge $timeout ]]; then
+            echo "Timeout waiting for service with label app=s3proxy"
+            exit 1
+        else
+            echo "Waiting for service with label app=s3proxy"
+            sleep 5
+        fi
+    fi
+done
+
+# Wait until pod becomes ready.
+kubectl wait --for=condition=Ready --timeout=2m pod -l app=s3proxy
+
+# Get the CA that signed the s3proxy's TLS certificate.
+kubectl get secret s3proxy-tls -o yaml | yq e '.data."ca.crt"' - | base64 -d > s3proxy-ca.crt
+if [[ "$?" -ne 0 ]]; then
+  echo "Error: failed to get s3proxy-tls secret"
+  exit 1
+fi
+
+# block for sudoers password so it is not requested for following command, which is put into the background.
+sudo -v
+
+port_forward_ip="172.30.0.1"
+# sudo -E kubectl port-forward --address "$port_forward_ip" svc/s3proxy-service 443:443 &
+sudo -E -b kubectl port-forward --address "$port_forward_ip" svc/s3proxy-service 443:443 > ./port-forward.log 2>&1
+
+# Kill port-forward on exit and print it's output.
+trap 'stop_port_forward "$port_forward_ip"' EXIT
+
+
+# wait for port-forward to be ready
+start_time=$(date +%s)
+timeout=300
+while true; do
+    # Check if a connection to the s3proxy can be established. Trust certificates signed by the CA fetched earlier.
+    echo | openssl s_client -connect "$port_forward_ip":443 -brief -verify_return_error -CAfile s3proxy-ca.crt
+    if [[ "$?" -eq 0 ]]; then
+        echo "Port-forward ready"
+        break
+    else
+        current_time=$(date +%s)
+        elapsed_time=$((current_time - start_time))
+        if [[ $elapsed_time -ge $timeout ]]; then
+            echo "Timeout waiting for port-forward"
+            exit 1
+        else
+            echo "Waiting for port-forward"
+            sleep 5
+        fi
+    fi
+done
+
+docker run -v $PWD/s3proxy-ca.crt:/etc/ssl/certs/kube-ca.crt \
+  --add-host s3.eu-west-1.amazonaws.com:"$port_forward_ip" \
+  -e AWS_CA_BUNDLE=/etc/ssl/certs/kube-ca.crt \
+  -e SERVER_REGION=eu-west-1 \
+  -e SERVER_ENDPOINT=s3.eu-west-1.amazonaws.com:443 \
+  -e ACCESS_KEY=$AWS_ACCESS_KEY_ID \
+  -e SECRET_KEY=$AWS_SECRET_ACCESS_KEY \
+  -e ENABLE_HTTPS=1 \
+  "$mint_image" aws-sdk-go versioning
+
+function stop_port_forward() {
+  local port_forward_ip="$1"
+  sudo pkill -f "kubectl port-forward --address $port_forward_ip svc/s3proxy-service 443:443"
+  echo "Output of kubectl port-forward:"
+  cat ./port-forward.log
+}

upload.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+if [[ ! "$1" =~ ^ghcr.io/edgelesssys/constellation/mint:v.*$ ]]; then
+  echo "Error: invalid tag, expected input to match pattern '^ghcr.io\/edgelesssys\/constellation\/mint:v*$'"
+  exit 1
+fi
+
+docker push "$1"