From 37a0a14b753c0b19a3f297379d694f74365fd945 Mon Sep 17 00:00:00 2001 From: Otto Bittner Date: Fri, 27 Oct 2023 09:12:10 +0200 Subject: [PATCH] build: add helper scripts and instructions --- README.md | 16 ++++++++ build.sh | 18 +++++++++ deploy.sh | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ run.sh | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++ upload.sh | 8 ++++ 5 files changed, 254 insertions(+) create mode 100755 build.sh create mode 100755 deploy.sh create mode 100755 run.sh create mode 100755 upload.sh diff --git a/README.md b/README.md index f71bd000..6e3a6ab9 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,19 @@ +# Notes for Constellation + +The e2e tests are shipped as a docker container that executes multiple payloads. +Each payload is a S3 client program that sends a list of requests to a configurable S3 server. + +Checkout `build.sh`, `upload.sh` and `run.sh` to see how the image is built, uploaded and run. + +To locally run e2e tests you will have to: +- deploy s3proxy using the instructions in `../deploy/README.md`. +- Run `build.sh && run.sh` + +You don't need to upload a new image manually as the CI will automatically build and upload mint images. + +The CI uses the `deploy.sh` script to run the test image as a Kubernetes job inside the cluster. +This is not done for local development so you don't have to upload the image for every change. + # Mint [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/mint.svg?maxAge=604800)](https://hub.docker.com/r/minio/mint/) Mint is a testing framework for Minio object server, available as a podman image. It runs correctness, benchmarking and stress tests. Following are the SDKs/tools used in correctness tests. diff --git a/build.sh b/build.sh new file mode 100755 index 00000000..51e1abbd --- /dev/null +++ b/build.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +script_path=$(realpath $0) + +bazel build //bazel/settings:tag +pseudo_version=$(cat ../../../bazel-bin/bazel/settings/_tag.tags.txt) + +docker build -t mint . -f Dockerfile +if [[ "$?" -ne 0 ]]; then + echo "Failed to build docker image" + exit 1 +fi + +tag=ghcr.io/edgelesssys/constellation/mint:"$pseudo_version" +docker tag mint:latest "$tag" + +if [[ "$?" -eq 0 ]]; then + echo "Successfully built docker image: $tag" +fi diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 00000000..85021733 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +function terminate_mint() { + kubectl logs job/mint-deploy + kubectl delete job mint-deploy +} + + +if [[ ! "$1" =~ ^ghcr.io/edgelesssys/constellation/mint:v.*$ ]]; then + echo "Error: invalid tag, expected input to match pattern '^ghcr.io\/edgelesssys\/constellation\/mint:v*$'" + exit 1 +fi + +if [[ -z "$KUBECONFIG" ]]; then + echo "Error: KUBECONFIG environment variable not set" + exit 1 +fi + +if [[ -z "$ACCESS_KEY" ]]; then + echo "Error: ACCESS_KEY environment variable not set" + exit 1 +fi + +if [[ -z "$SECRET_KEY" ]]; then + echo "Error: SECRET_KEY environment variable not set" + exit 1 +fi + +# Wait for the s3proxy service to be created. kubectl wait can not wait for resources to be created. +start_time=$(date +%s) +timeout=300 +while true; do + if [[ -n "$(kubectl get svc -l app=s3proxy -o jsonpath='{.items[*]}')" ]]; then + echo "Service with label app=s3proxy found" + service_ip=$(kubectl get svc s3proxy-service -o=jsonpath='{.spec.clusterIP}') + break + else + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + if [[ $elapsed_time -ge $timeout ]]; then + echo "Timeout waiting for service with label app=s3proxy" + exit 1 + else + echo "Waiting for service with label app=s3proxy" + sleep 5 + fi + fi +done + +kubectl delete job mint-deploy --ignore-not-found=true + +cat << EOF | kubectl apply -f - +apiVersion: batch/v1 +kind: Job +metadata: + name: mint-deploy +spec: + template: + metadata: + name: mint-deploy + spec: + restartPolicy: Never + hostAliases: + - ip: "$service_ip" + hostnames: + - "s3.eu-west-1.amazonaws.com" + containers: + - name: mint + image: "$1" + args: + - "aws-sdk-go" + - "versioning" + volumeMounts: + - name: ca-cert + mountPath: /etc/ssl/certs/kube-ca.crt + subPath: kube-ca.crt + env: + - name: SERVER_REGION + value: eu-west-1 + - name: SERVER_ENDPOINT + value: s3.eu-west-1.amazonaws.com:443 + - name: ENABLE_HTTPS + value: "1" + - name: AWS_CA_BUNDLE + value: /etc/ssl/certs/kube-ca.crt + - name: ACCESS_KEY + value: "$ACCESS_KEY" + - name: SECRET_KEY + value: "$SECRET_KEY" + volumes: + - name: ca-cert + secret: + secretName: s3proxy-tls + items: + - key: ca.crt + path: kube-ca.crt +EOF + +# Remove job before this script finishes. +trap "terminate_mint" EXIT + +# Tests have to complete within 10 minutes, otherwise they have failed. +if kubectl wait --for=condition=complete job/mint-deploy --timeout=600s; then + echo "Mint tests completed successfully" + exit 0 +else + echo "Mint tests failed" + exit 1 +fi diff --git a/run.sh b/run.sh new file mode 100755 index 00000000..585b37d2 --- /dev/null +++ b/run.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash + +# Validate inputs and environment variables. +if [[ ! "$1" =~ ^ghcr.io/edgelesssys/constellation/mint:v.*$ ]]; then + echo "Error: invalid tag, expected input to match pattern '^ghcr.io\/edgelesssys\/constellation\/mint:v*$'" + exit 1 +fi +mint_image=$1 + +if [[ -z "$KUBECONFIG" ]]; then + echo "Error: KUBECONFIG environment variable not set" + exit 1 +fi + +if [[ -z "$AWS_ACCESS_KEY_ID" ]]; then + echo "Error: AWS_ACCESS_KEY_ID environment variable not set" + exit 1 +fi + +if [[ -z "$AWS_SECRET_ACCESS_KEY" ]]; then + echo "Error: AWS_SECRET_ACCESS_KEY environment variable not set" + exit 1 +fi + +# Wait for the s3proxy pod to be created. kubectl wait can not wait for resources to be created. +start_time=$(date +%s) +timeout=300 +while true; do + if [[ -n "$(kubectl get svc -l app=s3proxy -o jsonpath='{.items[*]}')" ]]; then + echo "Service with label app=s3proxy found" + break + else + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + if [[ $elapsed_time -ge $timeout ]]; then + echo "Timeout waiting for service with label app=s3proxy" + exit 1 + else + echo "Waiting for service with label app=s3proxy" + sleep 5 + fi + fi +done + +# Wait until pod becomes ready. +kubectl wait --for=condition=Ready --timeout=2m pod -l app=s3proxy + +# Get the CA that signed the s3proxy's TLS certificate. +kubectl get secret s3proxy-tls -o yaml | yq e '.data."ca.crt"' - | base64 -d > s3proxy-ca.crt +if [[ "$?" -ne 0 ]]; then + echo "Error: failed to get s3proxy-tls secret" + exit 1 +fi + +# block for sudoers password so it is not requested for following command, which is put into the background. +sudo -v + +port_forward_ip="172.30.0.1" +# sudo -E kubectl port-forward --address "$port_forward_ip" svc/s3proxy-service 443:443 & +sudo -E -b kubectl port-forward --address "$port_forward_ip" svc/s3proxy-service 443:443 > ./port-forward.log 2>&1 + +# Kill port-forward on exit and print it's output. +trap 'stop_port_forward "$port_forward_ip"' EXIT + + +# wait for port-forward to be ready +start_time=$(date +%s) +timeout=300 +while true; do + # Check if a connection to the s3proxy can be established. Trust certificates signed by the CA fetched earlier. + echo | openssl s_client -connect "$port_forward_ip":443 -brief -verify_return_error -CAfile s3proxy-ca.crt + if [[ "$?" -eq 0 ]]; then + echo "Port-forward ready" + break + else + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + if [[ $elapsed_time -ge $timeout ]]; then + echo "Timeout waiting for port-forward" + exit 1 + else + echo "Waiting for port-forward" + sleep 5 + fi + fi +done + +docker run -v $PWD/s3proxy-ca.crt:/etc/ssl/certs/kube-ca.crt \ + --add-host s3.eu-west-1.amazonaws.com:"$port_forward_ip" \ + -e AWS_CA_BUNDLE=/etc/ssl/certs/kube-ca.crt \ + -e SERVER_REGION=eu-west-1 \ + -e SERVER_ENDPOINT=s3.eu-west-1.amazonaws.com:443 \ + -e ACCESS_KEY=$AWS_ACCESS_KEY_ID \ + -e SECRET_KEY=$AWS_SECRET_ACCESS_KEY \ + -e ENABLE_HTTPS=1 \ + "$mint_image" aws-sdk-go versioning + +function stop_port_forward() { + local port_forward_ip="$1" + sudo pkill -f "kubectl port-forward --address $port_forward_ip svc/s3proxy-service 443:443" + echo "Output of kubectl port-forward:" + cat ./port-forward.log +} diff --git a/upload.sh b/upload.sh new file mode 100755 index 00000000..f0fb9114 --- /dev/null +++ b/upload.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +if [[ ! "$1" =~ ^ghcr.io/edgelesssys/constellation/mint:v.*$ ]]; then + echo "Error: invalid tag, expected input to match pattern '^ghcr.io\/edgelesssys\/constellation\/mint:v*$'" + exit 1 +fi + +docker push "$1"