Skip to content

Commit

Permalink
ci: add e2e test for self-managed infrastructure (#2472)
Browse files Browse the repository at this point in the history
* add self-managed infra e2e test

* self-managed terminatio

Signed-off-by: Moritz Sanft <[email protected]>

* fix upgrade test

Signed-off-by: Moritz Sanft <[email protected]>

* fix indentation

Signed-off-by: Moritz Sanft <[email protected]>

* use -r when copying dir

Signed-off-by: Moritz Sanft <[email protected]>

* add terraform variable parsing

* copy constellation conf

Signed-off-by: Moritz Sanft <[email protected]>

* remove unnecessary line breaks

* add missing value

Signed-off-by: Moritz Sanft <[email protected]>

* add image fetching for CSP

Signed-off-by: Moritz Sanft <[email protected]>

* fix quoting

Signed-off-by: Moritz Sanft <[email protected]>

* add missing input to internal lb test

* normalize Azure URLs.. Of course

* tidy

Signed-off-by: Moritz Sanft <[email protected]>

* fix expressions

* initsecret to hex

* update hexdump cmd

* add build test

Signed-off-by: Moritz Sanft <[email protected]>

* add node / pod cidr outputs

Signed-off-by: Moritz Sanft <[email protected]>

* explicitly delete the state file

Signed-off-by: Moritz Sanft <[email protected]>

* add missing license header

Signed-off-by: Moritz Sanft <[email protected]>

* always write all outputs

Signed-off-by: Moritz Sanft <[email protected]>

* fix list output

Signed-off-by: Moritz Sanft <[email protected]>

* remove state-file and admin-conf on destroy

* dont use test payload

Signed-off-by: Moritz Sanft <[email protected]>

* [remove] use self managed infra in manual e2e for testing

Signed-off-by: Moritz Sanft <[email protected]>

* init: always skip infrastructure phase

* patch maa in workflow

Signed-off-by: Moritz Sanft <[email protected]>

* default to Constellation-created infra in e2e test

---------

Signed-off-by: Moritz Sanft <[email protected]>
  • Loading branch information
msanft authored Oct 27, 2023
1 parent f4bfbe3 commit 402a883
Show file tree
Hide file tree
Showing 24 changed files with 417 additions and 11 deletions.
18 changes: 16 additions & 2 deletions .github/actions/constellation_create/action.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
name: Constellation create
description: Create a new Constellation cluster using latest OS image.
description: Create a new Constellation cluster using the latest OS image.

inputs:
workerNodesCount:
Expand Down Expand Up @@ -50,6 +50,9 @@ inputs:
internalLoadBalancer:
description: "Whether to use an internal load balancer for the control plane"
required: false
selfManagedInfra:
description: "Use self-managed infrastructure instead of infrastructure created by the Constellation CLI."
required: true

outputs:
kubeconfig:
Expand Down Expand Up @@ -124,14 +127,25 @@ runs:
run: |
yq eval -i '(.internalLoadBalancer) = true' constellation-conf.yaml
- name: Constellation create
- name: Show Cluster Configuration
shell: bash
run: |
echo "Creating cluster using config:"
cat constellation-conf.yaml
sudo sh -c 'echo "127.0.0.1 license.confidential.cloud" >> /etc/hosts' || true
- name: Constellation create (CLI)
if : inputs.selfManagedInfra != 'true'
shell: bash
run: |
constellation create -y --debug --tf-log=DEBUG
- name: Constellation create (self-managed)
if : inputs.selfManagedInfra == 'true'
uses: ./.github/actions/self_managed_create
with:
cloudProvider: ${{ inputs.cloudProvider }}

- name: Cdbg deploy
if: inputs.isDebugImage == 'true'
uses: ./.github/actions/cdbg_deploy
Expand Down
15 changes: 15 additions & 0 deletions .github/actions/constellation_destroy/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ inputs:
kubeconfig:
description: "The kubeconfig for the cluster."
required: true
selfManagedInfra:
description: "Use self-managed infrastructure instead of infrastructure created by the Constellation CLI."
required: true

runs:
using: "composite"
Expand Down Expand Up @@ -39,6 +42,18 @@ runs:
echo "::endgroup::"
- name: Constellation terminate
if: inputs.selfManagedInfra != 'true'
shell: bash
run: |
constellation terminate --yes --tf-log=DEBUG
- name: Constellation terminate (self-managed)
if: inputs.selfManagedInfra == 'true'
shell: bash
working-directory: ${{ github.workspace }}/e2e-infra
run: |
terraform init
terraform destroy -auto-approve
# Explicitly delete the state file
rm ${{ github.workspace }}/constellation-state.yaml
rm ${{ github.workspace }}/constellation-admin.conf
5 changes: 5 additions & 0 deletions .github/actions/e2e_test/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,9 @@ inputs:
description: "Enable security policy for the cluster."
internalLoadBalancer:
description: "Enable internal load balancer for the cluster."
selfManagedInfra:
description: "Use self-managed infrastructure instead of infrastructure created by the Constellation CLI."
default: "false"

outputs:
kubeconfig:
Expand Down Expand Up @@ -260,6 +263,8 @@ runs:
kubernetesVersion: ${{ inputs.kubernetesVersion }}
refStream: ${{ inputs.refStream }}
internalLoadBalancer: ${{ inputs.internalLoadBalancer }}
test: ${{ inputs.test }}
selfManagedInfra: ${{ inputs.selfManagedInfra }}

- name: Deploy log- and metrics-collection (Kubernetes)
id: deploy-logcollection
Expand Down
111 changes: 111 additions & 0 deletions .github/actions/self_managed_create/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
name: Self-managed infrastructure creation
description: "Create the required infrastructure for a Constellation cluster manually."

inputs:
cloudProvider:
description: "The cloud provider the test runs on."
required: true

runs:
using: "composite"
steps:
- name: Copy Terraform configuration and Constellation config
shell: bash
working-directory:
run: |
cp -r ${{ github.workspace }}/cli/internal/terraform/terraform/${{ inputs.cloudProvider }} ${{ github.workspace }}/e2e-infra
cp ${{ github.workspace }}/constellation-conf.yaml ${{ github.workspace }}/e2e-infra
- name: Get CSP image reference
id: get_image
shell: bash
working-directory: ${{ github.workspace }}/e2e-infra
run: |
echo "image_ref=$(bazel run //hack/image-fetch:image-fetch)" >> $GITHUB_OUTPUT
- name: Write Terraform variables
shell: bash
working-directory: ${{ github.workspace }}/e2e-infra
run: |
echo "name = \"$(yq '.name' constellation-conf.yaml)\"" >> terraform.tfvars
echo "debug = $(yq '.debugCluster' constellation-conf.yaml)" >> terraform.tfvars
echo "custom_endpoint = \"$(yq '.customEndpoint' constellation-conf.yaml)\"" >> terraform.tfvars
echo "image_id = \"${{ steps.get_image.outputs.image_ref }}\"" >> terraform.tfvars
echo "node_groups = {
control_plane_default = {
role = \"$(yq '.nodeGroups.control_plane_default.role' constellation-conf.yaml)\"
zone = \"$(yq '.nodeGroups.control_plane_default.zone' constellation-conf.yaml)\"
instance_type = \"$(yq '.nodeGroups.control_plane_default.instanceType' constellation-conf.yaml)\"
disk_size = \"$(yq '.nodeGroups.control_plane_default.stateDiskSizeGB' constellation-conf.yaml)\"
disk_type = \"$(yq '.nodeGroups.control_plane_default.stateDiskType' constellation-conf.yaml)\"
initial_count = \"$(yq '.nodeGroups.control_plane_default.initialCount' constellation-conf.yaml)\"
}
worker_default = {
role = \"$(yq '.nodeGroups.worker_default.role' constellation-conf.yaml)\"
zone = \"$(yq '.nodeGroups.worker_default.zone' constellation-conf.yaml)\"
instance_type = \"$(yq '.nodeGroups.worker_default.instanceType' constellation-conf.yaml)\"
disk_size = \"$(yq '.nodeGroups.worker_default.stateDiskSizeGB' constellation-conf.yaml)\"
disk_type = \"$(yq '.nodeGroups.worker_default.stateDiskType' constellation-conf.yaml)\"
initial_count = \"$(yq '.nodeGroups.worker_default.initialCount' constellation-conf.yaml)\"
}
}" >> terraform.tfvars
if [[ "${{ inputs.cloudProvider }}" == 'aws' ]]; then
echo "iam_instance_profile_control_plane = \"$(yq '.provider.aws.iamProfileControlPlane' constellation-conf.yaml)\"" >> terraform.tfvars
echo "iam_instance_profile_worker_nodes = \"$(yq '.provider.aws.iamProfileWorkerNodes' constellation-conf.yaml)\"" >> terraform.tfvars
echo "region = \"$(yq '.provider.aws.region' constellation-conf.yaml)\"" >> terraform.tfvars
echo "zone = \"$(yq '.provider.aws.zone' constellation-conf.yaml)\"" >> terraform.tfvars
echo "ami = \"${{ steps.get_image.outputs.image_ref }}\"" >> terraform.tfvars
echo "enable_snp = $(yq '.attestation | has("awsSEVSNP")' constellation-conf.yaml)" >> terraform.tfvars
elif [[ "${{ inputs.cloudProvider }}" == 'azure' ]]; then
echo "location = \"$(yq '.provider.azure.location' constellation-conf.yaml)\"" >> terraform.tfvars
echo "create_maa = $(yq '.attestation | has("azureSEVSNP")' constellation-conf.yaml)" >> terraform.tfvars
echo "confidential_vm = $(yq '.attestation | has("azureSEVSNP")' constellation-conf.yaml)" >> terraform.tfvars
echo "secure_boot = $(yq '.provider.azure.secureBoot' constellation-conf.yaml)" >> terraform.tfvars
echo "resource_group = \"$(yq '.provider.azure.resourceGroup' constellation-conf.yaml)\"" >> terraform.tfvars
echo "user_assigned_identity = \"$(yq '.provider.azure.userAssignedIdentity' constellation-conf.yaml)\"" >> terraform.tfvars
elif [[ "${{ inputs.cloudProvider }}" == 'gcp' ]]; then
echo "project = \"$(yq '.provider.gcp.project' constellation-conf.yaml)\"" >> terraform.tfvars
echo "region = \"$(yq '.provider.gcp.region' constellation-conf.yaml)\"" >> terraform.tfvars
echo "zone = \"$(yq '.provider.gcp.zone' constellation-conf.yaml)\"" >> terraform.tfvars
fi
terraform fmt terraform.tfvars
echo "Using Terraform variables:"
cat terraform.tfvars
- name: Apply Terraform configuration
shell: bash
working-directory: ${{ github.workspace }}/e2e-infra
run: |
terraform init
terraform apply -auto-approve
- name: Patch MAA Policy
shell: bash
working-directory: ${{ github.workspace }}/e2e-infra
if: ${{ inputs.cloudProvider }} == 'azure'
run: |
bazel run //hack/maa-patch:maa-patch $(terraform output attestationURL | jq -r)
- name: Write outputs to state file
shell: bash
working-directory: ${{ github.workspace }}/e2e-infra
run: |
yq eval '.version ="v1"' --inplace ${{ github.workspace }}/constellation-state.yaml
yq eval ".infrastructure.initSecret =\"$(terraform output initSecret | jq -r | tr -d '\n' | hexdump -ve '/1 "%02x"' && echo '')\"" --inplace ${{ github.workspace }}/constellation-state.yaml
yq eval ".infrastructure.clusterEndpoint =\"$(terraform output out_of_cluster_endpoint | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
yq eval ".infrastructure.inClusterEndpoint =\"$(terraform output in_cluster_endpoint | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
yq eval ".infrastructure.ipCidrNode =\"$(terraform output ip_cidr_nodes | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
yq eval ".infrastructure.uid =\"$(terraform output uid | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
yq eval ".infrastructure.name =\"$(terraform output name | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
yq eval ".infrastructure.apiServerCertSANs =$(terraform output -json api_server_cert_sans)" --inplace ${{ github.workspace }}/constellation-state.yaml
if [[ "${{ inputs.cloudProvider }}" == 'azure' ]]; then
yq eval ".infrastructure.azure.resourceGroup =\"$(terraform output resource_group | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
yq eval ".infrastructure.azure.subscriptionID =\"$(terraform output subscription_id | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
yq eval ".infrastructure.azure.networkSecurityGroupName =\"$(terraform output network_security_group_name | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
yq eval ".infrastructure.azure.loadBalancerName =\"$(terraform output loadbalancer_name | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
yq eval ".infrastructure.azure.userAssignedIdentity =\"$(terraform output user_assigned_identity_client_id | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
yq eval ".infrastructure.azure.attestationURL =\"$(terraform output attestationURL | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
elif [[ "${{ inputs.cloudProvider }}" == 'gcp' ]]; then
yq eval ".infrastructure.gcp.projectID =\"$(terraform output project | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
yq eval ".infrastructure.gcp.ipCidrPod =\"$(terraform output ip_cidr_pods | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml
fi
2 changes: 2 additions & 0 deletions .github/workflows/e2e-test-daily.yml
Original file line number Diff line number Diff line change
Expand Up @@ -91,12 +91,14 @@ jobs:
awsOpenSearchDomain: ${{ secrets.AWS_OPENSEARCH_DOMAIN }}
awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }}
awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }}
selfManagedInfra: "false"

- name: Always terminate cluster
if: always()
uses: ./.github/actions/constellation_destroy
with:
kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
selfManagedInfra: "false"

- name: Always delete IAM configuration
if: always()
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/e2e-test-manual-internal.yml
Original file line number Diff line number Diff line change
Expand Up @@ -205,12 +205,14 @@ jobs:
cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
fetchMeasurements: ${{ contains(needs.find-latest-image.outputs.image, '/stream/stable/') }}
internalLoadBalancer: true
selfManagedInfra: "false"

- name: Always terminate cluster
if: always()
uses: ./.github/actions/constellation_destroy
with:
kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
selfManagedInfra: "false"

- name: Always delete IAM configuration
if: always()
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/e2e-test-manual.yml
Original file line number Diff line number Diff line change
Expand Up @@ -260,12 +260,14 @@ jobs:
cosignPassword: ${{ secrets.COSIGN_PASSWORD }}
cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
fetchMeasurements: ${{ contains(needs.find-latest-image.outputs.image, '/stream/stable/') }}
selfManagedInfra: "false"

- name: Always terminate cluster
if: always()
uses: ./.github/actions/constellation_destroy
with:
kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
selfManagedInfra: "false"

- name: Always delete IAM configuration
if: always()
Expand Down
20 changes: 20 additions & 0 deletions .github/workflows/e2e-test-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,24 @@ jobs:
kubernetes-version: "v1.28"
runner: "ubuntu-22.04"

# self-managed infra test on latest k8s version
# runs Sonobuoy full test
- test: "sonobuoy full"
provider: "gcp"
kubernetes-version: "v1.28"
runner: "ubuntu-22.04"
selfManagedInfra: "true"
- test: "sonobuoy full"
provider: "azure"
kubernetes-version: "v1.28"
runner: "ubuntu-22.04"
selfManagedInfra: "true"
- test: "sonobuoy full"
provider: "aws"
kubernetes-version: "v1.28"
runner: "ubuntu-22.04"
selfManagedInfra: "true"

#
# Tests on macOS runner
#
Expand Down Expand Up @@ -213,12 +231,14 @@ jobs:
cosignPassword: ${{ secrets.COSIGN_PASSWORD }}
cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
githubToken: ${{ secrets.GITHUB_TOKEN }}
selfManagedInfra: ${{ matrix.selfManagedInfra == 'true' }}

- name: Always terminate cluster
if: always()
uses: ./.github/actions/constellation_destroy
with:
kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
selfManagedInfra: ${{ matrix.selfManagedInfra == 'true' }}

- name: Always delete IAM configuration
if: always()
Expand Down
21 changes: 21 additions & 0 deletions .github/workflows/e2e-test-weekly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,24 @@ jobs:
provider: "aws"
kubernetes-version: "v1.28"

# self-managed infra test on latest k8s version
# with Sonobuoy full
- test: "sonobuoy full"
refStream: "ref/main/stream/debug/?"
provider: "gcp"
kubernetes-version: "v1.28"
selfManagedInfra: "true"
- test: "sonobuoy full"
refStream: "ref/main/stream/debug/?"
provider: "azure"
kubernetes-version: "v1.28"
selfManagedInfra: "true"
- test: "sonobuoy full"
provider: "aws"
refStream: "ref/main/stream/debug/?"
kubernetes-version: "v1.28"
selfManagedInfra: "true"

#
# Tests on release-stable refStream
#
Expand All @@ -188,6 +206,7 @@ jobs:
refStream: "ref/release/stream/stable/?"
provider: "aws"
kubernetes-version: "v1.27"

runs-on: ubuntu-22.04
permissions:
id-token: write
Expand Down Expand Up @@ -231,12 +250,14 @@ jobs:
cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
fetchMeasurements: ${{ matrix.refStream != 'ref/release/stream/stable/?' }}
azureSNPEnforcementPolicy: ${{ matrix.azureSNPEnforcementPolicy }}
selfManagedInfra: ${{ matrix.selfManagedInfra == 'true' }}

- name: Always terminate cluster
if: always()
uses: ./.github/actions/constellation_destroy
with:
kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
selfManagedInfra: ${{ matrix.selfManagedInfra == 'true' }}

- name: Always delete IAM configuration
if: always()
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/e2e-upgrade.yml
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,7 @@ jobs:
awsOpenSearchDomain: ${{ secrets.AWS_OPENSEARCH_DOMAIN }}
awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }}
awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }}
selfManagedInfra: "false"

- name: Build CLI
uses: ./.github/actions/build_cli
Expand Down Expand Up @@ -287,6 +288,7 @@ jobs:
uses: ./.github/actions/constellation_destroy
with:
kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
selfManagedInfra: "false"

- name: Always delete IAM configuration
if: always()
Expand Down
6 changes: 1 addition & 5 deletions cli/internal/cloudcmd/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ go_library(
"create.go",
"iam.go",
"iamupgrade.go",
"patch.go",
"rollback.go",
"serviceaccount.go",
"terminate.go",
Expand All @@ -36,10 +35,8 @@ go_library(
"//internal/constants",
"//internal/file",
"//internal/imagefetcher",
"//internal/maa",
"//internal/role",
"@com_github_azure_azure_sdk_for_go//profiles/latest/attestation/attestation",
"@com_github_azure_azure_sdk_for_go_sdk_azcore//policy",
"@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity",
"@com_github_spf13_cobra//:cobra",
],
)
Expand All @@ -51,7 +48,6 @@ go_test(
"clusterupgrade_test.go",
"create_test.go",
"iam_test.go",
"patch_test.go",
"rollback_test.go",
"terminate_test.go",
"tfupgrade_test.go",
Expand Down
3 changes: 2 additions & 1 deletion cli/internal/cloudcmd/clusterupgrade.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/maa"
)

// ClusterUpgrader is responsible for performing Terraform migrations on cluster upgrades.
Expand All @@ -43,7 +44,7 @@ func NewClusterUpgrader(ctx context.Context, existingWorkspace, upgradeWorkspace

return &ClusterUpgrader{
tf: tfClient,
policyPatcher: NewAzurePolicyPatcher(),
policyPatcher: maa.NewAzurePolicyPatcher(),
fileHandler: fileHandler,
existingWorkspace: existingWorkspace,
upgradeWorkspace: upgradeWorkspace,
Expand Down
Loading

0 comments on commit 402a883

Please sign in to comment.