ci: collect cluster metrics to OpenSearch (#2347)

* add Metricbeat deployment to debugd Signed-off-by: Moritz Sanft <[email protected]> * set metricbeat debugd image version Signed-off-by: Moritz Sanft <[email protected]> * fix k8s deployment Signed-off-by: Moritz Sanft <[email protected]> * use 2 separate deployments Signed-off-by: Moritz Sanft <[email protected]> * only deploy via k8s in non-debug-images Signed-off-by: Moritz Sanft <[email protected]> * add missing tilde * remove k8s metrics Signed-off-by: Moritz Sanft <[email protected]> * unify flag Signed-off-by: Moritz Sanft <[email protected]> * add cloud metadata processor to filebeat Signed-off-by: Moritz Sanft <[email protected]> * ci: fix debugd logcollection (#2355) * add missing keyvault access role Signed-off-by: Moritz Sanft <[email protected]> * bump logstash image version Signed-off-by: Moritz Sanft <[email protected]> * bump filebeat / metricbeat image version Signed-off-by: Moritz Sanft <[email protected]> * log used image version Signed-off-by: Moritz Sanft <[email protected]> * use debugging image versions Signed-off-by: Moritz Sanft <[email protected]> * increase wait timeout for image upload * add cloud metadata processor to filebeat Signed-off-by: Moritz Sanft <[email protected]> * fix template locations in container Signed-off-by: Moritz Sanft <[email protected]> * fix image version typo Signed-off-by: Moritz Sanft <[email protected]> * add filebeat / metricbeat users Signed-off-by: Moritz Sanft <[email protected]> * remove user additions Signed-off-by: Moritz Sanft <[email protected]> * update workflow step name Signed-off-by: Moritz Sanft <[email protected]> * only mount config files Signed-off-by: Moritz Sanft <[email protected]> * document potential rc Signed-off-by: Moritz Sanft <[email protected]> * fix IAM permissions in workflow Signed-off-by: Moritz Sanft <[email protected]> * fix AWS permissions Signed-off-by: Moritz Sanft <[email protected]> * tidy Signed-off-by: Moritz Sanft <[email protected]> * add missing workflow input Signed-off-by: Moritz Sanft <[email protected]> * rename action Signed-off-by: Moritz Sanft <[email protected]> * pin image versions Signed-off-by: Moritz Sanft <[email protected]> * remove unnecessary workflow inputs Signed-off-by: Moritz Sanft <[email protected]> --------- Signed-off-by: Moritz Sanft <[email protected]> * add refStream input Signed-off-by: Moritz Sanft <[email protected]> * remove inputs.yml dep Signed-off-by: Moritz Sanft <[email protected]> * increase system metric period Signed-off-by: Moritz Sanft <[email protected]> * fix linkchecker Signed-off-by: Moritz Sanft <[email protected]> --------- Signed-off-by: Moritz Sanft <[email protected]>
edgelesssys · Sep 27, 2023 · f4b2d02 · f4b2d02
1 parent c7b728f
commit f4b2d02
Show file tree

Hide file tree

Showing 27 changed files with 699 additions and 149 deletions.
diff --git a/.github/actions/cdbg_deploy/action.yml b/.github/actions/cdbg_deploy/action.yml
@@ -0,0 +1,100 @@
+name: Cdbg deploy
+description: Deploy the Constellation Bootstrapper to the cluster via the debugd.
+
+inputs:
+  test:
+    description: "The e2e test payload."
+    required: true
+  azureClusterCreateCredentials:
+    description: "Azure credentials authorized to create a Constellation cluster."
+    required: true
+  azureIAMCreateCredentials:
+    description: "Azure credentials authorized to create an IAM configuration."
+    required: true
+  cloudProvider:
+    description: "The cloud provider to use."
+    required: true
+  kubernetesVersion:
+    description: "Kubernetes version to create the cluster from."
+    required: true
+  refStream:
+    description: "The refStream of the image the test runs on."
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Login to Azure (IAM service principal)
+      if: inputs.cloudProvider == 'azure'
+      uses: ./.github/actions/login_azure
+      with:
+        azure_credentials: ${{ inputs.azureIAMCreateCredentials }}
+
+    - name: Add Azure Keyvault access role
+      if: inputs.cloudProvider == 'azure'
+      shell: bash
+      run: |
+        UAMI=$(yq eval ".provider.azure.userAssignedIdentity | upcase" constellation-conf.yaml)
+        PRINCIPAL_ID=$(az identity list | yq ".[] | select(.id | test(\"(?i)$UAMI\"; \"g\")) | .principalId")
+        az role assignment create --role "Key Vault Secrets User" \
+          --assignee "$PRINCIPAL_ID" \
+          --scope /subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/e2e-test-creds/providers/Microsoft.KeyVault/vaults/opensearch-creds
+
+    - name: Login to Azure (Cluster service principal)
+      if: inputs.cloudProvider == 'azure'
+      uses: ./.github/actions/login_azure
+      with:
+        azure_credentials: ${{ inputs.azureClusterCreateCredentials }}
+
+    - name: Login to AWS (IAM service principal)
+      if: inputs.cloudProvider == 'aws'
+      uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
+      with:
+        role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2EIAM
+        aws-region: eu-central-1
+        # extend token expiry to 6 hours to ensure constellation can terminate
+        role-duration-seconds: 21600
+
+    - name: Add AWS Secrets Manager access role
+      if: inputs.cloudProvider == 'aws'
+      shell: bash
+      run: |
+        INSTANCE_PROFILE=$(yq eval ".provider.aws.iamProfileControlPlane" constellation-conf.yaml)
+        ROLE_NAME=$(aws iam get-instance-profile --instance-profile-name "$INSTANCE_PROFILE" | yq ".InstanceProfile.Roles[0].RoleName")
+        aws iam attach-role-policy \
+          --role-name "$ROLE_NAME" \
+          --policy-arn arn:aws:iam::795746500882:policy/GitHubActionsOSCredAccess
+
+    - name: Login to AWS (Cluster service principal)
+      if: inputs.cloudProvider == 'aws'
+      uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
+      with:
+        role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2ECluster
+        aws-region: eu-central-1
+        # extend token expiry to 6 hours to ensure constellation can terminate
+        role-duration-seconds: 21600
+
+    - name: Cdbg deploy
+      shell: bash
+      run: |
+        echo "::group::cdbg deploy"
+        chmod +x $GITHUB_WORKSPACE/build/cdbg
+        cdbg deploy \
+          --bootstrapper "${{ github.workspace }}/build/bootstrapper" \
+          --upgrade-agent "${{ github.workspace }}/build/upgrade-agent" \
+          --info logcollect=true \
+          --info logcollect.github.actor="${{ github.triggering_actor }}" \
+          --info logcollect.github.workflow="${{ github.workflow }}" \
+          --info logcollect.github.run-id="${{ github.run_id }}" \
+          --info logcollect.github.run-attempt="${{ github.run_attempt }}" \
+          --info logcollect.github.ref-name="${{ github.ref_name }}" \
+          --info logcollect.github.sha="${{ github.sha }}" \
+          --info logcollect.github.runner-os="${{ runner.os }}" \
+          --info logcollect.github.e2e-test-payload="${{ inputs.test }}" \
+          --info logcollect.github.is-debug-cluster=false \
+          --info logcollect.github.ref-stream="${{ inputs.refStream }}" \
+          --info logcollect.github.kubernetes-version="${{ inputs.kubernetesVersion }}" \
+          --info logcollect.deployment-type="debugd" \
+          --verbosity=-1 \
+          --force
+        echo "::endgroup::"
diff --git a/.github/actions/constellation_create/action.yml b/.github/actions/constellation_create/action.yml
@@ -38,6 +38,15 @@ inputs:
   test:
     description: "The e2e test payload."
     required: true
+  azureClusterCreateCredentials:
+    description: "Azure credentials authorized to create a Constellation cluster."
+    required: true
+  azureIAMCreateCredentials:
+    description: "Azure credentials authorized to create an IAM configuration."
+    required: true
+  refStream:
+    description: "Reference and stream of the image in use"
+    required: false
 
 outputs:
   kubeconfig:
@@ -119,29 +128,14 @@ runs:
 
     - name: Cdbg deploy
       if: inputs.isDebugImage == 'true'
-      shell: bash
-      run: |
-        echo "::group::cdbg deploy"
-        chmod +x $GITHUB_WORKSPACE/build/cdbg
-        cdbg deploy \
-          --bootstrapper "${{ github.workspace }}/build/bootstrapper" \
-          --upgrade-agent "${{ github.workspace }}/build/upgrade-agent" \
-          --info logcollect=true \
-          --info logcollect.github.actor="${{ github.triggering_actor }}" \
-          --info logcollect.github.workflow="${{ github.workflow }}" \
-          --info logcollect.github.run-id="${{ github.run_id }}" \
-          --info logcollect.github.run-attempt="${{ github.run_attempt }}" \
-          --info logcollect.github.ref-name="${{ github.ref_name }}" \
-          --info logcollect.github.sha="${{ github.sha }}" \
-          --info logcollect.github.runner-os="${{ runner.os }}" \
-          --info logcollect.github.e2e-test-payload="${{ inputs.test }}" \
-          --info logcollect.github.is-debug-cluster=false \
-          --info logcollect.github.ref-stream="${{ inputs.refStream }}" \
-          --info logcollect.github.kubernetes-version="${{ inputs.kubernetesVersion }}" \
-          --info logcollect.deployment-type="debugd" \
-          --verbosity=-1 \
-          --force
-        echo "::endgroup::"
+      uses: ./.github/actions/cdbg_deploy
+      with:
+        cloudProvider: ${{ inputs.cloudProvider }}
+        test: ${{ inputs.test }}
+        azureClusterCreateCredentials: ${{ inputs.azureClusterCreateCredentials }}
+        azureIAMCreateCredentials: ${{ inputs.azureIAMCreateCredentials }}
+        refStream: ${{ inputs.refStream }}
+        kubernetesVersion: ${{ inputs.kubernetesVersion }}
 
     - name: Constellation init
       id: constellation-init

diff --git a/.github/actions/deploy_logcollection/action.yml b/.github/actions/deploy_logcollection/action.yml
@@ -50,7 +50,7 @@ runs:
          --fields github.sha="${{ github.sha }}" \
          --fields github.runner-os="${{ runner.os }}" \
          --fields github.e2e-test-payload="${{ inputs.test }}" \
-         --fields github.isDebugImage="${{ inputs.isDebugImage }}" \
+         --fields github.is-debug-cluster="${{ inputs.isDebugImage }}" \
          --fields github.e2e-test-provider="${{ inputs.provider }}" \
          --fields github.ref-stream="${{ inputs.refStream }}" \
          --fields github.kubernetes-version="${{ inputs.kubernetesVersion }}" \
@@ -86,3 +86,17 @@ runs:
         helm repo update
         helm install filebeat elastic/filebeat \
           --wait --timeout=1200s --values values.yml
+
+    - name: Deploy Metricbeat
+      id: deploy-metricbeat
+      shell: bash
+      working-directory: ./metricbeat
+      env:
+        KUBECONFIG: ${{ inputs.kubeconfig }}
+      run: |
+        helm repo add elastic https://helm.elastic.co
+        helm repo update
+        helm install metricbeat-k8s elastic/metricbeat \
+          --wait --timeout=1200s --values values-control-plane.yml
+        helm install metricbeat-system elastic/metricbeat \
+          --wait --timeout=1200s --values values-all-nodes.yml
diff --git a/.github/actions/e2e_test/action.yml b/.github/actions/e2e_test/action.yml
@@ -249,12 +249,14 @@ runs:
         fetchMeasurements: ${{ inputs.fetchMeasurements }}
         cliVersion: ${{ inputs.cliVersion }}
         azureSNPEnforcementPolicy: ${{ inputs.azureSNPEnforcementPolicy }}
+        azureIAMCreateCredentials: ${{ inputs.azureIAMCreateCredentials }}
+        azureClusterCreateCredentials: ${{ inputs.azureClusterCreateCredentials }}
+        kubernetesVersion: ${{ inputs.kubernetesVersion }}
+        refStream: ${{ inputs.refStream }}
 
-    - name: Deploy logcollection
+    - name: Deploy log- and metrics-collection (Kubernetes)
       id: deploy-logcollection
-      # TODO(msanft):temporarily deploy in debug clusters too to resolve "missing logs"-bug
-      # see https://dev.azure.com/Edgeless/Edgeless/_workitems/edit/3227
-      # if: inputs.isDebugImage == 'false'
+      if: inputs.isDebugImage == 'false'
       uses: ./.github/actions/deploy_logcollection
       with:
         kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}

diff --git a/.github/workflows/build-logcollector-images.yml b/.github/workflows/build-logcollector-images.yml
@@ -37,3 +37,10 @@ jobs:
           name: filebeat-debugd
           dockerfile: debugd/filebeat/Dockerfile
           githubToken: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and upload Metricbeat container image
+        uses: ./.github/actions/build_micro_service
+        with:
+          name: metricbeat-debugd
+          dockerfile: debugd/metricbeat/Dockerfile
+          githubToken: ${{ secrets.GITHUB_TOKEN }}
diff --git a/debugd/filebeat/BUILD.bazel b/debugd/filebeat/BUILD.bazel
@@ -3,10 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
 go_library(
     name = "filebeat",
     srcs = ["assets.go"],
-    embedsrcs = [
-        "templates/filebeat.yml",
-        "inputs.yml",
-    ],
+    embedsrcs = ["templates/filebeat.yml"],
     importpath = "github.com/edgelesssys/constellation/v2/debugd/filebeat",
     visibility = ["//visibility:public"],
 )
diff --git a/debugd/filebeat/Dockerfile b/debugd/filebeat/Dockerfile
@@ -2,7 +2,6 @@ FROM fedora:38@sha256:6fc00f83a1b6526b1c6562e30f552d109ba8e269259c6742a26efab1b7
 
 RUN dnf install -y https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-8.6.2-x86_64.rpm
 
-COPY debugd/filebeat/inputs.yml /usr/share/filebeat/inputs.yml
-COPY debugd/filebeat/templates/ /usr/share/filebeat/templates/
+COPY debugd/filebeat/templates/ /usr/share/constellogs/templates/
 
 ENTRYPOINT ["/usr/share/filebeat/bin/filebeat", "-e", "--path.home", "/usr/share/filebeat", "--path.data", "/usr/share/filebeat/data"]
diff --git a/debugd/filebeat/assets.go b/debugd/filebeat/assets.go
@@ -10,6 +10,5 @@ import "embed"
 
 // Assets are the exported Filebeat template files.
 //
-//go:embed *.yml
 //go:embed templates/*
 var Assets embed.FS
diff --git a/debugd/filebeat/inputs.yml b/debugd/filebeat/inputs.yml
diff --git a/debugd/filebeat/templates/filebeat.yml b/debugd/filebeat/templates/filebeat.yml
@@ -9,12 +9,15 @@ logging:
   metrics.enabled: false
   level: warning
 
-filebeat.config:
-  inputs:
+filebeat.inputs:
+  - type: journald
     enabled: true
-    path: /usr/share/filebeat/inputs.yml
-    # reload.enabled: true
-    # reload.period: 10s
+    id: journald
+  - type: filestream
+    enabled: true
+    id: container
+    paths:
+      - /var/log/pods/*/*/*.log
 
 timestamp.precision: nanosecond
 
@@ -27,3 +30,6 @@ processors:
       field: "log.file.path"
       target_prefix: "kubernetes"
       ignore_failure: true
+  {{ if .AddCloudMetadata }}
+  - add_cloud_metadata: ~
+  {{ end }}