Add agent server port for vending entity to fluent bit (#120)

* Added Agent Server and Fluent-bit client certificates to implement mtls on agent endpoint (#106) * added server and client certificates to implement mtls on agent endpoint * added latest fluent-bit config for application logs files to support sending entity (#118) * added flag to retrieve instance id behind entity flag in aws filter plugin for application logs (#122) * Increment fluentbit version for linux --------- Co-authored-by: POOJA REDDY NATHALA <[email protected]>
aws-observability · Nov 7, 2024 · 57c280b · 57c280b
1 parent a765bd3
commit 57c280b
Show file tree

Hide file tree

Showing 4 changed files with 144 additions and 1 deletion.
diff --git a/charts/amazon-cloudwatch-observability/templates/certmanager.yaml b/charts/amazon-cloudwatch-observability/templates/certmanager.yaml
@@ -63,6 +63,57 @@ spec:
     kind: Issuer
     name: "agent-ca"
   secretName: "amazon-cloudwatch-observability-agent-cert"
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  labels:
+    {{- include "amazon-cloudwatch-observability.labels" . | nindent 4 }}
+  name: "amazon-cloudwatch-observability-agent-server-cert"
+  namespace: {{ .Release.Namespace }}
+spec:
+  commonName: "agent-server"
+  dnsNames:
+    - "cloudwatch-agent"
+    - "cloudwatch-agent.amazon-cloudwatch.svc"
+  issuerRef:
+    kind: Issuer
+    name: "agent-ca"
+  secretName: "amazon-cloudwatch-observability-agent-server-cert"
+  usages:
+    - digital signature
+    - key encipherment
+    - cert sign
+  keyUsages:
+    critical: true
+    usages:
+      - digitalSignature
+      - keyEncipherment
+      - certSign
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  labels:
+    {{- include "amazon-cloudwatch-observability.labels" . | nindent 4 }}
+  name: "amazon-cloudwatch-observability-agent-client-cert"
+  namespace: {{ .Release.Namespace }}
+spec:
+  commonName: "agent-client"
+  issuerRef:
+    kind: Issuer
+    name: "agent-ca"
+  secretName:  "amazon-cloudwatch-observability-agent-client-cert"
+  usages:
+    - digital signature
+    - key encipherment
+    - cert sign
+  keyUsages:
+    critical: true
+    usages:
+      - digitalSignature
+      - keyEncipherment
+      - certSign
 {{- if not .Values.agent.certManager.issuerRef }}
 ---
 apiVersion: cert-manager.io/v1
@@ -87,5 +138,21 @@ metadata:
     {{- include "amazon-cloudwatch-observability.labels" . | nindent 4 }}
   name: "amazon-cloudwatch-observability-agent-cert"
   namespace: {{ .Release.Namespace }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    {{- include "amazon-cloudwatch-observability.labels" . | nindent 4 }}
+  name: "amazon-cloudwatch-observability-agent-server-cert"
+  namespace: {{ .Release.Namespace }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    {{- include "amazon-cloudwatch-observability.labels" . | nindent 4 }}
+  name:  "amazon-cloudwatch-observability-agent-client-cert"
+  namespace: {{ .Release.Namespace }}
 {{- end }}
 
diff --git a/charts/amazon-cloudwatch-observability/templates/linux/cloudwatch-agent-daemonset.yaml b/charts/amazon-cloudwatch-observability/templates/linux/cloudwatch-agent-daemonset.yaml
@@ -1,8 +1,11 @@
 {{- if .Values.agent.enabled }}
 {{- if and (.Values.agent.autoGenerateCert.enabled) (not .Values.agent.certManager.enabled) -}}
 {{- $altNames := list ( printf "%s-service" (include "dcgm-exporter.name" .) ) ( printf "%s-service" (include "neuron-monitor.name" .) ) ( printf "%s-service.%s.svc" (include "dcgm-exporter.name" .) .Release.Namespace ) ( printf "%s-service.%s.svc" (include "neuron-monitor.name" .) .Release.Namespace ) -}}
+{{- $agentAltNames := list ( printf "%s" (include "cloudwatch-agent.name" .) ) ( printf "%s.%s.svc" (include "cloudwatch-agent.name" .) .Release.Namespace ) -}}
 {{- $ca := genCA ("agent-ca")  ( .Values.agent.autoGenerateCert.expiryDays | int ) -}}
 {{- $cert := genSignedCert ("agent") nil $altNames ( .Values.admissionWebhooks.autoGenerateCert.expiryDays | int ) $ca -}}
+{{- $serverCert := genSignedCert ("agent-server") nil $agentAltNames ( .Values.admissionWebhooks.autoGenerateCert.expiryDays | int ) $ca -}}
+{{- $clientCert := genSignedCert ("agent-client") nil nil ( .Values.admissionWebhooks.autoGenerateCert.expiryDays | int ) $ca -}}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -15,6 +18,30 @@ data:
   tls.crt: {{ $cert.Cert | b64enc }}
   tls.key: {{ $cert.Key | b64enc }}
 ---
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+      {{- include "amazon-cloudwatch-observability.labels" . | nindent 4}}
+  name: "amazon-cloudwatch-observability-agent-server-cert"
+  namespace: {{ .Release.Namespace }}
+data:
+  ca.crt: {{ $ca.Cert | b64enc }}
+  tls.crt: {{ $serverCert.Cert | b64enc }}
+  tls.key: {{ $serverCert.Key | b64enc }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+      {{- include "amazon-cloudwatch-observability.labels" . | nindent 4}}
+  name: "amazon-cloudwatch-observability-agent-client-cert"
+  namespace: {{ .Release.Namespace }}
+data:
+  ca.crt: {{ $ca.Cert | b64enc }}
+  tls.crt: {{ $clientCert.Cert | b64enc }}
+  tls.key: {{ $clientCert.Key | b64enc }}
+---
 {{- end -}}
 
 {{- $clusterName := .Values.clusterName | required ".Values.clusterName is required." -}}
@@ -72,6 +99,12 @@ spec:
   - mountPath: /etc/amazon-cloudwatch-observability-agent-cert
     name: agenttls
     readOnly: true
+  - mountPath: /etc/amazon-cloudwatch-observability-agent-client-cert
+    name: agentclienttls
+    readOnly: true
+  - mountPath: /etc/amazon-cloudwatch-observability-agent-server-cert
+    name: agentservertls
+    readOnly: true
   - mountPath: /var/lib/kubelet/pod-resources
     name: kubelet-podresources
   volumes:
@@ -103,6 +136,20 @@ spec:
       items:
         - key: ca.crt
           path: tls-ca.crt
+  - name: agentclienttls
+    secret:
+      secretName: amazon-cloudwatch-observability-agent-client-cert
+      items:
+        - key: ca.crt
+          path: tls-ca.crt
+  - name: agentservertls
+    secret:
+      secretName: amazon-cloudwatch-observability-agent-server-cert
+      items:
+        - key: tls.crt
+          path: server.crt
+        - key: tls.key
+          path: server.key
   env:
   - name: K8S_NODE_NAME
     valueFrom:

diff --git a/charts/amazon-cloudwatch-observability/templates/linux/fluent-bit-daemonset.yaml b/charts/amazon-cloudwatch-observability/templates/linux/fluent-bit-daemonset.yaml
@@ -68,6 +68,12 @@ spec:
         - name: dmesg
           mountPath: /var/log/dmesg
           readOnly: true
+        - mountPath: /etc/amazon-cloudwatch-observability-agent-client-cert
+          name: agentclienttls
+          readOnly: true
+        - mountPath: /etc/amazon-cloudwatch-observability-agent-server-cert
+          name: agentservertls
+          readOnly: true
       terminationGracePeriodSeconds: 10
       hostNetwork: true
       dnsPolicy: ClusterFirstWithHostNet
@@ -90,6 +96,20 @@ spec:
       - name: dmesg
         hostPath:
           path: /var/log/dmesg
+      - name: agentclienttls
+        secret:
+          secretName: amazon-cloudwatch-observability-agent-client-cert
+          items:
+            - key: tls.crt
+              path: client.crt
+            - key: tls.key
+              path: client.key
+      - name: agentservertls
+        secret:
+          secretName: amazon-cloudwatch-observability-agent-server-cert
+          items:
+            - key: ca.crt
+              path: tls-ca.crt
       serviceAccountName: {{ template "cloudwatch-agent.serviceAccountName" . }}
       affinity:
         nodeAffinity:

diff --git a/charts/amazon-cloudwatch-observability/values.yaml b/charts/amazon-cloudwatch-observability/values.yaml
@@ -32,7 +32,7 @@ containerLogs:
   fluentBit:
     image:
       repository: aws-for-fluent-bit
-      tag: 2.32.2.20240627
+      tag: 2.32.4
       tagWindows: 2.31.12-windowsservercore
       repositoryDomainMap:
         public: public.ecr.aws/aws-observability
@@ -118,6 +118,13 @@ containerLogs:
             Refresh_Interval    10
             Read_from_Head      ${READ_FROM_HEAD}
           
+          [FILTER]
+            Name                aws
+            Match               application.*
+            az                  false
+            ec2_instance_id     false
+            Enable_Entity       true
+          
           [FILTER]
             Name                kubernetes
             Match               application.*
@@ -132,6 +139,7 @@ containerLogs:
             Use_Kubelet         On
             Kubelet_Port        10250
             Buffer_Size         0
+            Use_Pod_Association On
           
           [OUTPUT]
             Name                cloudwatch_logs
@@ -141,6 +149,7 @@ containerLogs:
             log_stream_prefix   ${HOST_NAME}-
             auto_create_group   true
             extra_user_agent    container-insights
+            add_entity          true
         dataplane-log.conf: |
           [INPUT]
             Name                systemd