From 50509459012fa8509418f4e1ea838f0db44e60ad Mon Sep 17 00:00:00 2001 From: Michael Chmielewski Date: Fri, 11 Feb 2022 11:55:08 -0500 Subject: [PATCH 1/3] Reintroduce logic to redeploy agent (daemonset and api-reader deployment) when secrets or configmap changes --- templates/daemonset.yaml | 11 +++++++++-- templates/deployment-api-reader.yaml | 7 +++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/templates/daemonset.yaml b/templates/daemonset.yaml index 7d7ccd0..69adcb4 100644 --- a/templates/daemonset.yaml +++ b/templates/daemonset.yaml @@ -22,9 +22,16 @@ spec: app.kubernetes.io/managed-by: {{ .Release.Service }} name: {{ include "threatstack-agent.name" . }} annotations: - {{- if .Values.daemonset.podAnnotations }} + # If configmap or secret files change, this will change the checksum annotations in the daemonset, forcing a redeploy. + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} +{{- if not .Values.agentSetupExternalSecretRef }} + checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} +{{- else }} + checksum/secrets: {{ .Values.agentSetupExternalSecretRef.checksum | sha256sum }} +{{- end }} +{{- if .Values.daemonset.podAnnotations }} {{ toYaml .Values.daemonset.podAnnotations | indent 8 }} - {{- end }} +{{- end }} spec: {{- if .Values.imagePullSecrets }} imagePullSecrets: diff --git a/templates/deployment-api-reader.yaml b/templates/deployment-api-reader.yaml index 7eec5ed..d4d20f6 100644 --- a/templates/deployment-api-reader.yaml +++ b/templates/deployment-api-reader.yaml @@ -23,6 +23,13 @@ spec: app.kubernetes.io/managed-by: {{ .Release.Service }} name: {{ include "threatstack-agent.name" .}}-kubernetes-api annotations: + # If configmap or secret files change, this will change the checksum annotations in the daemonset, forcing a redeploy. + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} +{{- if not .Values.agentSetupExternalSecretRef }} + checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} +{{- else }} + checksum/secrets: {{ .Values.agentSetupExternalSecretRef.checksum | sha256sum }} +{{- end }} spec: {{- if .Values.imagePullSecrets }} imagePullSecrets: From 5bb7fb242d8bc130aba96550f0495592a251fb86 Mon Sep 17 00:00:00 2001 From: Michael Chmielewski Date: Mon, 14 Feb 2022 11:00:13 -0500 Subject: [PATCH 2/3] Add checksum if externally-referenced Secret name or Secret entry name for the sensitive data changes. This won't cause the Deployment/DaemonSet to redeploy if the secret data itself changes, but it will redeploy if what the external secret reference is changes. Additional updates: * Brought over podAnnotations and priorityClass to the API Reader pod, for parity. * Updated documentation --- README.md | 10 +++++++++- templates/daemonset.yaml | 4 +++- templates/deployment-api-reader.yaml | 12 ++++++++++-- values.yaml | 16 ++++++++++++++++ 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 20acb10..3a54ad1 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,8 @@ The following kubernetes objects are created when the chart is installed: | apiReader.affinity | object | `{}` | | | apiReader.nodeSelector | object | `{}` | | | apiReader.tolerations | list | `[]` | | +| apiReader.podAnnotations | string | {} | | +| apiReader.priorityClassName | string | `""` | Optionally set the priority class name for the daemonset pods. Note that priority classes are not created via this helm chart. Ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ | | capabilities | string | `"[\"AUDIT_CONTROL\", \"SYS_ADMIN\", \"SYS_PTRACE\"]\n"` | Docker capabilites required for the proper operation of the agent | | customDaemonsetCmd | object | `{}` | Uncomment the `command` and `args` sub-attributes, and define them as desired to run custom commands in the daemonset. | | daemonset.additionalRuntimeConfig | string | `"log.level info"` | | @@ -148,13 +150,19 @@ Assuming you override the default values to match our environment in a `values.y ##### Using the `agentSetupExternalSecretRef` value block -> **WARNING:** Do not set the `agentSetupExternalSecretRef` block *and* the `agentDeployKey` settings at the same time. This will cause unnecessary kubernetes resource definitions to be created. If you had previously used the `agentDeployKey` value, the secret associated with it may be destroyed on deployment. +>>> +**IMPORTANT:** Using `agentSetupExternalSecretRef` decouples secret management from the helm chart. Therefore, if the value of the secret changes, the agent DaemonSet and Deployment will _not_ be redeployed/restarted. The user will need to force a redeployment of the helm chart explicitly. + +However, if the secret's name or secret's entry name changes in the `values.yaml` of the chart, helm will recognize this change with a new release, and trigger a redeployment of the DaemonsSet and Deployment. One way to take advantage of this is to update the secrets entry value name (what is defined at `agentSetupExternalSecretRef.value`) when changing the secret data, and doing a redeploy of the chart. The chart trigger a redeployment of the agent pods. +>>> An alternative to having the chart define the `ts-setup-args` secret itself, you can instead have it point to your own self-managed secret. Doing so requires the following three values to be set: * `agentSetupExternalSecretRef.name` :: This is the name of your self-managed secret. * `agentSetupExternalSecretRef.key` :: This is the key in your self-managed secret that is associated with the data you want to supply from the secret, to the Threat Stack agent setup registration. +Do not set the `agentSetupExternalSecretRef` block *and* the `agentDeployKey` settings at the same time. This will cause unnecessary kubernetes resource definitions to be created. If you had previously used the `agentDeployKey` value, the secret associated with it may be destroyed on deployment. + Using the `agentSetupExternalSecretRef` block will cause the chart to ignore the `agentDeployKey`, `rulesets`, and `additionalSetupConfig` values defined in `values.yaml` or any other values override file, until existing pods are terminated/rescheduled. The value defined in the secret by `agentSetupExternalSecretRef.name`/`agentSetupExternalSecretRef.key` should be defined as in the example below to properly setup up the agent. Failure to do so can cause the agent to not properly register itself with the Threat Stack platform. diff --git a/templates/daemonset.yaml b/templates/daemonset.yaml index 44d3711..2df8d5e 100644 --- a/templates/daemonset.yaml +++ b/templates/daemonset.yaml @@ -23,11 +23,13 @@ spec: name: {{ include "threatstack-agent.name" . }} annotations: # If configmap or secret files change, this will change the checksum annotations in the daemonset, forcing a redeploy. + # If using an external secret reference, then if external secret name or entry change, but NOT the actual secret data, + # this will change the checksum annotations in the deployment, forcing a redeploy. checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} {{- if not .Values.agentSetupExternalSecretRef }} checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} {{- else }} - checksum/secrets: {{ .Values.agentSetupExternalSecretRef.checksum | sha256sum }} + checksum/secrets: {{ .Values.agentSetupExternalSecretRef | toString | sha256sum }} {{- end }} {{- if .Values.daemonset.podAnnotations }} {{ toYaml .Values.daemonset.podAnnotations | indent 8 }} diff --git a/templates/deployment-api-reader.yaml b/templates/deployment-api-reader.yaml index 7c44d63..12b8043 100644 --- a/templates/deployment-api-reader.yaml +++ b/templates/deployment-api-reader.yaml @@ -23,12 +23,17 @@ spec: app.kubernetes.io/managed-by: {{ .Release.Service }} name: {{ include "threatstack-agent.name" .}}-kubernetes-api annotations: - # If configmap or secret files change, this will change the checksum annotations in the daemonset, forcing a redeploy. + # If configmap or secret files change, this will change the checksum annotations in the deployment, forcing a redeploy. + # If using an external secret reference, then if external secret name or entry change, but NOT the actual secret data, + # this will change the checksum annotations in the deployment, forcing a redeploy. checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} {{- if not .Values.agentSetupExternalSecretRef }} checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} {{- else }} - checksum/secrets: {{ .Values.agentSetupExternalSecretRef.checksum | sha256sum }} + checksum/secrets: {{ .Values.agentSetupExternalSecretRef | toString | sha256sum }} +{{- end }} +{{- if .Values.apiReader.podAnnotations }} +{{ toYaml .Values.apiReader.podAnnotations | indent 8 }} {{- end }} spec: {{- if .Values.imagePullSecrets }} @@ -50,6 +55,9 @@ spec: tolerations: {{ toYaml .Values.apiReader.tolerations | indent 8 }} {{- else }} +{{- end }} +{{- if .Values.apiReader.priorityClassName }} + priorityClassName: {{ .Values.apiReader.priorityClassName }} {{- end }} hostNetwork: true hostPID: true diff --git a/values.yaml b/values.yaml index 2686618..9eb859b 100644 --- a/values.yaml +++ b/values.yaml @@ -131,6 +131,22 @@ apiReader: # Optional tolerations: [] + # Optional + # Ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ + priorityClassName: "" + + ## Annotations to add to the threatstack api reader agent pod + # + # To remove the apparmor annotation, add a comment as the attribute value, + # Example: + # podAnnotations: + # # This comment triggers REMOVING any podAnnotations! + # + # podAnnotations: + # key: "value" + # Optional + podAnnotations: {} + securityContext: privileged: false From 48e02bc6850299403a4e75667df1a485701f9a5a Mon Sep 17 00:00:00 2001 From: Michael Chmielewski Date: Thu, 24 Feb 2022 08:25:50 -0500 Subject: [PATCH 3/3] Update chart version for release --- Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Chart.yaml b/Chart.yaml index f85d154..8541818 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: threatstack-agent -version: 2.3.0 +version: 2.4.0 appVersion: 2.5.0 description: A Helm chart for the Threat Stack Cloud Security Agent keywords: