From 0468a96aac223916d4b84b5f43f4db8cf48781d5 Mon Sep 17 00:00:00 2001 From: Kevin Conner Date: Wed, 7 Feb 2024 17:44:24 -0800 Subject: [PATCH 1/2] UD-1190: Add support for plugin pod annotations and plugin service account annotations Signed-off-by: Kevin Conner --- api/zora/v1alpha1/plugin_types.go | 4 +++ api/zora/v1alpha1/zz_generated.deepcopy.go | 7 +++++ charts/zora/README.md | 4 +++ .../zora/crds/zora.undistro.io_plugins.yaml | 6 +++++ .../zora/templates/operator/deployment.yaml | 3 +++ charts/zora/templates/plugins/marvin.yaml | 4 +++ charts/zora/templates/plugins/popeye.yaml | 4 +++ charts/zora/templates/plugins/trivy.yaml | 4 +++ charts/zora/values.yaml | 8 ++++++ cmd/main.go | 26 +++++++++++++++++++ .../crd/bases/zora.undistro.io_plugins.yaml | 6 +++++ .../aws-elastic-container-registry.md | 12 +++++++++ .../controller/zora/clusterscan_controller.go | 3 ++- mkdocs.yml | 1 + pkg/plugins/cronjob.go | 12 ++++++++- 15 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 docs/configuration/aws-elastic-container-registry.md diff --git a/api/zora/v1alpha1/plugin_types.go b/api/zora/v1alpha1/plugin_types.go index b78791ef..adaffe74 100644 --- a/api/zora/v1alpha1/plugin_types.go +++ b/api/zora/v1alpha1/plugin_types.go @@ -63,6 +63,10 @@ type PluginSpec struct { // Cannot be updated. Env []corev1.EnvVar `json:"env,omitempty"` + // Annotations to set in plugin and worker containers. + // Cannot be updated. + Annotations map[string]string `json:"annotations,omitempty"` + // Compute Resources required by this container. // Cannot be updated. // More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ diff --git a/api/zora/v1alpha1/zz_generated.deepcopy.go b/api/zora/v1alpha1/zz_generated.deepcopy.go index 66b72887..88e96371 100644 --- a/api/zora/v1alpha1/zz_generated.deepcopy.go +++ b/api/zora/v1alpha1/zz_generated.deepcopy.go @@ -776,6 +776,13 @@ func (in *PluginSpec) DeepCopyInto(out *PluginSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } in.Resources.DeepCopyInto(&out.Resources) if in.SecurityContext != nil { in, out := &in.SecurityContext, &out.SecurityContext diff --git a/charts/zora/README.md b/charts/zora/README.md index 6c9f2adc..3ab13331 100644 --- a/charts/zora/README.md +++ b/charts/zora/README.md @@ -103,16 +103,20 @@ The following table lists the configurable parameters of the Zora chart and thei | scan.vulnerability.plugins | list | `["trivy"]` | Vulnerability scanners plugins | | scan.worker.image.repository | string | `"ghcr.io/undistro/zora/worker"` | worker image repository | | scan.worker.image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion | +| scan.plugins.annotations | object | `{}` | Annotations added to the plugin service account | | scan.plugins.marvin.resources | object | `{"limits":{"cpu":"500m","memory":"500Mi"},"requests":{"cpu":"250m","memory":"256Mi"}}` | [Resources](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers) to add to `marvin` container | +| scan.plugins.marvin.podAnnotations | object | `{}` | Annotations added to the marvin pods | | scan.plugins.marvin.image.repository | string | `"ghcr.io/undistro/marvin"` | marvin plugin image repository | | scan.plugins.marvin.image.tag | string | `"v0.2.1"` | marvin plugin image tag | | scan.plugins.trivy.ignoreUnfixed | bool | `false` | Specifies whether only fixed vulnerabilities should be reported | | scan.plugins.trivy.ignoreDescriptions | bool | `false` | Specifies whether vulnerability descriptions should be ignored | | scan.plugins.trivy.resources | object | `{}` | [Resources](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers) to add to `trivy` container | +| scan.plugins.trivy.podAnnotations | object | `{}` | Annotations added to the trivy pods | | scan.plugins.trivy.image.repository | string | `"ghcr.io/aquasecurity/trivy"` | trivy plugin image repository | | scan.plugins.trivy.image.tag | string | `"0.48.2"` | trivy plugin image tag | | scan.plugins.popeye.skipInternalResources | bool | `false` | Specifies whether the following resources should be skipped by `popeye` scans. 1. resources from `kube-system`, `kube-public` and `kube-node-lease` namespaces; 2. kubernetes system reserved RBAC (prefixed with `system:`); 3. `kube-root-ca.crt` configmaps; 4. `default` namespace; 5. `default` serviceaccounts; 6. Helm secrets (prefixed with `sh.helm.release`); 7. Zora components. See `popeye` configuration file that is used for this case: https://github.com/undistro/zora/blob/main/charts/zora/templates/plugins/popeye-config.yaml | | scan.plugins.popeye.resources | object | `{"limits":{"cpu":"500m","memory":"500Mi"},"requests":{"cpu":"250m","memory":"256Mi"}}` | [Resources](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers) to add to `popeye` container | +| scan.plugins.popeye.podAnnotations | object | `{}` | Annotations added to the popeye pods | | scan.plugins.popeye.image.repository | string | `"ghcr.io/undistro/popeye"` | popeye plugin image repository | | scan.plugins.popeye.image.tag | string | `"pr252"` | popeye plugin image tag | | kubexnsImage.repository | string | `"ghcr.io/undistro/kubexns"` | kubexns image repository | diff --git a/charts/zora/crds/zora.undistro.io_plugins.yaml b/charts/zora/crds/zora.undistro.io_plugins.yaml index 61af8c2d..c41d4b97 100644 --- a/charts/zora/crds/zora.undistro.io_plugins.yaml +++ b/charts/zora/crds/zora.undistro.io_plugins.yaml @@ -58,6 +58,12 @@ spec: spec: description: PluginSpec defines the desired state of Plugin properties: + annotations: + additionalProperties: + type: string + description: Annotations to set in plugin and worker containers. Cannot + be updated. + type: object args: description: 'Arguments to the entrypoint. The docker image''s CMD is used if this is not provided. Variable references $(VAR_NAME) diff --git a/charts/zora/templates/operator/deployment.yaml b/charts/zora/templates/operator/deployment.yaml index 21a3ca84..cd473982 100644 --- a/charts/zora/templates/operator/deployment.yaml +++ b/charts/zora/templates/operator/deployment.yaml @@ -74,6 +74,9 @@ spec: - --worker-image={{ printf "%s:%s" .Values.scan.worker.image.repository (.Values.scan.worker.image.tag | default .Chart.AppVersion) }} - --cronjob-clusterrolebinding-name=zora-plugins-rolebinding - --cronjob-serviceaccount-name=zora-plugins +{{- if .Values.scan.plugins.annotations}} + - --cronjob-serviceaccount-annotations={{ $first := true }}{{- range $key, $value := .Values.scan.plugins.annotations }}{{if not $first}},{{else}}{{$first = false}}{{end}}{{ $key }}={{$value}}{{- end }} +{{- end }} - --saas-workspace-id={{ .Values.saas.workspaceID }} - --saas-server={{ .Values.saas.server }} - --version={{ .Chart.Version }} diff --git a/charts/zora/templates/plugins/marvin.yaml b/charts/zora/templates/plugins/marvin.yaml index fab9e67c..6993cbba 100644 --- a/charts/zora/templates/plugins/marvin.yaml +++ b/charts/zora/templates/plugins/marvin.yaml @@ -30,6 +30,10 @@ spec: runAsNonRoot: true readOnlyRootFilesystem: true allowPrivilegeEscalation: false + {{- if .Values.scan.plugins.marvin.podAnnotations }} + annotations: + {{- toYaml .Values.scan.plugins.marvin.podAnnotations | nindent 4 }} + {{- end }} command: - /bin/sh - -c diff --git a/charts/zora/templates/plugins/popeye.yaml b/charts/zora/templates/plugins/popeye.yaml index b0fdc987..dc01e61d 100644 --- a/charts/zora/templates/plugins/popeye.yaml +++ b/charts/zora/templates/plugins/popeye.yaml @@ -34,6 +34,10 @@ spec: securityContext: runAsNonRoot: true allowPrivilegeEscalation: false + {{- if .Values.scan.plugins.popeye.podAnnotations }} + annotations: + {{- toYaml .Values.scan.plugins.popeye.podAnnotations | nindent 4 }} + {{- end }} command: - /bin/sh - -c diff --git a/charts/zora/templates/plugins/trivy.yaml b/charts/zora/templates/plugins/trivy.yaml index cd9b025e..317909a9 100644 --- a/charts/zora/templates/plugins/trivy.yaml +++ b/charts/zora/templates/plugins/trivy.yaml @@ -31,6 +31,10 @@ spec: env: - name: TRIVY_IGNORE_VULN_DESCRIPTIONS value: {{ .Values.scan.plugins.trivy.ignoreDescriptions | quote }} + {{- if .Values.scan.plugins.trivy.podAnnotations }} + annotations: + {{- toYaml .Values.scan.plugins.trivy.podAnnotations | nindent 4 }} + {{- end }} command: - /bin/sh - -c diff --git a/charts/zora/values.yaml b/charts/zora/values.yaml index a9728226..1308f7d7 100644 --- a/charts/zora/values.yaml +++ b/charts/zora/values.yaml @@ -175,6 +175,8 @@ scan: # -- Overrides the image tag whose default is the chart appVersion tag: "" plugins: + # -- Annotations added to the plugin service account + annotations: {} marvin: # -- [Resources](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers) to add to `marvin` container resources: @@ -184,6 +186,8 @@ scan: limits: cpu: 500m memory: 500Mi + # -- Annotations added to the marvin pods + podAnnotations: {} image: # -- marvin plugin image repository repository: ghcr.io/undistro/marvin @@ -197,6 +201,8 @@ scan: ignoreDescriptions: false # -- [Resources](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers) to add to `trivy` container resources: {} + # -- Annotations added to the trivy pods + podAnnotations: {} image: # -- trivy plugin image repository repository: ghcr.io/aquasecurity/trivy @@ -221,6 +227,8 @@ scan: limits: cpu: 500m memory: 500Mi + # -- Annotations added to the popeye pods + podAnnotations: {} image: # -- popeye plugin image repository repository: ghcr.io/undistro/popeye diff --git a/cmd/main.go b/cmd/main.go index f8abbf7a..632da434 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -61,6 +61,7 @@ func main() { var workerImage string var cronJobClusterRoleBinding string var cronJobServiceAccount string + var cronJobAnnotations string var saasWorkspaceID string var saasServer string var version string @@ -78,6 +79,7 @@ func main() { flag.StringVar(&workerImage, "worker-image", "ghcr.io/undistro/zora/worker:latest", "Docker image name of Worker container") flag.StringVar(&cronJobClusterRoleBinding, "cronjob-clusterrolebinding-name", "zora-plugins-rolebinding", "Name of ClusterRoleBinding to append CronJob ServiceAccounts") flag.StringVar(&cronJobServiceAccount, "cronjob-serviceaccount-name", "zora-plugins", "Name of ServiceAccount to be configured, appended to ClusterRoleBinding and used by CronJobs") + flag.StringVar(&cronJobAnnotations, "cronjob-serviceaccount-annotations", "annotaion1=value1,annotation2=value2", "Annotations to be applied to the CronJob Service Account") flag.StringVar(&saasWorkspaceID, "saas-workspace-id", "", "Your workspace ID in Zora SaaS") flag.StringVar(&saasServer, "saas-server", "http://localhost:3003", "Address for Zora's saas server") flag.StringVar(&version, "version", "0.8.0", "Zora version") @@ -139,6 +141,11 @@ func main() { os.Exit(1) } + annotations, err := annotations(cronJobAnnotations) + if err != nil { + setupLog.Error(err, "unable to parse annotations") + os.Exit(1) + } if err = (&zoracontroller.ClusterScanReconciler{ Client: mgr.GetClient(), K8sClient: kcli, @@ -149,6 +156,7 @@ func main() { WorkerImage: workerImage, ClusterRoleBindingName: cronJobClusterRoleBinding, ServiceAccountName: cronJobServiceAccount, + Annotations: annotations, OnUpdate: onClusterScanUpdate, OnDelete: onClusterScanDelete, KubexnsImage: kubexnsImage, @@ -183,3 +191,21 @@ func main() { os.Exit(1) } } + +func annotations(cronJobAnnotations string) (map[string]string, error) { + cronJobAnnotations = strings.Trim(cronJobAnnotations, " ") + if len(cronJobAnnotations) == 0 { + return nil, nil + } + annotations := map[string]string{} + for _, annotation := range strings.Split(cronJobAnnotations, ",") { + index := strings.Index(annotation, "=") + if index == -1 || index == len(annotation) { + return nil, fmt.Errorf("Could not parse annotation %s", annotation) + } + key := annotation[:index] + value := annotation[index+1:] + annotations[key] = value + } + return annotations, nil +} diff --git a/config/crd/bases/zora.undistro.io_plugins.yaml b/config/crd/bases/zora.undistro.io_plugins.yaml index de84fd53..00506440 100644 --- a/config/crd/bases/zora.undistro.io_plugins.yaml +++ b/config/crd/bases/zora.undistro.io_plugins.yaml @@ -44,6 +44,12 @@ spec: spec: description: PluginSpec defines the desired state of Plugin properties: + annotations: + additionalProperties: + type: string + description: Annotations to set in plugin and worker containers. Cannot + be updated. + type: object args: description: 'Arguments to the entrypoint. The docker image''s CMD is used if this is not provided. Variable references $(VAR_NAME) diff --git a/docs/configuration/aws-elastic-container-registry.md b/docs/configuration/aws-elastic-container-registry.md new file mode 100644 index 00000000..1bf47924 --- /dev/null +++ b/docs/configuration/aws-elastic-container-registry.md @@ -0,0 +1,12 @@ +# AWS Elastic Container Registry + +If you are running within AWS, and making use of a private [Elastic Container Registry (ECR)](https://aws.amazon.com/ecr/) to host your application images, then the Trivy plugin will be unable to scan those images unless access is granted to the registry through an [Identity and Access Managemnent (IAM)](https://aws.amazon.com/iam/) role assigned to the service account running the Trivy plugins. + +Once an IAM role granting grant access to the ECR has been created, this can be assigned to the service account by including the following additional parameter when running the `helm upgrade --install` command. + +```shell +--set scan.plugins.annotations.eks\\.amazonaws\\.com/role-arn=arn:aws:iam:::role/ +``` +where `` should be replaced witth your AWS account ID, and `` should be replaced with the name of the role granting access to the ECR. + +This will now allow the Trivy plugin to scan your internal images for vulnerabilities. diff --git a/internal/controller/zora/clusterscan_controller.go b/internal/controller/zora/clusterscan_controller.go index e0b98a78..5b20a59c 100644 --- a/internal/controller/zora/clusterscan_controller.go +++ b/internal/controller/zora/clusterscan_controller.go @@ -63,6 +63,7 @@ type ClusterScanReconciler struct { ServiceAccountName string KubexnsImage string ChecksConfigMap string + Annotations map[string]string OnUpdate saas.ClusterScanHook OnDelete saas.ClusterScanHook } @@ -462,7 +463,7 @@ func (r *ClusterScanReconciler) applyRBAC(ctx context.Context, clusterscan *v1al return err } - sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: r.ServiceAccountName, Namespace: clusterscan.Namespace}} + sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: r.ServiceAccountName, Namespace: clusterscan.Namespace, Annotations: r.Annotations}} res, err := ctrl.CreateOrUpdate(ctx, r.Client, sa, func() error { return controllerutil.SetOwnerReference(clusterscan, sa, r.Scheme) }) diff --git a/mkdocs.yml b/mkdocs.yml index a727fa53..a5616ffc 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -87,6 +87,7 @@ nav: - Suspending scans: configuration/suspend-scan.md - Retain issues: configuration/retain-issues.md - Ignore unfixed vulnerabilities: plugins/trivy/#large-vulnerability-reports + - Scanning Images hosted in AWS Elastic Container Registry: configuration/aws-elastic-container-registry.md - "🔌 Plugins": - Overview: plugins/index.md - Misconfiguration: diff --git a/pkg/plugins/cronjob.go b/pkg/plugins/cronjob.go index db21e9ab..0fb83237 100644 --- a/pkg/plugins/cronjob.go +++ b/pkg/plugins/cronjob.go @@ -115,7 +115,7 @@ func (r *CronJobMutator) Mutate() error { r.Existing.Spec.JobTemplate.Spec.Template.Spec.RestartPolicy = corev1.RestartPolicyNever r.Existing.Spec.JobTemplate.Spec.BackoffLimit = pointer.Int32(0) r.Existing.Spec.JobTemplate.Spec.Template.Spec.ServiceAccountName = r.ServiceAccountName - r.Existing.Spec.JobTemplate.Spec.Template.Annotations = map[string]string{annotationDefaultContainer: r.Plugin.Name} + r.Existing.Spec.JobTemplate.Spec.Template.Annotations = r.annotations() r.Existing.Spec.JobTemplate.Spec.Template.Spec.Volumes = []corev1.Volume{ { Name: resultsVolumeName, @@ -320,3 +320,13 @@ func (r *CronJobMutator) workerEnv() []corev1.EnvVar { ) return p } + +func (r *CronJobMutator) annotations() map[string]string { + annotations := map[string]string{} + for key, value := range r.Plugin.Spec.Annotations { + annotations[key] = value + } + annotations[annotationDefaultContainer] = r.Plugin.Name + + return annotations +} From 3b6589df6c03286d8b379ece61b8898e1ce7d6dc Mon Sep 17 00:00:00 2001 From: Matheus Moraes Date: Fri, 9 Feb 2024 11:48:24 -0300 Subject: [PATCH 2/2] bump popeye version to 0.11.3 (#242) --- charts/zora/README.md | 2 +- charts/zora/values.yaml | 2 +- config/samples/zora_v1alpha1_plugin_popeye.yaml | 2 +- config/samples/zora_v1alpha1_plugin_popeye_all.yaml | 2 +- docs/plugins/index.md | 2 +- docs/plugins/popeye.md | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/zora/README.md b/charts/zora/README.md index 3ab13331..e5945d7d 100644 --- a/charts/zora/README.md +++ b/charts/zora/README.md @@ -118,7 +118,7 @@ The following table lists the configurable parameters of the Zora chart and thei | scan.plugins.popeye.resources | object | `{"limits":{"cpu":"500m","memory":"500Mi"},"requests":{"cpu":"250m","memory":"256Mi"}}` | [Resources](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers) to add to `popeye` container | | scan.plugins.popeye.podAnnotations | object | `{}` | Annotations added to the popeye pods | | scan.plugins.popeye.image.repository | string | `"ghcr.io/undistro/popeye"` | popeye plugin image repository | -| scan.plugins.popeye.image.tag | string | `"pr252"` | popeye plugin image tag | +| scan.plugins.popeye.image.tag | string | `"v0.11.3"` | popeye plugin image tag | | kubexnsImage.repository | string | `"ghcr.io/undistro/kubexns"` | kubexns image repository | | kubexnsImage.tag | string | `"v0.1.2"` | kubexns image tag | | customChecksConfigMap | string | `"zora-custom-checks"` | Custom checks ConfigMap name | diff --git a/charts/zora/values.yaml b/charts/zora/values.yaml index 1308f7d7..80171b1c 100644 --- a/charts/zora/values.yaml +++ b/charts/zora/values.yaml @@ -233,7 +233,7 @@ scan: # -- popeye plugin image repository repository: ghcr.io/undistro/popeye # -- popeye plugin image tag - tag: pr252 + tag: v0.11.3 kubexnsImage: # -- kubexns image repository diff --git a/config/samples/zora_v1alpha1_plugin_popeye.yaml b/config/samples/zora_v1alpha1_plugin_popeye.yaml index 83b79d06..005129e8 100644 --- a/config/samples/zora_v1alpha1_plugin_popeye.yaml +++ b/config/samples/zora_v1alpha1_plugin_popeye.yaml @@ -10,7 +10,7 @@ metadata: name: popeye spec: type: misconfiguration - image: ghcr.io/undistro/popeye:pr252 + image: ghcr.io/undistro/popeye:v0.11.3 resources: limits: cpu: 500m diff --git a/config/samples/zora_v1alpha1_plugin_popeye_all.yaml b/config/samples/zora_v1alpha1_plugin_popeye_all.yaml index b1aab56c..5fcc2f38 100644 --- a/config/samples/zora_v1alpha1_plugin_popeye_all.yaml +++ b/config/samples/zora_v1alpha1_plugin_popeye_all.yaml @@ -10,7 +10,7 @@ metadata: name: popeye spec: type: misconfiguration - image: ghcr.io/undistro/popeye:pr252 + image: ghcr.io/undistro/popeye:v0.11.3 resources: limits: cpu: 500m diff --git a/docs/plugins/index.md b/docs/plugins/index.md index 0f3ffd17..5bde7f65 100644 --- a/docs/plugins/index.md +++ b/docs/plugins/index.md @@ -16,7 +16,7 @@ kubectl get plugins -n zora-system ``` NAME IMAGE TYPE AGE marvin ghcr.io/undistro/marvin:v0.2.1 misconfiguration 14m -popeye ghcr.io/undistro/popeye:pr252 misconfiguration 14m +popeye ghcr.io/undistro/popeye:v0.11.3 misconfiguration 14m trivy ghcr.io/aquasecurity/trivy:0.48.2 vulnerability 14m ``` diff --git a/docs/plugins/popeye.md b/docs/plugins/popeye.md index bddc3d6c..a83055fc 100644 --- a/docs/plugins/popeye.md +++ b/docs/plugins/popeye.md @@ -8,7 +8,7 @@ Popeye is a utility that scans live Kubernetes cluster and reports potential iss :octicons-codescan-24: **Type**: `misconfiguration` -:simple-docker: **Image**: `ghcr.io/undistro/popeye:pr252` +:simple-docker: **Image**: `ghcr.io/undistro/popeye:v0.11.3` :simple-github: **GitHub repository**: [https://github.com/derailed/popeye](https://github.com/derailed/popeye){:target="_blank"}