From eef9c111ca1d5781bd6ab1174fe6aa822e8e7541 Mon Sep 17 00:00:00 2001 From: i325261 Date: Mon, 12 Feb 2024 14:06:08 +0100 Subject: [PATCH 01/13] added secret volume mounts --- .../reconcile-capapplicationversion.go | 23 ++++++++++--- .../reconcile-captenantoperation.go | 5 +++ internal/controller/reconcile.go | 32 +++++++++++++++++++ 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/internal/controller/reconcile-capapplicationversion.go b/internal/controller/reconcile-capapplicationversion.go index 45d46ea..42dd759 100644 --- a/internal/controller/reconcile-capapplicationversion.go +++ b/internal/controller/reconcile-capapplicationversion.go @@ -49,6 +49,8 @@ type DeploymentParameters struct { OwnerRef *metav1.OwnerReference WorkloadDetails v1alpha1.WorkloadDetails VCAPSecretName string + Volume []corev1.Volume + VolumeMount []corev1.VolumeMount } func (c *Controller) reconcileCAPApplicationVersion(ctx context.Context, item QueueItem, attempts int) (*ReconcileResult, error) { @@ -270,6 +272,9 @@ func newContentDeploymentJob(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPAppli AnnotationIstioSidecarInject: "false", }) + // Get ServiceInfos for consumed BTP services + consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), ca.Spec.BTP.Services) + return &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ Name: getContentJobName(workload.Name, cav), @@ -298,14 +303,14 @@ func newContentDeploymentJob(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPAppli {Name: EnvCAPOpAppVersion, Value: cav.Spec.Version}, }, workload.JobDefinition.Env...), EnvFrom: getEnvFrom(vcapSecretName), - VolumeMounts: workload.JobDefinition.VolumeMounts, + VolumeMounts: append(workload.JobDefinition.VolumeMounts, getVolumeMounts(consumedServiceInfos)...), Resources: workload.JobDefinition.Resources, SecurityContext: workload.JobDefinition.SecurityContext, }, }, SecurityContext: workload.JobDefinition.PodSecurityContext, ServiceAccountName: workload.JobDefinition.ServiceAccountName, - Volumes: workload.JobDefinition.Volumes, + Volumes: append(workload.JobDefinition.Volumes, getVolumes(consumedServiceInfos)...), ImagePullSecrets: convertToLocalObjectReferences(cav.Spec.RegistrySecrets), RestartPolicy: corev1.RestartPolicyOnFailure, NodeSelector: workload.JobDefinition.NodeSelector, @@ -557,12 +562,22 @@ func (c *Controller) updateDeployment(ca *v1alpha1.CAPApplication, cav *v1alpha1 // newDeployment creates a new generic Deployment for a CAV resource based on the type. It also sets the appropriate OwnerReferences. func newDeployment(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPApplicationVersion, workload *v1alpha1.WorkloadDetails, ownerRef metav1.OwnerReference, vcapSecretName string) *appsv1.Deployment { + // Get ServiceInfos for consumed BTP services + consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), ca.Spec.BTP.Services) + params := &DeploymentParameters{ CA: ca, CAV: cav, OwnerRef: &ownerRef, WorkloadDetails: *workload, VCAPSecretName: vcapSecretName, + Volume: getVolumes(consumedServiceInfos), + } + + if workload.DeploymentDefinition.Type == v1alpha1.DeploymentCAP { + params.VolumeMount = getCAPVolumeMounts(consumedServiceInfos, CDSVolMountPrefix) + } else { + params.VolumeMount = getVolumeMounts(consumedServiceInfos) } return createDeployment(params) @@ -597,7 +612,7 @@ func createDeployment(params *DeploymentParameters) *appsv1.Deployment { ImagePullSecrets: convertToLocalObjectReferences(params.CAV.Spec.RegistrySecrets), Containers: getContainer(params), ServiceAccountName: params.WorkloadDetails.DeploymentDefinition.ServiceAccountName, - Volumes: params.WorkloadDetails.DeploymentDefinition.Volumes, + Volumes: append(params.WorkloadDetails.DeploymentDefinition.Volumes, params.Volume...), SecurityContext: params.WorkloadDetails.DeploymentDefinition.PodSecurityContext, NodeSelector: params.WorkloadDetails.DeploymentDefinition.NodeSelector, NodeName: params.WorkloadDetails.DeploymentDefinition.NodeName, @@ -619,7 +634,7 @@ func getContainer(params *DeploymentParameters) []corev1.Container { Command: params.WorkloadDetails.DeploymentDefinition.Command, Env: getEnv(params), EnvFrom: getEnvFrom(params.VCAPSecretName), - VolumeMounts: params.WorkloadDetails.DeploymentDefinition.VolumeMounts, + VolumeMounts: append(params.WorkloadDetails.DeploymentDefinition.VolumeMounts, params.VolumeMount...), LivenessProbe: params.WorkloadDetails.DeploymentDefinition.LivenessProbe, ReadinessProbe: params.WorkloadDetails.DeploymentDefinition.ReadinessProbe, Resources: params.WorkloadDetails.DeploymentDefinition.Resources, diff --git a/internal/controller/reconcile-captenantoperation.go b/internal/controller/reconcile-captenantoperation.go index c01d377..fb53fce 100644 --- a/internal/controller/reconcile-captenantoperation.go +++ b/internal/controller/reconcile-captenantoperation.go @@ -423,6 +423,7 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c envFromVCAPSecret: getEnvFrom(vcapSecretName), imagePullSecrets: convertToLocalObjectReferences(relatedResources.CAPApplicationVersion.Spec.RegistrySecrets), version: relatedResources.CAPApplicationVersion.Spec.Version, + volume: getVolumes(consumedServiceInfos), } var job *batchv1.Job @@ -430,8 +431,10 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c if params.xsuaaInstanceName, err = getXSUAAInstanceName(consumedServiceInfos, relatedResources, c.kubeClient); err != nil { return } + params.volumeMount = getCAPVolumeMounts(consumedServiceInfos, CDSVolMountPrefix) job, err = c.createTenantOperationJob(ctx, ctop, workload, params) } else { // custom tenant operation + params.volumeMount = getVolumeMounts(consumedServiceInfos) job, err = c.createCustomTenantOperationJob(ctx, ctop, workload, params) } if err != nil { @@ -454,6 +457,8 @@ type jobCreateParams struct { imagePullSecrets []corev1.LocalObjectReference version string xsuaaInstanceName string + volume []corev1.Volume + volumeMount []corev1.VolumeMount } func (c *Controller) createTenantOperationJob(ctx context.Context, ctop *v1alpha1.CAPTenantOperation, workload *v1alpha1.WorkloadDetails, params *jobCreateParams) (*batchv1.Job, error) { diff --git a/internal/controller/reconcile.go b/internal/controller/reconcile.go index 582fa2f..c3d55a4 100644 --- a/internal/controller/reconcile.go +++ b/internal/controller/reconcile.go @@ -9,6 +9,7 @@ import ( "context" "encoding/json" "fmt" + "path" "github.com/sap/cap-operator/internal/util" "github.com/sap/cap-operator/pkg/apis/sme.sap.com/v1alpha1" @@ -51,6 +52,10 @@ const ( GardenerDNSClassIdentifier = "dns.gardener.cloud/class" ) +const ( + CDSVolMountPrefix = "/etc/secrets/cds" +) + const ( CertificateSuffix = "certificate" GardenerDNSClassValue = "garden" @@ -518,3 +523,30 @@ func copyMaps(originalMap map[string]string, additionalMap map[string]string) ma } return newMap } + +func getVolumeMounts(serviceInfos []v1alpha1.ServiceInfo) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{} + + for _, serviceInfo := range serviceInfos { + volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: serviceInfo.Name, MountPath: path.Join("/etc/secrets/sapcp", serviceInfo.Class, serviceInfo.Name), ReadOnly: true}) + } + return volumeMounts +} + +func getCAPVolumeMounts(serviceInfos []v1alpha1.ServiceInfo, mountPrefix string) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{} + + for _, serviceInfo := range serviceInfos { + volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: serviceInfo.Name, MountPath: path.Join(mountPrefix, "requires", serviceInfo.Class, "credentials"), ReadOnly: true}) + } + return volumeMounts +} + +func getVolumes(serviceInfos []v1alpha1.ServiceInfo) []corev1.Volume { + volumes := []corev1.Volume{} + + for _, serviceInfo := range serviceInfos { + volumes = append(volumes, corev1.Volume{Name: serviceInfo.Name, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: serviceInfo.Secret}}}) + } + return volumes +} From f3da4ee2300cf0d1e43fe74b05fe5c5e05b170ab Mon Sep 17 00:00:00 2001 From: i325261 Date: Thu, 7 Mar 2024 10:17:41 +0100 Subject: [PATCH 02/13] Added env check for volume mounts --- .../reconcile-capapplicationversion.go | 42 +++++++++++------ .../reconcile-captenantoperation.go | 45 ++++++++++++------- internal/controller/reconcile.go | 6 +++ 3 files changed, 64 insertions(+), 29 deletions(-) diff --git a/internal/controller/reconcile-capapplicationversion.go b/internal/controller/reconcile-capapplicationversion.go index 42dd759..d05b874 100644 --- a/internal/controller/reconcile-capapplicationversion.go +++ b/internal/controller/reconcile-capapplicationversion.go @@ -48,9 +48,9 @@ type DeploymentParameters struct { CAV *v1alpha1.CAPApplicationVersion OwnerRef *metav1.OwnerReference WorkloadDetails v1alpha1.WorkloadDetails - VCAPSecretName string - Volume []corev1.Volume - VolumeMount []corev1.VolumeMount + EnvFrom []corev1.EnvFromSource + Volumes []corev1.Volume + VolumeMounts []corev1.VolumeMount } func (c *Controller) reconcileCAPApplicationVersion(ctx context.Context, item QueueItem, attempts int) (*ReconcileResult, error) { @@ -275,6 +275,17 @@ func newContentDeploymentJob(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPAppli // Get ServiceInfos for consumed BTP services consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), ca.Spec.BTP.Services) + var envFrom []corev1.EnvFromSource + var serviceSecretVolumeMounts []corev1.VolumeMount + var serviceSecretVolumes []corev1.Volume + + if useVolumeMounts(workload.JobDefinition.Env) { + serviceSecretVolumeMounts = getVolumeMounts(consumedServiceInfos) + serviceSecretVolumes = getVolumes(consumedServiceInfos) + } else { + envFrom = getEnvFrom(vcapSecretName) + } + return &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ Name: getContentJobName(workload.Name, cav), @@ -302,15 +313,15 @@ func newContentDeploymentJob(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPAppli Env: append([]corev1.EnvVar{ {Name: EnvCAPOpAppVersion, Value: cav.Spec.Version}, }, workload.JobDefinition.Env...), - EnvFrom: getEnvFrom(vcapSecretName), - VolumeMounts: append(workload.JobDefinition.VolumeMounts, getVolumeMounts(consumedServiceInfos)...), + EnvFrom: envFrom, + VolumeMounts: append(workload.JobDefinition.VolumeMounts, serviceSecretVolumeMounts...), Resources: workload.JobDefinition.Resources, SecurityContext: workload.JobDefinition.SecurityContext, }, }, SecurityContext: workload.JobDefinition.PodSecurityContext, ServiceAccountName: workload.JobDefinition.ServiceAccountName, - Volumes: append(workload.JobDefinition.Volumes, getVolumes(consumedServiceInfos)...), + Volumes: append(workload.JobDefinition.Volumes, serviceSecretVolumes...), ImagePullSecrets: convertToLocalObjectReferences(cav.Spec.RegistrySecrets), RestartPolicy: corev1.RestartPolicyOnFailure, NodeSelector: workload.JobDefinition.NodeSelector, @@ -570,14 +581,17 @@ func newDeployment(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPApplicationVers CAV: cav, OwnerRef: &ownerRef, WorkloadDetails: *workload, - VCAPSecretName: vcapSecretName, - Volume: getVolumes(consumedServiceInfos), } - if workload.DeploymentDefinition.Type == v1alpha1.DeploymentCAP { - params.VolumeMount = getCAPVolumeMounts(consumedServiceInfos, CDSVolMountPrefix) + if useVolumeMounts(workload.DeploymentDefinition.Env) { + if workload.DeploymentDefinition.Type == v1alpha1.DeploymentCAP { + params.VolumeMounts = getCAPVolumeMounts(consumedServiceInfos, CDSVolMountPrefix) + } else { + params.VolumeMounts = getVolumeMounts(consumedServiceInfos) + } + params.Volumes = getVolumes(consumedServiceInfos) } else { - params.VolumeMount = getVolumeMounts(consumedServiceInfos) + params.EnvFrom = getEnvFrom(vcapSecretName) } return createDeployment(params) @@ -612,7 +626,7 @@ func createDeployment(params *DeploymentParameters) *appsv1.Deployment { ImagePullSecrets: convertToLocalObjectReferences(params.CAV.Spec.RegistrySecrets), Containers: getContainer(params), ServiceAccountName: params.WorkloadDetails.DeploymentDefinition.ServiceAccountName, - Volumes: append(params.WorkloadDetails.DeploymentDefinition.Volumes, params.Volume...), + Volumes: append(params.WorkloadDetails.DeploymentDefinition.Volumes, params.Volumes...), SecurityContext: params.WorkloadDetails.DeploymentDefinition.PodSecurityContext, NodeSelector: params.WorkloadDetails.DeploymentDefinition.NodeSelector, NodeName: params.WorkloadDetails.DeploymentDefinition.NodeName, @@ -633,8 +647,8 @@ func getContainer(params *DeploymentParameters) []corev1.Container { ImagePullPolicy: params.WorkloadDetails.DeploymentDefinition.ImagePullPolicy, Command: params.WorkloadDetails.DeploymentDefinition.Command, Env: getEnv(params), - EnvFrom: getEnvFrom(params.VCAPSecretName), - VolumeMounts: append(params.WorkloadDetails.DeploymentDefinition.VolumeMounts, params.VolumeMount...), + EnvFrom: params.EnvFrom, + VolumeMounts: append(params.WorkloadDetails.DeploymentDefinition.VolumeMounts, params.VolumeMounts...), LivenessProbe: params.WorkloadDetails.DeploymentDefinition.LivenessProbe, ReadinessProbe: params.WorkloadDetails.DeploymentDefinition.ReadinessProbe, Resources: params.WorkloadDetails.DeploymentDefinition.Resources, diff --git a/internal/controller/reconcile-captenantoperation.go b/internal/controller/reconcile-captenantoperation.go index fb53fce..cba5052 100644 --- a/internal/controller/reconcile-captenantoperation.go +++ b/internal/controller/reconcile-captenantoperation.go @@ -391,11 +391,26 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c return nil, fmt.Errorf("could not find workload %s in %s %s.%s", step.Name, v1alpha1.CAPApplicationVersionKind, relatedResources.CAPApplicationVersion.Namespace, relatedResources.CAPApplicationVersion.Name) } - // create VCAP secret from consumed BTP services consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), relatedResources.CAPApplication.Spec.BTP.Services) - vcapSecretName, err := createVCAPSecret(ctop.Name+"-"+strings.ToLower(workload.Name), ctop.Namespace, *metav1.NewControllerRef(ctop, v1alpha1.SchemeGroupVersion.WithKind(v1alpha1.CAPTenantOperationKind)), consumedServiceInfos, c.kubeClient) - if err != nil { - return nil, err + + var envFrom []corev1.EnvFromSource + var serviceSecretVolumeMounts []corev1.VolumeMount + var serviceSecretVolumes []corev1.Volume + + if (workload.JobDefinition != nil && useVolumeMounts(workload.JobDefinition.Env)) || (workload.DeploymentDefinition != nil && useVolumeMounts(workload.DeploymentDefinition.Env)) { + if ctop.Spec.Steps[*ctop.Status.CurrentStep-1].Type == v1alpha1.JobTenantOperation { + serviceSecretVolumeMounts = getCAPVolumeMounts(consumedServiceInfos, CDSVolMountPrefix) + } else { + serviceSecretVolumeMounts = getVolumeMounts(consumedServiceInfos) + } + serviceSecretVolumes = getVolumes(consumedServiceInfos) + } else { + // create VCAP secret from consumed BTP services + vcapSecretName, err := createVCAPSecret(ctop.Name+"-"+strings.ToLower(workload.Name), ctop.Namespace, *metav1.NewControllerRef(ctop, v1alpha1.SchemeGroupVersion.WithKind(v1alpha1.CAPTenantOperationKind)), consumedServiceInfos, c.kubeClient) + if err != nil { + return nil, err + } + envFrom = getEnvFrom(vcapSecretName) } annotations := copyMaps(workload.Annotations, map[string]string{ @@ -420,10 +435,11 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c namePrefix: relatedResources.CAPTenant.Name + "-" + workload.Name + "-", labels: labels, annotations: annotations, - envFromVCAPSecret: getEnvFrom(vcapSecretName), + envFromVCAPSecret: envFrom, imagePullSecrets: convertToLocalObjectReferences(relatedResources.CAPApplicationVersion.Spec.RegistrySecrets), version: relatedResources.CAPApplicationVersion.Spec.Version, - volume: getVolumes(consumedServiceInfos), + volumeMounts: serviceSecretVolumeMounts, + volumes: serviceSecretVolumes, } var job *batchv1.Job @@ -431,10 +447,8 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c if params.xsuaaInstanceName, err = getXSUAAInstanceName(consumedServiceInfos, relatedResources, c.kubeClient); err != nil { return } - params.volumeMount = getCAPVolumeMounts(consumedServiceInfos, CDSVolMountPrefix) job, err = c.createTenantOperationJob(ctx, ctop, workload, params) } else { // custom tenant operation - params.volumeMount = getVolumeMounts(consumedServiceInfos) job, err = c.createCustomTenantOperationJob(ctx, ctop, workload, params) } if err != nil { @@ -457,8 +471,8 @@ type jobCreateParams struct { imagePullSecrets []corev1.LocalObjectReference version string xsuaaInstanceName string - volume []corev1.Volume - volumeMount []corev1.VolumeMount + volumes []corev1.Volume + volumeMounts []corev1.VolumeMount } func (c *Controller) createTenantOperationJob(ctx context.Context, ctop *v1alpha1.CAPTenantOperation, workload *v1alpha1.WorkloadDetails, params *jobCreateParams) (*batchv1.Job, error) { @@ -501,7 +515,7 @@ func (c *Controller) createTenantOperationJob(ctx context.Context, ctop *v1alpha RestartPolicy: corev1.RestartPolicyNever, ImagePullSecrets: params.imagePullSecrets, Containers: getContainers(payload, ctop, derivedWorkload, workload, params), - Volumes: derivedWorkload.volumes, + Volumes: append(derivedWorkload.volumes, params.volumes...), ServiceAccountName: derivedWorkload.serviceAccountName, SecurityContext: derivedWorkload.podSecurityContext, NodeSelector: derivedWorkload.nodeSelector, @@ -531,7 +545,7 @@ func getContainers(payload []byte, ctop *v1alpha1.CAPTenantOperation, derivedWor {Name: EnvCAPOpAppVersion, Value: params.version}, {Name: EnvCAPOpTenantID, Value: ctop.Spec.TenantId}, {Name: EnvCAPOpTenantOperation, Value: string(ctop.Spec.Operation)}, {Name: EnvCAPOpTenantSubDomain, Value: string(ctop.Spec.SubDomain)}, }, derivedWorkload.env...), EnvFrom: params.envFromVCAPSecret, - VolumeMounts: derivedWorkload.volumeMounts, + VolumeMounts: append(derivedWorkload.volumeMounts, params.volumeMounts...), Resources: derivedWorkload.resources, SecurityContext: derivedWorkload.securityContext, } @@ -570,7 +584,8 @@ func getContainers(payload []byte, ctop *v1alpha1.CAPTenantOperation, derivedWor {Name: "MTX_TENANT_ID", Value: ctop.Spec.TenantId}, {Name: "MTX_REQUEST_PAYLOAD", Value: string(payload)}, }, - EnvFrom: params.envFromVCAPSecret, + EnvFrom: params.envFromVCAPSecret, + VolumeMounts: append(derivedWorkload.volumeMounts, params.volumeMounts...), }, *container, } @@ -646,7 +661,7 @@ func (c *Controller) createCustomTenantOperationJob(ctx context.Context, ctop *v Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyNever, SecurityContext: workload.JobDefinition.PodSecurityContext, - Volumes: workload.JobDefinition.Volumes, + Volumes: append(workload.JobDefinition.Volumes, params.volumes...), ServiceAccountName: workload.JobDefinition.ServiceAccountName, NodeSelector: workload.JobDefinition.NodeSelector, NodeName: workload.JobDefinition.NodeName, @@ -664,7 +679,7 @@ func (c *Controller) createCustomTenantOperationJob(ctx context.Context, ctop *v {Name: EnvCAPOpAppVersion, Value: params.version}, {Name: EnvCAPOpTenantID, Value: ctop.Spec.TenantId}, {Name: EnvCAPOpTenantOperation, Value: string(ctop.Spec.Operation)}, {Name: EnvCAPOpTenantSubDomain, Value: string(ctop.Spec.SubDomain)}, }, workload.JobDefinition.Env...), EnvFrom: params.envFromVCAPSecret, - VolumeMounts: workload.JobDefinition.VolumeMounts, + VolumeMounts: append(workload.JobDefinition.VolumeMounts, params.volumeMounts...), Command: workload.JobDefinition.Command, Resources: workload.JobDefinition.Resources, SecurityContext: workload.JobDefinition.SecurityContext, diff --git a/internal/controller/reconcile.go b/internal/controller/reconcile.go index c3d55a4..b8c61a4 100644 --- a/internal/controller/reconcile.go +++ b/internal/controller/reconcile.go @@ -13,6 +13,7 @@ import ( "github.com/sap/cap-operator/internal/util" "github.com/sap/cap-operator/pkg/apis/sme.sap.com/v1alpha1" + "golang.org/x/exp/slices" "golang.org/x/mod/semver" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -83,6 +84,7 @@ const ( EnvCAPOpTenantSubDomain = "CAPOP_TENANT_SUBDOMAIN" EnvCAPOpTenantOperation = "CAPOP_TENANT_OPERATION" EnvVCAPServices = "VCAP_SERVICES" + EnvUseVolumeMounts = "USE_VOLUME_MOUNTS" ) type JobState string @@ -550,3 +552,7 @@ func getVolumes(serviceInfos []v1alpha1.ServiceInfo) []corev1.Volume { } return volumes } + +func useVolumeMounts(envVars []corev1.EnvVar) bool { + return slices.ContainsFunc(envVars, func(env corev1.EnvVar) bool { return env.Name == EnvUseVolumeMounts && env.Value == "true" }) +} From bf3b034cb27b4a2b7bde63208d67d0a49f03843f Mon Sep 17 00:00:00 2001 From: i325261 Date: Mon, 12 Feb 2024 14:06:08 +0100 Subject: [PATCH 03/13] added secret volume mounts --- .../reconcile-capapplicationversion.go | 26 +++++++++++---- .../reconcile-captenantoperation.go | 5 +++ internal/controller/reconcile.go | 32 +++++++++++++++++++ 3 files changed, 57 insertions(+), 6 deletions(-) diff --git a/internal/controller/reconcile-capapplicationversion.go b/internal/controller/reconcile-capapplicationversion.go index 2116b5b..6517953 100644 --- a/internal/controller/reconcile-capapplicationversion.go +++ b/internal/controller/reconcile-capapplicationversion.go @@ -52,6 +52,8 @@ type DeploymentParameters struct { OwnerRef *metav1.OwnerReference WorkloadDetails v1alpha1.WorkloadDetails VCAPSecretName string + Volume []corev1.Volume + VolumeMount []corev1.VolumeMount } func (c *Controller) reconcileCAPApplicationVersion(ctx context.Context, item QueueItem, attempts int) (*ReconcileResult, error) { @@ -265,7 +267,7 @@ func (c *Controller) handleContentDeployJob(ca *v1alpha1.CAPApplication, cav *v1 vcapSecretName, err = createVCAPSecret(jobName, cav.Namespace, ownerRef, consumedServiceInfos, c.kubeClient) if err == nil { - contentDeployJob, err = c.kubeClient.BatchV1().Jobs(cav.Namespace).Create(context.TODO(), newContentDeploymentJob(cav, workload, ownerRef, vcapSecretName), metav1.CreateOptions{}) + contentDeployJob, err = c.kubeClient.BatchV1().Jobs(cav.Namespace).Create(context.TODO(), newContentDeploymentJob(ca, cav, workload, ownerRef, vcapSecretName), metav1.CreateOptions{}) if err == nil { util.LogInfo("Content job created successfully", string(Processing), cav, contentDeployJob, "version", cav.Spec.Version) } @@ -276,7 +278,7 @@ func (c *Controller) handleContentDeployJob(ca *v1alpha1.CAPApplication, cav *v1 } // newContentDeploymentJob creates a Content Deployment Job for the CAV resource. It also sets the appropriate OwnerReferences. -func newContentDeploymentJob(cav *v1alpha1.CAPApplicationVersion, workload *v1alpha1.WorkloadDetails, ownerRef metav1.OwnerReference, vcapSecretName string) *batchv1.Job { +func newContentDeploymentJob(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPApplicationVersion, workload *v1alpha1.WorkloadDetails, ownerRef metav1.OwnerReference, vcapSecretName string) *batchv1.Job { labels := copyMaps(workload.Labels, map[string]string{ LabelDisableKarydia: "true", }) @@ -288,6 +290,8 @@ func newContentDeploymentJob(cav *v1alpha1.CAPApplicationVersion, workload *v1al contentJobName := getContentJobName(workload.Name, cav) util.LogInfo("Creating content job", string(Processing), cav, nil, "contentJobName", contentJobName, "version", cav.Spec.Version) + // Get ServiceInfos for consumed BTP services + consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), ca.Spec.BTP.Services) return &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ @@ -317,7 +321,7 @@ func newContentDeploymentJob(cav *v1alpha1.CAPApplicationVersion, workload *v1al {Name: EnvCAPOpAppVersion, Value: cav.Spec.Version}, }, workload.JobDefinition.Env...), EnvFrom: getEnvFrom(vcapSecretName), - VolumeMounts: workload.JobDefinition.VolumeMounts, + VolumeMounts: append(workload.JobDefinition.VolumeMounts, getVolumeMounts(consumedServiceInfos)...), Resources: workload.JobDefinition.Resources, SecurityContext: workload.JobDefinition.SecurityContext, }, @@ -327,7 +331,7 @@ func newContentDeploymentJob(cav *v1alpha1.CAPApplicationVersion, workload *v1al }, vcapSecretName), SecurityContext: workload.JobDefinition.PodSecurityContext, ServiceAccountName: workload.JobDefinition.ServiceAccountName, - Volumes: workload.JobDefinition.Volumes, + Volumes: append(workload.JobDefinition.Volumes, getVolumes(consumedServiceInfos)...), ImagePullSecrets: convertToLocalObjectReferences(cav.Spec.RegistrySecrets), RestartPolicy: corev1.RestartPolicyOnFailure, NodeSelector: workload.JobDefinition.NodeSelector, @@ -689,12 +693,22 @@ func (c *Controller) updateDeployment(ca *v1alpha1.CAPApplication, cav *v1alpha1 // newDeployment creates a new generic Deployment for a CAV resource based on the type. It also sets the appropriate OwnerReferences. func newDeployment(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPApplicationVersion, workload *v1alpha1.WorkloadDetails, ownerRef metav1.OwnerReference, vcapSecretName string) *appsv1.Deployment { + // Get ServiceInfos for consumed BTP services + consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), ca.Spec.BTP.Services) + params := &DeploymentParameters{ CA: ca, CAV: cav, OwnerRef: &ownerRef, WorkloadDetails: *workload, VCAPSecretName: vcapSecretName, + Volume: getVolumes(consumedServiceInfos), + } + + if workload.DeploymentDefinition.Type == v1alpha1.DeploymentCAP { + params.VolumeMount = getCAPVolumeMounts(consumedServiceInfos, CDSVolMountPrefix) + } else { + params.VolumeMount = getVolumeMounts(consumedServiceInfos) } return createDeployment(params) @@ -734,7 +748,7 @@ func createDeployment(params *DeploymentParameters) *appsv1.Deployment { }, params.VCAPSecretName), Containers: getContainer(params), ServiceAccountName: params.WorkloadDetails.DeploymentDefinition.ServiceAccountName, - Volumes: params.WorkloadDetails.DeploymentDefinition.Volumes, + Volumes: append(params.WorkloadDetails.DeploymentDefinition.Volumes, params.Volume...), SecurityContext: params.WorkloadDetails.DeploymentDefinition.PodSecurityContext, NodeSelector: params.WorkloadDetails.DeploymentDefinition.NodeSelector, NodeName: params.WorkloadDetails.DeploymentDefinition.NodeName, @@ -756,7 +770,7 @@ func getContainer(params *DeploymentParameters) []corev1.Container { Command: params.WorkloadDetails.DeploymentDefinition.Command, Env: getEnv(params), EnvFrom: getEnvFrom(params.VCAPSecretName), - VolumeMounts: params.WorkloadDetails.DeploymentDefinition.VolumeMounts, + VolumeMounts: append(params.WorkloadDetails.DeploymentDefinition.VolumeMounts, params.VolumeMount...), LivenessProbe: params.WorkloadDetails.DeploymentDefinition.LivenessProbe, ReadinessProbe: params.WorkloadDetails.DeploymentDefinition.ReadinessProbe, Resources: params.WorkloadDetails.DeploymentDefinition.Resources, diff --git a/internal/controller/reconcile-captenantoperation.go b/internal/controller/reconcile-captenantoperation.go index 017b0e3..5e9381b 100644 --- a/internal/controller/reconcile-captenantoperation.go +++ b/internal/controller/reconcile-captenantoperation.go @@ -424,12 +424,15 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c providerTenantId: relatedResources.CAPApplication.Spec.Provider.TenantId, providerSubdomain: relatedResources.CAPApplication.Spec.Provider.SubDomain, tenantType: relatedResources.CAPTenant.Labels[LabelTenantType], + volume: getVolumes(consumedServiceInfos), } var job *batchv1.Job if ctop.Spec.Steps[*ctop.Status.CurrentStep-1].Type == v1alpha1.JobTenantOperation { + params.volumeMount = getCAPVolumeMounts(consumedServiceInfos, CDSVolMountPrefix) job, err = c.createTenantOperationJob(ctx, ctop, workload, params) } else { // custom tenant operation + params.volumeMount = getVolumeMounts(consumedServiceInfos) job, err = c.createCustomTenantOperationJob(ctx, ctop, workload, params) } if err != nil { @@ -458,6 +461,8 @@ type jobCreateParams struct { providerTenantId string providerSubdomain string tenantType string + volume []corev1.Volume + volumeMount []corev1.VolumeMount } func (c *Controller) createTenantOperationJob(ctx context.Context, ctop *v1alpha1.CAPTenantOperation, workload *v1alpha1.WorkloadDetails, params *jobCreateParams) (*batchv1.Job, error) { diff --git a/internal/controller/reconcile.go b/internal/controller/reconcile.go index 274cc37..a826b04 100644 --- a/internal/controller/reconcile.go +++ b/internal/controller/reconcile.go @@ -9,6 +9,7 @@ import ( "context" "encoding/json" "fmt" + "path" "strings" "github.com/sap/cap-operator/internal/util" @@ -55,6 +56,10 @@ const ( GardenerDNSClassIdentifier = "dns.gardener.cloud/class" ) +const ( + CDSVolMountPrefix = "/etc/secrets/cds" +) + const ( CertificateSuffix = "certificate" GardenerDNSClassValue = "garden" @@ -610,3 +615,30 @@ func updateInitContainers(initContainers []corev1.Container, additionalEnv []cor func getWorkloadName(cavName, workloadName string) string { return fmt.Sprintf("%s-%s", cavName, strings.ToLower(workloadName)) } + +func getVolumeMounts(serviceInfos []v1alpha1.ServiceInfo) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{} + + for _, serviceInfo := range serviceInfos { + volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: serviceInfo.Name, MountPath: path.Join("/etc/secrets/sapcp", serviceInfo.Class, serviceInfo.Name), ReadOnly: true}) + } + return volumeMounts +} + +func getCAPVolumeMounts(serviceInfos []v1alpha1.ServiceInfo, mountPrefix string) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{} + + for _, serviceInfo := range serviceInfos { + volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: serviceInfo.Name, MountPath: path.Join(mountPrefix, "requires", serviceInfo.Class, "credentials"), ReadOnly: true}) + } + return volumeMounts +} + +func getVolumes(serviceInfos []v1alpha1.ServiceInfo) []corev1.Volume { + volumes := []corev1.Volume{} + + for _, serviceInfo := range serviceInfos { + volumes = append(volumes, corev1.Volume{Name: serviceInfo.Name, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: serviceInfo.Secret}}}) + } + return volumes +} From 885298e34e368d5665fce55d47b6146af93b3fc2 Mon Sep 17 00:00:00 2001 From: i325261 Date: Thu, 7 Mar 2024 10:17:41 +0100 Subject: [PATCH 04/13] Added env check for volume mounts --- .../reconcile-capapplicationversion.go | 37 +++++++++++++------ .../reconcile-captenantoperation.go | 32 +++++++++++----- internal/controller/reconcile.go | 6 +++ 3 files changed, 53 insertions(+), 22 deletions(-) diff --git a/internal/controller/reconcile-capapplicationversion.go b/internal/controller/reconcile-capapplicationversion.go index 6517953..1c36202 100644 --- a/internal/controller/reconcile-capapplicationversion.go +++ b/internal/controller/reconcile-capapplicationversion.go @@ -52,8 +52,8 @@ type DeploymentParameters struct { OwnerRef *metav1.OwnerReference WorkloadDetails v1alpha1.WorkloadDetails VCAPSecretName string - Volume []corev1.Volume - VolumeMount []corev1.VolumeMount + Volumes []corev1.Volume + VolumeMounts []corev1.VolumeMount } func (c *Controller) reconcileCAPApplicationVersion(ctx context.Context, item QueueItem, attempts int) (*ReconcileResult, error) { @@ -293,6 +293,17 @@ func newContentDeploymentJob(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPAppli // Get ServiceInfos for consumed BTP services consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), ca.Spec.BTP.Services) + var envFrom []corev1.EnvFromSource + var serviceSecretVolumeMounts []corev1.VolumeMount + var serviceSecretVolumes []corev1.Volume + + if useVolumeMounts(workload.JobDefinition.Env) { + serviceSecretVolumeMounts = getVolumeMounts(consumedServiceInfos) + serviceSecretVolumes = getVolumes(consumedServiceInfos) + } else { + envFrom = getEnvFrom(vcapSecretName) + } + return &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ Name: contentJobName, @@ -320,8 +331,8 @@ func newContentDeploymentJob(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPAppli Env: append([]corev1.EnvVar{ {Name: EnvCAPOpAppVersion, Value: cav.Spec.Version}, }, workload.JobDefinition.Env...), - EnvFrom: getEnvFrom(vcapSecretName), - VolumeMounts: append(workload.JobDefinition.VolumeMounts, getVolumeMounts(consumedServiceInfos)...), + EnvFrom: envFrom, + VolumeMounts: append(workload.JobDefinition.VolumeMounts, serviceSecretVolumeMounts...), Resources: workload.JobDefinition.Resources, SecurityContext: workload.JobDefinition.SecurityContext, }, @@ -331,7 +342,7 @@ func newContentDeploymentJob(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPAppli }, vcapSecretName), SecurityContext: workload.JobDefinition.PodSecurityContext, ServiceAccountName: workload.JobDefinition.ServiceAccountName, - Volumes: append(workload.JobDefinition.Volumes, getVolumes(consumedServiceInfos)...), + Volumes: append(workload.JobDefinition.Volumes, serviceSecretVolumes...), ImagePullSecrets: convertToLocalObjectReferences(cav.Spec.RegistrySecrets), RestartPolicy: corev1.RestartPolicyOnFailure, NodeSelector: workload.JobDefinition.NodeSelector, @@ -702,13 +713,15 @@ func newDeployment(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPApplicationVers OwnerRef: &ownerRef, WorkloadDetails: *workload, VCAPSecretName: vcapSecretName, - Volume: getVolumes(consumedServiceInfos), } - if workload.DeploymentDefinition.Type == v1alpha1.DeploymentCAP { - params.VolumeMount = getCAPVolumeMounts(consumedServiceInfos, CDSVolMountPrefix) - } else { - params.VolumeMount = getVolumeMounts(consumedServiceInfos) + if useVolumeMounts(workload.DeploymentDefinition.Env) { + if workload.DeploymentDefinition.Type == v1alpha1.DeploymentCAP { + params.VolumeMounts = getCAPVolumeMounts(consumedServiceInfos, CDSVolMountPrefix) + } else { + params.VolumeMounts = getVolumeMounts(consumedServiceInfos) + } + params.Volumes = getVolumes(consumedServiceInfos) } return createDeployment(params) @@ -748,7 +761,7 @@ func createDeployment(params *DeploymentParameters) *appsv1.Deployment { }, params.VCAPSecretName), Containers: getContainer(params), ServiceAccountName: params.WorkloadDetails.DeploymentDefinition.ServiceAccountName, - Volumes: append(params.WorkloadDetails.DeploymentDefinition.Volumes, params.Volume...), + Volumes: append(params.WorkloadDetails.DeploymentDefinition.Volumes, params.Volumes...), SecurityContext: params.WorkloadDetails.DeploymentDefinition.PodSecurityContext, NodeSelector: params.WorkloadDetails.DeploymentDefinition.NodeSelector, NodeName: params.WorkloadDetails.DeploymentDefinition.NodeName, @@ -770,7 +783,7 @@ func getContainer(params *DeploymentParameters) []corev1.Container { Command: params.WorkloadDetails.DeploymentDefinition.Command, Env: getEnv(params), EnvFrom: getEnvFrom(params.VCAPSecretName), - VolumeMounts: append(params.WorkloadDetails.DeploymentDefinition.VolumeMounts, params.VolumeMount...), + VolumeMounts: append(params.WorkloadDetails.DeploymentDefinition.VolumeMounts, params.VolumeMounts...), LivenessProbe: params.WorkloadDetails.DeploymentDefinition.LivenessProbe, ReadinessProbe: params.WorkloadDetails.DeploymentDefinition.ReadinessProbe, Resources: params.WorkloadDetails.DeploymentDefinition.Resources, diff --git a/internal/controller/reconcile-captenantoperation.go b/internal/controller/reconcile-captenantoperation.go index 5e9381b..ffc995b 100644 --- a/internal/controller/reconcile-captenantoperation.go +++ b/internal/controller/reconcile-captenantoperation.go @@ -387,8 +387,21 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c return nil, fmt.Errorf("could not find workload %s in %s %s.%s", step.Name, v1alpha1.CAPApplicationVersionKind, relatedResources.CAPApplicationVersion.Namespace, relatedResources.CAPApplicationVersion.Name) } - // create VCAP secret from consumed BTP services consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), relatedResources.CAPApplication.Spec.BTP.Services) + + var serviceSecretVolumeMounts []corev1.VolumeMount + var serviceSecretVolumes []corev1.Volume + + if (workload.JobDefinition != nil && useVolumeMounts(workload.JobDefinition.Env)) || (workload.DeploymentDefinition != nil && useVolumeMounts(workload.DeploymentDefinition.Env)) { + if ctop.Spec.Steps[*ctop.Status.CurrentStep-1].Type == v1alpha1.JobTenantOperation { + serviceSecretVolumeMounts = getCAPVolumeMounts(consumedServiceInfos, CDSVolMountPrefix) + } else { + serviceSecretVolumeMounts = getVolumeMounts(consumedServiceInfos) + } + serviceSecretVolumes = getVolumes(consumedServiceInfos) + } + + // create VCAP secret from consumed BTP services vcapSecretName, err := createVCAPSecret(ctop.Name+"-"+strings.ToLower(workload.Name), ctop.Namespace, *metav1.NewControllerRef(ctop, v1alpha1.SchemeGroupVersion.WithKind(v1alpha1.CAPTenantOperationKind)), consumedServiceInfos, c.kubeClient) if err != nil { return nil, err @@ -424,15 +437,14 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c providerTenantId: relatedResources.CAPApplication.Spec.Provider.TenantId, providerSubdomain: relatedResources.CAPApplication.Spec.Provider.SubDomain, tenantType: relatedResources.CAPTenant.Labels[LabelTenantType], - volume: getVolumes(consumedServiceInfos), + volumeMounts: serviceSecretVolumeMounts, + volumes: serviceSecretVolumes, } var job *batchv1.Job if ctop.Spec.Steps[*ctop.Status.CurrentStep-1].Type == v1alpha1.JobTenantOperation { - params.volumeMount = getCAPVolumeMounts(consumedServiceInfos, CDSVolMountPrefix) job, err = c.createTenantOperationJob(ctx, ctop, workload, params) } else { // custom tenant operation - params.volumeMount = getVolumeMounts(consumedServiceInfos) job, err = c.createCustomTenantOperationJob(ctx, ctop, workload, params) } if err != nil { @@ -461,8 +473,8 @@ type jobCreateParams struct { providerTenantId string providerSubdomain string tenantType string - volume []corev1.Volume - volumeMount []corev1.VolumeMount + volumes []corev1.Volume + volumeMounts []corev1.VolumeMount } func (c *Controller) createTenantOperationJob(ctx context.Context, ctop *v1alpha1.CAPTenantOperation, workload *v1alpha1.WorkloadDetails, params *jobCreateParams) (*batchv1.Job, error) { @@ -490,7 +502,7 @@ func (c *Controller) createTenantOperationJob(ctx context.Context, ctop *v1alpha ImagePullSecrets: params.imagePullSecrets, Containers: getContainers(ctop, derivedWorkload, workload, params), InitContainers: *updateInitContainers(derivedWorkload.initContainers, getCTOPEnv(params, ctop), params.vcapSecretName), - Volumes: derivedWorkload.volumes, + Volumes: append(derivedWorkload.volumes, params.volumes...), ServiceAccountName: derivedWorkload.serviceAccountName, SecurityContext: derivedWorkload.podSecurityContext, NodeSelector: derivedWorkload.nodeSelector, @@ -515,7 +527,7 @@ func getContainers(ctop *v1alpha1.CAPTenantOperation, derivedWorkload tentantOpe ImagePullPolicy: derivedWorkload.imagePullPolicy, Env: append(getCTOPEnv(params, ctop), derivedWorkload.env...), EnvFrom: getEnvFrom(params.vcapSecretName), - VolumeMounts: derivedWorkload.volumeMounts, + VolumeMounts: append(derivedWorkload.volumeMounts, params.volumeMounts...), Resources: derivedWorkload.resources, SecurityContext: derivedWorkload.securityContext, } @@ -619,7 +631,7 @@ func (c *Controller) createCustomTenantOperationJob(ctx context.Context, ctop *v Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyNever, SecurityContext: workload.JobDefinition.PodSecurityContext, - Volumes: workload.JobDefinition.Volumes, + Volumes: append(workload.JobDefinition.Volumes, params.volumes...), ServiceAccountName: workload.JobDefinition.ServiceAccountName, NodeSelector: workload.JobDefinition.NodeSelector, NodeName: workload.JobDefinition.NodeName, @@ -635,7 +647,7 @@ func (c *Controller) createCustomTenantOperationJob(ctx context.Context, ctop *v ImagePullPolicy: workload.JobDefinition.ImagePullPolicy, Env: append(getCTOPEnv(params, ctop), workload.JobDefinition.Env...), EnvFrom: getEnvFrom(params.vcapSecretName), - VolumeMounts: workload.JobDefinition.VolumeMounts, + VolumeMounts: append(workload.JobDefinition.VolumeMounts, params.volumeMounts...), Command: workload.JobDefinition.Command, Resources: workload.JobDefinition.Resources, SecurityContext: workload.JobDefinition.SecurityContext, diff --git a/internal/controller/reconcile.go b/internal/controller/reconcile.go index a826b04..1735ae5 100644 --- a/internal/controller/reconcile.go +++ b/internal/controller/reconcile.go @@ -14,6 +14,7 @@ import ( "github.com/sap/cap-operator/internal/util" "github.com/sap/cap-operator/pkg/apis/sme.sap.com/v1alpha1" + "golang.org/x/exp/slices" "golang.org/x/mod/semver" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -93,6 +94,7 @@ const ( EnvCAPOpProviderSubDomain = "CAPOP_PROVIDER_SUBDOMAIN" EnvCAPOpSubscriptionPayload = "CAPOP_SUBSCRIPTION_PAYLOAD" EnvVCAPServices = "VCAP_SERVICES" + EnvUseVolumeMounts = "USE_VOLUME_MOUNTS" ) type JobState string @@ -642,3 +644,7 @@ func getVolumes(serviceInfos []v1alpha1.ServiceInfo) []corev1.Volume { } return volumes } + +func useVolumeMounts(envVars []corev1.EnvVar) bool { + return slices.ContainsFunc(envVars, func(env corev1.EnvVar) bool { return env.Name == EnvUseVolumeMounts && env.Value == "true" }) +} From 57eb28b0a9f7e49580e1dee9ab374b63557fc2d3 Mon Sep 17 00:00:00 2001 From: i325261 Date: Tue, 8 Oct 2024 09:09:12 +0200 Subject: [PATCH 05/13] use annotation check enhanced volume & volume mount propogation --- .../reconcile-capapplicationversion.go | 33 +++++++----- .../reconcile-captenantoperation.go | 53 ++++++++++--------- internal/controller/reconcile.go | 36 ++++++------- 3 files changed, 65 insertions(+), 57 deletions(-) diff --git a/internal/controller/reconcile-capapplicationversion.go b/internal/controller/reconcile-capapplicationversion.go index 1c36202..e02957c 100644 --- a/internal/controller/reconcile-capapplicationversion.go +++ b/internal/controller/reconcile-capapplicationversion.go @@ -51,7 +51,7 @@ type DeploymentParameters struct { CAV *v1alpha1.CAPApplicationVersion OwnerRef *metav1.OwnerReference WorkloadDetails v1alpha1.WorkloadDetails - VCAPSecretName string + EnvFrom []corev1.EnvFromSource Volumes []corev1.Volume VolumeMounts []corev1.VolumeMount } @@ -264,7 +264,10 @@ func (c *Controller) handleContentDeployJob(ca *v1alpha1.CAPApplication, cav *v1 ownerRef := *metav1.NewControllerRef(cav, v1alpha1.SchemeGroupVersion.WithKind(v1alpha1.CAPApplicationVersionKind)) // Get VCAP secret name - vcapSecretName, err = createVCAPSecret(jobName, cav.Namespace, ownerRef, consumedServiceInfos, c.kubeClient) + err = nil + if !useVolumeMounts(cav) { + vcapSecretName, err = createVCAPSecret(jobName, cav.Namespace, ownerRef, consumedServiceInfos, c.kubeClient) + } if err == nil { contentDeployJob, err = c.kubeClient.BatchV1().Jobs(cav.Namespace).Create(context.TODO(), newContentDeploymentJob(ca, cav, workload, ownerRef, vcapSecretName), metav1.CreateOptions{}) @@ -297,7 +300,8 @@ func newContentDeploymentJob(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPAppli var serviceSecretVolumeMounts []corev1.VolumeMount var serviceSecretVolumes []corev1.Volume - if useVolumeMounts(workload.JobDefinition.Env) { + if useVolumeMounts(cav) { + workload.JobDefinition.Env = updateServiceBindingRootEnv(workload.JobDefinition.Env) serviceSecretVolumeMounts = getVolumeMounts(consumedServiceInfos) serviceSecretVolumes = getVolumes(consumedServiceInfos) } else { @@ -339,7 +343,7 @@ func newContentDeploymentJob(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPAppli }, InitContainers: *updateInitContainers(workload.JobDefinition.InitContainers, []corev1.EnvVar{ {Name: EnvCAPOpAppVersion, Value: cav.Spec.Version}, - }, vcapSecretName), + }, envFrom), SecurityContext: workload.JobDefinition.PodSecurityContext, ServiceAccountName: workload.JobDefinition.ServiceAccountName, Volumes: append(workload.JobDefinition.Volumes, serviceSecretVolumes...), @@ -689,7 +693,10 @@ func (c *Controller) updateDeployment(ca *v1alpha1.CAPApplication, cav *v1alpha1 ownerRef := *metav1.NewControllerRef(cav, v1alpha1.SchemeGroupVersion.WithKind(v1alpha1.CAPApplicationVersionKind)) // Get VCAP secret name - vcapSecretName, err = createVCAPSecret(deploymentName, cav.Namespace, ownerRef, consumedServiceInfos, c.kubeClient) + err = nil + if !useVolumeMounts(cav) { + vcapSecretName, err = createVCAPSecret(deploymentName, cav.Namespace, ownerRef, consumedServiceInfos, c.kubeClient) + } if err == nil { workloadDeployment, err = c.kubeClient.AppsV1().Deployments(cav.Namespace).Create(context.TODO(), newDeployment(ca, cav, workload, ownerRef, vcapSecretName), metav1.CreateOptions{}) @@ -712,16 +719,14 @@ func newDeployment(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPApplicationVers CAV: cav, OwnerRef: &ownerRef, WorkloadDetails: *workload, - VCAPSecretName: vcapSecretName, } - if useVolumeMounts(workload.DeploymentDefinition.Env) { - if workload.DeploymentDefinition.Type == v1alpha1.DeploymentCAP { - params.VolumeMounts = getCAPVolumeMounts(consumedServiceInfos, CDSVolMountPrefix) - } else { - params.VolumeMounts = getVolumeMounts(consumedServiceInfos) - } + if useVolumeMounts(cav) { + params.WorkloadDetails.DeploymentDefinition.Env = updateServiceBindingRootEnv(params.WorkloadDetails.DeploymentDefinition.Env) + params.VolumeMounts = getVolumeMounts(consumedServiceInfos) params.Volumes = getVolumes(consumedServiceInfos) + } else { + params.EnvFrom = getEnvFrom(vcapSecretName) } return createDeployment(params) @@ -758,7 +763,7 @@ func createDeployment(params *DeploymentParameters) *appsv1.Deployment { ImagePullSecrets: convertToLocalObjectReferences(params.CAV.Spec.RegistrySecrets), InitContainers: *updateInitContainers(params.WorkloadDetails.DeploymentDefinition.InitContainers, []corev1.EnvVar{ {Name: EnvCAPOpAppVersion, Value: params.CAV.Spec.Version}, - }, params.VCAPSecretName), + }, params.EnvFrom), Containers: getContainer(params), ServiceAccountName: params.WorkloadDetails.DeploymentDefinition.ServiceAccountName, Volumes: append(params.WorkloadDetails.DeploymentDefinition.Volumes, params.Volumes...), @@ -782,7 +787,7 @@ func getContainer(params *DeploymentParameters) []corev1.Container { ImagePullPolicy: params.WorkloadDetails.DeploymentDefinition.ImagePullPolicy, Command: params.WorkloadDetails.DeploymentDefinition.Command, Env: getEnv(params), - EnvFrom: getEnvFrom(params.VCAPSecretName), + EnvFrom: params.EnvFrom, VolumeMounts: append(params.WorkloadDetails.DeploymentDefinition.VolumeMounts, params.VolumeMounts...), LivenessProbe: params.WorkloadDetails.DeploymentDefinition.LivenessProbe, ReadinessProbe: params.WorkloadDetails.DeploymentDefinition.ReadinessProbe, diff --git a/internal/controller/reconcile-captenantoperation.go b/internal/controller/reconcile-captenantoperation.go index ffc995b..1c9b069 100644 --- a/internal/controller/reconcile-captenantoperation.go +++ b/internal/controller/reconcile-captenantoperation.go @@ -389,22 +389,16 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), relatedResources.CAPApplication.Spec.BTP.Services) - var serviceSecretVolumeMounts []corev1.VolumeMount - var serviceSecretVolumes []corev1.Volume - - if (workload.JobDefinition != nil && useVolumeMounts(workload.JobDefinition.Env)) || (workload.DeploymentDefinition != nil && useVolumeMounts(workload.DeploymentDefinition.Env)) { - if ctop.Spec.Steps[*ctop.Status.CurrentStep-1].Type == v1alpha1.JobTenantOperation { - serviceSecretVolumeMounts = getCAPVolumeMounts(consumedServiceInfos, CDSVolMountPrefix) - } else { - serviceSecretVolumeMounts = getVolumeMounts(consumedServiceInfos) - } - serviceSecretVolumes = getVolumes(consumedServiceInfos) - } + // check volume mount annotation + useVolumeMount := useVolumeMounts(relatedResources.CAPApplicationVersion) // create VCAP secret from consumed BTP services - vcapSecretName, err := createVCAPSecret(ctop.Name+"-"+strings.ToLower(workload.Name), ctop.Namespace, *metav1.NewControllerRef(ctop, v1alpha1.SchemeGroupVersion.WithKind(v1alpha1.CAPTenantOperationKind)), consumedServiceInfos, c.kubeClient) - if err != nil { - return nil, err + var vcapSecretName string + if !useVolumeMount { + vcapSecretName, err = createVCAPSecret(ctop.Name+"-"+strings.ToLower(workload.Name), ctop.Namespace, *metav1.NewControllerRef(ctop, v1alpha1.SchemeGroupVersion.WithKind(v1alpha1.CAPTenantOperationKind)), consumedServiceInfos, c.kubeClient) + if err != nil { + return nil, err + } } annotations := copyMaps(workload.Annotations, map[string]string{ @@ -429,7 +423,6 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c namePrefix: relatedResources.CAPTenant.Name + "-" + workload.Name + "-", labels: labels, annotations: annotations, - vcapSecretName: vcapSecretName, imagePullSecrets: convertToLocalObjectReferences(relatedResources.CAPApplicationVersion.Spec.RegistrySecrets), version: relatedResources.CAPApplicationVersion.Spec.Version, appName: relatedResources.CAPApplication.Spec.BTPAppName, @@ -437,8 +430,19 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c providerTenantId: relatedResources.CAPApplication.Spec.Provider.TenantId, providerSubdomain: relatedResources.CAPApplication.Spec.Provider.SubDomain, tenantType: relatedResources.CAPTenant.Labels[LabelTenantType], - volumeMounts: serviceSecretVolumeMounts, - volumes: serviceSecretVolumes, + } + + if useVolumeMount { + if workload.DeploymentDefinition == nil { + workload.JobDefinition.Env = updateServiceBindingRootEnv(workload.JobDefinition.Env) + } else { + workload.DeploymentDefinition.Env = updateServiceBindingRootEnv(workload.DeploymentDefinition.Env) + } + params.env = []corev1.EnvVar{defaultServiceBindingRootEnv} + params.volumeMounts = getVolumeMounts(consumedServiceInfos) + params.volumes = getVolumes(consumedServiceInfos) + } else { + params.EnvFrom = getEnvFrom(vcapSecretName) } var job *batchv1.Job @@ -465,7 +469,6 @@ type jobCreateParams struct { namePrefix string labels map[string]string annotations map[string]string - vcapSecretName string imagePullSecrets []corev1.LocalObjectReference version string appName string @@ -473,6 +476,8 @@ type jobCreateParams struct { providerTenantId string providerSubdomain string tenantType string + env []corev1.EnvVar + EnvFrom []corev1.EnvFromSource volumes []corev1.Volume volumeMounts []corev1.VolumeMount } @@ -501,7 +506,7 @@ func (c *Controller) createTenantOperationJob(ctx context.Context, ctop *v1alpha RestartPolicy: corev1.RestartPolicyNever, ImagePullSecrets: params.imagePullSecrets, Containers: getContainers(ctop, derivedWorkload, workload, params), - InitContainers: *updateInitContainers(derivedWorkload.initContainers, getCTOPEnv(params, ctop), params.vcapSecretName), + InitContainers: *updateInitContainers(derivedWorkload.initContainers, getCTOPEnv(params, ctop), params.EnvFrom), Volumes: append(derivedWorkload.volumes, params.volumes...), ServiceAccountName: derivedWorkload.serviceAccountName, SecurityContext: derivedWorkload.podSecurityContext, @@ -525,8 +530,8 @@ func getContainers(ctop *v1alpha1.CAPTenantOperation, derivedWorkload tentantOpe Name: workload.Name, Image: derivedWorkload.image, ImagePullPolicy: derivedWorkload.imagePullPolicy, - Env: append(getCTOPEnv(params, ctop), derivedWorkload.env...), - EnvFrom: getEnvFrom(params.vcapSecretName), + Env: append(getCTOPEnv(params, ctop), append(derivedWorkload.env, params.env...)...), + EnvFrom: params.EnvFrom, VolumeMounts: append(derivedWorkload.volumeMounts, params.volumeMounts...), Resources: derivedWorkload.resources, SecurityContext: derivedWorkload.securityContext, @@ -645,15 +650,15 @@ func (c *Controller) createCustomTenantOperationJob(ctx context.Context, ctop *v Name: workload.Name, Image: workload.JobDefinition.Image, ImagePullPolicy: workload.JobDefinition.ImagePullPolicy, - Env: append(getCTOPEnv(params, ctop), workload.JobDefinition.Env...), - EnvFrom: getEnvFrom(params.vcapSecretName), + Env: append(getCTOPEnv(params, ctop), append(workload.JobDefinition.Env, params.env...)...), + EnvFrom: params.EnvFrom, VolumeMounts: append(workload.JobDefinition.VolumeMounts, params.volumeMounts...), Command: workload.JobDefinition.Command, Resources: workload.JobDefinition.Resources, SecurityContext: workload.JobDefinition.SecurityContext, }, }, - InitContainers: *updateInitContainers(workload.JobDefinition.InitContainers, getCTOPEnv(params, ctop), params.vcapSecretName), + InitContainers: *updateInitContainers(workload.JobDefinition.InitContainers, getCTOPEnv(params, ctop), params.EnvFrom), }, }, }, diff --git a/internal/controller/reconcile.go b/internal/controller/reconcile.go index 1735ae5..145642f 100644 --- a/internal/controller/reconcile.go +++ b/internal/controller/reconcile.go @@ -50,6 +50,7 @@ const ( AnnotationSubscriptionContextSecret = "sme.sap.com/subscription-context-secret" AnnotationProviderSubAccountId = "sme.sap.com/provider-sub-account-id" AnnotationEnableCleanupMonitoring = "sme.sap.com/enable-cleanup-monitoring" + AnnotationUseVolumeMount = "sme.sap.com/use-volume-mount" FinalizerCAPApplication = "sme.sap.com/capapplication" FinalizerCAPApplicationVersion = "sme.sap.com/capapplicationversion" FinalizerCAPTenant = "sme.sap.com/captenant" @@ -57,9 +58,7 @@ const ( GardenerDNSClassIdentifier = "dns.gardener.cloud/class" ) -const ( - CDSVolMountPrefix = "/etc/secrets/cds" -) +var defaultServiceBindingRootEnv = corev1.EnvVar{Name: "SERVICE_BINDING_ROOT", Value: "/etc/secrets"} const ( CertificateSuffix = "certificate" @@ -94,7 +93,6 @@ const ( EnvCAPOpProviderSubDomain = "CAPOP_PROVIDER_SUBDOMAIN" EnvCAPOpSubscriptionPayload = "CAPOP_SUBSCRIPTION_PAYLOAD" EnvVCAPServices = "VCAP_SERVICES" - EnvUseVolumeMounts = "USE_VOLUME_MOUNTS" ) type JobState string @@ -600,14 +598,14 @@ func copyMaps(originalMap map[string]string, additionalMap map[string]string) ma return newMap } -func updateInitContainers(initContainers []corev1.Container, additionalEnv []corev1.EnvVar, vcapSecretName string) *[]corev1.Container { +func updateInitContainers(initContainers []corev1.Container, additionalEnv []corev1.EnvVar, EnvFrom []corev1.EnvFromSource) *[]corev1.Container { var updatedInitContainers []corev1.Container if len(initContainers) > 0 { updatedInitContainers = []corev1.Container{} for _, container := range initContainers { updatedContainer := container.DeepCopy() updatedContainer.Env = append(updatedContainer.Env, additionalEnv...) - updatedContainer.EnvFrom = getEnvFrom(vcapSecretName) + updatedContainer.EnvFrom = EnvFrom updatedInitContainers = append(updatedInitContainers, *updatedContainer) } } @@ -620,18 +618,8 @@ func getWorkloadName(cavName, workloadName string) string { func getVolumeMounts(serviceInfos []v1alpha1.ServiceInfo) []corev1.VolumeMount { volumeMounts := []corev1.VolumeMount{} - for _, serviceInfo := range serviceInfos { - volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: serviceInfo.Name, MountPath: path.Join("/etc/secrets/sapcp", serviceInfo.Class, serviceInfo.Name), ReadOnly: true}) - } - return volumeMounts -} - -func getCAPVolumeMounts(serviceInfos []v1alpha1.ServiceInfo, mountPrefix string) []corev1.VolumeMount { - volumeMounts := []corev1.VolumeMount{} - - for _, serviceInfo := range serviceInfos { - volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: serviceInfo.Name, MountPath: path.Join(mountPrefix, "requires", serviceInfo.Class, "credentials"), ReadOnly: true}) + volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: serviceInfo.Name, MountPath: path.Join(defaultServiceBindingRootEnv.Value, serviceInfo.Class), ReadOnly: true}) } return volumeMounts } @@ -645,6 +633,16 @@ func getVolumes(serviceInfos []v1alpha1.ServiceInfo) []corev1.Volume { return volumes } -func useVolumeMounts(envVars []corev1.EnvVar) bool { - return slices.ContainsFunc(envVars, func(env corev1.EnvVar) bool { return env.Name == EnvUseVolumeMounts && env.Value == "true" }) +func useVolumeMounts(cav *v1alpha1.CAPApplicationVersion) bool { + value, exists := cav.Annotations[AnnotationUseVolumeMount] + return exists && value == "true" +} + +func updateServiceBindingRootEnv(envVars []corev1.EnvVar) []corev1.EnvVar { + if envIndex := slices.IndexFunc(envVars, func(currentEnv corev1.EnvVar) bool { return currentEnv.Name == defaultServiceBindingRootEnv.Name }); envIndex > -1 { + envVars[envIndex] = defaultServiceBindingRootEnv + } else { + envVars = append(envVars, defaultServiceBindingRootEnv) + } + return envVars } From 645f0cb41a20b8e63f4a044301991e70dfb2574d Mon Sep 17 00:00:00 2001 From: i325261 Date: Wed, 9 Oct 2024 15:11:51 +0200 Subject: [PATCH 06/13] fix env getting set on CAV pass volume mount to initcontainers added unit tests --- .../reconcile-capapplicationversion.go | 27 +- .../reconcile-capapplicationversion_test.go | 18 + .../reconcile-captenantoperation.go | 24 +- .../reconcile-captenantoperation_test.go | 44 ++ internal/controller/reconcile.go | 6 +- .../cav-use-vol-mount.yaml | 205 +++++++++ .../cav-processing-use-vol-mount.yaml | 432 ++++++++++++++++++ .../ctop-use-vol-mount-custom.expected.yaml | 262 +++++++++++ .../ctop-use-vol-mount-custom.initial.yaml | 38 ++ .../ctop-use-vol-mount.expected.yaml | 292 ++++++++++++ .../ctop-use-vol-mount.initial.yaml | 34 ++ ...cationversion-v1-use-vol-mount-custom.yaml | 183 ++++++++ ...apapplicationversion-v1-use-vol-mount.yaml | 157 +++++++ 13 files changed, 1699 insertions(+), 23 deletions(-) create mode 100644 internal/controller/testdata/capapplicationversion/cav-use-vol-mount.yaml create mode 100644 internal/controller/testdata/capapplicationversion/expected/cav-processing-use-vol-mount.yaml create mode 100644 internal/controller/testdata/captenantoperation/ctop-use-vol-mount-custom.expected.yaml create mode 100644 internal/controller/testdata/captenantoperation/ctop-use-vol-mount-custom.initial.yaml create mode 100644 internal/controller/testdata/captenantoperation/ctop-use-vol-mount.expected.yaml create mode 100644 internal/controller/testdata/captenantoperation/ctop-use-vol-mount.initial.yaml create mode 100644 internal/controller/testdata/common/capapplicationversion-v1-use-vol-mount-custom.yaml create mode 100644 internal/controller/testdata/common/capapplicationversion-v1-use-vol-mount.yaml diff --git a/internal/controller/reconcile-capapplicationversion.go b/internal/controller/reconcile-capapplicationversion.go index e02957c..4c5875c 100644 --- a/internal/controller/reconcile-capapplicationversion.go +++ b/internal/controller/reconcile-capapplicationversion.go @@ -51,6 +51,7 @@ type DeploymentParameters struct { CAV *v1alpha1.CAPApplicationVersion OwnerRef *metav1.OwnerReference WorkloadDetails v1alpha1.WorkloadDetails + Env []corev1.EnvVar EnvFrom []corev1.EnvFromSource Volumes []corev1.Volume VolumeMounts []corev1.VolumeMount @@ -293,15 +294,18 @@ func newContentDeploymentJob(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPAppli contentJobName := getContentJobName(workload.Name, cav) util.LogInfo("Creating content job", string(Processing), cav, nil, "contentJobName", contentJobName, "version", cav.Spec.Version) - // Get ServiceInfos for consumed BTP services - consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), ca.Spec.BTP.Services) var envFrom []corev1.EnvFromSource var serviceSecretVolumeMounts []corev1.VolumeMount var serviceSecretVolumes []corev1.Volume + env := workload.JobDefinition.Env + if useVolumeMounts(cav) { - workload.JobDefinition.Env = updateServiceBindingRootEnv(workload.JobDefinition.Env) + // Get ServiceInfos for consumed BTP services + consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), ca.Spec.BTP.Services) + + env = updateServiceBindingRootEnv(env) serviceSecretVolumeMounts = getVolumeMounts(consumedServiceInfos) serviceSecretVolumes = getVolumes(consumedServiceInfos) } else { @@ -334,7 +338,7 @@ func newContentDeploymentJob(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPAppli Command: workload.JobDefinition.Command, Env: append([]corev1.EnvVar{ {Name: EnvCAPOpAppVersion, Value: cav.Spec.Version}, - }, workload.JobDefinition.Env...), + }, env...), EnvFrom: envFrom, VolumeMounts: append(workload.JobDefinition.VolumeMounts, serviceSecretVolumeMounts...), Resources: workload.JobDefinition.Resources, @@ -343,7 +347,7 @@ func newContentDeploymentJob(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPAppli }, InitContainers: *updateInitContainers(workload.JobDefinition.InitContainers, []corev1.EnvVar{ {Name: EnvCAPOpAppVersion, Value: cav.Spec.Version}, - }, envFrom), + }, serviceSecretVolumeMounts, envFrom), SecurityContext: workload.JobDefinition.PodSecurityContext, ServiceAccountName: workload.JobDefinition.ServiceAccountName, Volumes: append(workload.JobDefinition.Volumes, serviceSecretVolumes...), @@ -711,18 +715,19 @@ func (c *Controller) updateDeployment(ca *v1alpha1.CAPApplication, cav *v1alpha1 // newDeployment creates a new generic Deployment for a CAV resource based on the type. It also sets the appropriate OwnerReferences. func newDeployment(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPApplicationVersion, workload *v1alpha1.WorkloadDetails, ownerRef metav1.OwnerReference, vcapSecretName string) *appsv1.Deployment { - // Get ServiceInfos for consumed BTP services - consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), ca.Spec.BTP.Services) - params := &DeploymentParameters{ CA: ca, CAV: cav, OwnerRef: &ownerRef, WorkloadDetails: *workload, + Env: workload.DeploymentDefinition.Env, } if useVolumeMounts(cav) { - params.WorkloadDetails.DeploymentDefinition.Env = updateServiceBindingRootEnv(params.WorkloadDetails.DeploymentDefinition.Env) + // Get ServiceInfos for consumed BTP services + consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), ca.Spec.BTP.Services) + + params.Env = updateServiceBindingRootEnv(params.Env) params.VolumeMounts = getVolumeMounts(consumedServiceInfos) params.Volumes = getVolumes(consumedServiceInfos) } else { @@ -763,7 +768,7 @@ func createDeployment(params *DeploymentParameters) *appsv1.Deployment { ImagePullSecrets: convertToLocalObjectReferences(params.CAV.Spec.RegistrySecrets), InitContainers: *updateInitContainers(params.WorkloadDetails.DeploymentDefinition.InitContainers, []corev1.EnvVar{ {Name: EnvCAPOpAppVersion, Value: params.CAV.Spec.Version}, - }, params.EnvFrom), + }, params.VolumeMounts, params.EnvFrom), Containers: getContainer(params), ServiceAccountName: params.WorkloadDetails.DeploymentDefinition.ServiceAccountName, Volumes: append(params.WorkloadDetails.DeploymentDefinition.Volumes, params.Volumes...), @@ -801,7 +806,7 @@ func getEnv(params *DeploymentParameters) []corev1.EnvVar { env := []corev1.EnvVar{ {Name: EnvCAPOpAppVersion, Value: params.CAV.Spec.Version}, } - env = append(env, params.WorkloadDetails.DeploymentDefinition.Env...) + env = append(env, params.Env...) if params.WorkloadDetails.DeploymentDefinition.Type == v1alpha1.DeploymentRouter { // Add destinations env for `Router` diff --git a/internal/controller/reconcile-capapplicationversion_test.go b/internal/controller/reconcile-capapplicationversion_test.go index 4f8c56d..8955ca5 100644 --- a/internal/controller/reconcile-capapplicationversion_test.go +++ b/internal/controller/reconcile-capapplicationversion_test.go @@ -846,3 +846,21 @@ func TestCAV_InvalidMonitoringConfig(t *testing.T) { } } + +func TestCAV_UseVolumeMount(t *testing.T) { + reconcileTestItem( + context.TODO(), t, + QueueItem{Key: ResourceCAPApplicationVersion, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01-cav-v1"}}, + TestData{ + description: "capapplication version with initContainers", + initialResources: []string{ + "testdata/common/capapplication.yaml", + "testdata/common/credential-secrets.yaml", + "testdata/capapplicationversion/cav-use-vol-mount.yaml", + }, + expectedResources: "testdata/capapplicationversion/expected/cav-processing-use-vol-mount.yaml", + expectedRequeue: map[int][]NamespacedResourceKey{ResourceCAPApplicationVersion: {{Namespace: "default", Name: "test-cap-01-cav-v1"}}}, + backlogItems: []string{}, + }, + ) +} diff --git a/internal/controller/reconcile-captenantoperation.go b/internal/controller/reconcile-captenantoperation.go index 1c9b069..eb2c08c 100644 --- a/internal/controller/reconcile-captenantoperation.go +++ b/internal/controller/reconcile-captenantoperation.go @@ -394,6 +394,7 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c // create VCAP secret from consumed BTP services var vcapSecretName string + err = nil if !useVolumeMount { vcapSecretName, err = createVCAPSecret(ctop.Name+"-"+strings.ToLower(workload.Name), ctop.Namespace, *metav1.NewControllerRef(ctop, v1alpha1.SchemeGroupVersion.WithKind(v1alpha1.CAPTenantOperationKind)), consumedServiceInfos, c.kubeClient) if err != nil { @@ -432,13 +433,14 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c tenantType: relatedResources.CAPTenant.Labels[LabelTenantType], } + if workload.DeploymentDefinition == nil { + params.Env = workload.JobDefinition.Env + } else { + params.Env = workload.DeploymentDefinition.Env + } + if useVolumeMount { - if workload.DeploymentDefinition == nil { - workload.JobDefinition.Env = updateServiceBindingRootEnv(workload.JobDefinition.Env) - } else { - workload.DeploymentDefinition.Env = updateServiceBindingRootEnv(workload.DeploymentDefinition.Env) - } - params.env = []corev1.EnvVar{defaultServiceBindingRootEnv} + params.Env = updateServiceBindingRootEnv(params.Env) params.volumeMounts = getVolumeMounts(consumedServiceInfos) params.volumes = getVolumes(consumedServiceInfos) } else { @@ -476,7 +478,7 @@ type jobCreateParams struct { providerTenantId string providerSubdomain string tenantType string - env []corev1.EnvVar + Env []corev1.EnvVar EnvFrom []corev1.EnvFromSource volumes []corev1.Volume volumeMounts []corev1.VolumeMount @@ -506,7 +508,7 @@ func (c *Controller) createTenantOperationJob(ctx context.Context, ctop *v1alpha RestartPolicy: corev1.RestartPolicyNever, ImagePullSecrets: params.imagePullSecrets, Containers: getContainers(ctop, derivedWorkload, workload, params), - InitContainers: *updateInitContainers(derivedWorkload.initContainers, getCTOPEnv(params, ctop), params.EnvFrom), + InitContainers: *updateInitContainers(derivedWorkload.initContainers, getCTOPEnv(params, ctop), params.volumeMounts, params.EnvFrom), Volumes: append(derivedWorkload.volumes, params.volumes...), ServiceAccountName: derivedWorkload.serviceAccountName, SecurityContext: derivedWorkload.podSecurityContext, @@ -530,7 +532,7 @@ func getContainers(ctop *v1alpha1.CAPTenantOperation, derivedWorkload tentantOpe Name: workload.Name, Image: derivedWorkload.image, ImagePullPolicy: derivedWorkload.imagePullPolicy, - Env: append(getCTOPEnv(params, ctop), append(derivedWorkload.env, params.env...)...), + Env: append(getCTOPEnv(params, ctop), params.Env...), EnvFrom: params.EnvFrom, VolumeMounts: append(derivedWorkload.volumeMounts, params.volumeMounts...), Resources: derivedWorkload.resources, @@ -650,7 +652,7 @@ func (c *Controller) createCustomTenantOperationJob(ctx context.Context, ctop *v Name: workload.Name, Image: workload.JobDefinition.Image, ImagePullPolicy: workload.JobDefinition.ImagePullPolicy, - Env: append(getCTOPEnv(params, ctop), append(workload.JobDefinition.Env, params.env...)...), + Env: append(getCTOPEnv(params, ctop), params.Env...), EnvFrom: params.EnvFrom, VolumeMounts: append(workload.JobDefinition.VolumeMounts, params.volumeMounts...), Command: workload.JobDefinition.Command, @@ -658,7 +660,7 @@ func (c *Controller) createCustomTenantOperationJob(ctx context.Context, ctop *v SecurityContext: workload.JobDefinition.SecurityContext, }, }, - InitContainers: *updateInitContainers(workload.JobDefinition.InitContainers, getCTOPEnv(params, ctop), params.EnvFrom), + InitContainers: *updateInitContainers(workload.JobDefinition.InitContainers, getCTOPEnv(params, ctop), params.volumeMounts, params.EnvFrom), }, }, }, diff --git a/internal/controller/reconcile-captenantoperation_test.go b/internal/controller/reconcile-captenantoperation_test.go index 7900188..3d6e9fd 100644 --- a/internal/controller/reconcile-captenantoperation_test.go +++ b/internal/controller/reconcile-captenantoperation_test.go @@ -726,3 +726,47 @@ func TestProvisioningWithInitContainersCustom(t *testing.T) { }, ) } + +func TestProvisioningUsingVolMountForServices(t *testing.T) { + _ = reconcileTestItem( + context.TODO(), t, + QueueItem{Key: ResourceCAPTenantOperation, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01-provider-abcd"}}, + TestData{ + backlogItems: []string{}, + description: "Provisioning - Using service volume mount for TenantOperation", + initialResources: []string{ + "testdata/common/capapplication.yaml", + "testdata/common/captenant-provider-ready.yaml", + "testdata/common/capapplicationversion-v1-use-vol-mount.yaml", + "testdata/common/credential-secrets.yaml", + "testdata/captenantoperation/ctop-use-vol-mount.initial.yaml", + }, + expectedResources: "testdata/captenantoperation/ctop-use-vol-mount.expected.yaml", + expectedRequeue: map[int][]NamespacedResourceKey{ + ResourceCAPTenantOperation: {{Namespace: "default", Name: "test-cap-01-provider-abcd"}}, + }, + }, + ) +} + +func TestProvisioningUsingVolMountForServicesCustom(t *testing.T) { + _ = reconcileTestItem( + context.TODO(), t, + QueueItem{Key: ResourceCAPTenantOperation, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01-provider-abcd"}}, + TestData{ + backlogItems: []string{}, + description: "Provisioning - Using service volume mount for CustomTenantOperation", + initialResources: []string{ + "testdata/common/capapplication.yaml", + "testdata/common/captenant-provider-ready.yaml", + "testdata/common/capapplicationversion-v1-use-vol-mount-custom.yaml", + "testdata/common/credential-secrets.yaml", + "testdata/captenantoperation/ctop-use-vol-mount-custom.initial.yaml", + }, + expectedResources: "testdata/captenantoperation/ctop-use-vol-mount-custom.expected.yaml", + expectedRequeue: map[int][]NamespacedResourceKey{ + ResourceCAPTenantOperation: {{Namespace: "default", Name: "test-cap-01-provider-abcd"}}, + }, + }, + ) +} diff --git a/internal/controller/reconcile.go b/internal/controller/reconcile.go index 145642f..29b3bfc 100644 --- a/internal/controller/reconcile.go +++ b/internal/controller/reconcile.go @@ -598,7 +598,7 @@ func copyMaps(originalMap map[string]string, additionalMap map[string]string) ma return newMap } -func updateInitContainers(initContainers []corev1.Container, additionalEnv []corev1.EnvVar, EnvFrom []corev1.EnvFromSource) *[]corev1.Container { +func updateInitContainers(initContainers []corev1.Container, additionalEnv []corev1.EnvVar, serviceSecretVolumeMounts []corev1.VolumeMount, EnvFrom []corev1.EnvFromSource) *[]corev1.Container { var updatedInitContainers []corev1.Container if len(initContainers) > 0 { updatedInitContainers = []corev1.Container{} @@ -606,6 +606,10 @@ func updateInitContainers(initContainers []corev1.Container, additionalEnv []cor updatedContainer := container.DeepCopy() updatedContainer.Env = append(updatedContainer.Env, additionalEnv...) updatedContainer.EnvFrom = EnvFrom + if len(serviceSecretVolumeMounts) > 0 { + updatedContainer.VolumeMounts = append(updatedContainer.VolumeMounts, serviceSecretVolumeMounts...) + updatedContainer.Env = append(updatedContainer.Env, defaultServiceBindingRootEnv) + } updatedInitContainers = append(updatedInitContainers, *updatedContainer) } } diff --git a/internal/controller/testdata/capapplicationversion/cav-use-vol-mount.yaml b/internal/controller/testdata/capapplicationversion/cav-use-vol-mount.yaml new file mode 100644 index 0000000..10fa396 --- /dev/null +++ b/internal/controller/testdata/capapplicationversion/cav-use-vol-mount.yaml @@ -0,0 +1,205 @@ +apiVersion: sme.sap.com/v1alpha1 +kind: CAPApplicationVersion +metadata: + creationTimestamp: "2022-07-18T06:13:52Z" + generation: 1 + annotations: + sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 + sme.sap.com/owner-identifier: default.test-cap-01 + sme.sap.com/use-volume-mount: "true" + labels: + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: 1f74ae2fbff71a708786a4df4bb2ca87ec603581 + name: test-cap-01-cav-v1 + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPApplication + name: test-cap-01 + uid: 3c7ba7cb-dc04-4fd1-be86-3eb3a5c64a98 + resourceVersion: "11371108" + uid: 5e64489b-7346-4984-8617-e8c37338b3d8 +spec: + capApplicationInstance: test-cap-01 + registrySecrets: + - regcred + version: 1.2.3 + workloads: + - name: cap-backend-srv + consumedBTPServices: + - cap-uaa + - cap-service-manager + - cap-saas-registry + deploymentDefinition: + image: docker.image.repo/srv/server:latest + type: CAP + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.PodIp + ports: + - name: "app-port" + port: 4004 + routerDestinationName: "srv-api" + appProtocol: http + - name: "app-tech-port" + port: 4005 + networkPolicy: "Cluster" + livenessProbe: + failureThreshold: 3 + httpGet: + path: / + port: 4004 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 2 + readinessProbe: + failureThreshold: 3 + httpGet: + path: / + port: 4004 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 2 + resources: + limits: + cpu: 200m + memory: 500Mi + requests: + cpu: 20m + memory: 50Mi + - name: app-router + consumedBTPServices: + - cap-uaa + - cap-saas-registry + deploymentDefinition: + image: docker.image.repo/approuter/approuter:latest + type: Router + volumeMounts: + - name: xsapp-vol + mountPath: /usr/app/xsapp.json + subPath: xsapp.json + env: + - name: debug + valueFrom: + configMapKeyRef: + key: someKey + name: someCM + optional: true + ports: + - name: "router-port" + port: 4000 + appProtocol: http + - name: "router-tech-port" + port: 4004 + networkPolicy: "Cluster" + - name: "router-metrics-port" + port: 4007 + networkPolicy: "Cluster" + livenessProbe: + failureThreshold: 3 + httpGet: + path: / + port: 4000 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 2 + readinessProbe: + failureThreshold: 3 + httpGet: + path: / + port: 4000 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 2 + resources: + limits: + cpu: 200m + memory: 500Mi + requests: + cpu: 20m + memory: 50Mi + nodeName: app + priorityClassName: some-prio + serviceAccountName: custom-router-sa + volumes: + - name: xsapp-vol + configMap: + name: xsapp-config + - name: log-data + emptyDir: {} + initContainers: + - name: init-container + image: docker.image.repo/init-container:latest + env: + - name: INIT_CONTAINER_ENV + value: "init-container-env" + resources: + limits: + cpu: 100m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: log-container + image: docker.image.repo/log-container:latest + restartPolicy: Always + command: ['sh', '-c', 'tail -F /log/logs.txt'] + volumeMounts: + - name: log-data + mountPath: /log + env: + - name: LOG_CONTAINER_ENV + value: "log-container-env" + resources: + limits: + cpu: 100m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: content-job + consumedBTPServices: + - cap-uaa + jobDefinition: + image: docker.image.repo/content/cap-content:latest + type: Content + env: + - name: SOME_VERSION + value: 0.0.1 + resources: + limits: + cpu: 100m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: mtx-job + consumedBTPServices: + - cap-uaa + - cap-service-manager + - cap-saas-registry + jobDefinition: + image: docker.image.repo/srv/server:latest + type: TenantOperation + - name: job-worker + consumedBTPServices: + - cap-uaa + - cap-service-manager + - cap-saas-registry + deploymentDefinition: + image: docker.image.repo/srv/server:latest + type: Additional + env: + - name: TEST_SEC_REF + valueFrom: + secretKeyRef: + key: someKey + name: someSecret + optional: true +status: + state: Processing diff --git a/internal/controller/testdata/capapplicationversion/expected/cav-processing-use-vol-mount.yaml b/internal/controller/testdata/capapplicationversion/expected/cav-processing-use-vol-mount.yaml new file mode 100644 index 0000000..d004f21 --- /dev/null +++ b/internal/controller/testdata/capapplicationversion/expected/cav-processing-use-vol-mount.yaml @@ -0,0 +1,432 @@ +apiVersion: sme.sap.com/v1alpha1 +kind: CAPApplicationVersion +metadata: + creationTimestamp: "2022-07-18T06:13:52Z" + generation: 1 + annotations: + sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 + sme.sap.com/owner-identifier: default.test-cap-01 + sme.sap.com/use-volume-mount: "true" + labels: + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: 1f74ae2fbff71a708786a4df4bb2ca87ec603581 + name: test-cap-01-cav-v1 + namespace: default + finalizers: + - "sme.sap.com/capapplicationversion" + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPApplication + name: test-cap-01 + uid: 3c7ba7cb-dc04-4fd1-be86-3eb3a5c64a98 + resourceVersion: "11371108" + uid: 5e64489b-7346-4984-8617-e8c37338b3d8 +spec: + capApplicationInstance: test-cap-01 + registrySecrets: + - regcred + version: 1.2.3 + workloads: + - name: cap-backend-srv + consumedBTPServices: + - cap-uaa + - cap-service-manager + - cap-saas-registry + deploymentDefinition: + image: docker.image.repo/srv/server:latest + type: CAP + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.PodIp + ports: + - name: "app-port" + port: 4004 + routerDestinationName: "srv-api" + appProtocol: http + - name: "app-tech-port" + port: 4005 + networkPolicy: "Cluster" + livenessProbe: + failureThreshold: 3 + httpGet: + path: / + port: 4004 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 2 + readinessProbe: + failureThreshold: 3 + httpGet: + path: / + port: 4004 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 2 + resources: + limits: + cpu: 200m + memory: 500Mi + requests: + cpu: 20m + memory: 50Mi + - name: app-router + consumedBTPServices: + - cap-uaa + - cap-saas-registry + deploymentDefinition: + image: docker.image.repo/approuter/approuter:latest + type: Router + volumeMounts: + - name: xsapp-vol + mountPath: /usr/app/xsapp.json + subPath: xsapp.json + env: + - name: debug + valueFrom: + configMapKeyRef: + key: someKey + name: someCM + optional: true + ports: + - name: "router-port" + port: 4000 + appProtocol: http + - name: "router-tech-port" + port: 4004 + networkPolicy: "Cluster" + - name: "router-metrics-port" + port: 4007 + networkPolicy: "Cluster" + livenessProbe: + failureThreshold: 3 + httpGet: + path: / + port: 4000 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 2 + readinessProbe: + failureThreshold: 3 + httpGet: + path: / + port: 4000 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 2 + resources: + limits: + cpu: 200m + memory: 500Mi + requests: + cpu: 20m + memory: 50Mi + nodeName: app + priorityClassName: some-prio + serviceAccountName: custom-router-sa + volumes: + - name: xsapp-vol + configMap: + name: xsapp-config + - name: log-data + emptyDir: {} + initContainers: + - name: init-container + image: docker.image.repo/init-container:latest + env: + - name: INIT_CONTAINER_ENV + value: "init-container-env" + resources: + limits: + cpu: 100m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: log-container + image: docker.image.repo/log-container:latest + restartPolicy: Always + command: ['sh', '-c', 'tail -F /log/logs.txt'] + volumeMounts: + - name: log-data + mountPath: /log + env: + - name: LOG_CONTAINER_ENV + value: "log-container-env" + resources: + limits: + cpu: 100m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: content-job + consumedBTPServices: + - cap-uaa + jobDefinition: + image: docker.image.repo/content/cap-content:latest + type: Content + env: + - name: SOME_VERSION + value: 0.0.1 + resources: + limits: + cpu: 100m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: mtx-job + consumedBTPServices: + - cap-uaa + - cap-service-manager + - cap-saas-registry + jobDefinition: + image: docker.image.repo/srv/server:latest + type: TenantOperation + - name: job-worker + consumedBTPServices: + - cap-uaa + - cap-service-manager + - cap-saas-registry + deploymentDefinition: + image: docker.image.repo/srv/server:latest + type: Additional + env: + - name: TEST_SEC_REF + valueFrom: + secretKeyRef: + key: someKey + name: someSecret + optional: true +status: + conditions: + - reason: WaitingForWorkloads + observedGeneration: 1 + status: "False" + type: Ready + observedGeneration: 1 + state: Processing +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: test-cap-01 + sme.sap.com/category: Workload + sme.sap.com/workload-name: test-cap-01-cav-v1-app-router + sme.sap.com/workload-type: Router + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + sme.sap.com/cav-version: "1.2.3" + sme.sap.com/owner-generation: "1" + sme.sap.com/owner-identifier-hash: e95e0682f33a657e75e1fc435972d19bd407ba3b + annotations: + sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 + sme.sap.com/owner-identifier: default.test-cap-01-cav-v1 + name: test-cap-01-cav-v1-app-router + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPApplicationVersion + name: test-cap-01-cav-v1 + uid: 5e64489b-7346-4984-8617-e8c37338b3d8 +spec: + selector: + matchLabels: + app: test-cap-01 + sme.sap.com/category: Workload + sme.sap.com/workload-name: test-cap-01-cav-v1-app-router + sme.sap.com/workload-type: Router + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + sme.sap.com/cav-version: "1.2.3" + sme.sap.com/owner-generation: "1" + sme.sap.com/owner-identifier-hash: e95e0682f33a657e75e1fc435972d19bd407ba3b + template: + metadata: + creationTimestamp: null + labels: + app: test-cap-01 + sme.sap.com/category: Workload + sme.sap.com/workload-name: test-cap-01-cav-v1-app-router + sme.sap.com/workload-type: Router + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + sme.sap.com/cav-version: "1.2.3" + sme.sap.com/owner-generation: "1" + sme.sap.com/owner-identifier-hash: e95e0682f33a657e75e1fc435972d19bd407ba3b + annotations: + sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 + sme.sap.com/owner-identifier: default.test-cap-01-cav-v1 + spec: + containers: + - env: + - name: CAPOP_APP_VERSION + value: "1.2.3" + - name: debug + valueFrom: + configMapKeyRef: + key: someKey + name: someCM + optional: true + - name: SERVICE_BINDING_ROOT + value: "/etc/secrets" + - name: destinations + value: '[{"name":"srv-api","url":"http://test-cap-01-cav-v1-cap-backend-srv-svc:4004","forwardAuthToken":true}]' + image: docker.image.repo/approuter/approuter:latest + volumeMounts: + - name: xsapp-vol + mountPath: /usr/app/xsapp.json + subPath: xsapp.json + - name: cap-uaa + readOnly: true + mountPath: "/etc/secrets/xsuaa" + - name: cap-saas-registry + readOnly: true + mountPath: "/etc/secrets/saas-registry" + livenessProbe: + failureThreshold: 3 + httpGet: + path: / + port: 4000 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 2 + readinessProbe: + failureThreshold: 3 + httpGet: + path: / + port: 4000 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 2 + resources: + limits: + cpu: 200m + memory: 500Mi + requests: + cpu: 20m + memory: 50Mi + name: app-router + imagePullSecrets: + - name: regcred + nodeName: app + priorityClassName: some-prio + serviceAccountName: custom-router-sa + volumes: + - name: xsapp-vol + configMap: + name: xsapp-config + - name: log-data + emptyDir: {} + - name: cap-uaa + secret: + secretName: cap-cap-01-uaa-bind-cf + - name: cap-saas-registry + secret: + secretName: cap-cap-01-saas-bind-cf + initContainers: + - name: init-container + image: docker.image.repo/init-container:latest + env: + - name: INIT_CONTAINER_ENV + value: "init-container-env" + - name: CAPOP_APP_VERSION + value: "1.2.3" + - name: SERVICE_BINDING_ROOT + value: "/etc/secrets" + volumeMounts: + - name: cap-uaa + readOnly: true + mountPath: "/etc/secrets/xsuaa" + - name: cap-saas-registry + readOnly: true + mountPath: "/etc/secrets/saas-registry" + resources: + limits: + cpu: 100m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: log-container + image: docker.image.repo/log-container:latest + restartPolicy: Always + command: ['sh', '-c', 'tail -F /log/logs.txt'] + volumeMounts: + - name: log-data + mountPath: /log + - name: cap-uaa + readOnly: true + mountPath: "/etc/secrets/xsuaa" + - name: cap-saas-registry + readOnly: true + mountPath: "/etc/secrets/saas-registry" + env: + - name: LOG_CONTAINER_ENV + value: "log-container-env" + - name: CAPOP_APP_VERSION + value: "1.2.3" + - name: SERVICE_BINDING_ROOT + value: "/etc/secrets" + resources: + limits: + cpu: 100m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-cap-01-cav-v1-content + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPApplicationVersion + name: test-cap-01-cav-v1 + uid: 5e64489b-7346-4984-8617-e8c37338b3d8 +spec: + template: + metadata: + annotations: + sidecar.istio.io/inject: "false" + creationTimestamp: null + labels: + x4.sap.com/disable-karydia: "true" + spec: + containers: + - env: + - name: CAPOP_APP_VERSION + value: 1.2.3 + - name: SOME_VERSION + value: 0.0.1 + - name: SERVICE_BINDING_ROOT + value: "/etc/secrets" + image: docker.image.repo/content/cap-content:latest + name: content-job + resources: + limits: + cpu: 100m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + volumeMounts: + - name: cap-uaa + readOnly: true + mountPath: "/etc/secrets/xsuaa" + imagePullSecrets: + - name: regcred + restartPolicy: "OnFailure" + volumes: + - name: cap-uaa + secret: + secretName: cap-cap-01-uaa-bind-cf diff --git a/internal/controller/testdata/captenantoperation/ctop-use-vol-mount-custom.expected.yaml b/internal/controller/testdata/captenantoperation/ctop-use-vol-mount-custom.expected.yaml new file mode 100644 index 0000000..ecc4c68 --- /dev/null +++ b/internal/controller/testdata/captenantoperation/ctop-use-vol-mount-custom.expected.yaml @@ -0,0 +1,262 @@ +apiVersion: sme.sap.com/v1alpha1 +kind: CAPTenantOperation +metadata: + name: test-cap-01-provider-abcd + namespace: default + generation: 1 + finalizers: + - sme.sap.com/captenantoperation + annotations: + sme.sap.com/owner-identifier: default.test-cap-01-provider + labels: + sme.sap.com/tenant-operation-type: upgrade + sme.sap.com/owner-generation: "0" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + sme.sap.com/cav-version: 8.9.10 + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + tenantId: tenant-id-for-provider + subDomain: my-provider + operation: upgrade + capApplicationVersionInstance: test-cap-01-cav-v2 + steps: + - name: custom-say + type: CustomTenantOperation + - name: ten-op + type: TenantOperation + - name: custom-say + type: CustomTenantOperation +status: + state: Processing + observedGeneration: 1 + conditions: + - type: Ready + status: "False" + observedGeneration: 1 + reason: StepInitiated + message: "step 1/3 : job default.test-cap-01-provider-custom-say-gen created" + currentStep: 1 + activeJob: "test-cap-01-provider-custom-say-gen" +--- +apiVersion: batch/v1 +kind: Job +metadata: + annotations: + sidecar.istio.io/inject: "false" + sme.sap.com/owner-identifier: default.test-cap-01-provider-abcd + sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 + labels: + sme.sap.com/tenant-operation-step: "1" + sme.sap.com/tenant-operation-type: upgrade + sme.sap.com/owner-generation: "1" + sme.sap.com/owner-identifier-hash: ce6bb3ae0b5ebbd0116259415ccac00bff0dc431 + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + app: test-cap-01 + sme.sap.com/category: "Workload" + sme.sap.com/workload-name: "custom-say" + sme.sap.com/workload-type: "CustomTenantOperation" + generateName: test-cap-01-provider-custom-say- + name: test-cap-01-provider-custom-say-gen + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenantOperation + name: test-cap-01-provider-abcd +spec: + backoffLimit: 1 + ttlSecondsAfterFinished: 150 + template: + metadata: + annotations: + sidecar.istio.io/inject: "false" + sme.sap.com/owner-identifier: default.test-cap-01-provider-abcd + sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 + labels: + sme.sap.com/tenant-operation-step: "1" + sme.sap.com/tenant-operation-type: upgrade + sme.sap.com/owner-generation: "1" + sme.sap.com/owner-identifier-hash: ce6bb3ae0b5ebbd0116259415ccac00bff0dc431 + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + app: test-cap-01 + sme.sap.com/category: "Workload" + sme.sap.com/workload-name: "custom-say" + sme.sap.com/workload-type: "CustomTenantOperation" + spec: + containers: + - env: + - name: CAPOP_APP_VERSION + value: 8.9.10 + - name: CAPOP_TENANT_ID + value: tenant-id-for-provider + - name: CAPOP_TENANT_OPERATION + value: upgrade + - name: CAPOP_TENANT_SUBDOMAIN + value: my-provider + - name: CAPOP_TENANT_TYPE + value: provider + - name: CAPOP_APP_NAME + value: test-cap-01 + - name: CAPOP_GLOBAL_ACCOUNT_ID + value: btp-glo-acc-id + - name: CAPOP_PROVIDER_TENANT_ID + value: tenant-id-for-provider + - name: CAPOP_PROVIDER_SUBDOMAIN + value: my-provider + - name: close + value: encounter + - name: SERVICE_BINDING_ROOT + value: "/etc/secrets" + securityContext: + runAsUser: 1000 + runAsGroup: 2000 + image: docker/whalesay + volumeMounts: + - mountPath: /cache + name: cache-vol + - name: cap-uaa + readOnly: true + mountPath: "/etc/secrets/xsuaa" + command: ["cowsay", "$(CAPOP_TENANT_OPERATION)", "$(CAPOP_TENANT_ID)"] + name: custom-say + imagePullSecrets: + - name: regcred + securityContext: + runAsUser: 2000 + runAsGroup: 2000 + nodeSelector: + disktype: ssd + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: + - antarctica-east1 + - antarctica-west1 + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: another-node-label-key + operator: In + values: + - another-node-label-value + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: someZone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + foo: bar + matchLabelKeys: + - dummy-test-1.27 + tolerations: + - key: "test" + operator: "Equal" + value: "foo" + effect: "NoExecute" + tolerationSeconds: 300 + - key: "never-exec" + operator: "Equal" + value: "noexec" + effect: "NoExecute" + nodeName: foo + priorityClassName: prio + volumes: + - name: cache-vol + emptyDir: + sizeLimit: 500Mi + - name: log-data + emptyDir: {} + - name: cap-uaa + secret: + secretName: cap-cap-01-uaa-bind-cf + initContainers: + - name: init-container + image: docker.image.repo/init-container:latest + volumeMounts: + - name: cap-uaa + readOnly: true + mountPath: "/etc/secrets/xsuaa" + env: + - name: INIT_CONTAINER_ENV + value: "init-container-env" + - name: CAPOP_APP_VERSION + value: 8.9.10 + - name: CAPOP_TENANT_ID + value: tenant-id-for-provider + - name: CAPOP_TENANT_OPERATION + value: upgrade + - name: CAPOP_TENANT_SUBDOMAIN + value: my-provider + - name: CAPOP_TENANT_TYPE + value: provider + - name: CAPOP_APP_NAME + value: test-cap-01 + - name: CAPOP_GLOBAL_ACCOUNT_ID + value: btp-glo-acc-id + - name: CAPOP_PROVIDER_TENANT_ID + value: tenant-id-for-provider + - name: CAPOP_PROVIDER_SUBDOMAIN + value: my-provider + - name: SERVICE_BINDING_ROOT + value: "/etc/secrets" + resources: + limits: + cpu: 100m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: log-container + image: docker.image.repo/log-container:latest + restartPolicy: Always + command: ['sh', '-c', 'tail -F /log/logs.txt'] + volumeMounts: + - name: log-data + mountPath: /log + - name: cap-uaa + readOnly: true + mountPath: "/etc/secrets/xsuaa" + env: + - name: LOG_CONTAINER_ENV + value: "log-container-env" + - name: CAPOP_APP_VERSION + value: 8.9.10 + - name: CAPOP_TENANT_ID + value: tenant-id-for-provider + - name: CAPOP_TENANT_OPERATION + value: upgrade + - name: CAPOP_TENANT_SUBDOMAIN + value: my-provider + - name: CAPOP_TENANT_TYPE + value: provider + - name: CAPOP_APP_NAME + value: test-cap-01 + - name: CAPOP_GLOBAL_ACCOUNT_ID + value: btp-glo-acc-id + - name: CAPOP_PROVIDER_TENANT_ID + value: tenant-id-for-provider + - name: CAPOP_PROVIDER_SUBDOMAIN + value: my-provider + - name: SERVICE_BINDING_ROOT + value: "/etc/secrets" + resources: + limits: + cpu: 100m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + serviceAccountName: custom-op-sa + restartPolicy: Never diff --git a/internal/controller/testdata/captenantoperation/ctop-use-vol-mount-custom.initial.yaml b/internal/controller/testdata/captenantoperation/ctop-use-vol-mount-custom.initial.yaml new file mode 100644 index 0000000..ce7a09b --- /dev/null +++ b/internal/controller/testdata/captenantoperation/ctop-use-vol-mount-custom.initial.yaml @@ -0,0 +1,38 @@ +apiVersion: sme.sap.com/v1alpha1 +kind: CAPTenantOperation +metadata: + name: test-cap-01-provider-abcd + namespace: default + generation: 1 + finalizers: + - sme.sap.com/captenantoperation + annotations: + sme.sap.com/owner-identifier: "default.test-cap-01-provider" + labels: + sme.sap.com/tenant-operation-type: upgrade + sme.sap.com/owner-generation: "0" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + sme.sap.com/cav-version: "8.9.10" + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + tenantId: tenant-id-for-provider + subDomain: my-provider + operation: upgrade + capApplicationVersionInstance: test-cap-01-cav-v2 + steps: + - name: custom-say + type: CustomTenantOperation + - name: ten-op + type: TenantOperation + - name: custom-say + type: CustomTenantOperation +status: + state: Processing + conditions: [] + currentStep: 1 diff --git a/internal/controller/testdata/captenantoperation/ctop-use-vol-mount.expected.yaml b/internal/controller/testdata/captenantoperation/ctop-use-vol-mount.expected.yaml new file mode 100644 index 0000000..ccb378b --- /dev/null +++ b/internal/controller/testdata/captenantoperation/ctop-use-vol-mount.expected.yaml @@ -0,0 +1,292 @@ +apiVersion: sme.sap.com/v1alpha1 +kind: CAPTenantOperation +metadata: + name: test-cap-01-provider-abcd + namespace: default + generation: 1 + finalizers: + - sme.sap.com/captenantoperation + annotations: + sme.sap.com/owner-identifier: default.test-cap-01-provider + sme.sap.com/subscription-context-secret: test-cap-01-gen + labels: + sme.sap.com/tenant-operation-type: provisioning + sme.sap.com/owner-generation: "0" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + sme.sap.com/cav-version: "5.6.7" + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + tenantId: tenant-id-for-provider + subDomain: my-provider + operation: provisioning + capApplicationVersionInstance: test-cap-01-cav-v1 + steps: + - name: mtx + type: TenantOperation +status: + state: Processing + observedGeneration: 1 + conditions: + - type: Ready + status: "False" + observedGeneration: 1 + reason: StepInitiated + message: "step 1/1 : job default.test-cap-01-provider-mtx-gen created" + currentStep: 1 + activeJob: "test-cap-01-provider-mtx-gen" +--- +apiVersion: batch/v1 +kind: Job +metadata: + annotations: + sidecar.istio.io/inject: "false" + sme.sap.com/owner-identifier: default.test-cap-01-provider-abcd + sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 + labels: + sme.sap.com/tenant-operation-step: "1" + sme.sap.com/tenant-operation-type: provisioning + sme.sap.com/owner-generation: "1" + sme.sap.com/owner-identifier-hash: ce6bb3ae0b5ebbd0116259415ccac00bff0dc431 + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + app: test-cap-01 + sme.sap.com/category: "Workload" + sme.sap.com/workload-name: "mtx" + sme.sap.com/workload-type: "TenantOperation" + generateName: test-cap-01-provider-mtx- + name: test-cap-01-provider-mtx-gen + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenantOperation + name: test-cap-01-provider-abcd +spec: + template: + metadata: + annotations: + sidecar.istio.io/inject: "false" + sme.sap.com/owner-identifier: default.test-cap-01-provider-abcd + sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 + labels: + sme.sap.com/tenant-operation-step: "1" + sme.sap.com/tenant-operation-type: provisioning + sme.sap.com/owner-generation: "1" + sme.sap.com/owner-identifier-hash: ce6bb3ae0b5ebbd0116259415ccac00bff0dc431 + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + app: test-cap-01 + sme.sap.com/category: "Workload" + sme.sap.com/workload-name: "mtx" + sme.sap.com/workload-type: "TenantOperation" + spec: + containers: + - env: + - name: CAPOP_APP_VERSION + value: 5.6.7 + - name: CAPOP_TENANT_ID + value: tenant-id-for-provider + - name: CAPOP_TENANT_OPERATION + value: provisioning + - name: CAPOP_TENANT_SUBDOMAIN + value: my-provider + - name: CAPOP_TENANT_TYPE + value: provider + - name: CAPOP_APP_NAME + value: test-cap-01 + - name: CAPOP_GLOBAL_ACCOUNT_ID + value: btp-glo-acc-id + - name: CAPOP_PROVIDER_TENANT_ID + value: tenant-id-for-provider + - name: CAPOP_PROVIDER_SUBDOMAIN + value: my-provider + - name: SERVICE_BINDING_ROOT + value: "/etc/secrets" + - name: CAPOP_SUBSCRIPTION_PAYLOAD + valueFrom: + secretKeyRef: + name: test-cap-01-gen + key: subscriptionContext + volumeMounts: + - mountPath: /cache + name: cache-vol + - name: cap-uaa + readOnly: true + mountPath: "/etc/secrets/xsuaa" + - name: cap-saas-registry + readOnly: true + mountPath: "/etc/secrets/saas-registry" + - name: cap-service-manager + readOnly: true + mountPath: "/etc/secrets/service-manager" + command: + - node + - ./node_modules/@sap/cds-mtxs/bin/cds-mtx + - subscribe + - tenant-id-for-provider + - --body + - $(CAPOP_SUBSCRIPTION_PAYLOAD) + resources: + limits: + cpu: 200m + memory: 200Mi + requests: + cpu: 20m + memory: 20Mi + securityContext: + runAsUser: 1000 + runAsGroup: 2000 + image: docker.image.repo/srv/server:latest + name: mtx + nodeSelector: + disktype: ssd + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: + - antarctica-east1 + - antarctica-west1 + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: another-node-label-key + operator: In + values: + - another-node-label-value + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: someZone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + foo: bar + matchLabelKeys: + - dummy-test-1.27 + tolerations: + - key: "test" + operator: "Equal" + value: "foo" + effect: "NoExecute" + tolerationSeconds: 300 + - key: "never-exec" + operator: "Equal" + value: "noexec" + effect: "NoExecute" + imagePullSecrets: + - name: regcred + volumes: + - name: cache-vol + emptyDir: + sizeLimit: 500Mi + - name: log-data + emptyDir: {} + - name: cap-uaa + secret: + secretName: cap-cap-01-uaa-bind-cf + - name: cap-saas-registry + secret: + secretName: cap-cap-01-saas-bind-cf + - name: cap-service-manager + secret: + secretName: cap-cap-01-svc-man-bind-cf + initContainers: + - name: init-container + image: docker.image.repo/init-container:latest + volumeMounts: + - name: cap-uaa + readOnly: true + mountPath: "/etc/secrets/xsuaa" + - name: cap-saas-registry + readOnly: true + mountPath: "/etc/secrets/saas-registry" + - name: cap-service-manager + readOnly: true + mountPath: "/etc/secrets/service-manager" + env: + - name: INIT_CONTAINER_ENV + value: "init-container-env" + - name: CAPOP_APP_VERSION + value: 5.6.7 + - name: CAPOP_TENANT_ID + value: tenant-id-for-provider + - name: CAPOP_TENANT_OPERATION + value: provisioning + - name: CAPOP_TENANT_SUBDOMAIN + value: my-provider + - name: CAPOP_TENANT_TYPE + value: provider + - name: CAPOP_APP_NAME + value: test-cap-01 + - name: CAPOP_GLOBAL_ACCOUNT_ID + value: btp-glo-acc-id + - name: CAPOP_PROVIDER_TENANT_ID + value: tenant-id-for-provider + - name: CAPOP_PROVIDER_SUBDOMAIN + value: my-provider + - name: SERVICE_BINDING_ROOT + value: "/etc/secrets" + resources: + limits: + cpu: 100m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: log-container + image: docker.image.repo/log-container:latest + restartPolicy: Always + command: ['sh', '-c', 'tail -F /log/logs.txt'] + volumeMounts: + - name: log-data + mountPath: /log + - name: cap-uaa + readOnly: true + mountPath: "/etc/secrets/xsuaa" + - name: cap-saas-registry + readOnly: true + mountPath: "/etc/secrets/saas-registry" + - name: cap-service-manager + readOnly: true + mountPath: "/etc/secrets/service-manager" + env: + - name: LOG_CONTAINER_ENV + value: "log-container-env" + - name: CAPOP_APP_VERSION + value: 5.6.7 + - name: CAPOP_TENANT_ID + value: tenant-id-for-provider + - name: CAPOP_TENANT_OPERATION + value: provisioning + - name: CAPOP_TENANT_SUBDOMAIN + value: my-provider + - name: CAPOP_TENANT_TYPE + value: provider + - name: CAPOP_APP_NAME + value: test-cap-01 + - name: CAPOP_GLOBAL_ACCOUNT_ID + value: btp-glo-acc-id + - name: CAPOP_PROVIDER_TENANT_ID + value: tenant-id-for-provider + - name: CAPOP_PROVIDER_SUBDOMAIN + value: my-provider + - name: SERVICE_BINDING_ROOT + value: "/etc/secrets" + resources: + limits: + cpu: 100m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + serviceAccountName: custom-op-sa + restartPolicy: Never diff --git a/internal/controller/testdata/captenantoperation/ctop-use-vol-mount.initial.yaml b/internal/controller/testdata/captenantoperation/ctop-use-vol-mount.initial.yaml new file mode 100644 index 0000000..3c5eb35 --- /dev/null +++ b/internal/controller/testdata/captenantoperation/ctop-use-vol-mount.initial.yaml @@ -0,0 +1,34 @@ +apiVersion: sme.sap.com/v1alpha1 +kind: CAPTenantOperation +metadata: + name: test-cap-01-provider-abcd + namespace: default + generation: 1 + finalizers: + - sme.sap.com/captenantoperation + annotations: + sme.sap.com/owner-identifier: default.test-cap-01-provider + labels: + sme.sap.com/tenant-operation-type: provisioning + sme.sap.com/owner-generation: "0" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + sme.sap.com/cav-version: "5.6.7" + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + tenantId: tenant-id-for-provider + subDomain: my-provider + operation: provisioning + capApplicationVersionInstance: test-cap-01-cav-v1 + steps: + - name: mtx + type: TenantOperation +status: + state: Processing + conditions: [] + currentStep: 1 diff --git a/internal/controller/testdata/common/capapplicationversion-v1-use-vol-mount-custom.yaml b/internal/controller/testdata/common/capapplicationversion-v1-use-vol-mount-custom.yaml new file mode 100644 index 0000000..6ad749c --- /dev/null +++ b/internal/controller/testdata/common/capapplicationversion-v1-use-vol-mount-custom.yaml @@ -0,0 +1,183 @@ +apiVersion: sme.sap.com/v1alpha1 +kind: CAPApplicationVersion +metadata: + generation: 1 + annotations: + sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 + sme.sap.com/owner-identifier: default.test-cap-01 + sme.sap.com/use-volume-mount: "true" + labels: + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: 1f74ae2fbff71a708786a4df4bb2ca87ec603581 + name: test-cap-01-cav-v2 + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPApplication + name: test-cap-01 + uid: 3c7ba7cb-dc04-4fd1-be86-3eb3a5c64a98 + resourceVersion: "11371108" + uid: 5e64489b-7346-4984-8617-e8c37338b3d8 +spec: + capApplicationInstance: test-cap-01 + version: 8.9.10 + registrySecrets: + - regcred + workloads: + - name: cap-backend + consumedBTPServices: + - cap-uaa + - cap-service-manager + - cap-saas-registry + deploymentDefinition: + type: CAP + image: docker.image.repo/srv/server:v2 + env: + - name: foo + value: bar + - name: content-job + consumedBTPServices: + - cap-uaa + jobDefinition: + type: Content + image: docker.image.repo/content/cap-content:v2 + - name: app-router + consumedBTPServices: + - cap-uaa + - cap-saas-registry + deploymentDefinition: + type: Router + image: docker.image.repo/approuter/approuter:v2 + - name: ten-op + consumedBTPServices: + - cap-uaa + - cap-service-manager + - cap-saas-registry + jobDefinition: + type: "TenantOperation" + backoffLimit: 1 + ttlSecondsAfterFinished: 150 + image: docker.image.repo/srv/server:latest + env: + - name: flow + value: glow + - name: custom-say + consumedBTPServices: + - cap-uaa + jobDefinition: + type: "CustomTenantOperation" + volumeMounts: + - mountPath: /cache + name: cache-vol + volumes: + - name: cache-vol + emptyDir: + sizeLimit: 500Mi + - name: log-data + emptyDir: {} + serviceAccountName: custom-op-sa + image: docker/whalesay + command: ["cowsay", "$(CAPOP_TENANT_OPERATION)", "$(CAPOP_TENANT_ID)"] + backoffLimit: 1 + ttlSecondsAfterFinished: 150 + securityContext: + runAsUser: 1000 + runAsGroup: 2000 + podSecurityContext: + runAsUser: 2000 + runAsGroup: 2000 + nodeSelector: + disktype: ssd + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: + - antarctica-east1 + - antarctica-west1 + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: another-node-label-key + operator: In + values: + - another-node-label-value + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: someZone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + foo: bar + matchLabelKeys: + - dummy-test-1.27 + tolerations: + - key: "test" + operator: "Equal" + value: "foo" + effect: "NoExecute" + tolerationSeconds: 300 + - key: "never-exec" + operator: "Equal" + value: "noexec" + effect: "NoExecute" + nodeName: foo + priorityClassName: prio + env: + - name: close + value: encounter + initContainers: + - name: init-container + image: docker.image.repo/init-container:latest + env: + - name: INIT_CONTAINER_ENV + value: "init-container-env" + resources: + limits: + cpu: 100m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: log-container + image: docker.image.repo/log-container:latest + restartPolicy: Always + command: ['sh', '-c', 'tail -F /log/logs.txt'] + volumeMounts: + - name: log-data + mountPath: /log + env: + - name: LOG_CONTAINER_ENV + value: "log-container-env" + resources: + limits: + cpu: 100m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + tenantOperations: + provisioning: + - workloadName: custom-say + - workloadName: content-job # this will be ignored + - workloadName: ten-op + upgrade: + - workloadName: custom-say + - workloadName: ten-op + - workloadName: custom-say +status: + conditions: + - lastTransitionTime: "2022-03-18T23:07:47Z" + lastUpdateTime: "2022-03-18T23:07:47Z" + reason: CreatedDeployments + status: "True" + type: Ready + observedGeneration: 1 + state: Ready diff --git a/internal/controller/testdata/common/capapplicationversion-v1-use-vol-mount.yaml b/internal/controller/testdata/common/capapplicationversion-v1-use-vol-mount.yaml new file mode 100644 index 0000000..588fb58 --- /dev/null +++ b/internal/controller/testdata/common/capapplicationversion-v1-use-vol-mount.yaml @@ -0,0 +1,157 @@ +apiVersion: sme.sap.com/v1alpha1 +kind: CAPApplicationVersion +metadata: + creationTimestamp: "2022-03-18T22:14:33Z" + generation: 1 + annotations: + sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 + sme.sap.com/owner-identifier: default.test-cap-01 + sme.sap.com/use-volume-mount: "true" + labels: + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: 1f74ae2fbff71a708786a4df4bb2ca87ec603581 + name: test-cap-01-cav-v1 + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPApplication + name: test-cap-01 + uid: 3c7ba7cb-dc04-4fd1-be86-3eb3a5c64a98 + resourceVersion: "11371108" + uid: 5e64489b-7346-4984-8617-e8c37338b3d8 +spec: + capApplicationInstance: test-cap-01 + registrySecrets: + - regcred + version: 5.6.7 + workloads: + - name: cap-backend + consumedBTPServices: + - cap-uaa + - cap-service-manager + - cap-saas-registry + deploymentDefinition: + type: CAP + image: docker.image.repo/srv/server:latest + - name: content-job + consumedBTPServices: + - cap-uaa + jobDefinition: + type: Content + image: docker.image.repo/content/cap-content:latest + - name: mtx + consumedBTPServices: + - cap-uaa + - cap-service-manager + - cap-saas-registry + jobDefinition: + type: "TenantOperation" + volumeMounts: + - mountPath: /cache + name: cache-vol + volumes: + - name: cache-vol + emptyDir: + sizeLimit: 500Mi + - name: log-data + emptyDir: {} + serviceAccountName: custom-op-sa + resources: + limits: + cpu: 200m + memory: 200Mi + requests: + cpu: 20m + memory: 20Mi + securityContext: + runAsUser: 1000 + runAsGroup: 2000 + nodeSelector: + disktype: ssd + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: + - antarctica-east1 + - antarctica-west1 + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: another-node-label-key + operator: In + values: + - another-node-label-value + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: someZone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + foo: bar + matchLabelKeys: + - dummy-test-1.27 + tolerations: + - key: "test" + operator: "Equal" + value: "foo" + effect: "NoExecute" + tolerationSeconds: 300 + - key: "never-exec" + operator: "Equal" + value: "noexec" + effect: "NoExecute" + image: docker.image.repo/srv/server:latest + initContainers: + - name: init-container + image: docker.image.repo/init-container:latest + env: + - name: INIT_CONTAINER_ENV + value: "init-container-env" + resources: + limits: + cpu: 100m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: log-container + image: docker.image.repo/log-container:latest + restartPolicy: Always + command: ["sh", "-c", "tail -F /log/logs.txt"] + volumeMounts: + - name: log-data + mountPath: /log + env: + - name: LOG_CONTAINER_ENV + value: "log-container-env" + resources: + limits: + cpu: 100m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: app-router + consumedBTPServices: + - cap-uaa + - cap-saas-registry + deploymentDefinition: + type: Router + image: docker.image.repo/approuter/approuter:latest +status: + conditions: + - lastTransitionTime: "2022-03-18T23:07:47Z" + lastUpdateTime: "2022-03-18T23:07:47Z" + reason: CreatedDeployments + status: "True" + type: Ready + observedGeneration: 1 + state: Ready From 663952898229e0410185c89289cccb6871642731 Mon Sep 17 00:00:00 2001 From: i325261 Date: Tue, 15 Oct 2024 14:09:34 +0200 Subject: [PATCH 07/13] minor corrections --- internal/controller/reconcile.go | 1 - .../expected/cav-processing-use-vol-mount.yaml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/controller/reconcile.go b/internal/controller/reconcile.go index 29b3bfc..5952669 100644 --- a/internal/controller/reconcile.go +++ b/internal/controller/reconcile.go @@ -630,7 +630,6 @@ func getVolumeMounts(serviceInfos []v1alpha1.ServiceInfo) []corev1.VolumeMount { func getVolumes(serviceInfos []v1alpha1.ServiceInfo) []corev1.Volume { volumes := []corev1.Volume{} - for _, serviceInfo := range serviceInfos { volumes = append(volumes, corev1.Volume{Name: serviceInfo.Name, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: serviceInfo.Secret}}}) } diff --git a/internal/controller/testdata/capapplicationversion/expected/cav-processing-use-vol-mount.yaml b/internal/controller/testdata/capapplicationversion/expected/cav-processing-use-vol-mount.yaml index d004f21..475b05b 100644 --- a/internal/controller/testdata/capapplicationversion/expected/cav-processing-use-vol-mount.yaml +++ b/internal/controller/testdata/capapplicationversion/expected/cav-processing-use-vol-mount.yaml @@ -6,7 +6,7 @@ metadata: annotations: sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 sme.sap.com/owner-identifier: default.test-cap-01 - sme.sap.com/use-volume-mount: "true" + sme.sap.com/use-volume-mount: "true" labels: sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 sme.sap.com/owner-generation: "2" From 422e152645a0ed67b34426799fde34616afefdfd Mon Sep 17 00:00:00 2001 From: i325261 Date: Tue, 15 Oct 2024 16:05:05 +0200 Subject: [PATCH 08/13] adopt test case --- .../ctop-use-vol-mount.expected.yaml | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/internal/controller/testdata/captenantoperation/ctop-use-vol-mount.expected.yaml b/internal/controller/testdata/captenantoperation/ctop-use-vol-mount.expected.yaml index ccb378b..46a3a8d 100644 --- a/internal/controller/testdata/captenantoperation/ctop-use-vol-mount.expected.yaml +++ b/internal/controller/testdata/captenantoperation/ctop-use-vol-mount.expected.yaml @@ -105,13 +105,15 @@ spec: value: tenant-id-for-provider - name: CAPOP_PROVIDER_SUBDOMAIN value: my-provider - - name: SERVICE_BINDING_ROOT - value: "/etc/secrets" - name: CAPOP_SUBSCRIPTION_PAYLOAD valueFrom: secretKeyRef: name: test-cap-01-gen key: subscriptionContext + - name: CAPOP_TENANT_MTXS_OPERATION + value: subscribe + - name: SERVICE_BINDING_ROOT + value: "/etc/secrets" volumeMounts: - mountPath: /cache name: cache-vol @@ -124,13 +126,8 @@ spec: - name: cap-service-manager readOnly: true mountPath: "/etc/secrets/service-manager" - command: - - node - - ./node_modules/@sap/cds-mtxs/bin/cds-mtx - - subscribe - - tenant-id-for-provider - - --body - - $(CAPOP_SUBSCRIPTION_PAYLOAD) + command: ["node", "./node_modules/@sap/cds-mtxs/bin/cds-mtx"] + args: ["$(CAPOP_TENANT_MTXS_OPERATION)", "tenant-id-for-provider", "--body", "$(CAPOP_SUBSCRIPTION_PAYLOAD)"] resources: limits: cpu: 200m @@ -233,6 +230,13 @@ spec: value: tenant-id-for-provider - name: CAPOP_PROVIDER_SUBDOMAIN value: my-provider + - name: CAPOP_SUBSCRIPTION_PAYLOAD + valueFrom: + secretKeyRef: + name: test-cap-01-gen + key: subscriptionContext + - name: CAPOP_TENANT_MTXS_OPERATION + value: subscribe - name: SERVICE_BINDING_ROOT value: "/etc/secrets" resources: @@ -279,6 +283,13 @@ spec: value: tenant-id-for-provider - name: CAPOP_PROVIDER_SUBDOMAIN value: my-provider + - name: CAPOP_SUBSCRIPTION_PAYLOAD + valueFrom: + secretKeyRef: + name: test-cap-01-gen + key: subscriptionContext + - name: CAPOP_TENANT_MTXS_OPERATION + value: subscribe - name: SERVICE_BINDING_ROOT value: "/etc/secrets" resources: From 460192d92c3fbb24be3dc6a86a4be5216dba0a78 Mon Sep 17 00:00:00 2001 From: i325261 Date: Wed, 16 Oct 2024 11:18:20 +0200 Subject: [PATCH 09/13] added documentation --- .../content/en/docs/usage/resources/capapplicationversion.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/website/content/en/docs/usage/resources/capapplicationversion.md b/website/content/en/docs/usage/resources/capapplicationversion.md index e766786..6e8553b 100644 --- a/website/content/en/docs/usage/resources/capapplicationversion.md +++ b/website/content/en/docs/usage/resources/capapplicationversion.md @@ -42,6 +42,8 @@ spec: > The `CAPApplicationVersion` resource is meant to be immutable - it's spec should not be modified once it is deployed. This is also prevented by our web-hooks which we recommend to always keep active (default). +> By default, credentials (from secrets) required to access SAP BTP services are automatically provided as the `VCAP_SERVICES` environment variable across all workloads. However, this behavior can be changed to use volume mounts by setting the annotation `sme.sap.com/use-volume-mount: "true"` on the `CAPApplicationVersion` resource. When this annotation is set, the `SERVICE_BINDING_ROOT` environment variable will be injected into all the workloads, pointing to the secret path and overriding any user-defined values, if specified. + ### Workloads with `deploymentDefinition` ```yaml @@ -364,4 +366,4 @@ spec: > > The supported configurations is kept minimal intentionally to keep the overall API simple by considering commonly used configurations. -> Note: For `initContainers` nearly the same environment variables as the main container are made available including VCAP_SERVICES environment. \ No newline at end of file +> Note: For `initContainers` nearly the same environment variables as the main container are made available including VCAP_SERVICES environment. From 6a3e6a89927827d4d353b1fa797218abe6099d32 Mon Sep 17 00:00:00 2001 From: i325261 Date: Thu, 17 Oct 2024 10:44:30 +0200 Subject: [PATCH 10/13] documentation updated volume mount annotation changed --- internal/controller/reconcile.go | 2 +- .../testdata/capapplicationversion/cav-use-vol-mount.yaml | 2 +- .../expected/cav-processing-use-vol-mount.yaml | 2 +- .../common/capapplicationversion-v1-use-vol-mount-custom.yaml | 2 +- .../common/capapplicationversion-v1-use-vol-mount.yaml | 2 +- .../content/en/docs/usage/resources/capapplicationversion.md | 4 ++-- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/controller/reconcile.go b/internal/controller/reconcile.go index 0697e36..bf1a724 100644 --- a/internal/controller/reconcile.go +++ b/internal/controller/reconcile.go @@ -50,7 +50,7 @@ const ( AnnotationSubscriptionContextSecret = "sme.sap.com/subscription-context-secret" AnnotationProviderSubAccountId = "sme.sap.com/provider-sub-account-id" AnnotationEnableCleanupMonitoring = "sme.sap.com/enable-cleanup-monitoring" - AnnotationUseVolumeMount = "sme.sap.com/use-volume-mount" + AnnotationUseVolumeMount = "sme.sap.com/use-credential-volume-mount" FinalizerCAPApplication = "sme.sap.com/capapplication" FinalizerCAPApplicationVersion = "sme.sap.com/capapplicationversion" FinalizerCAPTenant = "sme.sap.com/captenant" diff --git a/internal/controller/testdata/capapplicationversion/cav-use-vol-mount.yaml b/internal/controller/testdata/capapplicationversion/cav-use-vol-mount.yaml index 10fa396..a3c3f76 100644 --- a/internal/controller/testdata/capapplicationversion/cav-use-vol-mount.yaml +++ b/internal/controller/testdata/capapplicationversion/cav-use-vol-mount.yaml @@ -6,7 +6,7 @@ metadata: annotations: sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 sme.sap.com/owner-identifier: default.test-cap-01 - sme.sap.com/use-volume-mount: "true" + sme.sap.com/use-credential-volume-mount: "true" labels: sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 sme.sap.com/owner-generation: "2" diff --git a/internal/controller/testdata/capapplicationversion/expected/cav-processing-use-vol-mount.yaml b/internal/controller/testdata/capapplicationversion/expected/cav-processing-use-vol-mount.yaml index 475b05b..40dfb1d 100644 --- a/internal/controller/testdata/capapplicationversion/expected/cav-processing-use-vol-mount.yaml +++ b/internal/controller/testdata/capapplicationversion/expected/cav-processing-use-vol-mount.yaml @@ -6,7 +6,7 @@ metadata: annotations: sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 sme.sap.com/owner-identifier: default.test-cap-01 - sme.sap.com/use-volume-mount: "true" + sme.sap.com/use-credential-volume-mount: "true" labels: sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 sme.sap.com/owner-generation: "2" diff --git a/internal/controller/testdata/common/capapplicationversion-v1-use-vol-mount-custom.yaml b/internal/controller/testdata/common/capapplicationversion-v1-use-vol-mount-custom.yaml index 6ad749c..95fe5e1 100644 --- a/internal/controller/testdata/common/capapplicationversion-v1-use-vol-mount-custom.yaml +++ b/internal/controller/testdata/common/capapplicationversion-v1-use-vol-mount-custom.yaml @@ -5,7 +5,7 @@ metadata: annotations: sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 sme.sap.com/owner-identifier: default.test-cap-01 - sme.sap.com/use-volume-mount: "true" + sme.sap.com/use-credential-volume-mount: "true" labels: sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 sme.sap.com/owner-generation: "2" diff --git a/internal/controller/testdata/common/capapplicationversion-v1-use-vol-mount.yaml b/internal/controller/testdata/common/capapplicationversion-v1-use-vol-mount.yaml index 588fb58..3452646 100644 --- a/internal/controller/testdata/common/capapplicationversion-v1-use-vol-mount.yaml +++ b/internal/controller/testdata/common/capapplicationversion-v1-use-vol-mount.yaml @@ -6,7 +6,7 @@ metadata: annotations: sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 sme.sap.com/owner-identifier: default.test-cap-01 - sme.sap.com/use-volume-mount: "true" + sme.sap.com/use-credential-volume-mount: "true" labels: sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 sme.sap.com/owner-generation: "2" diff --git a/website/content/en/docs/usage/resources/capapplicationversion.md b/website/content/en/docs/usage/resources/capapplicationversion.md index 6e8553b..1a47ba6 100644 --- a/website/content/en/docs/usage/resources/capapplicationversion.md +++ b/website/content/en/docs/usage/resources/capapplicationversion.md @@ -42,7 +42,7 @@ spec: > The `CAPApplicationVersion` resource is meant to be immutable - it's spec should not be modified once it is deployed. This is also prevented by our web-hooks which we recommend to always keep active (default). -> By default, credentials (from secrets) required to access SAP BTP services are automatically provided as the `VCAP_SERVICES` environment variable across all workloads. However, this behavior can be changed to use volume mounts by setting the annotation `sme.sap.com/use-volume-mount: "true"` on the `CAPApplicationVersion` resource. When this annotation is set, the `SERVICE_BINDING_ROOT` environment variable will be injected into all the workloads, pointing to the secret path and overriding any user-defined values, if specified. +> By default, credentials (from secrets) required to access SAP BTP services are automatically provided as the `VCAP_SERVICES` environment variable across all workloads. However, this behavior can be changed to use volume mounts by setting the annotation `sme.sap.com/use-credential-volume-mount: "true"` on the `CAPApplicationVersion` resource. When this annotation is set, the `SERVICE_BINDING_ROOT` environment variable will be injected into all the workloads, pointing to the secret path and overriding user-defined values, if any. ### Workloads with `deploymentDefinition` @@ -366,4 +366,4 @@ spec: > > The supported configurations is kept minimal intentionally to keep the overall API simple by considering commonly used configurations. -> Note: For `initContainers` nearly the same environment variables as the main container are made available including VCAP_SERVICES environment. +> Note: For `initContainers` nearly the same environment variables as the main container are made available including VCAP_SERVICES environment. Additionally, if annotation `sme.sap.com/use-credential-volume-mount: "true"` is set on the `CAPApplicationVersion` resource, the environment variable `SERVICE_BINDING_ROOT` and the volume mounts for the service credentials will also be made available. From 388b7d57da907c9894f40faf9f48088ec0ba15cd Mon Sep 17 00:00:00 2001 From: i325261 Date: Thu, 17 Oct 2024 10:53:55 +0200 Subject: [PATCH 11/13] renamed method names to be more specific --- .../reconcile-capapplicationversion.go | 16 ++++++++-------- .../controller/reconcile-captenantoperation.go | 6 +++--- internal/controller/reconcile.go | 10 +++++----- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/internal/controller/reconcile-capapplicationversion.go b/internal/controller/reconcile-capapplicationversion.go index 191765e..c6e9154 100644 --- a/internal/controller/reconcile-capapplicationversion.go +++ b/internal/controller/reconcile-capapplicationversion.go @@ -266,7 +266,7 @@ func (c *Controller) handleContentDeployJob(ca *v1alpha1.CAPApplication, cav *v1 // Get VCAP secret name err = nil - if !useVolumeMounts(cav) { + if !useVolumeMountsForServiceCredentials(cav) { vcapSecretName, err = createVCAPSecret(jobName, cav.Namespace, ownerRef, consumedServiceInfos, c.kubeClient) } @@ -301,13 +301,13 @@ func newContentDeploymentJob(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPAppli env := workload.JobDefinition.Env - if useVolumeMounts(cav) { + if useVolumeMountsForServiceCredentials(cav) { // Get ServiceInfos for consumed BTP services consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), ca.Spec.BTP.Services) env = updateServiceBindingRootEnv(env) - serviceSecretVolumeMounts = getVolumeMounts(consumedServiceInfos) - serviceSecretVolumes = getVolumes(consumedServiceInfos) + serviceSecretVolumeMounts = getServiceCredentialVolumeMounts(consumedServiceInfos) + serviceSecretVolumes = getServiceCredentialVolumes(consumedServiceInfos) } else { envFrom = getEnvFrom(vcapSecretName) } @@ -699,7 +699,7 @@ func (c *Controller) updateDeployment(ca *v1alpha1.CAPApplication, cav *v1alpha1 // Get VCAP secret name err = nil - if !useVolumeMounts(cav) { + if !useVolumeMountsForServiceCredentials(cav) { vcapSecretName, err = createVCAPSecret(deploymentName, cav.Namespace, ownerRef, consumedServiceInfos, c.kubeClient) } @@ -724,13 +724,13 @@ func newDeployment(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPApplicationVers Env: workload.DeploymentDefinition.Env, } - if useVolumeMounts(cav) { + if useVolumeMountsForServiceCredentials(cav) { // Get ServiceInfos for consumed BTP services consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), ca.Spec.BTP.Services) params.Env = updateServiceBindingRootEnv(params.Env) - params.VolumeMounts = getVolumeMounts(consumedServiceInfos) - params.Volumes = getVolumes(consumedServiceInfos) + params.VolumeMounts = getServiceCredentialVolumeMounts(consumedServiceInfos) + params.Volumes = getServiceCredentialVolumes(consumedServiceInfos) } else { params.EnvFrom = getEnvFrom(vcapSecretName) } diff --git a/internal/controller/reconcile-captenantoperation.go b/internal/controller/reconcile-captenantoperation.go index fcf539c..52745ff 100644 --- a/internal/controller/reconcile-captenantoperation.go +++ b/internal/controller/reconcile-captenantoperation.go @@ -391,7 +391,7 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), relatedResources.CAPApplication.Spec.BTP.Services) // check volume mount annotation - useVolumeMount := useVolumeMounts(relatedResources.CAPApplicationVersion) + useVolumeMount := useVolumeMountsForServiceCredentials(relatedResources.CAPApplicationVersion) // create VCAP secret from consumed BTP services var vcapSecretName string @@ -442,8 +442,8 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c if useVolumeMount { params.Env = updateServiceBindingRootEnv(params.Env) - params.volumeMounts = getVolumeMounts(consumedServiceInfos) - params.volumes = getVolumes(consumedServiceInfos) + params.volumeMounts = getServiceCredentialVolumeMounts(consumedServiceInfos) + params.volumes = getServiceCredentialVolumes(consumedServiceInfos) } else { params.EnvFrom = getEnvFrom(vcapSecretName) } diff --git a/internal/controller/reconcile.go b/internal/controller/reconcile.go index bf1a724..8d0c390 100644 --- a/internal/controller/reconcile.go +++ b/internal/controller/reconcile.go @@ -50,7 +50,7 @@ const ( AnnotationSubscriptionContextSecret = "sme.sap.com/subscription-context-secret" AnnotationProviderSubAccountId = "sme.sap.com/provider-sub-account-id" AnnotationEnableCleanupMonitoring = "sme.sap.com/enable-cleanup-monitoring" - AnnotationUseVolumeMount = "sme.sap.com/use-credential-volume-mount" + AnnotationUseCredentialVolumeMount = "sme.sap.com/use-credential-volume-mount" FinalizerCAPApplication = "sme.sap.com/capapplication" FinalizerCAPApplicationVersion = "sme.sap.com/capapplicationversion" FinalizerCAPTenant = "sme.sap.com/captenant" @@ -621,7 +621,7 @@ func getWorkloadName(cavName, workloadName string) string { return fmt.Sprintf("%s-%s", cavName, strings.ToLower(workloadName)) } -func getVolumeMounts(serviceInfos []v1alpha1.ServiceInfo) []corev1.VolumeMount { +func getServiceCredentialVolumeMounts(serviceInfos []v1alpha1.ServiceInfo) []corev1.VolumeMount { volumeMounts := []corev1.VolumeMount{} for _, serviceInfo := range serviceInfos { volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: serviceInfo.Name, MountPath: path.Join(defaultServiceBindingRootEnv.Value, serviceInfo.Class), ReadOnly: true}) @@ -629,7 +629,7 @@ func getVolumeMounts(serviceInfos []v1alpha1.ServiceInfo) []corev1.VolumeMount { return volumeMounts } -func getVolumes(serviceInfos []v1alpha1.ServiceInfo) []corev1.Volume { +func getServiceCredentialVolumes(serviceInfos []v1alpha1.ServiceInfo) []corev1.Volume { volumes := []corev1.Volume{} for _, serviceInfo := range serviceInfos { volumes = append(volumes, corev1.Volume{Name: serviceInfo.Name, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: serviceInfo.Secret}}}) @@ -637,8 +637,8 @@ func getVolumes(serviceInfos []v1alpha1.ServiceInfo) []corev1.Volume { return volumes } -func useVolumeMounts(cav *v1alpha1.CAPApplicationVersion) bool { - value, exists := cav.Annotations[AnnotationUseVolumeMount] +func useVolumeMountsForServiceCredentials(cav *v1alpha1.CAPApplicationVersion) bool { + value, exists := cav.Annotations[AnnotationUseCredentialVolumeMount] return exists && value == "true" } From 89525e4390b8ac539c9621593746d1bbec6e649f Mon Sep 17 00:00:00 2001 From: i325261 Date: Thu, 24 Oct 2024 16:09:42 +0200 Subject: [PATCH 12/13] use service binding name instead of class name in the volume mounts --- internal/controller/reconcile-captenantoperation.go | 6 +++--- internal/controller/reconcile.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/controller/reconcile-captenantoperation.go b/internal/controller/reconcile-captenantoperation.go index 52745ff..26dba97 100644 --- a/internal/controller/reconcile-captenantoperation.go +++ b/internal/controller/reconcile-captenantoperation.go @@ -391,12 +391,12 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), relatedResources.CAPApplication.Spec.BTP.Services) // check volume mount annotation - useVolumeMount := useVolumeMountsForServiceCredentials(relatedResources.CAPApplicationVersion) + useVolumeMountsForServiceCredentials := useVolumeMountsForServiceCredentials(relatedResources.CAPApplicationVersion) // create VCAP secret from consumed BTP services var vcapSecretName string err = nil - if !useVolumeMount { + if !useVolumeMountsForServiceCredentials { vcapSecretName, err = createVCAPSecret(ctop.Name+"-"+strings.ToLower(workload.Name), ctop.Namespace, *metav1.NewControllerRef(ctop, v1alpha1.SchemeGroupVersion.WithKind(v1alpha1.CAPTenantOperationKind)), consumedServiceInfos, c.kubeClient) if err != nil { return nil, err @@ -440,7 +440,7 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c params.Env = workload.DeploymentDefinition.Env } - if useVolumeMount { + if useVolumeMountsForServiceCredentials { params.Env = updateServiceBindingRootEnv(params.Env) params.volumeMounts = getServiceCredentialVolumeMounts(consumedServiceInfos) params.volumes = getServiceCredentialVolumes(consumedServiceInfos) diff --git a/internal/controller/reconcile.go b/internal/controller/reconcile.go index 8d0c390..b86fb9e 100644 --- a/internal/controller/reconcile.go +++ b/internal/controller/reconcile.go @@ -624,7 +624,7 @@ func getWorkloadName(cavName, workloadName string) string { func getServiceCredentialVolumeMounts(serviceInfos []v1alpha1.ServiceInfo) []corev1.VolumeMount { volumeMounts := []corev1.VolumeMount{} for _, serviceInfo := range serviceInfos { - volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: serviceInfo.Name, MountPath: path.Join(defaultServiceBindingRootEnv.Value, serviceInfo.Class), ReadOnly: true}) + volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: serviceInfo.Name, MountPath: path.Join(defaultServiceBindingRootEnv.Value, serviceInfo.Name), ReadOnly: true}) } return volumeMounts } From 314a5a9c0cb261dc3c4c2c9aa2945c4de8774c5b Mon Sep 17 00:00:00 2001 From: i325261 Date: Thu, 24 Oct 2024 16:17:44 +0200 Subject: [PATCH 13/13] test cases adopted to use service name in volume mount path --- .../expected/cav-processing-use-vol-mount.yaml | 14 +++++++------- .../ctop-use-vol-mount-custom.expected.yaml | 6 +++--- .../ctop-use-vol-mount.expected.yaml | 18 +++++++++--------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/internal/controller/testdata/capapplicationversion/expected/cav-processing-use-vol-mount.yaml b/internal/controller/testdata/capapplicationversion/expected/cav-processing-use-vol-mount.yaml index 40dfb1d..78a6a41 100644 --- a/internal/controller/testdata/capapplicationversion/expected/cav-processing-use-vol-mount.yaml +++ b/internal/controller/testdata/capapplicationversion/expected/cav-processing-use-vol-mount.yaml @@ -284,10 +284,10 @@ spec: subPath: xsapp.json - name: cap-uaa readOnly: true - mountPath: "/etc/secrets/xsuaa" + mountPath: "/etc/secrets/cap-uaa" - name: cap-saas-registry readOnly: true - mountPath: "/etc/secrets/saas-registry" + mountPath: "/etc/secrets/cap-saas-registry" livenessProbe: failureThreshold: 3 httpGet: @@ -342,10 +342,10 @@ spec: volumeMounts: - name: cap-uaa readOnly: true - mountPath: "/etc/secrets/xsuaa" + mountPath: "/etc/secrets/cap-uaa" - name: cap-saas-registry readOnly: true - mountPath: "/etc/secrets/saas-registry" + mountPath: "/etc/secrets/cap-saas-registry" resources: limits: cpu: 100m @@ -362,10 +362,10 @@ spec: mountPath: /log - name: cap-uaa readOnly: true - mountPath: "/etc/secrets/xsuaa" + mountPath: "/etc/secrets/cap-uaa" - name: cap-saas-registry readOnly: true - mountPath: "/etc/secrets/saas-registry" + mountPath: "/etc/secrets/cap-saas-registry" env: - name: LOG_CONTAINER_ENV value: "log-container-env" @@ -422,7 +422,7 @@ spec: volumeMounts: - name: cap-uaa readOnly: true - mountPath: "/etc/secrets/xsuaa" + mountPath: "/etc/secrets/cap-uaa" imagePullSecrets: - name: regcred restartPolicy: "OnFailure" diff --git a/internal/controller/testdata/captenantoperation/ctop-use-vol-mount-custom.expected.yaml b/internal/controller/testdata/captenantoperation/ctop-use-vol-mount-custom.expected.yaml index ecc4c68..3edb7f2 100644 --- a/internal/controller/testdata/captenantoperation/ctop-use-vol-mount-custom.expected.yaml +++ b/internal/controller/testdata/captenantoperation/ctop-use-vol-mount-custom.expected.yaml @@ -123,7 +123,7 @@ spec: name: cache-vol - name: cap-uaa readOnly: true - mountPath: "/etc/secrets/xsuaa" + mountPath: "/etc/secrets/cap-uaa" command: ["cowsay", "$(CAPOP_TENANT_OPERATION)", "$(CAPOP_TENANT_ID)"] name: custom-say imagePullSecrets: @@ -187,7 +187,7 @@ spec: volumeMounts: - name: cap-uaa readOnly: true - mountPath: "/etc/secrets/xsuaa" + mountPath: "/etc/secrets/cap-uaa" env: - name: INIT_CONTAINER_ENV value: "init-container-env" @@ -227,7 +227,7 @@ spec: mountPath: /log - name: cap-uaa readOnly: true - mountPath: "/etc/secrets/xsuaa" + mountPath: "/etc/secrets/cap-uaa" env: - name: LOG_CONTAINER_ENV value: "log-container-env" diff --git a/internal/controller/testdata/captenantoperation/ctop-use-vol-mount.expected.yaml b/internal/controller/testdata/captenantoperation/ctop-use-vol-mount.expected.yaml index 46a3a8d..28641f8 100644 --- a/internal/controller/testdata/captenantoperation/ctop-use-vol-mount.expected.yaml +++ b/internal/controller/testdata/captenantoperation/ctop-use-vol-mount.expected.yaml @@ -119,13 +119,13 @@ spec: name: cache-vol - name: cap-uaa readOnly: true - mountPath: "/etc/secrets/xsuaa" + mountPath: "/etc/secrets/cap-uaa" - name: cap-saas-registry readOnly: true - mountPath: "/etc/secrets/saas-registry" + mountPath: "/etc/secrets/cap-saas-registry" - name: cap-service-manager readOnly: true - mountPath: "/etc/secrets/service-manager" + mountPath: "/etc/secrets/cap-service-manager" command: ["node", "./node_modules/@sap/cds-mtxs/bin/cds-mtx"] args: ["$(CAPOP_TENANT_MTXS_OPERATION)", "tenant-id-for-provider", "--body", "$(CAPOP_SUBSCRIPTION_PAYLOAD)"] resources: @@ -202,13 +202,13 @@ spec: volumeMounts: - name: cap-uaa readOnly: true - mountPath: "/etc/secrets/xsuaa" + mountPath: "/etc/secrets/cap-uaa" - name: cap-saas-registry readOnly: true - mountPath: "/etc/secrets/saas-registry" + mountPath: "/etc/secrets/cap-saas-registry" - name: cap-service-manager readOnly: true - mountPath: "/etc/secrets/service-manager" + mountPath: "/etc/secrets/cap-service-manager" env: - name: INIT_CONTAINER_ENV value: "init-container-env" @@ -255,13 +255,13 @@ spec: mountPath: /log - name: cap-uaa readOnly: true - mountPath: "/etc/secrets/xsuaa" + mountPath: "/etc/secrets/cap-uaa" - name: cap-saas-registry readOnly: true - mountPath: "/etc/secrets/saas-registry" + mountPath: "/etc/secrets/cap-saas-registry" - name: cap-service-manager readOnly: true - mountPath: "/etc/secrets/service-manager" + mountPath: "/etc/secrets/cap-service-manager" env: - name: LOG_CONTAINER_ENV value: "log-container-env"