Skip to content

Commit

Permalink
use annotation check
Browse files Browse the repository at this point in the history 
enhanced volume & volume mount propogation
  • Loading branch information
@anirudhprasad-sap
anirudhprasad-sap committed Oct 8, 2024
1 parent bb3938d commit 57eb28b
Show file tree
Hide file tree
Showing 3 changed files with 65 additions and 57 deletions.
33 changes: 19 additions & 14 deletions internal/controller/reconcile-capapplicationversion.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ type DeploymentParameters struct {
CAV *v1alpha1.CAPApplicationVersion
OwnerRef *metav1.OwnerReference
WorkloadDetails v1alpha1.WorkloadDetails
VCAPSecretName string
EnvFrom []corev1.EnvFromSource
Volumes []corev1.Volume
VolumeMounts []corev1.VolumeMount
}
Expand Down Expand Up @@ -264,7 +264,10 @@ func (c *Controller) handleContentDeployJob(ca *v1alpha1.CAPApplication, cav *v1
ownerRef := *metav1.NewControllerRef(cav, v1alpha1.SchemeGroupVersion.WithKind(v1alpha1.CAPApplicationVersionKind))

// Get VCAP secret name
vcapSecretName, err = createVCAPSecret(jobName, cav.Namespace, ownerRef, consumedServiceInfos, c.kubeClient)
err = nil
if !useVolumeMounts(cav) {
vcapSecretName, err = createVCAPSecret(jobName, cav.Namespace, ownerRef, consumedServiceInfos, c.kubeClient)
}

if err == nil {
contentDeployJob, err = c.kubeClient.BatchV1().Jobs(cav.Namespace).Create(context.TODO(), newContentDeploymentJob(ca, cav, workload, ownerRef, vcapSecretName), metav1.CreateOptions{})
Expand Down Expand Up @@ -297,7 +300,8 @@ func newContentDeploymentJob(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPAppli
var serviceSecretVolumeMounts []corev1.VolumeMount
var serviceSecretVolumes []corev1.Volume

if useVolumeMounts(workload.JobDefinition.Env) {
if useVolumeMounts(cav) {
workload.JobDefinition.Env = updateServiceBindingRootEnv(workload.JobDefinition.Env)
serviceSecretVolumeMounts = getVolumeMounts(consumedServiceInfos)
serviceSecretVolumes = getVolumes(consumedServiceInfos)
} else {
Expand Down Expand Up @@ -339,7 +343,7 @@ func newContentDeploymentJob(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPAppli
},
InitContainers: *updateInitContainers(workload.JobDefinition.InitContainers, []corev1.EnvVar{
{Name: EnvCAPOpAppVersion, Value: cav.Spec.Version},
}, vcapSecretName),
}, envFrom),
SecurityContext: workload.JobDefinition.PodSecurityContext,
ServiceAccountName: workload.JobDefinition.ServiceAccountName,
Volumes: append(workload.JobDefinition.Volumes, serviceSecretVolumes...),
Expand Down Expand Up @@ -689,7 +693,10 @@ func (c *Controller) updateDeployment(ca *v1alpha1.CAPApplication, cav *v1alpha1
ownerRef := *metav1.NewControllerRef(cav, v1alpha1.SchemeGroupVersion.WithKind(v1alpha1.CAPApplicationVersionKind))

// Get VCAP secret name
vcapSecretName, err = createVCAPSecret(deploymentName, cav.Namespace, ownerRef, consumedServiceInfos, c.kubeClient)
err = nil
if !useVolumeMounts(cav) {
vcapSecretName, err = createVCAPSecret(deploymentName, cav.Namespace, ownerRef, consumedServiceInfos, c.kubeClient)
}

if err == nil {
workloadDeployment, err = c.kubeClient.AppsV1().Deployments(cav.Namespace).Create(context.TODO(), newDeployment(ca, cav, workload, ownerRef, vcapSecretName), metav1.CreateOptions{})
Expand All @@ -712,16 +719,14 @@ func newDeployment(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPApplicationVers
CAV: cav,
OwnerRef: &ownerRef,
WorkloadDetails: *workload,
VCAPSecretName: vcapSecretName,
}

if useVolumeMounts(workload.DeploymentDefinition.Env) {
if workload.DeploymentDefinition.Type == v1alpha1.DeploymentCAP {
params.VolumeMounts = getCAPVolumeMounts(consumedServiceInfos, CDSVolMountPrefix)
} else {
params.VolumeMounts = getVolumeMounts(consumedServiceInfos)
}
if useVolumeMounts(cav) {
params.WorkloadDetails.DeploymentDefinition.Env = updateServiceBindingRootEnv(params.WorkloadDetails.DeploymentDefinition.Env)
params.VolumeMounts = getVolumeMounts(consumedServiceInfos)
params.Volumes = getVolumes(consumedServiceInfos)
} else {
params.EnvFrom = getEnvFrom(vcapSecretName)
}

return createDeployment(params)
Expand Down Expand Up @@ -758,7 +763,7 @@ func createDeployment(params *DeploymentParameters) *appsv1.Deployment {
ImagePullSecrets: convertToLocalObjectReferences(params.CAV.Spec.RegistrySecrets),
InitContainers: *updateInitContainers(params.WorkloadDetails.DeploymentDefinition.InitContainers, []corev1.EnvVar{
{Name: EnvCAPOpAppVersion, Value: params.CAV.Spec.Version},
}, params.VCAPSecretName),
}, params.EnvFrom),
Containers: getContainer(params),
ServiceAccountName: params.WorkloadDetails.DeploymentDefinition.ServiceAccountName,
Volumes: append(params.WorkloadDetails.DeploymentDefinition.Volumes, params.Volumes...),
Expand All @@ -782,7 +787,7 @@ func getContainer(params *DeploymentParameters) []corev1.Container {
ImagePullPolicy: params.WorkloadDetails.DeploymentDefinition.ImagePullPolicy,
Command: params.WorkloadDetails.DeploymentDefinition.Command,
Env: getEnv(params),
EnvFrom: getEnvFrom(params.VCAPSecretName),
EnvFrom: params.EnvFrom,
VolumeMounts: append(params.WorkloadDetails.DeploymentDefinition.VolumeMounts, params.VolumeMounts...),
LivenessProbe: params.WorkloadDetails.DeploymentDefinition.LivenessProbe,
ReadinessProbe: params.WorkloadDetails.DeploymentDefinition.ReadinessProbe,
Expand Down
53 changes: 29 additions & 24 deletions internal/controller/reconcile-captenantoperation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -389,22 +389,16 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c

consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), relatedResources.CAPApplication.Spec.BTP.Services)

var serviceSecretVolumeMounts []corev1.VolumeMount
var serviceSecretVolumes []corev1.Volume

if (workload.JobDefinition != nil && useVolumeMounts(workload.JobDefinition.Env)) || (workload.DeploymentDefinition != nil && useVolumeMounts(workload.DeploymentDefinition.Env)) {
if ctop.Spec.Steps[*ctop.Status.CurrentStep-1].Type == v1alpha1.JobTenantOperation {
serviceSecretVolumeMounts = getCAPVolumeMounts(consumedServiceInfos, CDSVolMountPrefix)
} else {
serviceSecretVolumeMounts = getVolumeMounts(consumedServiceInfos)
}
serviceSecretVolumes = getVolumes(consumedServiceInfos)
}
// check volume mount annotation
useVolumeMount := useVolumeMounts(relatedResources.CAPApplicationVersion)

// create VCAP secret from consumed BTP services
vcapSecretName, err := createVCAPSecret(ctop.Name+"-"+strings.ToLower(workload.Name), ctop.Namespace, *metav1.NewControllerRef(ctop, v1alpha1.SchemeGroupVersion.WithKind(v1alpha1.CAPTenantOperationKind)), consumedServiceInfos, c.kubeClient)
if err != nil {
return nil, err
var vcapSecretName string
if !useVolumeMount {
vcapSecretName, err = createVCAPSecret(ctop.Name+"-"+strings.ToLower(workload.Name), ctop.Namespace, *metav1.NewControllerRef(ctop, v1alpha1.SchemeGroupVersion.WithKind(v1alpha1.CAPTenantOperationKind)), consumedServiceInfos, c.kubeClient)
if err != nil {
return nil, err
}
}

annotations := copyMaps(workload.Annotations, map[string]string{
Expand All @@ -429,16 +423,26 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c
namePrefix: relatedResources.CAPTenant.Name + "-" + workload.Name + "-",
labels: labels,
annotations: annotations,
vcapSecretName: vcapSecretName,
imagePullSecrets: convertToLocalObjectReferences(relatedResources.CAPApplicationVersion.Spec.RegistrySecrets),
version: relatedResources.CAPApplicationVersion.Spec.Version,
appName: relatedResources.CAPApplication.Spec.BTPAppName,
globalAccountId: relatedResources.CAPApplication.Spec.GlobalAccountId,
providerTenantId: relatedResources.CAPApplication.Spec.Provider.TenantId,
providerSubdomain: relatedResources.CAPApplication.Spec.Provider.SubDomain,
tenantType: relatedResources.CAPTenant.Labels[LabelTenantType],
volumeMounts: serviceSecretVolumeMounts,
volumes: serviceSecretVolumes,
}

if useVolumeMount {
if workload.DeploymentDefinition == nil {
workload.JobDefinition.Env = updateServiceBindingRootEnv(workload.JobDefinition.Env)
} else {
workload.DeploymentDefinition.Env = updateServiceBindingRootEnv(workload.DeploymentDefinition.Env)
}
params.env = []corev1.EnvVar{defaultServiceBindingRootEnv}
params.volumeMounts = getVolumeMounts(consumedServiceInfos)
params.volumes = getVolumes(consumedServiceInfos)
} else {
params.EnvFrom = getEnvFrom(vcapSecretName)
}

var job *batchv1.Job
Expand All @@ -465,14 +469,15 @@ type jobCreateParams struct {
namePrefix string
labels map[string]string
annotations map[string]string
vcapSecretName string
imagePullSecrets []corev1.LocalObjectReference
version string
appName string
globalAccountId string
providerTenantId string
providerSubdomain string
tenantType string
env []corev1.EnvVar
EnvFrom []corev1.EnvFromSource
volumes []corev1.Volume
volumeMounts []corev1.VolumeMount
}
Expand Down Expand Up @@ -501,7 +506,7 @@ func (c *Controller) createTenantOperationJob(ctx context.Context, ctop *v1alpha
RestartPolicy: corev1.RestartPolicyNever,
ImagePullSecrets: params.imagePullSecrets,
Containers: getContainers(ctop, derivedWorkload, workload, params),
InitContainers: *updateInitContainers(derivedWorkload.initContainers, getCTOPEnv(params, ctop), params.vcapSecretName),
InitContainers: *updateInitContainers(derivedWorkload.initContainers, getCTOPEnv(params, ctop), params.EnvFrom),
Volumes: append(derivedWorkload.volumes, params.volumes...),
ServiceAccountName: derivedWorkload.serviceAccountName,
SecurityContext: derivedWorkload.podSecurityContext,
Expand All @@ -525,8 +530,8 @@ func getContainers(ctop *v1alpha1.CAPTenantOperation, derivedWorkload tentantOpe
Name: workload.Name,
Image: derivedWorkload.image,
ImagePullPolicy: derivedWorkload.imagePullPolicy,
Env: append(getCTOPEnv(params, ctop), derivedWorkload.env...),
EnvFrom: getEnvFrom(params.vcapSecretName),
Env: append(getCTOPEnv(params, ctop), append(derivedWorkload.env, params.env...)...),
EnvFrom: params.EnvFrom,
VolumeMounts: append(derivedWorkload.volumeMounts, params.volumeMounts...),
Resources: derivedWorkload.resources,
SecurityContext: derivedWorkload.securityContext,
Expand Down Expand Up @@ -645,15 +650,15 @@ func (c *Controller) createCustomTenantOperationJob(ctx context.Context, ctop *v
Name: workload.Name,
Image: workload.JobDefinition.Image,
ImagePullPolicy: workload.JobDefinition.ImagePullPolicy,
Env: append(getCTOPEnv(params, ctop), workload.JobDefinition.Env...),
EnvFrom: getEnvFrom(params.vcapSecretName),
Env: append(getCTOPEnv(params, ctop), append(workload.JobDefinition.Env, params.env...)...),
EnvFrom: params.EnvFrom,
VolumeMounts: append(workload.JobDefinition.VolumeMounts, params.volumeMounts...),
Command: workload.JobDefinition.Command,
Resources: workload.JobDefinition.Resources,
SecurityContext: workload.JobDefinition.SecurityContext,
},
},
InitContainers: *updateInitContainers(workload.JobDefinition.InitContainers, getCTOPEnv(params, ctop), params.vcapSecretName),
InitContainers: *updateInitContainers(workload.JobDefinition.InitContainers, getCTOPEnv(params, ctop), params.EnvFrom),
},
},
},
Expand Down
36 changes: 17 additions & 19 deletions internal/controller/reconcile.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,16 +50,15 @@ const (
AnnotationSubscriptionContextSecret = "sme.sap.com/subscription-context-secret"
AnnotationProviderSubAccountId = "sme.sap.com/provider-sub-account-id"
AnnotationEnableCleanupMonitoring = "sme.sap.com/enable-cleanup-monitoring"
AnnotationUseVolumeMount = "sme.sap.com/use-volume-mount"
FinalizerCAPApplication = "sme.sap.com/capapplication"
FinalizerCAPApplicationVersion = "sme.sap.com/capapplicationversion"
FinalizerCAPTenant = "sme.sap.com/captenant"
FinalizerCAPTenantOperation = "sme.sap.com/captenantoperation"
GardenerDNSClassIdentifier = "dns.gardener.cloud/class"
)

const (
CDSVolMountPrefix = "/etc/secrets/cds"
)
var defaultServiceBindingRootEnv = corev1.EnvVar{Name: "SERVICE_BINDING_ROOT", Value: "/etc/secrets"}

const (
CertificateSuffix = "certificate"
Expand Down Expand Up @@ -94,7 +93,6 @@ const (
EnvCAPOpProviderSubDomain = "CAPOP_PROVIDER_SUBDOMAIN"
EnvCAPOpSubscriptionPayload = "CAPOP_SUBSCRIPTION_PAYLOAD"
EnvVCAPServices = "VCAP_SERVICES"
EnvUseVolumeMounts = "USE_VOLUME_MOUNTS"
)

type JobState string
Expand Down Expand Up @@ -600,14 +598,14 @@ func copyMaps(originalMap map[string]string, additionalMap map[string]string) ma
return newMap
}

func updateInitContainers(initContainers []corev1.Container, additionalEnv []corev1.EnvVar, vcapSecretName string) *[]corev1.Container {
func updateInitContainers(initContainers []corev1.Container, additionalEnv []corev1.EnvVar, EnvFrom []corev1.EnvFromSource) *[]corev1.Container {
var updatedInitContainers []corev1.Container
if len(initContainers) > 0 {
updatedInitContainers = []corev1.Container{}
for _, container := range initContainers {
updatedContainer := container.DeepCopy()
updatedContainer.Env = append(updatedContainer.Env, additionalEnv...)
updatedContainer.EnvFrom = getEnvFrom(vcapSecretName)
updatedContainer.EnvFrom = EnvFrom
updatedInitContainers = append(updatedInitContainers, *updatedContainer)
}
}
Expand All @@ -620,18 +618,8 @@ func getWorkloadName(cavName, workloadName string) string {

func getVolumeMounts(serviceInfos []v1alpha1.ServiceInfo) []corev1.VolumeMount {
volumeMounts := []corev1.VolumeMount{}

for _, serviceInfo := range serviceInfos {
volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: serviceInfo.Name, MountPath: path.Join("/etc/secrets/sapcp", serviceInfo.Class, serviceInfo.Name), ReadOnly: true})
}
return volumeMounts
}

func getCAPVolumeMounts(serviceInfos []v1alpha1.ServiceInfo, mountPrefix string) []corev1.VolumeMount {
volumeMounts := []corev1.VolumeMount{}

for _, serviceInfo := range serviceInfos {
volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: serviceInfo.Name, MountPath: path.Join(mountPrefix, "requires", serviceInfo.Class, "credentials"), ReadOnly: true})
volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: serviceInfo.Name, MountPath: path.Join(defaultServiceBindingRootEnv.Value, serviceInfo.Class), ReadOnly: true})
}
return volumeMounts
}
Expand All @@ -645,6 +633,16 @@ func getVolumes(serviceInfos []v1alpha1.ServiceInfo) []corev1.Volume {
return volumes
}

func useVolumeMounts(envVars []corev1.EnvVar) bool {
return slices.ContainsFunc(envVars, func(env corev1.EnvVar) bool { return env.Name == EnvUseVolumeMounts && env.Value == "true" })
func useVolumeMounts(cav *v1alpha1.CAPApplicationVersion) bool {
value, exists := cav.Annotations[AnnotationUseVolumeMount]
return exists && value == "true"
}

func updateServiceBindingRootEnv(envVars []corev1.EnvVar) []corev1.EnvVar {
if envIndex := slices.IndexFunc(envVars, func(currentEnv corev1.EnvVar) bool { return currentEnv.Name == defaultServiceBindingRootEnv.Name }); envIndex > -1 {
envVars[envIndex] = defaultServiceBindingRootEnv
} else {
envVars = append(envVars, defaultServiceBindingRootEnv)
}
return envVars
}

0 comments on commit 57eb28b

Please sign in to comment.