From dcb59eedd2aa42b84038519c959b6b0410657967 Mon Sep 17 00:00:00 2001 From: Timothy Potter Date: Thu, 28 Jul 2022 14:26:27 -0600 Subject: [PATCH 1/3] Option to merge the JVM truststore with user-supplied truststore --- api/v1beta1/solrcloud_types.go | 8 ++ .../crd/bases/solr.apache.org_solrclouds.yaml | 12 ++ ...lr.apache.org_solrprometheusexporters.yaml | 6 + controllers/solrcloud_controller_tls_test.go | 94 ++++++++++++++-- ...rprometheusexporter_controller_tls_test.go | 32 ++++++ controllers/util/solr_tls_util.go | 104 +++++++++++++++++- helm/solr-operator/crds/crds.yaml | 18 +++ 7 files changed, 266 insertions(+), 8 deletions(-) diff --git a/api/v1beta1/solrcloud_types.go b/api/v1beta1/solrcloud_types.go index 29933f56..d19581b5 100644 --- a/api/v1beta1/solrcloud_types.go +++ b/api/v1beta1/solrcloud_types.go @@ -1523,6 +1523,14 @@ type SolrTLSOptions struct { // This option is typically used with `spec.updateStrategy.restartSchedule` to restart Solr pods before the mounted TLS cert expires. // +optional MountedTLSDir *MountedTLSDirectory `json:"mountedTLSDir,omitempty"` + + // Path on the Solr image to your JVM's truststore to merge with an external truststore. + // If supplied, Solr will be configured to use the merged truststore. + // The truststore for the JVM in the default Solr image is: $JAVA_HOME/lib/security/cacerts + MergeJavaTruststore string `json:"mergeJavaTrustStore,omitempty"` + + // Password for the Java truststore to merge; defaults to "changeit" + MergeJavaTruststorePass string `json:"mergeJavaTrustStorePass,omitempty"` } // +kubebuilder:validation:Enum=Basic diff --git a/config/crd/bases/solr.apache.org_solrclouds.yaml b/config/crd/bases/solr.apache.org_solrclouds.yaml index 71d81b5d..1aa5ae86 100644 --- a/config/crd/bases/solr.apache.org_solrclouds.yaml +++ b/config/crd/bases/solr.apache.org_solrclouds.yaml @@ -4926,6 +4926,12 @@ spec: required: - key type: object + mergeJavaTrustStore: + description: 'Path on the Solr image to your JVM''s truststore to merge with an external truststore. If supplied, Solr will be configured to use the merged truststore. The truststore for the JVM in the default Solr image is: $JAVA_HOME/lib/security/cacerts' + type: string + mergeJavaTrustStorePass: + description: Password for the Java truststore to merge; defaults to "changeit" + type: string mountedTLSDir: description: Used to specify a path where the keystore, truststore, and password files for the TLS certificate are mounted by an external agent or CSI driver. This option is typically used with `spec.updateStrategy.restartSchedule` to restart Solr pods before the mounted TLS cert expires. properties: @@ -5087,6 +5093,12 @@ spec: required: - key type: object + mergeJavaTrustStore: + description: 'Path on the Solr image to your JVM''s truststore to merge with an external truststore. If supplied, Solr will be configured to use the merged truststore. The truststore for the JVM in the default Solr image is: $JAVA_HOME/lib/security/cacerts' + type: string + mergeJavaTrustStorePass: + description: Password for the Java truststore to merge; defaults to "changeit" + type: string mountedTLSDir: description: Used to specify a path where the keystore, truststore, and password files for the TLS certificate are mounted by an external agent or CSI driver. This option is typically used with `spec.updateStrategy.restartSchedule` to restart Solr pods before the mounted TLS cert expires. properties: diff --git a/config/crd/bases/solr.apache.org_solrprometheusexporters.yaml b/config/crd/bases/solr.apache.org_solrprometheusexporters.yaml index ee12b317..90772e50 100644 --- a/config/crd/bases/solr.apache.org_solrprometheusexporters.yaml +++ b/config/crd/bases/solr.apache.org_solrprometheusexporters.yaml @@ -3688,6 +3688,12 @@ spec: required: - key type: object + mergeJavaTrustStore: + description: 'Path on the Solr image to your JVM''s truststore to merge with an external truststore. If supplied, Solr will be configured to use the merged truststore. The truststore for the JVM in the default Solr image is: $JAVA_HOME/lib/security/cacerts' + type: string + mergeJavaTrustStorePass: + description: Password for the Java truststore to merge; defaults to "changeit" + type: string mountedTLSDir: description: Used to specify a path where the keystore, truststore, and password files for the TLS certificate are mounted by an external agent or CSI driver. This option is typically used with `spec.updateStrategy.restartSchedule` to restart Solr pods before the mounted TLS cert expires. properties: diff --git a/controllers/solrcloud_controller_tls_test.go b/controllers/solrcloud_controller_tls_test.go index f07cb11d..fe715df0 100644 --- a/controllers/solrcloud_controller_tls_test.go +++ b/controllers/solrcloud_controller_tls_test.go @@ -270,6 +270,64 @@ var _ = FDescribe("SolrCloud controller - TLS", func() { }) }) + FContext("Secret TLS - Merge Truststore", func() { + tlsSecretName := "tls-cert-secret-from-user" + keystorePassKey := "some-password-key-thingy" + trustStoreSecretName := "custom-truststore-secret" + trustStoreFile := "truststore.p12" + BeforeEach(func() { + solrCloud.Spec.SolrSecurity = &solrv1beta1.SolrSecurityOptions{AuthenticationType: solrv1beta1.Basic} + solrCloud.Spec.SolrTLS = createTLSOptions(tlsSecretName, keystorePassKey, false) + solrCloud.Spec.SolrTLS.TrustStoreSecret = &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: trustStoreSecretName}, + Key: trustStoreFile, + } + solrCloud.Spec.SolrTLS.TrustStorePasswordSecret = &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: trustStoreSecretName}, + Key: "truststore-pass", + } + solrCloud.Spec.SolrTLS.ClientAuth = solrv1beta1.Need // require client auth too (mTLS between the pods) + solrCloud.Spec.SolrTLS.MergeJavaTruststore = util.DefaultJvmTruststore + }) + FIt("has the correct resources", func() { + verifyReconcileUserSuppliedTLS(ctx, solrCloud, false, false) + }) + }) + + FContext("Secret TLS - Merge Truststores for Server & Client", func() { + serverCertSecret := "tls-server-cert" + clientCertSecret := "tls-client-cert" + keystorePassKey := "some-password-key-thingy" + BeforeEach(func() { + solrCloud.Spec.SolrTLS = createTLSOptions(serverCertSecret, keystorePassKey, false) + solrCloud.Spec.SolrTLS.MergeJavaTruststore = util.DefaultJvmTruststore + + solrCloud.Spec.SolrTLS.ClientAuth = solrv1beta1.Need + + // Additional client cert + solrCloud.Spec.SolrClientTLS = &solrv1beta1.SolrTLSOptions{ + KeyStorePasswordSecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: clientCertSecret}, + Key: keystorePassKey, + }, + PKCS12Secret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: clientCertSecret}, + Key: util.DefaultPkcs12KeystoreFile, + }, + TrustStoreSecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: clientCertSecret}, + Key: util.DefaultPkcs12TruststoreFile, + }, + RestartOnTLSSecretUpdate: false, + MergeJavaTruststore: util.DefaultJvmTruststore, + } + }) + FIt("has the correct resources", func() { + By("checking that the User supplied TLS Config is correct in the generated StatefulSet") + verifyReconcileUserSuppliedTLS(ctx, solrCloud, false, false) + }) + }) + FContext("Secret TLS - Pkcs12 Conversion", func() { tlsSecretName := "tls-cert-secret-from-user-no-pkcs12" keystorePassKey := "some-password-key-thingy" @@ -466,6 +524,11 @@ func expectTLSConfigOnPodTemplateWithGomega(g Gomega, tls *solrv1beta1.SolrTLSOp expectedTrustStorePath = util.DefaultTrustStorePath + "/" + tls.TrustStoreSecret.Key } + // did we merge the Java truststore with the user-supplied? + if tls.MergeJavaTruststore != "" { + expectedTrustStorePath = "/var/solr/tls-merged/truststore.p12" + } + if tls.PKCS12Secret != nil { expectTLSEnvVarsWithGomega(g, mainContainer.Env, tls.KeyStorePasswordSecret.Name, tls.KeyStorePasswordSecret.Key, needsPkcs12InitContainer, expectedTrustStorePath, clientOnly, clientTLS) } else if tls.TrustStoreSecret != nil { @@ -477,7 +540,11 @@ func expectTLSConfigOnPodTemplateWithGomega(g Gomega, tls *solrv1beta1.SolrTLSOp g.Expect(len(envVars)).To(Equal(3), "Wrong number of SSL env vars for Pod") for _, envVar := range envVars { if envVar.Name == "SOLR_SSL_CLIENT_TRUST_STORE" { - g.Expect(envVar.Value).To(Equal(expectedTrustStorePath), "Wrong envVar value for %s", envVar.Name) + expectedPath := expectedTrustStorePath + if clientTLS != nil && clientTLS.MergeJavaTruststore != "" { + expectedPath = "/var/solr/tls-client-merged/truststore.p12" + } + g.Expect(envVar.Value).To(Equal(expectedPath), "Wrong envVar value for %s", envVar.Name) } if envVar.Name == "SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD" { g.Expect(envVar.Value).To(BeEmpty(), "EnvVar %s should not use an explicit Value, since it is populated from a secret", envVar.Name) @@ -531,6 +598,20 @@ func expectTLSConfigOnPodTemplateWithGomega(g Gomega, tls *solrv1beta1.SolrTLSOp g.Expect(expInitContainer.Command[2]).To(Equal(expCmd), "Wrong TLS initContainer command") } + if tls.MergeJavaTruststore != "" { + // verify the merge-truststore initContainer was created as well + g.Expect(podTemplate.Spec.InitContainers).To(Not(BeEmpty()), "An init container should exist to merge truststores") + var expInitContainer *corev1.Container = nil + for _, cnt := range podTemplate.Spec.InitContainers { + if cnt.Name == "merge-truststore" { + expInitContainer = &cnt + break + } + } + g.Expect(expInitContainer).To(Not(BeNil()), "Didn't find the merge-truststore InitContainer in the sts!") + g.Expect(expInitContainer.Command[2]).To(ContainSubstring("keytool"), "Wrong merge initContainer command") + } + if tls.ClientAuth == solrv1beta1.Need { // verify the probes use a command with SSL opts tlsProps := "" @@ -611,11 +692,6 @@ func expectMountedTLSDirEnvVars(envVars []corev1.EnvVar, solrCloud *solrv1beta1. } } -// ensure the TLS related env vars are set for the Solr pod -func expectTLSEnvVars(envVars []corev1.EnvVar, expectedKeystorePasswordSecretName string, expectedKeystorePasswordSecretKey string, needsPkcs12InitContainer bool, expectedTruststorePath string, clientOnly bool, clientTLS *solrv1beta1.SolrTLSOptions) { - expectTLSEnvVarsWithGomega(Default, envVars, expectedKeystorePasswordSecretName, expectedKeystorePasswordSecretKey, needsPkcs12InitContainer, expectedTruststorePath, clientOnly, clientTLS) -} - // ensure the TLS related env vars are set for the Solr pod func expectTLSEnvVarsWithGomega(g Gomega, envVars []corev1.EnvVar, expectedKeystorePasswordSecretName string, expectedKeystorePasswordSecretKey string, needsPkcs12InitContainer bool, expectedTruststorePath string, clientOnly bool, clientTLS *solrv1beta1.SolrTLSOptions) { g.Expect(envVars).To(Not(BeNil()), "Env Vars should not be nil") @@ -678,7 +754,11 @@ func expectTLSEnvVarsWithGomega(g Gomega, envVars []corev1.EnvVar, expectedKeyst } if envVar.Name == "SOLR_SSL_CLIENT_TRUST_STORE" { - g.Expect(envVar.Value).To(Equal("/var/solr/client-tls/truststore.p12"), "Wrong envVar value for %s", envVar.Name) + expectedPath := "/var/solr/client-tls/truststore.p12" + if clientTLS.MergeJavaTruststore != "" { + expectedPath = "/var/solr/tls-client-merged/truststore.p12" + } + g.Expect(envVar.Value).To(Equal(expectedPath), "Wrong envVar value for %s", envVar.Name) } if envVar.Name == "SOLR_SSL_CLIENT_KEY_STORE_PASSWORD" || envVar.Name == "SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD" { diff --git a/controllers/solrprometheusexporter_controller_tls_test.go b/controllers/solrprometheusexporter_controller_tls_test.go index 73b6d44b..8ec5a1c2 100644 --- a/controllers/solrprometheusexporter_controller_tls_test.go +++ b/controllers/solrprometheusexporter_controller_tls_test.go @@ -228,6 +228,38 @@ var _ = FDescribe("SolrPrometheusExporter controller - TLS", func() { }) }) + FContext("TLS Secret - Merge TrustStore Only", func() { + tlsSecretName := "tls-cert-secret-from-user" + BeforeEach(func() { + solrPrometheusExporter.Spec = solrv1beta1.SolrPrometheusExporterSpec{ + SolrReference: solrv1beta1.SolrReference{ + Cloud: &solrv1beta1.SolrCloudReference{ + ZookeeperConnectionInfo: &solrv1beta1.ZookeeperConnectionInfo{ + InternalConnectionString: "host:2181", + ChRoot: "/this/path", + }, + }, + SolrTLS: &solrv1beta1.SolrTLSOptions{ + TrustStorePasswordSecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: tlsSecretName}, + Key: "keystore-passwords-are-important", + }, + TrustStoreSecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: tlsSecretName}, + Key: util.DefaultPkcs12TruststoreFile, + }, + RestartOnTLSSecretUpdate: false, + MergeJavaTruststore: util.DefaultJvmTruststore, + }, + }, + } + }) + FIt("has the correct resources", func() { + By("testing the SolrPrometheusExporter Deployment") + testReconcileWithTruststoreOnly(ctx, solrPrometheusExporter, tlsSecretName) + }) + }) + FContext("TLS Secret - TrustStore Only - Restart on Secret Update", func() { tlsSecretName := "tls-cert-secret-from-user" BeforeEach(func() { diff --git a/controllers/util/solr_tls_util.go b/controllers/util/solr_tls_util.go index 14d9251b..fed20d6d 100644 --- a/controllers/util/solr_tls_util.go +++ b/controllers/util/solr_tls_util.go @@ -43,6 +43,7 @@ const ( DefaultPkcs12KeystoreFile = "keystore.p12" DefaultPkcs12TruststoreFile = "truststore.p12" DefaultKeystorePasswordFile = "keystore-password" + DefaultJvmTruststore = "$JAVA_HOME/lib/security/cacerts" ) // Helper struct for holding server and/or client cert config @@ -72,6 +73,7 @@ type TLSConfig struct { TruststorePath string VolumePrefix string Namespace string + EnvVarPrefix string } // Get a TLSCerts struct for reconciling TLS on a SolrCloud @@ -83,6 +85,7 @@ func TLSCertsForSolrCloud(instance *solr.SolrCloud) *TLSCerts { TruststorePath: DefaultTrustStorePath, CertMd5Annotation: SolrTlsCertMd5Annotation, Namespace: instance.Namespace, + EnvVarPrefix: "SOLR_SSL", }, InitContainerImage: instance.Spec.BusyBoxImage, } @@ -94,6 +97,7 @@ func TLSCertsForSolrCloud(instance *solr.SolrCloud) *TLSCerts { VolumePrefix: "client-", CertMd5Annotation: SolrClientTlsCertMd5Annotation, Namespace: instance.Namespace, + EnvVarPrefix: "SOLR_SSL_CLIENT", } } return tls @@ -117,6 +121,7 @@ func TLSCertsForExporter(prometheusExporter *solr.SolrPrometheusExporter) *TLSCe TruststorePath: DefaultTrustStorePath, CertMd5Annotation: SolrClientTlsCertMd5Annotation, Namespace: prometheusExporter.Namespace, + EnvVarPrefix: "SOLR_SSL_CLIENT", }, InitContainerImage: bbImage, } @@ -129,8 +134,9 @@ func TLSCertsForExporter(prometheusExporter *solr.SolrPrometheusExporter) *TLSCe func (tls *TLSCerts) enableTLSOnSolrCloudStatefulSet(stateful *appsv1.StatefulSet) { serverCert := tls.ServerConfig - // Add the SOLR_SSL_* vars to the main container's environment mainContainer := &stateful.Spec.Template.Spec.Containers[0] + + // Add the SOLR_SSL_* vars to the main container's environment mainContainer.Env = append(mainContainer.Env, serverCert.serverEnvVars()...) // Was a client cert mounted too? If so, add the client env vars to the main container as well if tls.ClientConfig != nil { @@ -151,6 +157,13 @@ func (tls *TLSCerts) enableTLSOnSolrCloudStatefulSet(stateful *appsv1.StatefulSe // use an initContainer to create the wrapper script in the initdb stateful.Spec.Template.Spec.InitContainers = append(stateful.Spec.Template.Spec.InitContainers, tls.generateTLSInitdbScriptInitContainer()) } + + if serverCert.Options.MergeJavaTruststore != "" { + serverCert.addMergeTruststoreInitContainer(&stateful.Spec.Template) + } + if tls.ClientConfig != nil && tls.ClientConfig.Options.MergeJavaTruststore != "" { + tls.ClientConfig.addMergeTruststoreInitContainer(&stateful.Spec.Template) + } } // Enrich the config for a Prometheus Exporter Deployment to allow the exporter to make requests to TLS enabled Solr pods @@ -172,6 +185,11 @@ func (tls *TLSCerts) enableTLSOnExporterDeployment(deployment *appsv1.Deployment // volumes and mounts for TLS when using the mounted dir option clientCert.mountTLSWrapperScriptAndInitContainer(deployment, tls.InitContainerImage) } + + if clientCert.Options.MergeJavaTruststore != "" { + // add an initContainer that merges the truststores together + clientCert.addMergeTruststoreInitContainer(&deployment.Spec.Template) + } } // Configures a pod template (either StatefulSet or Deployment) to mount the TLS files from a secret @@ -807,3 +825,87 @@ func verifyTLSSecretConfig(client *client.Client, secretName string, secretNames return foundTLSSecret, nil } + +// Adds an initContainer that merges the JVM's truststore with the user-supplied truststore +func (tls *TLSConfig) addMergeTruststoreInitContainer(template *corev1.PodTemplateSpec) { + mainContainer := &template.Spec.Containers[0] + + // supports either client or server truststore env var names + envVar := tls.trustStoreEnvVarName() + + // build an initContainer that merges the truststores together + initContainer, mergeVol, mergeMount := + tls.buildMergeTruststoreInitContainer(mainContainer.Image, mainContainer.ImagePullPolicy, mainContainer.Env) + template.Spec.InitContainers = append(template.Spec.InitContainers, *initContainer) + template.Spec.Volumes = append(template.Spec.Volumes, *mergeVol) + mainContainer.VolumeMounts = append(mainContainer.VolumeMounts, *mergeMount) + // point the truststore to the merged + updateEnvVars := []corev1.EnvVar{} + for _, n := range mainContainer.Env { + // copy over all but the one we're swapping out ... + if n.Name != envVar { + updateEnvVars = append(updateEnvVars, n) + } + } + mainContainer.Env = append(updateEnvVars, corev1.EnvVar{Name: envVar, Value: mergeMount.MountPath + "/truststore.p12"}) +} + +func (tls *TLSConfig) buildMergeTruststoreInitContainer(solrImageName string, imagePullPolicy corev1.PullPolicy, serverEnvVars []corev1.EnvVar) (*corev1.Container, *corev1.Volume, *corev1.VolumeMount) { + // StatefulSet might have a client and server SSL config, so need to vary the initContainer and vol mount names + envVar := tls.trustStoreEnvVarName() + passEnvVar := envVar + "_PASSWORD" + volName := fmt.Sprintf("merge-%struststore", tls.VolumePrefix) + mergedEnvVar := tls.mergedTruststoreEnvVarName() + mountPath := tls.mergedTruststoreMountPath() + envVars := []corev1.EnvVar{ + { + Name: mergedEnvVar, + Value: mountPath + "/truststore.p12", + }, + } + envVars = append(envVars, serverEnvVars...) + + // the default truststore pass for most JVM is "changeit" + javaTruststorePass := tls.Options.MergeJavaTruststorePass + if javaTruststorePass == "" { + javaTruststorePass = "changeit" + } + + // Use Java's keytool to merge the JVM's truststore with the user-supplied truststore + cmd := fmt.Sprintf(`keytool -importkeystore -srckeystore %s -srcstorepass %s -destkeystore $%s -deststoretype pkcs12 -deststorepass $%s; +keytool -importkeystore -srckeystore $%s -srcstorepass $%s -destkeystore $%s -deststoretype pkcs12 -deststorepass $%s`, + tls.Options.MergeJavaTruststore, javaTruststorePass, mergedEnvVar, passEnvVar, envVar, passEnvVar, mergedEnvVar, passEnvVar) + + // Need volume mounts from mainContainer (to get access to the user-supplied truststore) + _, mounts := tls.volumesAndMounts() + // Mount a shared dir between the initContainer and mainContainer to store the merged truststore + mergeVol := &corev1.Volume{Name: volName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}} + mergeMount := &corev1.VolumeMount{Name: volName, ReadOnly: false, MountPath: mountPath} + mounts = append(mounts, *mergeMount) + + return &corev1.Container{ + Name: volName, + Image: solrImageName, // we use the Solr image for the initContainer since it has the truststore and keytool + ImagePullPolicy: imagePullPolicy, + TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: "File", + Command: []string{"sh", "-c", cmd}, + VolumeMounts: mounts, + Env: envVars, + }, mergeVol, mergeMount +} + +func (tls *TLSConfig) trustStoreEnvVarName() string { + return tls.EnvVarPrefix + "_TRUST_STORE" +} + +func (tls *TLSConfig) mergedTruststoreEnvVarName() string { + if tls.VolumePrefix == "client-" { + return "MERGED_CLIENT_TRUST_STORE" + } + return "MERGED_TRUST_STORE" +} + +func (tls *TLSConfig) mergedTruststoreMountPath() string { + return fmt.Sprintf("/var/solr/tls-%smerged", tls.VolumePrefix) +} diff --git a/helm/solr-operator/crds/crds.yaml b/helm/solr-operator/crds/crds.yaml index 995f81c5..77fa5fd2 100644 --- a/helm/solr-operator/crds/crds.yaml +++ b/helm/solr-operator/crds/crds.yaml @@ -5158,6 +5158,12 @@ spec: required: - key type: object + mergeJavaTrustStore: + description: 'Path on the Solr image to your JVM''s truststore to merge with an external truststore. If supplied, Solr will be configured to use the merged truststore. The truststore for the JVM in the default Solr image is: $JAVA_HOME/lib/security/cacerts' + type: string + mergeJavaTrustStorePass: + description: Password for the Java truststore to merge; defaults to "changeit" + type: string mountedTLSDir: description: Used to specify a path where the keystore, truststore, and password files for the TLS certificate are mounted by an external agent or CSI driver. This option is typically used with `spec.updateStrategy.restartSchedule` to restart Solr pods before the mounted TLS cert expires. properties: @@ -5319,6 +5325,12 @@ spec: required: - key type: object + mergeJavaTrustStore: + description: 'Path on the Solr image to your JVM''s truststore to merge with an external truststore. If supplied, Solr will be configured to use the merged truststore. The truststore for the JVM in the default Solr image is: $JAVA_HOME/lib/security/cacerts' + type: string + mergeJavaTrustStorePass: + description: Password for the Java truststore to merge; defaults to "changeit" + type: string mountedTLSDir: description: Used to specify a path where the keystore, truststore, and password files for the TLS certificate are mounted by an external agent or CSI driver. This option is typically used with `spec.updateStrategy.restartSchedule` to restart Solr pods before the mounted TLS cert expires. properties: @@ -10119,6 +10131,12 @@ spec: required: - key type: object + mergeJavaTrustStore: + description: 'Path on the Solr image to your JVM''s truststore to merge with an external truststore. If supplied, Solr will be configured to use the merged truststore. The truststore for the JVM in the default Solr image is: $JAVA_HOME/lib/security/cacerts' + type: string + mergeJavaTrustStorePass: + description: Password for the Java truststore to merge; defaults to "changeit" + type: string mountedTLSDir: description: Used to specify a path where the keystore, truststore, and password files for the TLS certificate are mounted by an external agent or CSI driver. This option is typically used with `spec.updateStrategy.restartSchedule` to restart Solr pods before the mounted TLS cert expires. properties: From 042097029616bd737c5d9467089d7371f2ba4c6f Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 1 Aug 2022 16:49:16 -0400 Subject: [PATCH 2/3] Add some bulk to the tests --- controllers/solrcloud_controller_tls_test.go | 88 +++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/controllers/solrcloud_controller_tls_test.go b/controllers/solrcloud_controller_tls_test.go index fe715df0..4d810bb9 100644 --- a/controllers/solrcloud_controller_tls_test.go +++ b/controllers/solrcloud_controller_tls_test.go @@ -609,7 +609,93 @@ func expectTLSConfigOnPodTemplateWithGomega(g Gomega, tls *solrv1beta1.SolrTLSOp } } g.Expect(expInitContainer).To(Not(BeNil()), "Didn't find the merge-truststore InitContainer in the sts!") - g.Expect(expInitContainer.Command[2]).To(ContainSubstring("keytool"), "Wrong merge initContainer command") + g.Expect(expInitContainer.Command[2]).To(ContainSubstring("keytool"), "Wrong merge-truststore initContainer command") + providedTruststorePath := util.DefaultKeyStorePath + trustStoreMount := 0 + if tls.TrustStoreSecret != nil && (tls.PKCS12Secret == nil || tls.PKCS12Secret.Name != tls.TrustStoreSecret.Name) { + providedTruststorePath = util.DefaultTrustStorePath + if tls.PKCS12Secret != nil { + trustStoreMount = 1 + } + } + g.Expect(expInitContainer.VolumeMounts).To(Not(BeEmpty()), "There are no volume mounts for the merge-truststore InitContainer!") + g.Expect(expInitContainer.VolumeMounts[trustStoreMount].MountPath).To(Equal(providedTruststorePath), "Wrong mountPath for the provided truststore in the merge-truststore InitContainer!") + g.Expect(expInitContainer.VolumeMounts[trustStoreMount+1].MountPath).To(Equal("/var/solr/tls-merged"), "Wrong mountPath for the merge-truststore InitContainer!") + sslTrustStoreVar := "SOLR_SSL_TRUST_STORE" + if clientOnly { + sslTrustStoreVar = "SOLR_SSL_CLIENT_TRUST_STORE" + } + g.Expect(expInitContainer.Command[2]).To( + And( + ContainSubstring("-srckeystore "+tls.MergeJavaTruststore), + ContainSubstring("-srckeystore $"+sslTrustStoreVar), + ), "Wrong paths in the merge-truststore InitContainer command!") + + // verify the mountpath in the Solr container + var mergedTruststoreVolumeMount *corev1.VolumeMount = nil + for _, vm := range podTemplate.Spec.Containers[0].VolumeMounts { + if vm.Name == "merge-truststore" { + mergedTruststoreVolumeMount = &vm + break + } + } + g.Expect(mergedTruststoreVolumeMount).To(Not(BeNil()), "Didn't find the merge-truststore VolumeMount in the solr container!") + g.Expect(mergedTruststoreVolumeMount.MountPath).To(Equal("/var/solr/tls-merged"), "Wrong merge-truststore VolumeMount in the solr container") + + for _, envVar := range expInitContainer.Env { + if envVar.Name == "MERGED_TRUST_STORE" { + g.Expect(envVar.Value).To(Equal("/var/solr/tls-merged/truststore.p12"), "EnvVar %s has the wrong string value", envVar.Name) + g.Expect(envVar.ValueFrom).To(BeNil(), "EnvVar %s must not have a ValueFrom", envVar.Name) + } + } + } + + if clientTLS != nil && clientTLS.MergeJavaTruststore != "" { + // verify the merge-truststore initContainer was created as well + g.Expect(podTemplate.Spec.InitContainers).To(Not(BeEmpty()), "An init container should exist to merge client truststores") + var expInitContainer *corev1.Container = nil + for _, cnt := range podTemplate.Spec.InitContainers { + if cnt.Name == "merge-client-truststore" { + expInitContainer = &cnt + break + } + } + g.Expect(expInitContainer).To(Not(BeNil()), "Didn't find the merge-client-truststore InitContainer in the sts!") + g.Expect(expInitContainer.Command[2]).To(ContainSubstring("keytool"), "Wrong merge-client-truststore initContainer command") + g.Expect(expInitContainer.VolumeMounts).To(Not(BeEmpty()), "There are no volume mounts for the merge-client-truststore InitContainer!") + providedTruststorePath := util.DefaultClientKeyStorePath + trustStoreMount := 0 + if clientTLS.TrustStoreSecret != nil && (clientTLS.PKCS12Secret == nil || clientTLS.PKCS12Secret.Name != clientTLS.TrustStoreSecret.Name) { + providedTruststorePath = util.DefaultClientTrustStorePath + if clientTLS.PKCS12Secret != nil && clientTLS.PKCS12Secret.Name != clientTLS.TrustStoreSecret.Name { + trustStoreMount = 1 + } + } + g.Expect(expInitContainer.VolumeMounts[trustStoreMount].MountPath).To(Equal(providedTruststorePath), "Wrong mountPath for the provided truststore in the merge-client-truststore InitContainer!") + g.Expect(expInitContainer.VolumeMounts[trustStoreMount+1].MountPath).To(Equal("/var/solr/tls-client-merged"), "Wrong mountPath for the merge-client-truststore InitContainer!") + g.Expect(expInitContainer.Command[2]).To( + And( + ContainSubstring("-srckeystore "+clientTLS.MergeJavaTruststore), + ContainSubstring("-srckeystore $SOLR_SSL_CLIENT_TRUST_STORE"), + ), "Wrong paths in the merge-client-truststore InitContainer command!") + + // verify the mountpath in the Solr container + var mergedTruststoreVolumeMount *corev1.VolumeMount = nil + for _, vm := range podTemplate.Spec.Containers[0].VolumeMounts { + if vm.Name == "merge-client-truststore" { + mergedTruststoreVolumeMount = &vm + break + } + } + g.Expect(mergedTruststoreVolumeMount).To(Not(BeNil()), "Didn't find the merge-client-truststore VolumeMount in the solr container!") + g.Expect(mergedTruststoreVolumeMount.MountPath).To(Equal("/var/solr/tls-client-merged"), "Wrong merge-client-truststore VolumeMount in the solr container") + + for _, envVar := range expInitContainer.Env { + if envVar.Name == "MERGED_TRUST_STORE" { + g.Expect(envVar.Value).To(Equal("/var/solr/tls-client-merged/truststore.p12"), "EnvVar %s has the wrong string value", envVar.Name) + g.Expect(envVar.ValueFrom).To(BeNil(), "EnvVar %s must not have a ValueFrom", envVar.Name) + } + } } if tls.ClientAuth == solrv1beta1.Need { From f99abc04a251292f3bffdfc98fa594b55aeadba7 Mon Sep 17 00:00:00 2001 From: Timothy Potter Date: Mon, 1 Aug 2022 16:26:08 -0600 Subject: [PATCH 3/3] Add resource requests for the merge initContainer --- api/v1beta1/solrcloud_types.go | 2 ++ controllers/util/solr_tls_util.go | 51 ++++++++++++++++++++++++++----- helm/solr-operator/Chart.yaml | 8 +++++ 3 files changed, 54 insertions(+), 7 deletions(-) diff --git a/api/v1beta1/solrcloud_types.go b/api/v1beta1/solrcloud_types.go index d19581b5..cb3bae63 100644 --- a/api/v1beta1/solrcloud_types.go +++ b/api/v1beta1/solrcloud_types.go @@ -1527,9 +1527,11 @@ type SolrTLSOptions struct { // Path on the Solr image to your JVM's truststore to merge with an external truststore. // If supplied, Solr will be configured to use the merged truststore. // The truststore for the JVM in the default Solr image is: $JAVA_HOME/lib/security/cacerts + // +optional MergeJavaTruststore string `json:"mergeJavaTrustStore,omitempty"` // Password for the Java truststore to merge; defaults to "changeit" + // +optional MergeJavaTruststorePass string `json:"mergeJavaTrustStorePass,omitempty"` } diff --git a/controllers/util/solr_tls_util.go b/controllers/util/solr_tls_util.go index fed20d6d..9eab7220 100644 --- a/controllers/util/solr_tls_util.go +++ b/controllers/util/solr_tls_util.go @@ -24,6 +24,7 @@ import ( solr "github.com/apache/solr-operator/api/v1beta1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "strconv" @@ -46,6 +47,11 @@ const ( DefaultJvmTruststore = "$JAVA_HOME/lib/security/cacerts" ) +var ( + DefaultMergeTruststoreInitContainerMemory = resource.NewScaledQuantity(64, 6) + DefaultMergeTruststoreInitContainerCPU = resource.NewMilliQuantity(50, resource.DecimalExponent) +) + // Helper struct for holding server and/or client cert config // This struct is intended for internal use only and is only exposed outside the package so that the controllers can access type TLSCerts struct { @@ -55,6 +61,8 @@ type TLSCerts struct { ClientConfig *TLSConfig // Image used for initContainers that help configure the TLS settings InitContainerImage *solr.ContainerImage + // Holds custom resource requests for the initContainer if the defaults are overridden by config + InitContainerResources *corev1.ResourceRequirements } // Holds TLS options from the user config as well as other config properties determined during reconciliation @@ -78,6 +86,7 @@ type TLSConfig struct { // Get a TLSCerts struct for reconciling TLS on a SolrCloud func TLSCertsForSolrCloud(instance *solr.SolrCloud) *TLSCerts { + tls := &TLSCerts{ ServerConfig: &TLSConfig{ Options: instance.Spec.SolrTLS.DeepCopy(), @@ -100,6 +109,14 @@ func TLSCertsForSolrCloud(instance *solr.SolrCloud) *TLSCerts { EnvVarPrefix: "SOLR_SSL_CLIENT", } } + + if instance.Spec.CustomSolrKubeOptions.PodOptions != nil { + resources := instance.Spec.CustomSolrKubeOptions.PodOptions.DefaultInitContainerResources + if resources.Limits != nil || resources.Requests != nil { + tls.InitContainerResources = &resources + } + } + return tls } @@ -114,7 +131,7 @@ func TLSCertsForExporter(prometheusExporter *solr.SolrPrometheusExporter) *TLSCe PullPolicy: solr.DefaultPullPolicy, } } - return &TLSCerts{ + tls := &TLSCerts{ ClientConfig: &TLSConfig{ Options: prometheusExporter.Spec.SolrReference.SolrTLS.DeepCopy(), KeystorePath: DefaultKeyStorePath, @@ -125,6 +142,15 @@ func TLSCertsForExporter(prometheusExporter *solr.SolrPrometheusExporter) *TLSCe }, InitContainerImage: bbImage, } + + if prometheusExporter.Spec.CustomKubeOptions.PodOptions != nil { + resources := prometheusExporter.Spec.CustomKubeOptions.PodOptions.DefaultInitContainerResources + if resources.Limits != nil || resources.Requests != nil { + tls.InitContainerResources = &resources + } + } + + return tls } // Enrich the config for a SolrCloud StatefulSet to enable TLS, either loaded from a secret or @@ -159,10 +185,10 @@ func (tls *TLSCerts) enableTLSOnSolrCloudStatefulSet(stateful *appsv1.StatefulSe } if serverCert.Options.MergeJavaTruststore != "" { - serverCert.addMergeTruststoreInitContainer(&stateful.Spec.Template) + serverCert.addMergeTruststoreInitContainer(&stateful.Spec.Template, tls.InitContainerResources) } if tls.ClientConfig != nil && tls.ClientConfig.Options.MergeJavaTruststore != "" { - tls.ClientConfig.addMergeTruststoreInitContainer(&stateful.Spec.Template) + tls.ClientConfig.addMergeTruststoreInitContainer(&stateful.Spec.Template, tls.InitContainerResources) } } @@ -188,7 +214,7 @@ func (tls *TLSCerts) enableTLSOnExporterDeployment(deployment *appsv1.Deployment if clientCert.Options.MergeJavaTruststore != "" { // add an initContainer that merges the truststores together - clientCert.addMergeTruststoreInitContainer(&deployment.Spec.Template) + clientCert.addMergeTruststoreInitContainer(&deployment.Spec.Template, tls.InitContainerResources) } } @@ -827,7 +853,7 @@ func verifyTLSSecretConfig(client *client.Client, secretName string, secretNames } // Adds an initContainer that merges the JVM's truststore with the user-supplied truststore -func (tls *TLSConfig) addMergeTruststoreInitContainer(template *corev1.PodTemplateSpec) { +func (tls *TLSConfig) addMergeTruststoreInitContainer(template *corev1.PodTemplateSpec, initContRes *corev1.ResourceRequirements) { mainContainer := &template.Spec.Containers[0] // supports either client or server truststore env var names @@ -836,6 +862,9 @@ func (tls *TLSConfig) addMergeTruststoreInitContainer(template *corev1.PodTempla // build an initContainer that merges the truststores together initContainer, mergeVol, mergeMount := tls.buildMergeTruststoreInitContainer(mainContainer.Image, mainContainer.ImagePullPolicy, mainContainer.Env) + if initContRes != nil { + initContainer.Resources = *initContRes + } template.Spec.InitContainers = append(template.Spec.InitContainers, *initContainer) template.Spec.Volumes = append(template.Spec.Volumes, *mergeVol) mainContainer.VolumeMounts = append(mainContainer.VolumeMounts, *mergeMount) @@ -883,7 +912,12 @@ keytool -importkeystore -srckeystore $%s -srcstorepass $%s -destkeystore $%s -de mergeMount := &corev1.VolumeMount{Name: volName, ReadOnly: false, MountPath: mountPath} mounts = append(mounts, *mergeMount) - return &corev1.Container{ + volumePrepResources := corev1.ResourceList{ + corev1.ResourceCPU: *DefaultMergeTruststoreInitContainerCPU, + corev1.ResourceMemory: *DefaultMergeTruststoreInitContainerMemory, + } + + initCont := &corev1.Container{ Name: volName, Image: solrImageName, // we use the Solr image for the initContainer since it has the truststore and keytool ImagePullPolicy: imagePullPolicy, @@ -892,7 +926,10 @@ keytool -importkeystore -srckeystore $%s -srcstorepass $%s -destkeystore $%s -de Command: []string{"sh", "-c", cmd}, VolumeMounts: mounts, Env: envVars, - }, mergeVol, mergeMount + Resources: corev1.ResourceRequirements{Requests: volumePrepResources, Limits: volumePrepResources}, + } + + return initCont, mergeVol, mergeMount } func (tls *TLSConfig) trustStoreEnvVarName() string { diff --git a/helm/solr-operator/Chart.yaml b/helm/solr-operator/Chart.yaml index 64150f8f..3fabfad8 100644 --- a/helm/solr-operator/Chart.yaml +++ b/helm/solr-operator/Chart.yaml @@ -175,6 +175,14 @@ annotations: url: https://github.com/apache/solr-operator/issues/435 - name: Github PR url: https://github.com/apache/solr-operator/pull/456 + - kind: added + description: TLS config provides an option to merge the JVM truststore from the Solr image with a user-supplied truststore. + links: + - name: Github Issue + url: https://github.com/apache/solr-operator/issues/390 + - name: Github PR + url: https://github.com/apache/solr-operator/pull/461 + artifacthub.io/images: | - name: solr-operator image: apache/solr-operator:v0.6.0-prerelease