chore: for rebase main

apecloud · Oct 11, 2023 · a759732 · a759732
1 parent 85e282c
commit a759732
Show file tree

Hide file tree

Showing 20 changed files with 2,062 additions and 199 deletions.
diff --git a/cmd/oteld/manager/main.go b/cmd/oteld/manager/main.go
@@ -56,7 +56,7 @@ func init() {
 
 	viper.SetConfigName("config")
 	viper.SetConfigType("yaml")
-	viper.SetDefault("OTELD_NAMESPACE", "default")
+	viper.SetDefault(constant.MonitorNamespaceEnvName, "default")
 	viper.AddConfigPath(fmt.Sprintf("/etc/%s/", constant.OTeldAppName))
 	viper.AddConfigPath(fmt.Sprintf("$HOME/.%s", constant.OTeldAppName))
 	viper.AddConfigPath(".")

diff --git a/config/crd/bases/monitor.kubeblocks.io_collectordatasources.yaml b/config/crd/bases/monitor.kubeblocks.io_collectordatasources.yaml
@@ -1,10 +1,11 @@
----
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.9.0
   creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: kubeblocks
   name: collectordatasources.monitor.kubeblocks.io
 spec:
   group: monitor.kubeblocks.io
@@ -66,16 +67,12 @@ spec:
                 - sidecar
                 - statefulset
                 type: string
-              name:
-                description: Name is the name of the data source
-                type: string
               type:
                 description: Type is the type of the data source
                 type: string
             required:
             - dataSourceList
             - exporterRef
-            - name
             - type
             type: object
           status:

diff --git a/config/crd/bases/monitor.kubeblocks.io_logsexportersinks.yaml b/config/crd/bases/monitor.kubeblocks.io_logsexportersinks.yaml
@@ -1,10 +1,11 @@
----
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.9.0
   creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: kubeblocks
   name: logsexportersinks.monitor.kubeblocks.io
 spec:
   group: monitor.kubeblocks.io

diff --git a/config/crd/bases/monitor.kubeblocks.io_metricsexportersinks.yaml b/config/crd/bases/monitor.kubeblocks.io_metricsexportersinks.yaml
@@ -1,10 +1,11 @@
----
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.9.0
   creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: kubeblocks
   name: metricsexportersinks.monitor.kubeblocks.io
 spec:
   group: monitor.kubeblocks.io

diff --git a/config/crd/bases/monitor.kubeblocks.io_oteldcollectortemplates.yaml b/config/crd/bases/monitor.kubeblocks.io_oteldcollectortemplates.yaml
diff --git a/controllers/monitor/builder/builder.go b/controllers/monitor/builder/builder.go
@@ -86,7 +86,10 @@ func MergeValMapFromYamlStr(defaultMap map[string]any, yamlStr string) {
 		defaultMap = map[string]any{}
 	}
 	valMap := map[string]any{}
-	yaml.Unmarshal([]byte(yamlStr), &valMap)
+	err := yaml.Unmarshal([]byte(yamlStr), &valMap)
+	if err != nil {
+		return
+	}
 	for k, v := range valMap {
 		defaultMap[k] = v
 	}

diff --git a/controllers/monitor/oteld_controller.go b/controllers/monitor/oteld_controller.go
@@ -76,11 +76,13 @@ type OTeldReconciler struct {
 // - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile
 func (r *OTeldReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	reqCtx := monitortypes.ReconcileCtx{
-		Ctx: ctx,
-		Req: req,
-		Log: log.FromContext(ctx).WithName("OTeldCollectorReconciler"),
+		Ctx:       ctx,
+		Req:       req,
+		Log:       log.FromContext(ctx).WithName("OTeldCollectorReconciler"),
+		Namespace: viper.GetString(constant.MonitorNamespaceEnvName),
 
-		Config: r.Config,
+		Config:      r.Config,
+		OteldCfgRef: &monitortypes.OteldCfgRef{},
 	}
 
 	// TODO prepare required resources

diff --git a/controllers/monitor/reconcile/configmap.go b/controllers/monitor/reconcile/configmap.go
@@ -32,15 +32,19 @@ import (
 func ConfigMap(reqCtx types.ReconcileCtx, params types.OTeldParams) error {
 	desired := []*corev1.ConfigMap{}
 
+	cg := reqCtx.GetConfigGenerator()
+	if cg == nil {
+		return fmt.Errorf("config generator is nil")
+	}
 	if reqCtx.GetOteldInstance(monitorv1alpha1.ModeDaemonSet) != nil {
-		configmap, _ := buildConfigMapForOteld(reqCtx.Config, reqCtx.GetOteldInstance(monitorv1alpha1.ModeDaemonSet), reqCtx.Namespace, reqCtx.GetExporters(), params.ConfigGenerator)
+		configmap, _ := buildConfigMapForOteld(reqCtx.Config, reqCtx.GetOteldInstance(monitorv1alpha1.ModeDaemonSet), reqCtx.Namespace, reqCtx.GetExporters(), cg)
 		if configmap != nil {
 			desired = append(desired, configmap)
 		}
 	}
 
 	if reqCtx.GetOteldInstance(monitorv1alpha1.ModeDeployment) != nil {
-		configmap, _ := buildConfigMapForOteld(reqCtx.Config, reqCtx.GetOteldInstance(monitorv1alpha1.ModeDeployment), reqCtx.Namespace, reqCtx.GetExporters(), nil)
+		configmap, _ := buildConfigMapForOteld(reqCtx.Config, reqCtx.GetOteldInstance(monitorv1alpha1.ModeDeployment), reqCtx.Namespace, reqCtx.GetExporters(), cg)
 		if configmap != nil {
 			desired = append(desired, configmap)
 		}

diff --git a/controllers/monitor/reconcile/deployment.go b/controllers/monitor/reconcile/deployment.go
@@ -41,9 +41,9 @@ func Deployment(reqCtx monitortypes.ReconcileCtx, params monitortypes.OTeldParam
 		namespace = viper.GetString(constant.MonitorNamespaceEnvName)
 	)
 
-	instance := reqCtx.GetOteldInstance(monitorv1alpha1.ModeDaemonSet)
+	instance := reqCtx.GetOteldInstance(monitorv1alpha1.ModeDeployment)
 
-	oteldDaemonset := buildDeploymentForOteld(reqCtx.Config, instance, namespace, OTeldName)
+	oteldDeployment := buildDeploymentForOteld(reqCtx.Config, instance, namespace, OTeldName)
 
 	existingDeployment := &appsv1.Deployment{}
 	err := k8sClient.Get(reqCtx.Ctx, client.ObjectKey{Name: OTeldName, Namespace: namespace}, existingDeployment)
@@ -53,26 +53,26 @@ func Deployment(reqCtx monitortypes.ReconcileCtx, params monitortypes.OTeldParam
 			params.Recorder.Eventf(existingDeployment, corev1.EventTypeWarning, "Failed to find secret", err.Error())
 			return err
 		}
-		if existingDeployment == nil {
+		if oteldDeployment == nil {
 			return nil
 		}
-		return k8sClient.Create(reqCtx.Ctx, oteldDaemonset)
+		return k8sClient.Create(reqCtx.Ctx, oteldDeployment)
 	}
 
-	if oteldDaemonset == nil {
+	if oteldDeployment == nil {
 		return k8sClient.Delete(reqCtx.Ctx, existingDeployment)
 	}
 
-	if reflect.DeepEqual(existingDeployment.Spec, oteldDaemonset.Spec) {
+	if reflect.DeepEqual(existingDeployment.Spec, oteldDeployment.Spec) {
 		return nil
 	}
 
 	updatedDeployment := existingDeployment.DeepCopy()
-	updatedDeployment.Spec = oteldDaemonset.Spec
-	updatedDeployment.Labels = oteldDaemonset.Labels
-	updatedDeployment.Annotations = oteldDaemonset.Annotations
+	updatedDeployment.Spec = oteldDeployment.Spec
+	updatedDeployment.Labels = oteldDeployment.Labels
+	updatedDeployment.Annotations = oteldDeployment.Annotations
 	reqCtx.Log.Info("updating existing daemonset", "daemonset", client.ObjectKeyFromObject(updatedDeployment))
-	return k8sClient.Update(reqCtx.Ctx, oteldDaemonset)
+	return k8sClient.Update(reqCtx.Ctx, oteldDeployment)
 }
 
 func buildDeploymentForOteld(config *monitortypes.Config, instance *monitortypes.OteldInstance, namespace, name string) *appsv1.Deployment {

diff --git a/controllers/monitor/reconcile/oteld.go b/controllers/monitor/reconcile/oteld.go
@@ -33,61 +33,59 @@ func OTeld(reqCtx monitortype.ReconcileCtx, params monitortype.OTeldParams) (err
 		k8sClient = params.Client
 	)
 
-	params.ConfigGenerator = monitortype.NewConfigGenerator()
+	reqCtx.SetConfigGenerator(monitortype.NewConfigGenerator())
 
 	exporter := monitortype.Exporters{}
 	metricsExporters := &v1alpha1.MetricsExporterSinkList{}
 	if err = k8sClient.List(ctx, metricsExporters); err != nil {
-		return
+		return err
 	}
 	exporter.Metricsexporter = metricsExporters.Items
 
 	logsExporters := &v1alpha1.LogsExporterSinkList{}
 	if err = k8sClient.List(ctx, logsExporters); err != nil {
-		return
+		return err
 	}
 	exporter.Logsexporter = logsExporters.Items
 
 	reqCtx.SetExporters(&exporter)
 
 	datasources := &v1alpha1.CollectorDataSourceList{}
 	if err = k8sClient.List(ctx, datasources); err != nil {
-		return
+		return err
 	}
 
 	oteldTemplates := &v1alpha1.OTeldCollectorTemplateList{}
 	if err = k8sClient.List(ctx, oteldTemplates); err != nil {
-		return
-	}
-	buildOteldInstance(reqCtx, metricsExporters, logsExporters, datasources, oteldTemplates)
-	return
-}
-
-func buildOteldInstance(
-	reqCtx monitortype.ReconcileCtx,
-	metricsExporterList *v1alpha1.MetricsExporterSinkList,
-	logsExporterList *v1alpha1.LogsExporterSinkList,
-	datasources *v1alpha1.CollectorDataSourceList,
-	templates *v1alpha1.OTeldCollectorTemplateList,
-) error {
-	instanceMap, err := BuildInstanceMapForPipline(datasources)
-	if err != nil {
 		return err
 	}
+	instanceMap, err := buildOteldInstance(datasources, oteldTemplates)
 	reqCtx.SetOteldInstanceMap(instanceMap)
 
-	for _, template := range templates.Items {
+	for _, template := range oteldTemplates.Items {
 		instance := reqCtx.GetOteldInstance(template.Spec.Mode)
 		if instance == nil {
 			continue
 		}
 		instance.OteldTemplate = &template
 	}
+	if err != nil {
+		return err
+	}
 
-	if err := reqCtx.VerifyOteldInstance(metricsExporterList, logsExporterList); err != nil {
+	if err := reqCtx.VerifyOteldInstance(metricsExporters, logsExporters); err != nil {
 		return err
 	}
-	return nil
+	return
+}
+
+func buildOteldInstance(datasources *v1alpha1.CollectorDataSourceList, templates *v1alpha1.OTeldCollectorTemplateList) (map[v1alpha1.Mode]*monitortype.OteldInstance, error) {
+	instanceMap, err := BuildInstanceMapForPipline(datasources)
+	if err != nil {
+		return nil, err
+	}
+
+	return instanceMap, nil
 }
 
 func BuildInstanceMapForPipline(datasources *v1alpha1.CollectorDataSourceList) (map[v1alpha1.Mode]*monitortype.OteldInstance, error) {
@@ -114,6 +112,10 @@ func BuildInstanceMapForPipline(datasources *v1alpha1.CollectorDataSourceList) (
 					CollectionInterval: dataSource.Spec.CollectionInterval,
 				}
 			}
+
+			for _, exporter := range dataSource.Spec.ExporterNames {
+				pipline.ExporterMap[exporter] = true
+			}
 			oteldInstance.MetricsPipline = append(oteldInstance.MetricsPipline, pipline)
 
 		case v1alpha1.LogsDataSourceType:

diff --git a/controllers/monitor/reconcile/secret.go b/controllers/monitor/reconcile/secret.go
@@ -36,15 +36,21 @@ const OteldSecretName = "oteld-secret"
 func Secret(reqCtx types.ReconcileCtx, params types.OTeldParams) error {
 	desired := []*corev1.Secret{}
 
-	if reqCtx.GetOteldInstance(monitorv1alpha1.ModeDaemonSet) != nil {
-		secret, _ := buildSecretForOteld(reqCtx.Config, reqCtx.GetOteldInstance(monitorv1alpha1.ModeDaemonSet), reqCtx.Namespace, reqCtx.GetExporters(), params.ConfigGenerator)
+	cg := reqCtx.GetConfigGenerator()
+	if cg == nil {
+		return fmt.Errorf("config generator is nil")
+	}
+	daemonsetInstance := reqCtx.GetOteldInstance(monitorv1alpha1.ModeDaemonSet)
+	if daemonsetInstance != nil {
+		secret, _ := buildSecretForOteld(reqCtx.Config, daemonsetInstance, reqCtx.Namespace, reqCtx.GetExporters(), cg)
 		if secret != nil {
 			desired = append(desired, secret)
 		}
 	}
 
-	if reqCtx.GetOteldInstance(monitorv1alpha1.ModeDeployment) != nil {
-		secret, _ := buildSecretForOteld(reqCtx.Config, reqCtx.GetOteldInstance(monitorv1alpha1.ModeDeployment), reqCtx.Namespace, reqCtx.GetExporters(), params.ConfigGenerator)
+	deploymentInstance := reqCtx.GetOteldInstance(monitorv1alpha1.ModeDeployment)
+	if deploymentInstance != nil {
+		secret, _ := buildSecretForOteld(reqCtx.Config, deploymentInstance, reqCtx.Namespace, reqCtx.GetExporters(), cg)
 		if secret != nil {
 			desired = append(desired, secret)
 		}

diff --git a/controllers/monitor/reconcile/service.go b/controllers/monitor/reconcile/service.go
@@ -35,7 +35,10 @@ func Service(reqCtx types.ReconcileCtx, params types.OTeldParams) error {
 	desired := []*corev1.Service{}
 
 	if reqCtx.GetOteldInstance(monitorv1alpha1.ModeDaemonSet) != nil {
-		svc, _ := buildSvcForOtel(reqCtx.Config, reqCtx.GetOteldInstance(monitorv1alpha1.ModeDaemonSet), reqCtx.Namespace)
+		svc, err := buildSvcForOtel(reqCtx.Config, reqCtx.GetOteldInstance(monitorv1alpha1.ModeDaemonSet), reqCtx.Namespace)
+		if err != nil {
+			return err
+		}
 		if svc != nil {
 			desired = append(desired, svc)
 		}