feat: allow to override sidecar configuration (#551)

* feat(operator): expose sidecar configuration Signed-off-by: Dominik Rosiek <[email protected]> * feat(helm): expose sidecar configuration Signed-off-by: Dominik Rosiek <[email protected]> * tests(operator): add e2e tests Signed-off-by: Dominik Rosiek <[email protected]> * chore: set default mountPath Signed-off-by: Dominik Rosiek <[email protected]> * tests(e2e): fix test with custom configuration Signed-off-by: Dominik Rosiek <[email protected]> * refactor: use const for volume mount name Signed-off-by: Dominik Rosiek <[email protected]> * docs: add information how to override sidecar configuration Signed-off-by: Dominik Rosiek <[email protected]> * feat: use configMap as exemplar for all namespaces Signed-off-by: Dominik Rosiek <[email protected]> * fix: fix rebase Signed-off-by: Dominik Rosiek <[email protected]> * refactor: move configmap to values * tests: fix e2e Signed-off-by: Dominik Rosiek <[email protected]> * chor: install kuttl in vagrant Signed-off-by: Dominik Rosiek <[email protected]> * refactor(operator): move deletion logic to new function Signed-off-by: Dominik Rosiek <[email protected]> * tests: adjust e2e test for custom configuration Signed-off-by: Dominik Rosiek <[email protected]> * feat(helm): update clusterRole to support configmaps management Signed-off-by: Dominik Rosiek <[email protected]> * tests: fix sidecar test for custom configuration test Signed-off-by: Dominik Rosiek <[email protected]> * tests: remove resource requests Signed-off-by: Dominik Rosiek <[email protected]> * tests: set resources to 0 Signed-off-by: Dominik Rosiek <[email protected]> * feat(helm): make sidecar configuration optional Signed-off-by: Dominik Rosiek <[email protected]> * chore(helm): use empty for conditional checks in templates Signed-off-by: Dominik Rosiek <[email protected]> * docs: update overriding tailing sidecar configuration Signed-off-by: Dominik Rosiek <[email protected]> --------- Signed-off-by: Dominik Rosiek <[email protected]>
SumoLogic · Jul 4, 2023 · d0f16a3 · d0f16a3
1 parent f21acb4
commit d0f16a3
Show file tree

Hide file tree

Showing 37 changed files with 951 additions and 27 deletions.
diff --git a/.github/workflows/pull_requests.yml b/.github/workflows/pull_requests.yml
@@ -102,3 +102,21 @@ jobs:
           chmod +x /usr/local/bin/kubectl-kuttl
       - name: Run e2e tests
         run: make e2e-helm-certmanager
+
+  test-helm-chart-with-custom-configuration:
+    name: Test Helm chart with custom configuration
+    runs-on: ubuntu-20.04
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.20'
+      - uses: imranismail/setup-kustomize@v2
+      - name: Install kuttl
+        run: |
+          curl -Lo /usr/local/bin/kubectl-kuttl https://github.com/kudobuilder/kuttl/releases/download/v${{ env.KUTTL_VERSION }}/kubectl-kuttl_${{ env.KUTTL_VERSION }}_linux_x86_64
+          chmod +x /usr/local/bin/kubectl-kuttl
+      - name: Run e2e tests
+        run: make e2e-helm-custom-configuration
diff --git a/Makefile b/Makefile
@@ -38,6 +38,10 @@ e2e-helm: e2e
 e2e-helm-certmanager: KUTTL_CONFIG = kuttl-test-helm-certmanager.yaml
 e2e-helm-certmanager: e2e
 
+.PHONY: e2e-helm-custom-configuration
+e2e-helm-custom-configuration: KUTTL_CONFIG = kuttl-test-helm-custom-configuration.yaml
+e2e-helm-custom-configuration: e2e
+
 build-push-deploy: build-push-sidecar build-push-deploy-operator
 
 build-push-sidecar:

diff --git a/helm/tailing-sidecar-operator/README.md b/helm/tailing-sidecar-operator/README.md
@@ -48,3 +48,22 @@ The generated certificate is valid for 365 days after issuing, i.e. after chart
 
 If you have [cert-manager](https://cert-manager.io/) installed in your cluster,
 you can make the chart use it for certificate management by setting the property `certManager.enabled` to `true`.
+
+### Overriding Tailing Sidecar configuration
+
+In order to override tailing sidecar configuration, the following properties may be used:
+
+```yaml
+sidecar:
+  config:
+    mountPath: /fluent-bit/etc/
+    content:
+      file-1.conf: |
+        content of file-1.conf
+      file-2.conf: |
+        content of file-2.conf
+```
+
+The above configuration is going to create `file-1.conf` and `file-2.conf` in `/fluent-bit/etc/` directory.
+
+**All existing content of `/fluent-bit/etc/` directory will be replaced with the `sidecar.config.content`.**
diff --git a/helm/tailing-sidecar-operator/conf/operator.yaml b/helm/tailing-sidecar-operator/conf/operator.yaml
@@ -2,6 +2,12 @@ sidecar:
   image: {{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag | default .Chart.AppVersion }}
   resources:
     {{- .Values.sidecar.resources | toYaml | nindent 4 }}
+{{- if not (empty .Values.sidecar.config.content) }}
+  config:
+    name: {{ template "tailing-sidecar.configMap.name" . }}
+    mountPath: {{ .Values.sidecar.config.mountPath }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}
 leaderElection:
   leaseDuration: {{ .Values.operator.leaderElection.leaseDuration }}
   renewDeadline: {{ .Values.operator.leaderElection.renewDeadline }}

diff --git a/helm/tailing-sidecar-operator/templates/_helpers.tpl b/helm/tailing-sidecar-operator/templates/_helpers.tpl
@@ -56,3 +56,10 @@ Configmap name
 {{- define "tailing-sidecar-operator.configMap.name" -}}
 {{- printf "%s-%s" .Release.Name "operator-config" | trunc 63 | trimSuffix "-" }}
 {{- end }}
+
+{{/*
+Tailing sidecar configmap name
+*/}}
+{{- define "tailing-sidecar.configMap.name" -}}
+{{- printf "%s-%s" .Release.Name "sidecar-config" | trunc 63 | trimSuffix "-" }}
+{{- end }}
diff --git a/helm/tailing-sidecar-operator/templates/resources.yaml b/helm/tailing-sidecar-operator/templates/resources.yaml
@@ -302,6 +302,19 @@ rules:
   - tailingsidecars/finalizers
   verbs:
   - update
+{{- if not (empty .Values.sidecar.config.content) }}
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+{{- end }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole

diff --git a/helm/tailing-sidecar-operator/templates/sidecar-configmap.yaml b/helm/tailing-sidecar-operator/templates/sidecar-configmap.yaml
@@ -0,0 +1,10 @@
+{{- if not (empty .Values.sidecar.config.content) }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "tailing-sidecar.configMap.name" . }}
+  labels:
+    {{- include "tailing-sidecar-operator.labels" . | nindent 4 }}
+data:
+{{- toYaml .Values.sidecar.config.content | nindent 2 }}
+{{- end }}
diff --git a/helm/tailing-sidecar-operator/values.yaml b/helm/tailing-sidecar-operator/values.yaml
@@ -44,6 +44,18 @@ sidecar:
       cpu: 100m
       memory: 200Mi
 
+  # Overrides the sidecar configuration
+  config:
+    mountPath: /fluent-bit/etc/
+    # map of files which are going to be created in mountPath
+    # for example to create three empty files:
+    #
+    # content:
+    #  fluent-bit.conf: |
+    #  parsers.conf: |
+    #  plugins.conf
+    content: {}
+
 kubeRbacProxy:
   image:
     pullPolicy: IfNotPresent

diff --git a/helm/tests/values.withCertManager.yaml b/helm/tests/values.withCertManager.yaml
@@ -7,6 +7,13 @@ sidecar:
   image:
     repository: registry.localhost:5000/sumologic/tailing-sidecar
     tag: test
+  resources:
+    limits:
+      cpu: "0"
+      memory: "0"
+    requests:
+      cpu: "0"
+      memory: "0"
 
 certManager:
   enabled: true
diff --git a/helm/tests/values.withCustomConfiguration.yaml b/helm/tests/values.withCustomConfiguration.yaml
@@ -0,0 +1,127 @@
+operator:
+  image:
+    repository: registry.localhost:5000/sumologic/tailing-sidecar-operator
+    tag: test
+
+sidecar:
+  image:
+    repository: registry.localhost:5000/sumologic/tailing-sidecar
+    tag: test
+
+  resources:
+    limits:
+      cpu: "0"
+      memory: "0"
+    requests:
+      cpu: "0"
+      memory: "0"
+
+  config:
+    content:
+      fluent-bit.conf: |
+        [SERVICE]
+            # Flush
+            # =====
+            # set an interval of seconds before to flush records to a destination
+            flush        1
+
+            # Daemon
+            # ======
+            # instruct Fluent Bit to run in foreground or background mode.
+            daemon       Off
+
+            # Log_Level
+            # =========
+            # Set the verbosity level of the service, values can be:
+            #
+            # - error
+            # - warning
+            # - info
+            # - debug
+            # - trace
+            #
+            # e.g. when 'info' is set, that means it includes 'error' and 'warning'.
+            log_level    ${LOG_LEVEL}
+
+            # Parsers File
+            # ============
+            # specify an optional 'Parsers' configuration file
+            parsers_file parsers.conf
+
+            # Plugins File
+            # ============
+            # specify an optional 'Plugins' configuration file to load external plugins.
+            plugins_file plugins.conf
+
+            # HTTP Server
+            # ===========
+            # Enable/Disable the built-in HTTP Server for metrics
+            http_server  Off
+            http_listen  0.0.0.0
+            http_port    2020
+
+            # Storage
+            # =======
+            # Fluent Bit can use memory and filesystem buffering based mechanisms
+            #
+            # - https://docs.fluentbit.io/manual/administration/buffering-and-storage
+            #
+            # storage metrics
+            # ---------------
+            # publish storage pipeline metrics in '/api/v1/storage'. The metrics are
+            # exported only if the 'http_server' option is enabled.
+            #
+            storage.metrics on
+
+            # storage.path
+            # ------------
+            # absolute file system path to store filesystem data buffers (chunks).
+            #
+            # storage.path /tmp/storage
+
+            # storage.sync
+            # ------------
+            # configure the synchronization mode used to store the data into the
+            # filesystem. It can take the values normal or full.
+            #
+            # storage.sync normal
+
+            # storage.checksum
+            # ----------------
+            # enable the data integrity check when writing and reading data from the
+            # filesystem. The storage layer uses the CRC32 algorithm.
+            #
+            # storage.checksum off
+
+            # storage.backlog.mem_limit
+            # -------------------------
+            # if storage.path is set, Fluent Bit will look for data chunks that were
+            # not delivered and are still in the storage layer, these are called
+            # backlog data. This option configure a hint of maximum value of memory
+            # to use when processing these records.
+            #
+            # storage.backlog.mem_limit 5M
+
+        [INPUT]
+            name                tail
+            path                ${PATH_TO_TAIL}
+            Read_from_Head      On
+            Refresh_Interval    1
+            Rotate_Wait         60
+            Mem_Buf_Limit       5MB
+            DB                  /tailing-sidecar/var/fluent-bit-state.db
+            DB.Sync             Normal
+        
+        [FILTER]
+            Name modify
+            Match *
+            Set log modified
+
+        [OUTPUT]
+            name  gstdout
+
+      plugins.conf: |
+        [PLUGINS]
+            Path /tailing-sidecar/lib/out_gstdout.so
+      
+      parsers.conf: |
diff --git a/helm/tests/values.yaml b/helm/tests/values.yaml
@@ -7,3 +7,11 @@ sidecar:
   image:
     repository: registry.localhost:5000/sumologic/tailing-sidecar
     tag: test
+
+  resources:
+    limits:
+      cpu: "0"
+      memory: "0"
+    requests:
+      cpu: "0"
+      memory: "0"
diff --git a/kuttl-test-helm-custom-configuration.yaml b/kuttl-test-helm-custom-configuration.yaml
@@ -0,0 +1,17 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestSuite
+artifactsDir: ./tests/_build/artifacts/
+testDirs:
+  - ./tests/modified/sidecar/
+  - ./tests/modified/operator/
+timeout: 150
+parallel: 4
+startKIND: true
+kindNodeCache: true
+kindContainers:
+  - registry.localhost:5000/sumologic/tailing-sidecar-operator:test
+  - registry.localhost:5000/sumologic/tailing-sidecar:test
+commands: 
+  - command: helm upgrade --install test-release ./helm/tailing-sidecar-operator -f ./helm/tests/values.withCustomConfiguration.yaml -n tailing-sidecar-system --create-namespace
+  - command: kubectl wait --for=condition=available --timeout 300s deploy -l app.kubernetes.io/name=tailing-sidecar-operator -n tailing-sidecar-system
+  - command: kubectl wait --for=condition=ready --timeout 300s pod -l app.kubernetes.io/name=tailing-sidecar-operator -n tailing-sidecar-system
diff --git a/operator/config.go b/operator/config.go
@@ -19,6 +19,7 @@ type Config struct {
 type SidecarConfig struct {
 	Image     string                      `yaml:"image,omitempty"`
 	Resources corev1.ResourceRequirements `yaml:"resources,omitempty"`
+	Config    SidecarConfigConfig         `yaml:"config,omitempty"`
 }
 
 type LeaderElectionConfig struct {
@@ -27,6 +28,12 @@ type LeaderElectionConfig struct {
 	RetryPeriod   Duration `yaml:"retryPeriod,omitempty"`
 }
 
+type SidecarConfigConfig struct {
+	Name      string `yaml:"name,omitempty"`
+	MountPath string `yaml:"mountPath,omitempty"`
+	Namespace string `yaml:"namespace,omitempty"`
+}
+
 // Duration sigs.k8s.io/yaml not support time.Duration:https://github.com/kubernetes-sigs/yaml/issues/64
 type Duration time.Duration
 
@@ -60,6 +67,7 @@ func ReadConfig(configPath string, config *Config) error {
 	if err != nil {
 		return err
 	}
+
 	err = yaml.Unmarshal(content, config)
 	if err != nil {
 		return err