From c0b69fa2844d0c0ef5ed063b0b9ec6c20cd5560b Mon Sep 17 00:00:00 2001 From: rohit-bandlamudi-nr Date: Thu, 30 Jan 2025 17:47:27 +0530 Subject: [PATCH] fix: enable metrics and fb metrics config by default --- charts/newrelic-logging/Chart.yaml | 2 +- charts/newrelic-logging/README.md | 1 - .../newrelic-logging/templates/_helpers.tpl | 33 +++++++++++++++++++ .../newrelic-logging/templates/configmap.yaml | 4 +-- .../templates/daemonset-windows.yaml | 2 -- .../newrelic-logging/templates/daemonset.yaml | 2 -- .../tests/fluentbit_sendmetrics_test.yaml | 13 -------- charts/newrelic-logging/values.yaml | 32 ++---------------- 8 files changed, 38 insertions(+), 51 deletions(-) diff --git a/charts/newrelic-logging/Chart.yaml b/charts/newrelic-logging/Chart.yaml index a8bd0d57f..931d25a94 100644 --- a/charts/newrelic-logging/Chart.yaml +++ b/charts/newrelic-logging/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 description: A Helm chart to deploy New Relic Kubernetes Logging as a DaemonSet, supporting both Linux and Windows nodes and containers name: newrelic-logging -version: 1.23.5 +version: 1.23.6-beta appVersion: 2.1.0 home: https://github.com/newrelic/kubernetes-logging icon: https://newrelic.com/assets/newrelic/source/NewRelic-logo-square.svg diff --git a/charts/newrelic-logging/README.md b/charts/newrelic-logging/README.md index 1635b0d86..584fdbeff 100644 --- a/charts/newrelic-logging/README.md +++ b/charts/newrelic-logging/README.md @@ -176,7 +176,6 @@ See [values.yaml](values.yaml) for the default values | `fluentBit.config.extraOutputs` | Contains extra fluent-bit.conf Outputs config | | | `fluentBit.config.parsers` | Contains parsers.conf Parsers config | | | `fluentBit.retryLimit` | Amount of times to retry sending a given batch of logs to New Relic. This prevents data loss if there is a temporary network disruption, if a request to the Logs API is lost or when receiving a recoverable HTTP response. Set it to "False" for unlimited retries. | 5 | -| `fluentBit.sendMetrics` | Enable the collection of Fluent Bit internal metrics in Prometheus format as well as newrelic-fluent-bit-output internal plugin metrics. See [this documentation page](https://docs.newrelic.com/docs/logs/forward-logs/kubernetes-plugin-log-forwarding/#troubleshoot-installation) for more details. | `false` | | `dnsConfig` | [DNS configuration](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-dns-config) that will be added to the pods. Can be configured also with `global.dnsConfig`. | `{}` | | `fluentBit.criEnabled` | We assume that `kubelet`directly communicates with the container engine using the [CRI](https://kubernetes.io/docs/concepts/overview/components/#container-runtime) specification. Set this to `false` if your K8s installation uses [dockershim](https://kubernetes.io/docs/tasks/administer-cluster/migrating-from-dockershim/) instead, in order to get the logs properly parsed. | `true` | diff --git a/charts/newrelic-logging/templates/_helpers.tpl b/charts/newrelic-logging/templates/_helpers.tpl index a02ff74b2..262629b45 100644 --- a/charts/newrelic-logging/templates/_helpers.tpl +++ b/charts/newrelic-logging/templates/_helpers.tpl @@ -178,6 +178,39 @@ https://log-api.newrelic.com/log/v1 {{- end -}} {{- end -}} +{{/* +Returns fluentbit config to collect and forward its metrics to New Relic +*/}} +{{- define "newrelic-logging.fluentBit.monitoring.config" -}} +[INPUT] + name prometheus_scrape + Alias fb-metrics-collector + host 127.0.0.1 + port 2020 + tag fb_metrics + metrics_path /api/v2/metrics/prometheus + scrape_interval 10s + +[OUTPUT] + Name prometheus_remote_write + Match fb_metrics + Alias fb-metrics-forwarder + Host ${METRICS_HOST} + Port 443 + Uri /prometheus/v1/write?prometheus_server=${CLUSTER_NAME} + Header Authorization Bearer ${LICENSE_KEY} + Tls On + Tls.verify Off + add_label app fluent-bit + add_label source kubernetes + add_label pod_name ${HOSTNAME} + add_label node_name ${NODE_NAME} + {{- printf "add_label cluster_name %s" (include "newrelic-logging.cluster" .) | nindent 4 -}} + {{- printf "add_label namespace %s" .Release.Namespace | nindent 4 -}} + {{- printf "add_label daemonset_name %s" (include "newrelic-logging.fullname" .) | nindent 4 -}} +{{- end -}} + + {{/* Returns metricsHost */}} diff --git a/charts/newrelic-logging/templates/configmap.yaml b/charts/newrelic-logging/templates/configmap.yaml index 4b1d89014..dc9b77850 100644 --- a/charts/newrelic-logging/templates/configmap.yaml +++ b/charts/newrelic-logging/templates/configmap.yaml @@ -29,9 +29,7 @@ data: {{- if .Values.fluentBit.config.extraOutputs }} {{- .Values.fluentBit.config.extraOutputs | nindent 4}} {{- end }} - {{- if and (.Values.fluentBit.sendMetrics) (.Values.fluentBit.config.metricInstrumentation) }} - {{- .Values.fluentBit.config.metricInstrumentation | nindent 4}} - {{- end }} + {{- (include "newrelic-logging.fluentBit.monitoring.config" .) | nindent 4}} parsers.conf: | {{- if .Values.fluentBit.config.parsers }} {{- .Values.fluentBit.config.parsers | nindent 4}} diff --git a/charts/newrelic-logging/templates/daemonset-windows.yaml b/charts/newrelic-logging/templates/daemonset-windows.yaml index 6a3145d13..5ffe60883 100644 --- a/charts/newrelic-logging/templates/daemonset-windows.yaml +++ b/charts/newrelic-logging/templates/daemonset-windows.yaml @@ -111,8 +111,6 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - - name: SEND_OUTPUT_PLUGIN_METRICS - value: {{ $.Values.fluentBit.sendMetrics | default "false" | quote }} - name: METRICS_HOST value: {{ include "newrelic-logging.metricsHost" $ | quote }} {{- include "newrelic-logging.extraEnv" $ | nindent 12 }} diff --git a/charts/newrelic-logging/templates/daemonset.yaml b/charts/newrelic-logging/templates/daemonset.yaml index 1e087b6cb..35431ae71 100644 --- a/charts/newrelic-logging/templates/daemonset.yaml +++ b/charts/newrelic-logging/templates/daemonset.yaml @@ -125,8 +125,6 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - - name: SEND_OUTPUT_PLUGIN_METRICS - value: {{ $.Values.fluentBit.sendMetrics | default "false" | quote }} - name: METRICS_HOST value: {{ include "newrelic-logging.metricsHost" . | quote }} {{- include "newrelic-logging.extraEnv" . | nindent 12 }} diff --git a/charts/newrelic-logging/tests/fluentbit_sendmetrics_test.yaml b/charts/newrelic-logging/tests/fluentbit_sendmetrics_test.yaml index f320172cb..f7e69c1b6 100644 --- a/charts/newrelic-logging/tests/fluentbit_sendmetrics_test.yaml +++ b/charts/newrelic-logging/tests/fluentbit_sendmetrics_test.yaml @@ -12,7 +12,6 @@ tests: set: licenseKey: nr_license_key enableWindows: true - fluentBit.sendMetrics: true asserts: # Linux - contains: @@ -31,12 +30,6 @@ tests: fieldRef: fieldPath: metadata.name template: templates/daemonset.yaml - - contains: - path: spec.template.spec.containers[0].env - content: - name: SEND_OUTPUT_PLUGIN_METRICS - value: "true" - template: templates/daemonset.yaml - contains: path: spec.template.spec.containers[0].env content: @@ -60,12 +53,6 @@ tests: fieldRef: fieldPath: metadata.name template: templates/daemonset-windows.yaml - - contains: - path: spec.template.spec.containers[0].env - content: - name: SEND_OUTPUT_PLUGIN_METRICS - value: "true" - template: templates/daemonset-windows.yaml - contains: path: spec.template.spec.containers[0].env content: diff --git a/charts/newrelic-logging/values.yaml b/charts/newrelic-logging/values.yaml index ca6336cb6..47ef1a30d 100644 --- a/charts/newrelic-logging/values.yaml +++ b/charts/newrelic-logging/values.yaml @@ -36,7 +36,6 @@ fluentBit: k8sBufferSize: "32k" k8sLoggingExclude: "false" retryLimit: 5 - sendMetrics: false extraEnv: [] # extraEnv: # - name: HTTPS_PROXY @@ -166,6 +165,8 @@ fluentBit: Allowlist_key message Allowlist_key log +# sendMetrics is set to false by default, this has to be defaulted to true in the future, in the newrelic-fluent-bit-output plugin + outputs: | [OUTPUT] Name newrelic @@ -174,7 +175,7 @@ fluentBit: licenseKey ${LICENSE_KEY} endpoint ${ENDPOINT} lowDataMode ${LOW_DATA_MODE} - sendMetrics ${SEND_OUTPUT_PLUGIN_METRICS} + sendMetrics true Retry_Limit ${RETRY_LIMIT} # extraOutputs: | @@ -189,33 +190,6 @@ fluentBit: # Time_Key time # Time_Format %Y-%m-%dT%H:%M:%S.%L # Time_Keep On - metricInstrumentation: | - [INPUT] - name prometheus_scrape - Alias fb-metrics-collector - host 127.0.0.1 - port 2020 - tag fb_metrics - metrics_path /api/v2/metrics/prometheus - scrape_interval 10s - - [OUTPUT] - Name prometheus_remote_write - Match fb_metrics - Alias fb-metrics-forwarder - Host ${METRICS_HOST} - Port 443 - Uri /prometheus/v1/write?prometheus_server=${CLUSTER_NAME} - Header Authorization Bearer ${LICENSE_KEY} - Tls On - # Windows pods using prometheus_remote_write currently have issues if TLS verify is On - Tls.verify Off - # User-defined labels - add_label app fluent-bit - add_label cluster_name "${CLUSTER_NAME}" - add_label pod_name ${HOSTNAME} - add_label node_name ${NODE_NAME} - add_label source kubernetes image: repository: newrelic/newrelic-fluentbit-output