From 4f822dc6849ddb2ffee19166477be1eed34b8c3d Mon Sep 17 00:00:00 2001 From: Greg Smith <65406958+gsmith-sas@users.noreply.github.com> Date: Thu, 14 Nov 2024 15:25:54 -0500 Subject: [PATCH] [UPGRADE] Log monitoring stack (#691) * [UPGRADE] OpenSearch/OpenSearch Dashboards 2.17.1; Fluent Bit 3.1.9; related Helm charts * [UPGRADE] Elasticsearch Exporter and associated serviceMonitor * [UPGRADE] OpenSearch Data Source Plugin to Grafana upgraded from 2.18.0 to 2.21.1 --- CHANGELOG.md | 8 ++++++++ component_versions.env | 18 +++++++++--------- logging/bin/deploy_esexporter.sh | 17 +++++++++++------ monitoring/bin/deploy_monitoring_cluster.sh | 4 +++- 4 files changed, 31 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1f2566f..985bb1a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # SAS Viya Monitoring for Kubernetes +## Version 1.2.30 (11OCT2024) +* **Logging** + * [UPGRADE] OpenSearch and OpenSearch Dashboards upgraded from 2.15.0 to 2.17.1 + * [UPGRADE] Elasticsearch Exporter upgraded from 1.7.0 to 1.8.0. Note that this included a change to the pod labels that +required a new serviceMonitor (elasticsearch-v2) be deployed. + * [UPGRADE] Fluent Bit upgraded from 3.1.3 to 3.1.9 + * [UPGRADE] OpenSearch Data Source Plugin to Grafana upgraded from 2.18.0 to 2.21.1 + ## Version 1.2.30 (11OCT2024) * **Logging** * [SECURITY] OpenSearch Dashboards pod `securityContext` updated to set allowPrivilegeEscalation to 'false' diff --git a/component_versions.env b/component_versions.env index 2e4b7a83..2166f210 100644 --- a/component_versions.env +++ b/component_versions.env @@ -11,27 +11,27 @@ #Elasticsearch Exporter ESEXPORTER_HELM_CHART_REPO=prometheus-community ESEXPORTER_HELM_CHART_NAME=prometheus-elasticsearch-exporter -ESEXPORTER_HELM_CHART_VERSION=5.6.0 -ES_EXPORTER_FULL_IMAGE="quay.io/prometheuscommunity/elasticsearch-exporter:v1.7.0" +ESEXPORTER_HELM_CHART_VERSION=6.5.0 +ES_EXPORTER_FULL_IMAGE="quay.io/prometheuscommunity/elasticsearch-exporter:v1.8.0" #Fluent Bit FLUENTBIT_HELM_CHART_REPO=fluent FLUENTBIT_HELM_CHART_NAME=fluent-bit -FLUENTBIT_HELM_CHART_VERSION=0.47.2 -FB_FULL_IMAGE="cr.fluentbit.io/fluent/fluent-bit:3.1.3" +FLUENTBIT_HELM_CHART_VERSION=0.47.10 +FB_FULL_IMAGE="cr.fluentbit.io/fluent/fluent-bit:3.1.9" #OpenSearch OPENSEARCH_HELM_CHART_REPO=opensearch OPENSEARCH_HELM_CHART_NAME=opensearch -OPENSEARCH_HELM_CHART_VERSION=2.21.0 -OS_FULL_IMAGE="docker.io/opensearchproject/opensearch:2.15.0" +OPENSEARCH_HELM_CHART_VERSION=2.26.0 +OS_FULL_IMAGE="docker.io/opensearchproject/opensearch:2.17.1" OS_SYSCTL_FULL_IMAGE="docker.io/library/busybox:latest" #OpenSearch Dashboards OSD_HELM_CHART_REPO=opensearch OSD_HELM_CHART_NAME=opensearch-dashboards -OSD_HELM_CHART_VERSION=2.19.0 -OSD_FULL_IMAGE="docker.io/opensearchproject/opensearch-dashboards:2.15.0" +OSD_HELM_CHART_VERSION=2.24.0 +OSD_FULL_IMAGE="docker.io/opensearchproject/opensearch-dashboards:2.17.1" #Grafana (when deployed on OpenShift) OPENSHIFT_GRAFANA_CHART_REPO=grafana @@ -42,7 +42,7 @@ OPENSHIFT_OAUTHPROXY_FULL_IMAGE="registry.redhat.io/openshift4/ose-oauth-proxy:l #Grafana (everywhere) GRAFANA_FULL_IMAGE="docker.io/grafana/grafana:11.2.0" GRAFANA_SIDECAR_FULL_IMAGE="quay.io/kiwigrid/k8s-sidecar:1.27.4" -GRAFANA_DATASOURCE_PLUGIN_VERSION="2.18.0" +GRAFANA_DATASOURCE_PLUGIN_VERSION="2.21.1" #Kube-Prometheus Stack KUBE_PROM_STACK_CHART_REPO=prometheus-community diff --git a/logging/bin/deploy_esexporter.sh b/logging/bin/deploy_esexporter.sh index f797641a..a7ffdbd7 100755 --- a/logging/bin/deploy_esexporter.sh +++ b/logging/bin/deploy_esexporter.sh @@ -40,11 +40,20 @@ if [ "$rc" != "0" ] ;then log_debug "RC=$rc"; exit $rc;fi if helm3ReleaseExists es-exporter $LOG_NS; then - #remove an existing instance if it does NOT target OPENSEARCH (i.e. targets ODFE) - if [ -z $(kubectl -n $LOG_NS get pods -l "app=prometheus-elasticsearch-exporter,searchbackend=opensearch" -o name 2>/dev/null) ]; then + #remove an existing instance if it does NOT have the most current set of labels + # NOTE: pod label 'app' changed to 'app.kubernetes.io/name' w/Helm chart 6.x + if [ -z $(kubectl -n $LOG_NS get pods -l "app.kubernetes.io/name=prometheus-elasticsearch-exporter,searchbackend=opensearch" -o name 2>/dev/null) ]; then log_debug "Removing an outdated version of Helm release [es-exporter]" helm -n $LOG_NS delete es-exporter fi + + monNamespace=$(kubectl get servicemonitor -A --field-selector=metadata.name=elasticsearch -l sas.com/monitoring-base=kube-viya-monitoring -o=custom-columns=NAMESPACE:.metadata.namespace --no-headers) + if [ -n "$monNamespace" ]; then + log_debug "Removing obsolete serviceMonitor [$monNamespace/elasticsearch]" + kubectl delete -n $monNamespace servicemonitor elasticsearch + log_debug "Deploying an updated serviceMonitor for Elasticsearch [$monNamespace/elasticsearch-v2]" + kubectl apply -n $monNamespace -f monitoring/monitors/logging/serviceMonitor-elasticsearch-v2.yaml + fi else log_debug "No existing Helm release [es-exporter] found." fi @@ -58,10 +67,6 @@ fi helmRepoAdd prometheus-community https://prometheus-community.github.io/helm-charts -## Commenting out because it might be redundant code. -# log_verbose "Updating Helm repositories" -# helm repo update - primaryValuesFile="logging/esexporter/values-es-exporter.yaml" log_debug "Deploying Elasticsearch Exporter" diff --git a/monitoring/bin/deploy_monitoring_cluster.sh b/monitoring/bin/deploy_monitoring_cluster.sh index 4964833d..f485e7c0 100755 --- a/monitoring/bin/deploy_monitoring_cluster.sh +++ b/monitoring/bin/deploy_monitoring_cluster.sh @@ -291,7 +291,9 @@ fi kubectl apply -n $MON_NS -f monitoring/monitors/kube/podMonitor-eventrouter.yaml 2>/dev/null # Elasticsearch ServiceMonitor -kubectl apply -n $MON_NS -f monitoring/monitors/logging/serviceMonitor-elasticsearch.yaml +## remove obsolete version, if installed +kubectl delete -n $MON_NS servicemonitor elasticsearch --ignore-not-found +kubectl apply -n $MON_NS -f monitoring/monitors/logging/serviceMonitor-elasticsearch-v2.yaml # Fluent Bit ServiceMonitors kubectl apply -n $MON_NS -f monitoring/monitors/logging/serviceMonitor-fluent-bit.yaml