From abc2e19a3610e0857d30047570b24441ad90c372 Mon Sep 17 00:00:00 2001 From: David Martin Date: Mon, 30 Sep 2024 15:02:36 +0100 Subject: [PATCH] Update local observability stack steps (#886) * Update cmds to install observability stack locally Signed-off-by: David Martin * Use docker.io prefix for kfilt image Signed-off-by: David Martin --------- Signed-off-by: David Martin --- config/observability/README.md | 13 ++- .../grafana/grafana_deployment_patch.yaml | 86 +------------------ hack/quickstart-setup.sh | 4 +- make/observability.mk | 4 +- 4 files changed, 16 insertions(+), 91 deletions(-) diff --git a/config/observability/README.md b/config/observability/README.md index c096375c8..654c3052e 100644 --- a/config/observability/README.md +++ b/config/observability/README.md @@ -2,14 +2,23 @@ ## Deploying the observabilty stack +If you run the `quickstart-setup.sh` script, the observability stack should already be set up. +In that case, you can skip the below commands. +If however you have run `make local-setup` and would like to install the observability stack, these commands will install the stack and example dashboards & alerts. + ```bash -./bin/kustomize build ./config/observability/| docker run --rm -i ryane/kfilt -i kind=CustomResourceDefinition | kubectl apply --server-side -f - -./bin/kustomize build ./config/observability/| docker run --rm -i ryane/kfilt -x kind=CustomResourceDefinition | kubectl apply -f - +./bin/kustomize build ./config/observability/| docker run --rm -i docker.io/ryane/kfilt -i kind=CustomResourceDefinition | kubectl apply --server-side -f - +./bin/kustomize build ./config/observability/| docker run --rm -i docker.io/ryane/kfilt -x kind=CustomResourceDefinition | kubectl apply -f - +./bin/kustomize build ./config/thanos | kubectl apply -f - ./bin/kustomize build ./examples/dashboards | kubectl apply -f - +./bin/kustomize build ./examples/alerts | kubectl apply -f - +THANOS_RECEIVE_ROUTER_IP=$(kubectl -n monitoring get svc thanos-receive-router-lb -o jsonpath='{.status.loadBalancer.ingress[0].ip}') +kubectl -n monitoring patch prometheus k8s --type='merge' -p '{"spec":{"remoteWrite":[{"url":"http://'"$THANOS_RECEIVE_ROUTER_IP"':19291/api/v1/receive", "writeRelabelConfigs":[{"action":"replace", "replacement":"'"$KUADRANT_CLUSTER_NAME"'", "targetLabel":"cluster_id"}]}]}}' ``` This will deploy prometheus, alertmanager and grafana into the `monitoring` namespace, along with metrics scrape configuration for Istio and Envoy. +Thanos will also be deployed with prometheus configured to remote write to it. ## Accessing Grafana & Prometheus diff --git a/config/observability/grafana/grafana_deployment_patch.yaml b/config/observability/grafana/grafana_deployment_patch.yaml index 0bd1dbd2a..39725f9a2 100644 --- a/config/observability/grafana/grafana_deployment_patch.yaml +++ b/config/observability/grafana/grafana_deployment_patch.yaml @@ -1,52 +1,3 @@ -- op: add - path: /spec/template/spec/volumes/- - value: - name: grafana-gatewayclasses - configMap: - defaultMode: 420 - name: grafana-gatewayclasses -- op: add - path: /spec/template/spec/volumes/- - value: - name: grafana-gateways - configMap: - defaultMode: 420 - name: grafana-gateways -- op: add - path: /spec/template/spec/volumes/- - value: - name: grafana-httproutes - configMap: - defaultMode: 420 - name: grafana-httproutes -- op: add - path: /spec/template/spec/volumes/- - value: - name: grafana-grpcroutes - configMap: - defaultMode: 420 - name: grafana-grpcroutes -- op: add - path: /spec/template/spec/volumes/- - value: - name: grafana-tlsroutes - configMap: - defaultMode: 420 - name: grafana-tlsroutes -- op: add - path: /spec/template/spec/volumes/- - value: - name: grafana-tcproutes - configMap: - defaultMode: 420 - name: grafana-tcproutes -- op: add - path: /spec/template/spec/volumes/- - value: - name: grafana-udproutes - configMap: - defaultMode: 420 - name: grafana-udproutes - op: add path: /spec/template/spec/volumes/- value: @@ -81,42 +32,7 @@ name: grafana-controller-runtime configMap: defaultMode: 420 - name: grafana-controller-runtime -- op: add - path: /spec/template/spec/containers/0/volumeMounts/- - value: - name: grafana-gatewayclasses - mountPath: /grafana-dashboard-definitions/0/grafana-gatewayclasses -- op: add - path: /spec/template/spec/containers/0/volumeMounts/- - value: - name: grafana-gateways - mountPath: /grafana-dashboard-definitions/0/grafana-gateways -- op: add - path: /spec/template/spec/containers/0/volumeMounts/- - value: - name: grafana-httproutes - mountPath: /grafana-dashboard-definitions/0/grafana-httproutes -- op: add - path: /spec/template/spec/containers/0/volumeMounts/- - value: - name: grafana-grpcroutes - mountPath: /grafana-dashboard-definitions/0/grafana-grpcroutes -- op: add - path: /spec/template/spec/containers/0/volumeMounts/- - value: - name: grafana-tlsroutes - mountPath: /grafana-dashboard-definitions/0/grafana-tlsroutes -- op: add - path: /spec/template/spec/containers/0/volumeMounts/- - value: - name: grafana-tcproutes - mountPath: /grafana-dashboard-definitions/0/grafana-tcproutes -- op: add - path: /spec/template/spec/containers/0/volumeMounts/- - value: - name: grafana-udproutes - mountPath: /grafana-dashboard-definitions/0/grafana-udproutes + name: grafana-controller-runtime - op: add path: /spec/template/spec/containers/0/volumeMounts/- value: diff --git a/hack/quickstart-setup.sh b/hack/quickstart-setup.sh index e6c517db6..06a63cb4a 100755 --- a/hack/quickstart-setup.sh +++ b/hack/quickstart-setup.sh @@ -460,8 +460,8 @@ fi # Install observability stack info "Installing observability stack in ${KUADRANT_CLUSTER_NAME}..." -kubectl kustomize ${KUADARNT_OBSERVABILITY_KUSTOMIZATION} | $CONTAINER_RUNTIME_BIN run --rm -i ryane/kfilt -i kind=CustomResourceDefinition | kubectl apply --server-side -f - -kubectl kustomize ${KUADARNT_OBSERVABILITY_KUSTOMIZATION} | $CONTAINER_RUNTIME_BIN run --rm -i ryane/kfilt -x kind=CustomResourceDefinition | kubectl apply -f - +kubectl kustomize ${KUADARNT_OBSERVABILITY_KUSTOMIZATION} | $CONTAINER_RUNTIME_BIN run --rm -i docker.io/ryane/kfilt -i kind=CustomResourceDefinition | kubectl apply --server-side -f - +kubectl kustomize ${KUADARNT_OBSERVABILITY_KUSTOMIZATION} | $CONTAINER_RUNTIME_BIN run --rm -i docker.io/ryane/kfilt -x kind=CustomResourceDefinition | kubectl apply -f - kubectl kustomize ${KUADRANT_DASHBOARDS_KUSTOMIZATION} | kubectl apply --server-side -f - kubectl kustomize ${KUADRANT_ALERTS_KUSTOMIZATION} | kubectl apply --server-side -f - success "observability stack installed successfully." diff --git a/make/observability.mk b/make/observability.mk index cefa5e1b1..c6f5a201e 100644 --- a/make/observability.mk +++ b/make/observability.mk @@ -1,8 +1,8 @@ .PHONY: deploy-observability deploy-observability: kustomize - $(KUSTOMIZE) build config/observability | docker run --rm -i ryane/kfilt -i kind=CustomResourceDefinition | kubectl apply --server-side -f - - $(KUSTOMIZE) build config/observability | docker run --rm -i ryane/kfilt -x kind=CustomResourceDefinition | kubectl apply -f - + $(KUSTOMIZE) build config/observability | docker run --rm -i docker.io/ryane/kfilt -i kind=CustomResourceDefinition | kubectl apply --server-side -f - + $(KUSTOMIZE) build config/observability | docker run --rm -i docker.io/ryane/kfilt -x kind=CustomResourceDefinition | kubectl apply -f - .PHONY: thanos-manifests thanos-manifests: ./hack/thanos/thanos_build.sh ./hack/thanos/thanos.jsonnet