From 091dfba9fa6aa4391c28b15578f47681c737b18e Mon Sep 17 00:00:00 2001 From: Matthias Theuermann Date: Fri, 22 Dec 2023 22:18:24 +0100 Subject: [PATCH] feat: calendar working and deployable with observability in kubernetes --- README.md | 89 +++++++------ .../fluent/fluentd-config-map.yaml | 105 +++++++++++++++ .../fluent/fluentd-dapr-with-rbac.yaml | 100 ++++++++++++++ dapr-distributed-calendar/go/Dockerfile | 27 +++- dapr-distributed-calendar/go/go_events.go | 6 +- .../kubernetes-deploy.sh | 34 ++--- .../kubernetes/appconfig.yaml | 13 -- .../kubernetes/go-events.yaml | 10 +- .../kubernetes/node-controller.yaml | 10 +- .../kubernetes/python-messages.yaml | 13 +- dapr-distributed-calendar/node/Dockerfile | 17 ++- .../node/instrumentation.js | 65 ++++++++++ .../node/node_controller.js | 10 ++ .../otel/instrumentation.yaml | 98 +++++++------- .../otel/otel-collector-values.yaml | 122 +++--------------- .../kube-prometheus-stack-values.yaml | 8 -- dapr-distributed-calendar/python/Dockerfile | 15 ++- 17 files changed, 498 insertions(+), 244 deletions(-) create mode 100644 dapr-distributed-calendar/fluent/fluentd-config-map.yaml create mode 100644 dapr-distributed-calendar/fluent/fluentd-dapr-with-rbac.yaml create mode 100644 dapr-distributed-calendar/node/instrumentation.js diff --git a/README.md b/README.md index 1502e5d..7debaea 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,63 @@ ## Original repos ```shell +# dapr-distributed-calendar +https://github.com/dapr/samples + # distributed-calculator https://github.com/dapr/quickstarts +``` -# dapr-distributed-calendar -https://github.com/dapr/samples +## dapr-distributed-calendar + +The dapr-distributed-calendar is now working perfectly. It is possible to do POST, DELETE and GET events. + +### Possible operations with Postman are + +POST: +Body: + +```json +{ + "data": { + "name": "Uninstall Event", + "date": "TBD", + "id": "1" + } +} +``` + +DELETE: + +GET: + +### Setup with docker-compose + +```shell +cd 12-factor-app/dapr-distributed-calendar +docker-compose up +``` + +### Setup with Kubernetes + +```shell +cd 12-factor-app/dapr-distributed-calendar +./kubernetes-deploy.sh ``` +Or pick specific parts of the deployment. +Every part of the deployment process that is not required, has a `OPTIONAL` comment! + +**Traces** are gathered and send to the OpenTelemetry Collector with the build in dapr integration and then send to Jaeger. + +**Metrics** are scraped by the OpenTelemetry Collector and then send to Prometheus. In addition, the nodejs application is instrumented manually to send additional non dapr metrics to the collector, but currently the metrics are only send to console output, which will be fixed in the next Sprint. The other microservices are yet to be instrumented. + +**Logs** are currently send directly over fluentd to elasticsearch. Due to most logging implementations for OpenTelemetry still being under development, being experimental or not implemented yet, this is the most logical resolution. (Go not implemented yet, Python experimental, Node in development) + +### About Auto-Instrumentation + +It is theoretically possible to use auto-instrumentation for kubernetes, but sadly this is very buggy especially Go and Python. and even those that do work do not provide a very good instrumentation when it comes to metrics, traces and logs. Therefore thy have been uncommented in the code, but can still be found within the folder `12-factor-app/dapr-distributed-calendar/otel`. + ## distributed-calculator The distributed-calculator works, and I added DELETE and PUT to the already existing POST and GET requests: @@ -45,38 +95,3 @@ curl -s -X PUT http://localhost:8000/state -H 'Content-Type: application/json' - ```bash curl -s -X DELETE http://localhost:8000/state ``` - -## dapr-distributed-calendar - -The dapr-distributed-calendar is not working perfectly. It is possible to do PUT and DELETE but not GET. In addition no put statement directly to the state store is possible like described in the documentation and same redis retryable errors. - -### Possible operations with Postman are - -PUT: -Body: - -```json -{ - "data": { - "name": "Uninstall Event", - "date": "TBD", - "id": "1" - } -} -``` - -DELETE: - -### What is not working is - -PUT: -Body: - -```json -{ - "key": "1", - "value": "Test Postman" -} -``` - -GET: diff --git a/dapr-distributed-calendar/fluent/fluentd-config-map.yaml b/dapr-distributed-calendar/fluent/fluentd-config-map.yaml new file mode 100644 index 0000000..4d91ee5 --- /dev/null +++ b/dapr-distributed-calendar/fluent/fluentd-config-map.yaml @@ -0,0 +1,105 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: fluentd-config + namespace: kube-system +data: + fluent.conf: | + + @type null + + + + @type null + + + + @type null + + + + @type null + + + + @type tail + path /var/log/containers/*.log + pos_file fluentd-docker.pos + time_format %Y-%m-%dT%H:%M:%S + tag kubernetes.* + + @type multi_format + + format json + time_key time + time_type string + time_format "%Y-%m-%dT%H:%M:%S.%NZ" + keep_time_key false + + + format regexp + expression /^(? + + + + + @type kubernetes_metadata + @id filter_kube_metadata + + + + @type parser + + @type json + format json + time_key time + time_type string + time_format "%Y-%m-%dT%H:%M:%S.%NZ" + keep_time_key false + + key_name log + replace_invalid_sequence true + emit_invalid_record_to_error true + reserve_data true + + + + @type elasticsearch + @id out_es + @log_level info + include_tag_key true + host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}" + port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}" + path "#{ENV['FLUENT_ELASTICSEARCH_PATH']}" + scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME'] || 'http'}" + ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY'] || 'true'}" + ssl_version "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERSION'] || 'TLSv1_2'}" + user "#{ENV['FLUENT_ELASTICSEARCH_USER'] || use_default}" + password "#{ENV['FLUENT_ELASTICSEARCH_PASSWORD'] || use_default}" + reload_connections "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS'] || 'false'}" + reconnect_on_error "#{ENV['FLUENT_ELASTICSEARCH_RECONNECT_ON_ERROR'] || 'true'}" + reload_on_failure "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_ON_FAILURE'] || 'true'}" + log_es_400_reason "#{ENV['FLUENT_ELASTICSEARCH_LOG_ES_400_REASON'] || 'false'}" + logstash_prefix "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_PREFIX'] || 'dapr'}" + logstash_dateformat "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_DATEFORMAT'] || '%Y.%m.%d'}" + logstash_format "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_FORMAT'] || 'true'}" + index_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_INDEX_NAME'] || 'dapr'}" + type_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_TYPE_NAME'] || 'fluentd'}" + include_timestamp "#{ENV['FLUENT_ELASTICSEARCH_INCLUDE_TIMESTAMP'] || 'false'}" + template_name "#{ENV['FLUENT_ELASTICSEARCH_TEMPLATE_NAME'] || use_nil}" + template_file "#{ENV['FLUENT_ELASTICSEARCH_TEMPLATE_FILE'] || use_nil}" + template_overwrite "#{ENV['FLUENT_ELASTICSEARCH_TEMPLATE_OVERWRITE'] || use_default}" + sniffer_class_name "#{ENV['FLUENT_SNIFFER_CLASS_NAME'] || 'Fluent::Plugin::ElasticsearchSimpleSniffer'}" + request_timeout "#{ENV['FLUENT_ELASTICSEARCH_REQUEST_TIMEOUT'] || '5s'}" + + flush_thread_count "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_THREAD_COUNT'] || '8'}" + flush_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_INTERVAL'] || '5s'}" + chunk_limit_size "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_CHUNK_LIMIT_SIZE'] || '2M'}" + queue_limit_length "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_QUEUE_LIMIT_LENGTH'] || '32'}" + retry_max_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_RETRY_MAX_INTERVAL'] || '30'}" + retry_forever true + + diff --git a/dapr-distributed-calendar/fluent/fluentd-dapr-with-rbac.yaml b/dapr-distributed-calendar/fluent/fluentd-dapr-with-rbac.yaml new file mode 100644 index 0000000..9995e9f --- /dev/null +++ b/dapr-distributed-calendar/fluent/fluentd-dapr-with-rbac.yaml @@ -0,0 +1,100 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fluentd + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fluentd + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - pods + - namespaces + verbs: + - get + - list + - watch + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: fluentd + namespace: default +roleRef: + kind: ClusterRole + name: fluentd + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: fluentd + namespace: kube-system +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: fluentd + namespace: kube-system + labels: + k8s-app: fluentd-logging + version: v1 +spec: + selector: + matchLabels: + k8s-app: fluentd-logging + version: v1 + template: + metadata: + labels: + k8s-app: fluentd-logging + version: v1 + spec: + serviceAccount: fluentd + serviceAccountName: fluentd + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: fluentd + image: fluent/fluentd-kubernetes-daemonset:v1.9.2-debian-elasticsearch7-1.0 + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: "elasticsearch-master.observability" + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" + - name: FLUENT_ELASTICSEARCH_SCHEME + value: "http" + - name: FLUENT_UID + value: "0" + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 200Mi + volumeMounts: + - name: varlog + mountPath: /var/log + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: fluentd-config + mountPath: /fluentd/etc + terminationGracePeriodSeconds: 30 + volumes: + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: fluentd-config + configMap: + name: fluentd-config diff --git a/dapr-distributed-calendar/go/Dockerfile b/dapr-distributed-calendar/go/Dockerfile index 842d727..57257d1 100644 --- a/dapr-distributed-calendar/go/Dockerfile +++ b/dapr-distributed-calendar/go/Dockerfile @@ -1,7 +1,32 @@ +# #first stage - builder +# FROM golang:1.15-buster as builder +# WORKDIR /dir +# COPY go_events.go . +# RUN go get -d -v +# RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o app . +# #second stage +# FROM debian:buster-slim +# WORKDIR /root/ +# COPY --from=builder /dir/app . +# EXPOSE 6000 +# CMD ["./app"] + +# FROM golang:1.21-alpine +# WORKDIR /root +# COPY go_events.go . +# RUN go mod init go_events +# RUN go mod tidy +# RUN go get -d -v +# RUN go build go_events . +# EXPOSE 6000 +# CMD ["./go_events"] + #first stage - builder -FROM golang:1.15-buster as builder +FROM golang:1.20.12-alpine as builder WORKDIR /dir COPY go_events.go . +RUN go mod init go_events +RUN go mod tidy RUN go get -d -v RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o app . #second stage diff --git a/dapr-distributed-calendar/go/go_events.go b/dapr-distributed-calendar/go/go_events.go index 7ab9b3d..3907886 100644 --- a/dapr-distributed-calendar/go/go_events.go +++ b/dapr-distributed-calendar/go/go_events.go @@ -4,7 +4,7 @@ import ( "bytes" "encoding/json" "fmt" - "io/ioutil" + "io" "log" "net/http" "os" @@ -77,7 +77,7 @@ func deleteEvent(w http.ResponseWriter, r *http.Request) { log.Printf("Response after delete call: %s", resp.Status) defer resp.Body.Close() - bodyBytes, _ := ioutil.ReadAll(resp.Body) + bodyBytes, _ := io.ReadAll(resp.Body) log.Printf(string(bodyBytes)) } @@ -108,7 +108,7 @@ func getEvent(w http.ResponseWriter, r *http.Request) { log.Printf("Response after get call: %s", resp.Status) defer resp.Body.Close() - bodyBytes, err := ioutil.ReadAll(resp.Body) + bodyBytes, err := io.ReadAll(resp.Body) if err != nil { log.Printf("Error reading response body: %v", err) return diff --git a/dapr-distributed-calendar/kubernetes-deploy.sh b/dapr-distributed-calendar/kubernetes-deploy.sh index 6085f12..60cab08 100755 --- a/dapr-distributed-calendar/kubernetes-deploy.sh +++ b/dapr-distributed-calendar/kubernetes-deploy.sh @@ -18,26 +18,19 @@ # create namespace # kubectl create namespace 12-factor-app -# create OTel collector -# helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts -# helm repo update -# helm upgrade --install my-opentelemetry-collector open-telemetry/opentelemetry-collector \ -# --set mode=deployment \ -# --values otel/otel-collector-values.yaml -# --namespace 12-factor-app - # install OTel Operator helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts helm repo update helm install my-opentelemetry-operator open-telemetry/opentelemetry-operator \ --set admissionWebhooks.certManager.enabled=false \ --set admissionWebhooks.certManager.autoGenerateCert.enabled=true \ + --set manager.featureGates='operator.autoinstrumentation.go' \ --wait # create OTel collector and instrumentation kubectl apply -f otel/. -# install cert-manager +# install cert-manager OPTIONAL helm repo add jetstack https://charts.jetstack.io helm repo update helm upgrade --install \ @@ -48,12 +41,13 @@ helm upgrade --install \ --set installCRDs=true \ --wait -# install jaeger +# install jaeger (requires cert-manager) OPTIONAL kubectl create namespace observability -kubectl create -f https://github.com/jaegertracing/jaeger-operator/releases/download/v1.38.0/jaeger-operator.yaml -n observability --wait -kubectl apply -f jaeger/simplest.yaml --wait +kubectl create -f https://github.com/jaegertracing/jaeger-operator/releases/download/v1.38.0/jaeger-operator.yaml -n observability +kubectl wait --for=condition=ready pod --all --timeout=200s -n observability +kubectl apply -f jaeger/simplest.yaml -# install prometheus +# install prometheus OPTIONAL helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm repo update helm install prometheus prometheus-community/kube-prometheus-stack \ @@ -62,6 +56,15 @@ helm install prometheus prometheus-community/kube-prometheus-stack \ --values prometheus/kube-prometheus-stack-values.yaml \ --wait +# install elastic (requires namespace 'observability') OPTIONAL +helm repo add elastic https://helm.elastic.co +helm repo update +helm install elasticsearch elastic/elasticsearch --version 7.17.3 -n observability --set replicas=1 --wait +helm install kibana elastic/kibana --version 7.17.3 -n observability --wait +kubectl apply -f ./fluent/fluentd-config-map.yaml +kubectl apply -f ./fluent/fluentd-dapr-with-rbac.yaml +kubectl wait --for=condition=ready pod --all --timeout=200s -n observability + # install dapr helm repo add dapr https://dapr.github.io/helm-charts/ helm repo update @@ -71,17 +74,16 @@ helm upgrade --install dapr dapr/dapr \ --set global.logAsJson=true \ --wait -# install dapr dashboard +# install dapr dashboard OPTIONAL helm install dapr-dashboard dapr/dapr-dashboard --namespace dapr-system --wait # install redis helm repo add bitnami https://charts.bitnami.com/bitnami helm repo update helm install redis bitnami/redis --wait -# helm install redis bitnami/redis --set image.tag=6.2 -f redis/values.yaml --wait # deploy the 12-factor-app kubectl apply -f kubernetes/. -# get redis password +# get redis password (for manual interactions with the redis cli) OPTIONAL kubectl get secret redis -o jsonpath='{.data.redis-password}' | base64 --decode \ No newline at end of file diff --git a/dapr-distributed-calendar/kubernetes/appconfig.yaml b/dapr-distributed-calendar/kubernetes/appconfig.yaml index 2431380..836d727 100644 --- a/dapr-distributed-calendar/kubernetes/appconfig.yaml +++ b/dapr-distributed-calendar/kubernetes/appconfig.yaml @@ -11,16 +11,3 @@ spec: endpointAddress: otel-dapr-collector:4317 isSecure: false protocol: grpc - # metrics: - # enabled: true - # rules: - # - name: dapr_runtime_service_invocation_req_sent_total - # labels: - # - name: method - # regex: - # "orders/": "orders/.+" # match all methods under orders - # logging: - # apiLogging: - # enabled: true - # obfuscateURLs: true - # omitHealthChecks: true diff --git a/dapr-distributed-calendar/kubernetes/go-events.yaml b/dapr-distributed-calendar/kubernetes/go-events.yaml index a37ea88..266bdd0 100644 --- a/dapr-distributed-calendar/kubernetes/go-events.yaml +++ b/dapr-distributed-calendar/kubernetes/go-events.yaml @@ -15,8 +15,8 @@ spec: labels: app: go-events annotations: - instrumentation.opentelemetry.io/inject-go: "go-instrumentation" - instrumentation.opentelemetry.io/otel-go-auto-target-exe: '/root/' + # instrumentation.opentelemetry.io/inject-go: "go-instrumentation" + # instrumentation.opentelemetry.io/otel-go-auto-target-exe: '/root/app' dapr.io/enabled: "true" dapr.io/app-id: "go-events" dapr.io/app-port: "6000" @@ -30,15 +30,15 @@ spec: - name: go-events image: mati007thm/dapr-distributed-calendar_go-events:latest env: + # - name: OTEL_SERVICE_NAME + # value: "go-events" - name: APP_PORT value: "6000" - name: DAPR_HTTP_PORT value: "3500" - - name: OTEL_SERVICE_NAME - value: "go-events" ports: - containerPort: 6000 imagePullPolicy: Always securityContext: - runAsUser: 0 privileged: true + runAsUser: 0 diff --git a/dapr-distributed-calendar/kubernetes/node-controller.yaml b/dapr-distributed-calendar/kubernetes/node-controller.yaml index 5bf8e7a..f68e58b 100644 --- a/dapr-distributed-calendar/kubernetes/node-controller.yaml +++ b/dapr-distributed-calendar/kubernetes/node-controller.yaml @@ -31,7 +31,7 @@ spec: labels: app: controller annotations: - instrumentation.opentelemetry.io/inject-nodejs: "node-instrumentation" + # instrumentation.opentelemetry.io/inject-nodejs: "node-instrumentation" dapr.io/enabled: "true" dapr.io/app-id: "controller" dapr.io/app-port: "3000" @@ -43,12 +43,14 @@ spec: spec: containers: - name: controller - image: mati007thm/dapr-distributed-calendar_controller:latest + image: mati007thm/dapr-distributed-calendar_controller:manual env: - name: APP_PORT value: "3000" - - name: OTEL_SERVICE_NAME - value: "controller" + # - name: OTEL_EXPORTER_OTLP_METRICS_ENDPOINT + # value: "otel-dapr-collector:4317" + # - name: OTEL_SERVICE_NAME + # value: "controller" ports: - containerPort: 3000 imagePullPolicy: Always diff --git a/dapr-distributed-calendar/kubernetes/python-messages.yaml b/dapr-distributed-calendar/kubernetes/python-messages.yaml index abe804c..7649076 100644 --- a/dapr-distributed-calendar/kubernetes/python-messages.yaml +++ b/dapr-distributed-calendar/kubernetes/python-messages.yaml @@ -28,13 +28,22 @@ spec: containers: - name: messages image: mati007thm/dapr-distributed-calendar_messages:latest + # image: mati007thm/dapr-distributed-calendar_messages:woOtel env: + # - name: OTEL_SERVICE_NAME + # value: "messages" + # - name: OTEL_TRACES_EXPORTER + # value: console,otlp + # - name: OTEL_METRICS_EXPORTER + # value: console,otlp + # - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + # value: http://otel-dapr-collector:4317 + # - name: OTEL_EXPORTER_OTLP_METRICS_ENDPOINT + # value: http://otel-dapr-collector:4318 - name: "APP_PORT" value: "5000" - name: FLASK_RUN_PORT value: "5000" - # - name: OTEL_SERVICE_NAME - # value: "messages" ports: - containerPort: 5000 imagePullPolicy: Always diff --git a/dapr-distributed-calendar/node/Dockerfile b/dapr-distributed-calendar/node/Dockerfile index fb29596..f4c9a12 100644 --- a/dapr-distributed-calendar/node/Dockerfile +++ b/dapr-distributed-calendar/node/Dockerfile @@ -1,7 +1,22 @@ +# without instrumentation +# FROM node:17-alpine +# WORKDIR /app +# COPY . . +# RUN npm install +# # RUN npm install cloudevents +# EXPOSE 3000 +# CMD [ "node", "node_controller.js" ] + +# with manual instrumenatation FROM node:17-alpine WORKDIR /app COPY . . RUN npm install +RUN npm install @opentelemetry/api @opentelemetry/resources @opentelemetry/semantic-conventions +RUN npm install @opentelemetry/sdk-node +RUN npm install @opentelemetry/sdk-metrics +# RUN npm install @opentelemetry/exporter-metrics-otlp-proto +RUN npm install @opentelemetry/exporter-metrics-otlp-grpc # RUN npm install cloudevents EXPOSE 3000 -CMD [ "node", "node_controller.js" ] \ No newline at end of file +CMD [ "node", "--require", "./instrumentation.js", "node_controller.js"] \ No newline at end of file diff --git a/dapr-distributed-calendar/node/instrumentation.js b/dapr-distributed-calendar/node/instrumentation.js new file mode 100644 index 0000000..42ac4e6 --- /dev/null +++ b/dapr-distributed-calendar/node/instrumentation.js @@ -0,0 +1,65 @@ +/*instrumentation.js*/ +// only console exporter seems to work +const opentelemetry = require('@opentelemetry/api'); +const { + MeterProvider, + ConsoleMetricExporter, + PeriodicExportingMetricReader, +} = require('@opentelemetry/sdk-metrics'); +const { + OTLPMetricExporter, +} = require('@opentelemetry/exporter-metrics-otlp-grpc'); + +const { Resource } = require('@opentelemetry/resources'); +const { + SemanticResourceAttributes, +} = require('@opentelemetry/semantic-conventions'); + +const resource = Resource.default().merge( + new Resource({ + [SemanticResourceAttributes.SERVICE_NAME]: 'controller', + [SemanticResourceAttributes.SERVICE_VERSION]: '0.1.0', + }), +); + +const metricReader = new PeriodicExportingMetricReader({ + exporter: new OTLPMetricExporter({ + url: 'http://otel-dapr-collector:4317', + }), + exporter: new ConsoleMetricExporter(), +}); + +const myServiceMeterProvider = new MeterProvider({ + resource: resource, +}); + +myServiceMeterProvider.addMetricReader(metricReader); + +// Set this MeterProvider to be global to the app being instrumented. +opentelemetry.metrics.setGlobalMeterProvider(myServiceMeterProvider); + + + + +// possible other solution dont forget to set OTEL_EXPORTER_OTLP_METRICS_ENDPOINT env in pod +// const { MeterProvider, PeriodicExportingMetricReader } = require('@opentelemetry/sdk-metrics'); +// const { OTLPMetricExporter } = require('@opentelemetry/exporter-metrics-otlp-grpc'); + +// const metricExporter = new OTLPMetricExporter(); +// const resource = Resource.default().merge( +// new Resource({ +// [SemanticResourceAttributes.SERVICE_NAME]: 'controller', +// [SemanticResourceAttributes.SERVICE_VERSION]: '0.1.0', +// }), +// ); +// const meterProvider = new MeterProvider({ +// resource: resource +// }); + +// meterProvider.addMetricReader(new PeriodicExportingMetricReader({ +// exporter: metricExporter, +// exportIntervalMillis: 1000, +// })); + +// // Set this MeterProvider to be global to the app being instrumented. +// opentelemetry.metrics.setGlobalMeterProvider(meterProvider); diff --git a/dapr-distributed-calendar/node/node_controller.js b/dapr-distributed-calendar/node/node_controller.js index 7fb1a6e..0082491 100644 --- a/dapr-distributed-calendar/node/node_controller.js +++ b/dapr-distributed-calendar/node/node_controller.js @@ -18,6 +18,13 @@ const publishUrl = `http://localhost:${daprPort}/v1.0/publish/${pubsub_name}/${t const port = 3000; +const opentelemetry = require('@opentelemetry/api'); + +const myMeter = opentelemetry.metrics.getMeter('controller'); + +const newEventCounter = myMeter.createCounter('newEvents.counter'); +const getEventCounter = myMeter.createCounter('getEvents.counter'); +const deleteEventCounter = myMeter.createCounter('deleteEvents.counter'); // app.get('/dapr/subscribe', (_req, res) => { // res.json([ // { @@ -61,6 +68,7 @@ function send_notif(data) { } app.post('/newevent', (req, res) => { + newEventCounter.add(1); const data = req.body.data; const eventId = data.id; console.log("New event registration! Event ID: " + eventId); @@ -87,6 +95,7 @@ app.post('/newevent', (req, res) => { }); app.delete('/event/:id', (req, res) => { + deleteEventCounter.add(1); const key = req.params.id; console.log('Invoke Delete for ID ' + key); @@ -112,6 +121,7 @@ app.delete('/event/:id', (req, res) => { }); app.get('/event/:id', (req, res) =>{ + getEventCounter.add(1); const key = req.params.id; console.log('Invoke Get for ID ' + key); diff --git a/dapr-distributed-calendar/otel/instrumentation.yaml b/dapr-distributed-calendar/otel/instrumentation.yaml index 5e93b53..4c7b6b3 100644 --- a/dapr-distributed-calendar/otel/instrumentation.yaml +++ b/dapr-distributed-calendar/otel/instrumentation.yaml @@ -1,48 +1,50 @@ -apiVersion: opentelemetry.io/v1alpha1 -kind: Instrumentation -metadata: - name: node-instrumentation -spec: - exporter: - endpoint: otel-dapr-collector:4317 - propagators: - - tracecontext - - baggage - sampler: - type: parentbased_traceidratio - argument: "1" ---- -apiVersion: opentelemetry.io/v1alpha1 -kind: Instrumentation -metadata: - name: python-instrumentation -spec: - exporter: - endpoint: otel-dapr-collector:4318 - env: - propagators: - - tracecontext - - baggage - sampler: - type: parentbased_traceidratio - argument: "1" - python: - env: - - name: OTEL_LOGS_EXPORTER - value: otlp_proto_http - - name: OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED - value: 'true' ---- -apiVersion: opentelemetry.io/v1alpha1 -kind: Instrumentation -metadata: - name: go-instrumentation -spec: - exporter: - endpoint: otel-dapr-collector:4318 - propagators: - - tracecontext - - baggage - sampler: - type: parentbased_traceidratio - argument: "1" \ No newline at end of file +# apiVersion: opentelemetry.io/v1alpha1 +# kind: Instrumentation +# metadata: +# name: node-instrumentation +# spec: +# exporter: +# endpoint: http://otel-dapr-collector:4317 +# propagators: +# - tracecontext +# - baggage +# sampler: +# type: parentbased_traceidratio +# argument: "1" +# --- +# # Auto instrumentation for python is not working yet due to version mismatch +# apiVersion: opentelemetry.io/v1alpha1 +# kind: Instrumentation +# metadata: +# name: python-instrumentation +# spec: +# exporter: +# endpoint: http://otel-dapr-collector:4317 +# env: +# propagators: +# - tracecontext +# - baggage +# sampler: +# type: parentbased_traceidratio +# argument: "1" +# python: +# env: +# - name: OTEL_LOGS_EXPORTER +# value: otlp_proto_http +# - name: OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED +# value: 'true' +# --- +# # Auto instrumentation for go is not working yet +# apiVersion: opentelemetry.io/v1alpha1 +# kind: Instrumentation +# metadata: +# name: go-instrumentation +# spec: +# exporter: +# endpoint: http://otel-dapr-collector:4318 +# propagators: +# - tracecontext +# - baggage +# sampler: +# type: parentbased_traceidratio +# argument: "1" diff --git a/dapr-distributed-calendar/otel/otel-collector-values.yaml b/dapr-distributed-calendar/otel/otel-collector-values.yaml index 52d4a0f..2e8fb9b 100644 --- a/dapr-distributed-calendar/otel/otel-collector-values.yaml +++ b/dapr-distributed-calendar/otel/otel-collector-values.yaml @@ -1,109 +1,9 @@ -# mode: deployment -# replicaCount: 1 -# podAnnotations: -# dapr_collector_demo: "true" -# config: -# exporters: -# # NOTE: Prior to v0.86.0 use `logging` instead of `debug`. -# debug: {} -# otlp/jaeger: -# endpoint: simplest-collector:4317 -# tls: -# insecure: true -# prometheus: -# enable_open_metrics: true -# endpoint: 0.0.0.0:9464 -# resource_to_telemetry_conversion: -# enabled: true - -# extensions: -# health_check: {} -# memory_ballast: -# size_in_percentage: 40 -# processors: -# batch: {} -# memory_limiter: -# check_interval: 5s -# limit_percentage: 80 -# spike_limit_percentage: 25 -# receivers: -# jaeger: -# protocols: -# grpc: -# endpoint: ${env:MY_POD_IP}:14250 -# thrift_compact: -# endpoint: ${env:MY_POD_IP}:6831 -# thrift_http: -# endpoint: ${env:MY_POD_IP}:14268 -# otlp: -# protocols: -# grpc: -# endpoint: ${env:MY_POD_IP}:4317 -# http: -# endpoint: ${env:MY_POD_IP}:4318 -# prometheus: -# config: -# scrape_configs: -# - job_name: opentelemetry-collector -# scrape_interval: 10s -# static_configs: -# - targets: -# - ${env:MY_POD_IP}:8888 -# zipkin: -# endpoint: ${env:MY_POD_IP}:9411 -# service: -# extensions: -# - health_check -# - memory_ballast -# pipelines: -# logs: -# exporters: -# - debug -# processors: -# - memory_limiter -# - batch -# receivers: -# - otlp -# metrics: -# exporters: -# - prometheus -# processors: -# - memory_limiter -# - batch -# receivers: -# - otlp -# - prometheus -# traces: -# exporters: -# - debug -# - otlp/jaeger -# processors: -# - memory_limiter -# - batch -# receivers: -# - otlp -# - jaeger -# - zipkin -# telemetry: -# metrics: -# address: ${env:MY_POD_IP}:8888 - -# kind: ServiceAccount -# apiVersion: v1 -# metadata: -# name: otel-dapr-collector -# labels: -# app.kubernetes.io/component: opentelemetry-collector -# app.kubernetes.io/instance: default.otel-dapr -# app.kubernetes.io/managed-by: opentelemetry-operator -# app.kubernetes.io/name: otel-dapr-collector -# app.kubernetes.io/part-of: opentelemetry -# app.kubernetes.io/version: latest -# --- apiVersion: opentelemetry.io/v1alpha1 kind: OpenTelemetryCollector metadata: name: otel-dapr + annotations: + dapr_collector_demo: "true" spec: config: | receivers: @@ -111,6 +11,14 @@ spec: protocols: grpc: http: + prometheus: + config: + scrape_configs: + - job_name: otel-collector + scrape_interval: 5s + static_configs: + - targets: [controller-dapr:9090, messages-dapr:9090, go-events-dapr:9090] + processors: memory_limiter: check_interval: 1s @@ -123,10 +31,16 @@ spec: exporters: # NOTE: Prior to v0.86.0 use `logging` instead of `debug`. debug: + verbosity: detailed otlp/jaeger: endpoint: simplest-collector:4317 tls: insecure: true + prometheus: + enable_open_metrics: true + endpoint: 0.0.0.0:9464 + resource_to_telemetry_conversion: + enabled: true service: pipelines: @@ -135,9 +49,9 @@ spec: processors: [memory_limiter, batch] exporters: [debug, otlp/jaeger] metrics: - receivers: [otlp] + receivers: [otlp, prometheus] processors: [memory_limiter, batch] - exporters: [debug] + exporters: [debug, prometheus] logs: receivers: [otlp] processors: [memory_limiter, batch] diff --git a/dapr-distributed-calendar/prometheus/kube-prometheus-stack-values.yaml b/dapr-distributed-calendar/prometheus/kube-prometheus-stack-values.yaml index 2b212ba..67e40bc 100644 --- a/dapr-distributed-calendar/prometheus/kube-prometheus-stack-values.yaml +++ b/dapr-distributed-calendar/prometheus/kube-prometheus-stack-values.yaml @@ -15,16 +15,8 @@ prometheus: # names: [12-factor-app] relabel_configs: - source_labels: [__meta_kubernetes_pod_annotation_dapr_collector_demo] - # - source_labels: [meta_kubernetes_pod_label_dapr_io_metrics_enabled] action: keep regex: true - - job_name: dapr-demo - honor_labels: true - kubernetes_sd_configs: - - namespaces: - own_namespace: true - names: [dapr-system, kube-system] - role: pod - job_name: 'traefik' honor_labels: true kubernetes_sd_configs: diff --git a/dapr-distributed-calendar/python/Dockerfile b/dapr-distributed-calendar/python/Dockerfile index 6ef6aad..3e2c057 100644 --- a/dapr-distributed-calendar/python/Dockerfile +++ b/dapr-distributed-calendar/python/Dockerfile @@ -1,11 +1,22 @@ -FROM python:3.12-alpine +FROM python:3.11-alpine COPY . /app WORKDIR /app # Install build dependencies -RUN apk add --no-cache build-base +RUN apk add python3-dev +RUN apk add build-base +# RUN apk add --no-cache build-base # Install Python dependencies +RUN pip install --upgrade pip +# RUN apk add gcc python3-dev +# RUN apk add linux-headers +# RUN apk add py3-psutil +# RUN pip install --upgrade psutil RUN pip3 install wheel python-dotenv flask_cors flask RUN pip install dapr +# https://github.com/open-telemetry/opentelemetry-python-contrib/issues/2053#issuecomment-1845408543 +# RUN pip install opentelemetry-distro==0.41b0 opentelemetry-exporter-otlp==1.20.0 +# RUN pip install opentelemetry-distro opentelemetry-exporter-otlp +# RUN opentelemetry-bootstrap -a install ENTRYPOINT ["python"] EXPOSE 5000 CMD ["messages.py"] \ No newline at end of file