From a40e162ec865aa9acdcdeaa09b94f0265f269f63 Mon Sep 17 00:00:00 2001 From: Morven Cao Date: Mon, 17 Feb 2025 13:29:40 +0800 Subject: [PATCH] enhance log level and environment (#201) * consistent environment settings. Signed-off-by: morvencao * add comments for log level mapping. Signed-off-by: morvencao * rebase code. Signed-off-by: morvencao --------- Signed-off-by: morvencao --- Makefile | 18 ++++++++--------- cmd/maestro/environments/types.go | 2 +- cmd/maestro/main.go | 23 ++++++++++++++++++++-- go.mod | 2 +- go.sum | 4 ++-- templates/agent-template-aro-hcp.yml | 13 ++++++++++++ templates/agent-template-rosa.yml | 14 ++++++++++--- templates/agent-template.yml | 14 +++++++++++++ templates/agent-tls-template.yml | 14 +++++++++++++ templates/service-template-aro-hcp.yml | 6 +++--- templates/service-template-rosa.yml | 4 ++-- templates/service-template.yml | 8 ++++---- templates/service-tls-template.yml | 8 ++++---- test/integration/resource_test.go | 21 ++++++++++++++++++++ test/integration/status_dispatcher_test.go | 2 ++ 15 files changed, 122 insertions(+), 31 deletions(-) diff --git a/Makefile b/Makefile index 2c085e34..c90c7813 100755 --- a/Makefile +++ b/Makefile @@ -63,7 +63,7 @@ mqtt_client_cert ?= "" mqtt_client_key ?= "" # Log verbosity level -klog_v:=10 +klog_v:=4 # Location of the JSON web key set used to verify tokens: jwks_url:=https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/certs @@ -144,11 +144,11 @@ GOLANGCI_LINT_BIN:=$(shell go env GOPATH)/bin/golangci-lint ### Envrionment-sourced variables with defaults # Can be overriden by setting environment var before running # Example: -# OCM_ENV=testing make run -# export OCM_ENV=testing; make run +# MAESTRO_ENV=testing make run +# export MAESTRO_ENV=testing; make run # Set the environment to development by default -ifndef OCM_ENV - OCM_ENV:=development +ifndef MAESTRO_ENV + MAESTRO_ENV:=development endif ifndef TEST_SUMMARY_FORMAT @@ -221,7 +221,7 @@ install: check-gopath # Examples: # make test TESTFLAGS="-run TestSomething" test: - OCM_ENV=testing gotestsum --jsonfile-timing-events=$(unit_test_json_output) --format $(TEST_SUMMARY_FORMAT) -- -p 1 -v $(TESTFLAGS) \ + MAESTRO_ENV=testing gotestsum --jsonfile-timing-events=$(unit_test_json_output) --format $(TEST_SUMMARY_FORMAT) -- -p 1 -v $(TESTFLAGS) \ ./pkg/... \ ./cmd/... .PHONY: test @@ -240,12 +240,12 @@ test-integration: test-integration-mqtt test-integration-grpc .PHONY: test-integration test-integration-mqtt: - BROKER=mqtt OCM_ENV=testing gotestsum --jsonfile-timing-events=$(mqtt_integration_test_json_output) --format $(TEST_SUMMARY_FORMAT) -- -p 1 -ldflags -s -v -timeout 1h $(TESTFLAGS) \ + BROKER=mqtt MAESTRO_ENV=testing gotestsum --jsonfile-timing-events=$(mqtt_integration_test_json_output) --format $(TEST_SUMMARY_FORMAT) -- -p 1 -ldflags -s -v -timeout 1h $(TESTFLAGS) \ ./test/integration .PHONY: test-integration-mqtt test-integration-grpc: - BROKER=grpc OCM_ENV=testing gotestsum --jsonfile-timing-events=$(grpc_integration_test_json_output) --format $(TEST_SUMMARY_FORMAT) -- -p 1 -ldflags -s -v -timeout 1h $(TESTFLAGS) \ + BROKER=grpc MAESTRO_ENV=testing gotestsum --jsonfile-timing-events=$(grpc_integration_test_json_output) --format $(TEST_SUMMARY_FORMAT) -- -p 1 -ldflags -s -v -timeout 1h $(TESTFLAGS) \ ./test/integration .PHONY: test-integration-grpc @@ -295,7 +295,7 @@ cmds: --filename="templates/$*-template.yml" \ --local="true" \ --ignore-unknown-parameters="true" \ - --param="ENVIRONMENT=$(OCM_ENV)" \ + --param="ENVIRONMENT=$(MAESTRO_ENV)" \ --param="KLOG_V=$(klog_v)" \ --param="SERVER_REPLICAS=$(SERVER_REPLICAS)" \ --param="DATABASE_HOST=$(db_host)" \ diff --git a/cmd/maestro/environments/types.go b/cmd/maestro/environments/types.go index 53fbf2ce..2f3c0c71 100755 --- a/cmd/maestro/environments/types.go +++ b/cmd/maestro/environments/types.go @@ -16,7 +16,7 @@ const ( DevelopmentEnv string = "development" ProductionEnv string = "production" - EnvironmentStringKey string = "OCM_ENV" + EnvironmentStringKey string = "MAESTRO_ENV" EnvironmentDefault string = DevelopmentEnv ) diff --git a/cmd/maestro/main.go b/cmd/maestro/main.go index 5cecb502..9fd1b384 100755 --- a/cmd/maestro/main.go +++ b/cmd/maestro/main.go @@ -7,6 +7,7 @@ import ( "github.com/go-logr/zapr" "github.com/openshift-online/maestro/cmd/maestro/agent" + "github.com/openshift-online/maestro/cmd/maestro/environments" "github.com/openshift-online/maestro/cmd/maestro/migrate" "github.com/openshift-online/maestro/cmd/maestro/servecmd" "github.com/spf13/cobra" @@ -36,11 +37,29 @@ func main() { klog.Fatalf("can't parse log level: %v", err) } - // Initialize zap logger - zc := zap.NewDevelopmentConfig() + // Initialize zap logger based on environment + var zc zap.Config + env := environments.GetEnvironmentStrFromEnv() + switch env { + case environments.DevelopmentEnv: + zc = zap.NewDevelopmentConfig() + case environments.ProductionEnv: + zc = zap.NewProductionConfig() + default: + zc = zap.NewDevelopmentConfig() + } + // zap log level is the inverse of klog log level, for more details refer to: // https://github.com/go-logr/zapr?tab=readme-ov-file#increasing-verbosity + // zap level mapping to klog level: + // fatal: -5, panic: -4, dpanic: -3, error: -2, warn: -1, info: 0, debug: 1 + // the higher the log level, the more verbose the logs: + // if the log level is 0, it will log everything except debug logs; + // if the log level is 1, it will log everything including debug logs; + // if the log level is 2 or higher, it will log everything including klog verbose logs. zc.Level = zap.NewAtomicLevelAt(zapcore.Level(0 - logLevel)) + // Disable stacktrace to reduce logs size + zc.DisableStacktrace = true zapLog, err := zc.Build() if err != nil { klog.Fatalf("can't initialize zap logger: %v", err) diff --git a/go.mod b/go.mod index 02171025..6fbb495f 100755 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( k8s.io/klog/v2 v2.130.1 open-cluster-management.io/api v0.15.1-0.20241210025410-0ba6809d0ae2 open-cluster-management.io/ocm v0.15.1-0.20250108154653-2397c4e91119 - open-cluster-management.io/sdk-go v0.15.1-0.20250212140629-c9cbca9e52cb + open-cluster-management.io/sdk-go v0.15.1-0.20250217031345-04acb74ee5ae sigs.k8s.io/yaml v1.4.0 ) diff --git a/go.sum b/go.sum index 1dbfd962..312e4184 100644 --- a/go.sum +++ b/go.sum @@ -893,8 +893,8 @@ open-cluster-management.io/api v0.15.1-0.20241210025410-0ba6809d0ae2 h1:zkp3VJnv open-cluster-management.io/api v0.15.1-0.20241210025410-0ba6809d0ae2/go.mod h1:9erZEWEn4bEqh0nIX2wA7f/s3KCuFycQdBrPrRzi0QM= open-cluster-management.io/ocm v0.15.1-0.20250108154653-2397c4e91119 h1:Ftx7vxDumTB9d4+ZdcqYwQavTOVzgF5h6vAXJ/gh0IE= open-cluster-management.io/ocm v0.15.1-0.20250108154653-2397c4e91119/go.mod h1:T9pfSm3EYHnysEP9JYfCojV2pI44IYMz3zaZNylulz8= -open-cluster-management.io/sdk-go v0.15.1-0.20250212140629-c9cbca9e52cb h1:w8ZV4DTTyIeRPR9OIgdNGLtC4YGmfSaq9lZeiJd7qZI= -open-cluster-management.io/sdk-go v0.15.1-0.20250212140629-c9cbca9e52cb/go.mod h1:fi5WBsbC5K3txKb8eRLuP0Sim/Oqz/PHX18skAEyjiA= +open-cluster-management.io/sdk-go v0.15.1-0.20250217031345-04acb74ee5ae h1:jxNiZbTXMFbdLUIFiowpSCSegXPWfGML0jlaEBCnq5o= +open-cluster-management.io/sdk-go v0.15.1-0.20250217031345-04acb74ee5ae/go.mod h1:fi5WBsbC5K3txKb8eRLuP0Sim/Oqz/PHX18skAEyjiA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsAtVhSeUFseziht227YAWYHLGNM8QPwY= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.19.3 h1:XO2GvC9OPftRst6xWCpTgBZO04S2cbp0Qqkj8bX1sPw= diff --git a/templates/agent-template-aro-hcp.yml b/templates/agent-template-aro-hcp.yml index 7e3d462a..a71ae161 100644 --- a/templates/agent-template-aro-hcp.yml +++ b/templates/agent-template-aro-hcp.yml @@ -14,6 +14,11 @@ labels: template: maestro-agent parameters: +- name: ENVIRONMENT + displayName: Environment + description: Which maestro environment to use for this deployment + value: production + - name: AGENT_NAMESPACE description: namespace of maestro agent @@ -33,6 +38,11 @@ parameters: displayName: Image tag value: latest +- name: KLOG_V + displayName: KLOG V Level + description: Log verbosity level + value: "4" + objects: - apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -271,7 +281,10 @@ objects: - --workload-source-driver=mqtt - --workload-source-config=/secrets/mqtt/config.yaml - --cloudevents-client-id=$(CONSUMER_NAME)-work-agent + - -v=${KLOG_V} env: + - name: "MAESTRO_ENV" + value: "${ENVIRONMENT}" - name: CONSUMER_NAME valueFrom: secretKeyRef: diff --git a/templates/agent-template-rosa.yml b/templates/agent-template-rosa.yml index bb63bd4f..99b37745 100644 --- a/templates/agent-template-rosa.yml +++ b/templates/agent-template-rosa.yml @@ -14,6 +14,11 @@ labels: template: maestro-agent parameters: +- name: ENVIRONMENT + displayName: Environment + description: Which maestro environment to use for this deployment + value: production + - name: AGENT_NAMESPACE description: namespace of maestro agent @@ -33,14 +38,14 @@ parameters: displayName: Image tag value: latest -- name: MQTT_HOST - description: Hostname for the mqtt broker - - name: KLOG_V displayName: KLOG V Level description: Log verbosity level value: "4" +- name: MQTT_HOST + description: Hostname for the mqtt broker. + objects: - apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -273,6 +278,9 @@ objects: - name: maestro-agent image: ${IMAGE_REGISTRY}/${IMAGE_REPOSITORY}:${IMAGE_TAG} imagePullPolicy: IfNotPresent + env: + - name: "MAESTRO_ENV" + value: "${ENVIRONMENT}" command: - /usr/local/bin/maestro - agent diff --git a/templates/agent-template.yml b/templates/agent-template.yml index 7350c8f5..035784b3 100644 --- a/templates/agent-template.yml +++ b/templates/agent-template.yml @@ -14,6 +14,11 @@ labels: template: maestro-agent parameters: +- name: ENVIRONMENT + displayName: Environment + description: Which maestro environment to use for this deployment + value: production + - name: AGENT_NAMESPACE description: namespace of maestro agent @@ -33,6 +38,11 @@ parameters: displayName: Image tag value: latest +- name: KLOG_V + displayName: KLOG V Level + description: Log verbosity level + value: "4" + - name: MESSAGE_DRIVER_TYPE displayName: Message Driver Type description: Message driver type, mqtt or grpc. @@ -290,6 +300,9 @@ objects: - name: maestro-agent image: ${IMAGE_REGISTRY}/${IMAGE_REPOSITORY}:${IMAGE_TAG} imagePullPolicy: IfNotPresent + env: + - name: "MAESTRO_ENV" + value: "${ENVIRONMENT}" command: - /usr/local/bin/maestro - agent @@ -297,6 +310,7 @@ objects: - --workload-source-driver=${MESSAGE_DRIVER_TYPE} - --workload-source-config=/secrets/${MESSAGE_DRIVER_TYPE}/config.yaml - --cloudevents-client-id=${CONSUMER_NAME}-work-agent + - -v=${KLOG_V} volumeMounts: - name: ${MESSAGE_DRIVER_TYPE} mountPath: /secrets/${MESSAGE_DRIVER_TYPE} diff --git a/templates/agent-tls-template.yml b/templates/agent-tls-template.yml index 06f0d71b..4ed955d0 100644 --- a/templates/agent-tls-template.yml +++ b/templates/agent-tls-template.yml @@ -14,6 +14,11 @@ labels: template: maestro-agent parameters: +- name: ENVIRONMENT + displayName: Environment + description: Which maestro environment to use for this deployment + value: production + - name: AGENT_NAMESPACE description: namespace of maestro agent @@ -33,6 +38,11 @@ parameters: displayName: Image tag value: latest +- name: KLOG_V + displayName: KLOG V Level + description: Log verbosity level + value: "4" + - name: MESSAGE_DRIVER_TYPE displayName: Message Driver Type description: Message driver type, mqtt or grpc. @@ -290,6 +300,9 @@ objects: - name: maestro-agent image: ${IMAGE_REGISTRY}/${IMAGE_REPOSITORY}:${IMAGE_TAG} imagePullPolicy: IfNotPresent + env: + - name: "MAESTRO_ENV" + value: "${ENVIRONMENT}" command: - /usr/local/bin/maestro - agent @@ -298,6 +311,7 @@ objects: - --workload-source-config=/secrets/${MESSAGE_DRIVER_TYPE}/config.yaml - --cloudevents-client-id=${CONSUMER_NAME}-work-agent - --appliedmanifestwork-eviction-grace-period=30s + - -v=${KLOG_V} volumeMounts: - name: ${MESSAGE_DRIVER_TYPE} mountPath: /secrets/${MESSAGE_DRIVER_TYPE} diff --git a/templates/service-template-aro-hcp.yml b/templates/service-template-aro-hcp.yml index 441cfa89..c5d9c2ce 100755 --- a/templates/service-template-aro-hcp.yml +++ b/templates/service-template-aro-hcp.yml @@ -16,7 +16,7 @@ parameters: - name: ENVIRONMENT displayName: Environment - description: Which Account Manager environment to use for this deployment + description: Which maestro environment to use for this deployment value: production - name: IMAGE_REGISTRY @@ -38,7 +38,7 @@ parameters: - name: KLOG_V displayName: KLOG V Level description: Log verbosity level - value: "10" + value: "4" - name: MEMORY_REQUEST description: Memory request for the API pods. @@ -231,7 +231,7 @@ objects: mountPath: /secrets/mqtt-creds readOnly: true env: - - name: "AMS_ENV" + - name: "MAESTRO_ENV" value: "${ENVIRONMENT}" - name: POD_NAME valueFrom: diff --git a/templates/service-template-rosa.yml b/templates/service-template-rosa.yml index 366abc47..7ffccbef 100644 --- a/templates/service-template-rosa.yml +++ b/templates/service-template-rosa.yml @@ -16,7 +16,7 @@ parameters: - name: ENVIRONMENT displayName: Environment - description: Which Account Manager environment to use for this deployment + description: Which maestro environment to use for this deployment value: production - name: IMAGE_REGISTRY @@ -213,7 +213,7 @@ objects: mountPath: /secrets/mqtt-creds readOnly: true env: - - name: "AMS_ENV" + - name: "MAESTRO_ENV" value: "${ENVIRONMENT}" - name: POD_NAME valueFrom: diff --git a/templates/service-template.yml b/templates/service-template.yml index c548061b..cbe0b8fd 100755 --- a/templates/service-template.yml +++ b/templates/service-template.yml @@ -16,7 +16,7 @@ parameters: - name: ENVIRONMENT displayName: Environment - description: Which Account Manager environment to use for this deployment + description: Which maestro environment to use for this deployment value: production - name: IMAGE_REGISTRY @@ -47,7 +47,7 @@ parameters: - name: KLOG_V displayName: KLOG V Level description: Log verbosity level - value: "10" + value: "4" - name: MEMORY_REQUEST description: Memory request for the API pods. @@ -324,7 +324,7 @@ objects: - name: authentication mountPath: /configs/authentication env: - - name: "AMS_ENV" + - name: "MAESTRO_ENV" value: "${ENVIRONMENT}" - name: POD_NAME valueFrom: @@ -415,7 +415,7 @@ objects: app: maestro port: api annotations: - description: Exposes and load balances the account manager pods + description: Exposes and load balances the maestro pods service.alpha.openshift.io/serving-cert-secret-name: maestro-tls spec: selector: diff --git a/templates/service-tls-template.yml b/templates/service-tls-template.yml index 5966d4dc..5cca62f3 100755 --- a/templates/service-tls-template.yml +++ b/templates/service-tls-template.yml @@ -16,7 +16,7 @@ parameters: - name: ENVIRONMENT displayName: Environment - description: Which Account Manager environment to use for this deployment + description: Which maestro environment to use for this deployment value: production - name: IMAGE_REGISTRY @@ -34,7 +34,7 @@ parameters: - name: KLOG_V displayName: KLOG V Level description: Log verbosity level - value: "10" + value: "4" - name: MEMORY_REQUEST description: Memory request for the API pods. @@ -366,7 +366,7 @@ objects: - name: maestro-grpc-cert mountPath: /secrets/maestro-grpc-cert env: - - name: "AMS_ENV" + - name: "MAESTRO_ENV" value: "${ENVIRONMENT}" - name: POD_NAME valueFrom: @@ -460,7 +460,7 @@ objects: app: maestro port: api annotations: - description: Exposes and load balances the account manager pods + description: Exposes and load balances the maestro pods service.alpha.openshift.io/serving-cert-secret-name: maestro-tls spec: selector: diff --git a/test/integration/resource_test.go b/test/integration/resource_test.go index 4ad6e251..d0afbc06 100755 --- a/test/integration/resource_test.go +++ b/test/integration/resource_test.go @@ -186,12 +186,30 @@ func TestResourcePost(t *testing.T) { families := getServerMetrics(t, "http://localhost:8080/metrics") labels := []*prommodel.LabelPair{ {Name: strPtr("source"), Value: strPtr("maestro")}, + {Name: strPtr("original_source"), Value: strPtr("none")}, {Name: strPtr("cluster"), Value: strPtr(clusterName)}, {Name: strPtr("type"), Value: strPtr("io.open-cluster-management.works.v1alpha1.manifestbundles")}, {Name: strPtr("subresource"), Value: strPtr(string(types.SubResourceSpec))}, {Name: strPtr("action"), Value: strPtr("create_request")}, } checkServerCounterMetric(t, families, "cloudevents_sent_total", labels, 1.0) + labels = []*prommodel.LabelPair{ + {Name: strPtr("source"), Value: strPtr("maestro")}, + {Name: strPtr("cluster"), Value: strPtr(clusterName)}, + {Name: strPtr("type"), Value: strPtr("io.open-cluster-management.works.v1alpha1.manifestbundles")}, + {Name: strPtr("subresource"), Value: strPtr(string(types.SubResourceSpec))}, + {Name: strPtr("action"), Value: strPtr("create_request")}, + } + checkServerCounterMetric(t, families, "cloudevents_received_total", labels, 1.0) + labels = []*prommodel.LabelPair{ + {Name: strPtr("source"), Value: strPtr(clusterName)}, + {Name: strPtr("original_source"), Value: strPtr("maestro")}, + {Name: strPtr("cluster"), Value: strPtr(clusterName)}, + {Name: strPtr("type"), Value: strPtr("io.open-cluster-management.works.v1alpha1.manifestbundles")}, + {Name: strPtr("subresource"), Value: strPtr(string(types.SubResourceStatus))}, + {Name: strPtr("action"), Value: strPtr("update_request")}, + } + checkServerCounterMetric(t, families, "cloudevents_sent_total", labels, 1.0) labels = []*prommodel.LabelPair{ {Name: strPtr("source"), Value: strPtr(clusterName)}, {Name: strPtr("cluster"), Value: strPtr(clusterName)}, @@ -824,6 +842,7 @@ func TestResourceFromGRPC(t *testing.T) { if h.Broker != "grpc" { labels = []*prommodel.LabelPair{ {Name: strPtr("source"), Value: strPtr("maestro")}, + {Name: strPtr("original_source"), Value: strPtr("none")}, {Name: strPtr("cluster"), Value: strPtr(clusterName)}, {Name: strPtr("type"), Value: strPtr("io.open-cluster-management.works.v1alpha1.manifestbundles")}, {Name: strPtr("subresource"), Value: strPtr(string(types.SubResourceSpec))}, @@ -832,6 +851,7 @@ func TestResourceFromGRPC(t *testing.T) { checkServerCounterMetric(t, families, "cloudevents_sent_total", labels, 2.0) labels = []*prommodel.LabelPair{ {Name: strPtr("source"), Value: strPtr("maestro")}, + {Name: strPtr("original_source"), Value: strPtr("none")}, {Name: strPtr("cluster"), Value: strPtr(clusterName)}, {Name: strPtr("type"), Value: strPtr("io.open-cluster-management.works.v1alpha1.manifestbundles")}, {Name: strPtr("subresource"), Value: strPtr(string(types.SubResourceSpec))}, @@ -840,6 +860,7 @@ func TestResourceFromGRPC(t *testing.T) { checkServerCounterMetric(t, families, "cloudevents_sent_total", labels, 2.0) labels = []*prommodel.LabelPair{ {Name: strPtr("source"), Value: strPtr("maestro")}, + {Name: strPtr("original_source"), Value: strPtr("none")}, {Name: strPtr("cluster"), Value: strPtr(clusterName)}, {Name: strPtr("type"), Value: strPtr("io.open-cluster-management.works.v1alpha1.manifestbundles")}, {Name: strPtr("subresource"), Value: strPtr(string(types.SubResourceSpec))}, diff --git a/test/integration/status_dispatcher_test.go b/test/integration/status_dispatcher_test.go index eec946c1..2c3d1236 100644 --- a/test/integration/status_dispatcher_test.go +++ b/test/integration/status_dispatcher_test.go @@ -67,6 +67,7 @@ func TestStatusDispatcher(t *testing.T) { families := getServerMetrics(t, "http://localhost:8080/metrics") labels := []*prommodel.LabelPair{ {Name: strPtr("source"), Value: strPtr("maestro")}, + {Name: strPtr("original_source"), Value: strPtr("none")}, {Name: strPtr("cluster"), Value: strPtr(consumer1)}, {Name: strPtr("type"), Value: strPtr("io.open-cluster-management.works.v1alpha1.manifestbundles")}, {Name: strPtr("subresource"), Value: strPtr(string(types.SubResourceStatus))}, @@ -75,6 +76,7 @@ func TestStatusDispatcher(t *testing.T) { checkServerCounterMetric(t, families, "cloudevents_sent_total", labels, 1.0) labels = []*prommodel.LabelPair{ {Name: strPtr("source"), Value: strPtr("maestro")}, + {Name: strPtr("original_source"), Value: strPtr("none")}, {Name: strPtr("cluster"), Value: strPtr(consumer2)}, {Name: strPtr("type"), Value: strPtr("io.open-cluster-management.works.v1alpha1.manifestbundles")}, {Name: strPtr("subresource"), Value: strPtr(string(types.SubResourceStatus))},