From 8b54821b76c82cc864b44f3ba6e54548a91d1f1b Mon Sep 17 00:00:00 2001 From: Jacob Aronoff Date: Thu, 22 Aug 2024 15:01:25 -0400 Subject: [PATCH 1/4] fix release --- .github/workflows/publish-image.yml | 3 + arrow/otelcolarrow-build.yaml | 6 +- cno-oks-values.yaml | 166 ++++++++++++++++++++++++++++ 3 files changed, 172 insertions(+), 3 deletions(-) create mode 100644 cno-oks-values.yaml diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index 2161ce9..d9c9af4 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -3,6 +3,9 @@ name: Create and publish a Docker image for Otel Collectors with Arrow on: workflow_dispatch: + push: + branches: + - main # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: diff --git a/arrow/otelcolarrow-build.yaml b/arrow/otelcolarrow-build.yaml index 5ece441..0616a4c 100644 --- a/arrow/otelcolarrow-build.yaml +++ b/arrow/otelcolarrow-build.yaml @@ -37,7 +37,7 @@ dist: # description: My Organization telemetry collector # name: otelarrowcol - version: 0.25.0 + version: 0.26.0 description: ServiceNow Cloud Observability OpenTelemetry Protocol with Apache Arrow gateway collector # This indicates which version of the core collector components as @@ -126,8 +126,8 @@ processors: # The tail sampling processor is included, by request. Please note that this component # does not provide the necessary information to correctly adjust counts in metrics derived # from sampled spans. - - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor v0.103.0 - + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor v0.107.0 + # You may be interested in other transform components in the # Collector-Contrib repository. Here are some that might be useful: # diff --git a/cno-oks-values.yaml b/cno-oks-values.yaml new file mode 100644 index 0000000..ce43c22 --- /dev/null +++ b/cno-oks-values.yaml @@ -0,0 +1,166 @@ +clusterName: demo +opentelemetry-operator: + enabled: true +collectors: + daemon: + image: + repository: ghcr.io/lightstep/sn-collector/sn-collector-experimental + tag: "latest" + env: + - name: LS_TOKEN + valueFrom: + secretKeyRef: + key: LS_TOKEN + name: otel-collector-secret + config: + exporters: + otlp: + endpoint: ingest.lightstep.com:443 + headers: + lightstep-access-token: ${LS_TOKEN} + service: + pipelines: + metrics: + exporters: [otlp] + traces: + exporters: [otlp] + logs: + exporters: [otlp] + cluster: + image: + repository: ghcr.io/lightstep/sn-collector/sn-collector-experimental + tag: "latest" + volumes: + - name: credentials + secret: + secretName: k8s-informer-cred-myinstance + securityContext: + runAsNonRoot: true + additionalContainers: + - command: + - /k8s_informer + args: ["--kubeconfig", "in_cluster", "--standalone", "true"] + # The image of the informer program. Change this value if you place the image in your company repository + image: "servicenowdocker/informer:2.2.0" + name: k8sinformer + imagePullPolicy: Always + volumeMounts: + - name: credentials + readOnly: true + mountPath: "/etc/credentials" + env: + # The Kubernetes cluster name as should be reflected in the CMDB + - name: CLUSTER_NAME + value: "My Cluster" + # How frequently in minutes by which the informer calls the API server and list all relevant resources. 0 means only during initialization + - name: RESYNC_MIN + value: "0" + # The time window in seconds for accumulating changes arriving from the API server before preparing payload to be sent to the instance + - name: SEND_INTERVAL_SEC + value: "10" + # The maximum number of items in a single accumulated payload. When we reach this number, we prepare the payload for sending and start a new payload + - name: MAX_CI_IN_MESSAGE + value: "50" + # If the time between now and the last time the item was reported exceeds this value, we re-send it even if there was no change. 0 means no refresh of identical items + - name: REFRESH_TIME_MIN + value: "0" + # The time interval in minutes for sending to the instance a full inventory of items + - name: FULL_DISCOVERY_MIN + value: "1440" + # The time interval in seconds during which the informer looks for command coming from the instance + - name: READ_COMMAND_SEC + value: "60" + # The time interval for sending a payload of the cluster CI, if no other change has happened. Relevant only to the standalone mode + - name: HEARTBEAT_MIN + value: "5" + # The max allowed size of the in-memory queue holding outgoing messages to the instance in standalone mode. If the queue is overflowed, we start to drop messages + - name: MAX_QUEUE_SIZE_BYTES + value: "104755200" + # The retry count on failures to send messages to the instance in standalone mode. If we exceed this number, the message is dropped + - name: MAX_RETRY + value: "5" + # The maximum size of the payload on a single ECC input message. Relevant to the standalone mode only. + - name: MAX_ECC_PAYLOAD_SIZE_BYTES + value: "1048576" + # If access from the cluster to the internet requires a proxy, the proxy URL should be in the format http://proxy_host:proxy_port + - name: PROXY_URL + value: "" + # The instance to which the informer reports to + - name: INSTANCE_URL + value: "https://myinstance.service-now.com" + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + # Should the informer report to the instance on changes in real-time mode. If false, the informer will report only during full-discovery. + - name: CONTINUOUS_DISCOVERY + value: "true" + # The interval (in seconds) in which the ECC sender may insert records to the ECC queue + - name: ECC_SENDER_INTERVAL_SEC + value: "30" + # When the elapsed time of sending ECC message exceeds this number, the informer will assume the instance is loaded and will back off + - name: MAX_ELAPSED_TIME_MS + value: "10000" + # When true, the informer prints messages per every object processed + - name: VERBOSE_LOGGING + value: "false" + # The image of the informer program. Change this value if you place the image in your company repository + - name: INFORMER_VERSION + value: "servicenowdocker/informer:2.2.0" + resources: + limits: + cpu: 500m + # Memory limit for the informer pod. The recommended memory limit is the (Number of pods)/8 Megabytes, with a minimum of 200Mi + memory: 250Mi + requests: + cpu: 100m + # Memory requests + memory: 200Mi + env: + - name: LS_TOKEN + valueFrom: + secretKeyRef: + key: LS_TOKEN + name: otel-collector-secret + config: + exporters: + otlp: + endpoint: ingest.lightstep.com:443 + headers: + lightstep-access-token: ${LS_TOKEN} + service: + pipelines: + metrics: + exporters: [otlp] +instrumentation: + enabled: true + env: + - name: OTEL_K8S_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + exporter: + endpoint: http://${OTEL_K8S_NODE_NAME}:4317 + python: + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://${OTEL_K8S_NODE_NAME}:4318 +opAMPBridge: + enabled: true + addReportingLabel: true + image: + tag: "0.104.0" + endpoint: https://opamp.lightstep.com/v1/opamp + headers: + "Authorization": "bearer ${LS_OPAMP_API_KEY}" + env: + - name: LS_OPAMP_API_KEY + valueFrom: + secretKeyRef: + key: LS_OPAMP_API_KEY + name: otel-opamp-bridge-secret + optional: true From 9c98c9d9b28ee06a560d81463dfb31d7c8b51ae8 Mon Sep 17 00:00:00 2001 From: Jacob Aronoff Date: Thu, 22 Aug 2024 15:01:33 -0400 Subject: [PATCH 2/4] oop --- cno-oks-values.yaml | 166 -------------------------------------------- 1 file changed, 166 deletions(-) delete mode 100644 cno-oks-values.yaml diff --git a/cno-oks-values.yaml b/cno-oks-values.yaml deleted file mode 100644 index ce43c22..0000000 --- a/cno-oks-values.yaml +++ /dev/null @@ -1,166 +0,0 @@ -clusterName: demo -opentelemetry-operator: - enabled: true -collectors: - daemon: - image: - repository: ghcr.io/lightstep/sn-collector/sn-collector-experimental - tag: "latest" - env: - - name: LS_TOKEN - valueFrom: - secretKeyRef: - key: LS_TOKEN - name: otel-collector-secret - config: - exporters: - otlp: - endpoint: ingest.lightstep.com:443 - headers: - lightstep-access-token: ${LS_TOKEN} - service: - pipelines: - metrics: - exporters: [otlp] - traces: - exporters: [otlp] - logs: - exporters: [otlp] - cluster: - image: - repository: ghcr.io/lightstep/sn-collector/sn-collector-experimental - tag: "latest" - volumes: - - name: credentials - secret: - secretName: k8s-informer-cred-myinstance - securityContext: - runAsNonRoot: true - additionalContainers: - - command: - - /k8s_informer - args: ["--kubeconfig", "in_cluster", "--standalone", "true"] - # The image of the informer program. Change this value if you place the image in your company repository - image: "servicenowdocker/informer:2.2.0" - name: k8sinformer - imagePullPolicy: Always - volumeMounts: - - name: credentials - readOnly: true - mountPath: "/etc/credentials" - env: - # The Kubernetes cluster name as should be reflected in the CMDB - - name: CLUSTER_NAME - value: "My Cluster" - # How frequently in minutes by which the informer calls the API server and list all relevant resources. 0 means only during initialization - - name: RESYNC_MIN - value: "0" - # The time window in seconds for accumulating changes arriving from the API server before preparing payload to be sent to the instance - - name: SEND_INTERVAL_SEC - value: "10" - # The maximum number of items in a single accumulated payload. When we reach this number, we prepare the payload for sending and start a new payload - - name: MAX_CI_IN_MESSAGE - value: "50" - # If the time between now and the last time the item was reported exceeds this value, we re-send it even if there was no change. 0 means no refresh of identical items - - name: REFRESH_TIME_MIN - value: "0" - # The time interval in minutes for sending to the instance a full inventory of items - - name: FULL_DISCOVERY_MIN - value: "1440" - # The time interval in seconds during which the informer looks for command coming from the instance - - name: READ_COMMAND_SEC - value: "60" - # The time interval for sending a payload of the cluster CI, if no other change has happened. Relevant only to the standalone mode - - name: HEARTBEAT_MIN - value: "5" - # The max allowed size of the in-memory queue holding outgoing messages to the instance in standalone mode. If the queue is overflowed, we start to drop messages - - name: MAX_QUEUE_SIZE_BYTES - value: "104755200" - # The retry count on failures to send messages to the instance in standalone mode. If we exceed this number, the message is dropped - - name: MAX_RETRY - value: "5" - # The maximum size of the payload on a single ECC input message. Relevant to the standalone mode only. - - name: MAX_ECC_PAYLOAD_SIZE_BYTES - value: "1048576" - # If access from the cluster to the internet requires a proxy, the proxy URL should be in the format http://proxy_host:proxy_port - - name: PROXY_URL - value: "" - # The instance to which the informer reports to - - name: INSTANCE_URL - value: "https://myinstance.service-now.com" - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - # Should the informer report to the instance on changes in real-time mode. If false, the informer will report only during full-discovery. - - name: CONTINUOUS_DISCOVERY - value: "true" - # The interval (in seconds) in which the ECC sender may insert records to the ECC queue - - name: ECC_SENDER_INTERVAL_SEC - value: "30" - # When the elapsed time of sending ECC message exceeds this number, the informer will assume the instance is loaded and will back off - - name: MAX_ELAPSED_TIME_MS - value: "10000" - # When true, the informer prints messages per every object processed - - name: VERBOSE_LOGGING - value: "false" - # The image of the informer program. Change this value if you place the image in your company repository - - name: INFORMER_VERSION - value: "servicenowdocker/informer:2.2.0" - resources: - limits: - cpu: 500m - # Memory limit for the informer pod. The recommended memory limit is the (Number of pods)/8 Megabytes, with a minimum of 200Mi - memory: 250Mi - requests: - cpu: 100m - # Memory requests - memory: 200Mi - env: - - name: LS_TOKEN - valueFrom: - secretKeyRef: - key: LS_TOKEN - name: otel-collector-secret - config: - exporters: - otlp: - endpoint: ingest.lightstep.com:443 - headers: - lightstep-access-token: ${LS_TOKEN} - service: - pipelines: - metrics: - exporters: [otlp] -instrumentation: - enabled: true - env: - - name: OTEL_K8S_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - exporter: - endpoint: http://${OTEL_K8S_NODE_NAME}:4317 - python: - env: - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: http://${OTEL_K8S_NODE_NAME}:4318 -opAMPBridge: - enabled: true - addReportingLabel: true - image: - tag: "0.104.0" - endpoint: https://opamp.lightstep.com/v1/opamp - headers: - "Authorization": "bearer ${LS_OPAMP_API_KEY}" - env: - - name: LS_OPAMP_API_KEY - valueFrom: - secretKeyRef: - key: LS_OPAMP_API_KEY - name: otel-opamp-bridge-secret - optional: true From da7d07324962c42642a3a7450bf289db818c0e85 Mon Sep 17 00:00:00 2001 From: Jacob Aronoff Date: Thu, 22 Aug 2024 15:41:51 -0400 Subject: [PATCH 3/4] run ci --- .github/workflows/publish-image.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index d9c9af4..594e413 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -3,9 +3,16 @@ name: Create and publish a Docker image for Otel Collectors with Arrow on: workflow_dispatch: + pull_request: + branches: + - "**" + paths: + - "arrow/**" push: branches: - main + paths: + - "arrow/**" # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: From 3383ee206a9747c9b964bbda6186dc8203db48be Mon Sep 17 00:00:00 2001 From: Jacob Aronoff Date: Thu, 22 Aug 2024 15:42:25 -0400 Subject: [PATCH 4/4] change --- arrow/otelcolarrow-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arrow/otelcolarrow-build.yaml b/arrow/otelcolarrow-build.yaml index 0616a4c..7bd2779 100644 --- a/arrow/otelcolarrow-build.yaml +++ b/arrow/otelcolarrow-build.yaml @@ -11,7 +11,7 @@ # # Note: This is a relatively advanced operation, as it requires installing a Golang # toolchain to execute and build the collector this way. When you have this file -# configured to your liking, +# configured to your liking: # # go install go.opentelemetry.io/collector/cmd/builder@latest # builder --config otelarrowcol-build.yaml