Skip to content

Commit

Permalink
feat: Add k8s distro to pipelines
Browse files Browse the repository at this point in the history
  • Loading branch information
mailo-nr committed Feb 6, 2025
1 parent d6f5feb commit 44583f8
Show file tree
Hide file tree
Showing 10 changed files with 266 additions and 19 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/ci-base.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: 🧩 CI Base
name: 🧩 CI | Base

on:
workflow_call:
Expand Down
10 changes: 6 additions & 4 deletions .github/workflows/ci-nightly.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,9 @@ jobs:
strategy:
matrix:
distribution:
- nr-otel-collector
- nrdot-collector-host
# - nr-otel-collector
# - nrdot-collector-host
- nrdot-collector-k8s
uses: ./.github/workflows/ci-base.yaml
with:
nightly: true
Expand All @@ -46,8 +47,9 @@ jobs:
strategy:
matrix:
distribution:
- nr-otel-collector
- nrdot-collector-host
# - nr-otel-collector
# - nrdot-collector-host
- nrdot-collector-k8s
steps:
- name: Checkout
uses: actions/checkout@v4
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ jobs:
distribution:
- nr-otel-collector
- nrdot-collector-host
- nrdot-collector-k8s
uses: ./.github/workflows/ci-base.yaml
with:
distribution: ${{ matrix.distribution }}
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ TOOLS_BIN_NAMES := $(addprefix $(TOOLS_BIN_DIR)/, $(notdir $(shell echo $(TOOLS_
GO_LICENCE_DETECTOR := $(TOOLS_BIN_DIR)/go-licence-detector
GO_LICENCE_DETECTOR_CONFIG := $(SRC_ROOT)/internal/assets/license/rules.json

DISTRIBUTIONS ?= "nr-otel-collector,nrdot-collector-host"
DISTRIBUTIONS ?= "nr-otel-collector,nrdot-collector-host,nrdot-collector-k8s"

ci: check build licenses-check
check: ensure-goreleaser-up-to-date
Expand Down
2 changes: 1 addition & 1 deletion cmd/goreleaser/internal/configure.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ var (
NightlyImagePrefixes = []string{EnvRegistry}

Architectures = []string{"amd64", "arm64"}
DefaultConfigDists = map[string]bool{LegacyDistro: true, HostDistro: true}
DefaultConfigDists = map[string]bool{LegacyDistro: true, HostDistro: true, K8sDistro: true}
K8sDockerSkipArchs = map[string]bool{"arm": true, "386": true}
K8sGoos = []string{"linux"}
K8sArchs = []string{"amd64", "arm64"}
Expand Down
12 changes: 6 additions & 6 deletions distributions/nrdot-collector-k8s/.goreleaser-nightly.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
version: 2
project_name: nrdot-collector-releases-nightly
release:
disable: "true"
builds:
- id: nrdot-collector-k8s
goos:
Expand Down Expand Up @@ -38,6 +40,8 @@ dockers:
image_templates:
- '{{ .Env.REGISTRY }}/nrdot-collector-k8s:{{ .Version }}-nightly-amd64'
- '{{ .Env.REGISTRY }}/nrdot-collector-k8s:nightly-amd64'
extra_files:
- config.yaml
build_flag_templates:
- --pull
- --platform=linux/amd64
Expand All @@ -54,6 +58,8 @@ dockers:
image_templates:
- '{{ .Env.REGISTRY }}/nrdot-collector-k8s:{{ .Version }}-nightly-arm64'
- '{{ .Env.REGISTRY }}/nrdot-collector-k8s:nightly-arm64'
extra_files:
- config.yaml
build_flag_templates:
- --pull
- --platform=linux/arm64
Expand Down Expand Up @@ -82,9 +88,3 @@ signs:
- ${artifact}
signature: ${artifact}.sig
artifacts: all
certificate: ${artifact}.pem
docker_signs:
- args:
- sign
- ${artifact}
artifacts: all
12 changes: 6 additions & 6 deletions distributions/nrdot-collector-k8s/.goreleaser.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
version: 2
project_name: nrdot-collector-releases
release:
disable: "true"
builds:
- id: nrdot-collector-k8s
goos:
Expand Down Expand Up @@ -38,6 +40,8 @@ dockers:
image_templates:
- newrelic/nrdot-collector-k8s:{{ .Version }}-amd64
- newrelic/nrdot-collector-k8s:latest-amd64
extra_files:
- config.yaml
build_flag_templates:
- --pull
- --platform=linux/amd64
Expand All @@ -54,6 +58,8 @@ dockers:
image_templates:
- newrelic/nrdot-collector-k8s:{{ .Version }}-arm64
- newrelic/nrdot-collector-k8s:latest-arm64
extra_files:
- config.yaml
build_flag_templates:
- --pull
- --platform=linux/arm64
Expand Down Expand Up @@ -86,9 +92,3 @@ signs:
- ${artifact}
signature: ${artifact}.sig
artifacts: all
certificate: ${artifact}.pem
docker_signs:
- args:
- sign
- ${artifact}
artifacts: all
2 changes: 2 additions & 0 deletions distributions/nrdot-collector-k8s/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ USER ${USER_UID}

COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
COPY --chmod=755 nrdot-collector-k8s /nrdot-collector-k8s
COPY config.yaml /etc/nrdot-collector-k8s/config.yaml
ENTRYPOINT ["/nrdot-collector-k8s"]
CMD ["--config", "/etc/nrdot-collector-k8s/config.yaml"]
# `4137` and `4318`: OTLP
EXPOSE 4317 4318
242 changes: 242 additions & 0 deletions distributions/nrdot-collector-k8s/config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,242 @@
extensions:
health_check:

receivers:
otlp:
protocols:
grpc:
http:

hostmetrics:
# Default collection interval is 60s. Lower if you need finer granularity.
collection_interval: 60s
scrapers:
cpu:
metrics:
system.cpu.time:
enabled: false
system.cpu.utilization:
enabled: true
load:
memory:
metrics:
system.memory.utilization:
enabled: true
paging:
metrics:
system.paging.utilization:
enabled: false
system.paging.faults:
enabled: false
filesystem:
metrics:
system.filesystem.utilization:
enabled: true
disk:
metrics:
system.disk.merged:
enabled: false
system.disk.pending_operations:
enabled: false
system.disk.weighted_io_time:
enabled: false
network:
metrics:
system.network.connections:
enabled: false
# Uncomment to enable process metrics, which can be noisy but valuable.
# processes:
# process:
# metrics:
# process.cpu.utilization:
# enabled: true
# process.cpu.time:
# enabled: false

filelog:
include:
- /var/log/alternatives.log
- /var/log/cloud-init.log
- /var/log/auth.log
- /var/log/dpkg.log
- /var/log/syslog
- /var/log/messages
- /var/log/secure
- /var/log/yum.log

processors:
# group system.cpu metrics by cpu
metricstransform:
transforms:
- include: system.cpu.utilization
action: update
operations:
- action: aggregate_labels
label_set: [ state ]
aggregation_type: mean
- include: system.paging.operations
action: update
operations:
- action: aggregate_labels
label_set: [ direction ]
aggregation_type: sum
# remove system.cpu metrics for states
filter/exclude_cpu_utilization:
metrics:
datapoint:
- 'metric.name == "system.cpu.utilization" and attributes["state"] == "interrupt"'
- 'metric.name == "system.cpu.utilization" and attributes["state"] == "nice"'
- 'metric.name == "system.cpu.utilization" and attributes["state"] == "softirq"'
filter/exclude_memory_utilization:
metrics:
datapoint:
- 'metric.name == "system.memory.utilization" and attributes["state"] == "slab_unreclaimable"'
- 'metric.name == "system.memory.utilization" and attributes["state"] == "inactive"'
- 'metric.name == "system.memory.utilization" and attributes["state"] == "cached"'
- 'metric.name == "system.memory.utilization" and attributes["state"] == "buffered"'
- 'metric.name == "system.memory.utilization" and attributes["state"] == "slab_reclaimable"'
filter/exclude_memory_usage:
metrics:
datapoint:
- 'metric.name == "system.memory.usage" and attributes["state"] == "slab_unreclaimable"'
- 'metric.name == "system.memory.usage" and attributes["state"] == "inactive"'
filter/exclude_filesystem_utilization:
metrics:
datapoint:
- 'metric.name == "system.filesystem.utilization" and attributes["type"] == "squashfs"'
filter/exclude_filesystem_usage:
metrics:
datapoint:
- 'metric.name == "system.filesystem.usage" and attributes["type"] == "squashfs"'
- 'metric.name == "system.filesystem.usage" and attributes["state"] == "reserved"'
filter/exclude_filesystem_inodes_usage:
metrics:
datapoint:
- 'metric.name == "system.filesystem.inodes.usage" and attributes["type"] == "squashfs"'
- 'metric.name == "system.filesystem.inodes.usage" and attributes["state"] == "reserved"'
filter/exclude_system_disk:
metrics:
datapoint:
- 'metric.name == "system.disk.operations" and IsMatch(attributes["device"], "^loop.*") == true'
- 'metric.name == "system.disk.merged" and IsMatch(attributes["device"], "^loop.*") == true'
- 'metric.name == "system.disk.io" and IsMatch(attributes["device"], "^loop.*") == true'
- 'metric.name == "system.disk.io_time" and IsMatch(attributes["device"], "^loop.*") == true'
- 'metric.name == "system.disk.operation_time" and IsMatch(attributes["device"], "^loop.*") == true'
filter/exclude_system_paging:
metrics:
datapoint:
- 'metric.name == "system.paging.usage" and attributes["state"] == "cached"'
- 'metric.name == "system.paging.operations" and attributes["type"] == "cached"'
filter/exclude_network:
metrics:
datapoint:
- 'IsMatch(metric.name, "^system.network.*") == true and attributes["device"] == "lo"'

attributes/exclude_system_paging:
include:
match_type: strict
metric_names:
- system.paging.operations
actions:
- key: type
action: delete

cumulativetodelta:

transform/host:
metric_statements:
- context: metric
statements:
- set(description, "")
- set(unit, "")

transform:
trace_statements:
- context: span
statements:
- truncate_all(attributes, 4095)
- truncate_all(resource.attributes, 4095)
log_statements:
- context: log
statements:
- truncate_all(attributes, 4095)
- truncate_all(resource.attributes, 4095)

# used to prevent out of memory situations on the collector
memory_limiter:
check_interval: 1s
limit_mib: ${NEW_RELIC_MEMORY_LIMIT_MIB}

batch:

resourcedetection:
detectors: ["system"]
system:
hostname_sources: ["os"]
resource_attributes:
host.id:
enabled: true

resourcedetection/cloud:
detectors: ["gcp", "ec2", "azure"]
timeout: 2s
ec2:
resource_attributes:
host.name:
enabled: false

# Gives OTEL_RESOURCE_ATTRIBUTES precedence over other sources.
# host.id is set from env whenever the collector is orchestrated by NR Agents.
resourcedetection/env:
detectors: ["env"]
timeout: 2s
override: true

exporters:
debug:
otlphttp:
endpoint: ${OTEL_EXPORTER_OTLP_ENDPOINT}
headers:
api-key: ${NEW_RELIC_LICENSE_KEY}

service:
pipelines:
metrics/host:
receivers: [hostmetrics]
processors:
- memory_limiter
- metricstransform
- filter/exclude_cpu_utilization
- filter/exclude_memory_utilization
- filter/exclude_memory_usage
- filter/exclude_filesystem_utilization
- filter/exclude_filesystem_usage
- filter/exclude_filesystem_inodes_usage
- filter/exclude_system_disk
- filter/exclude_network
- attributes/exclude_system_paging
- transform/host
- resourcedetection
- resourcedetection/cloud
- resourcedetection/env
- cumulativetodelta
- batch
exporters: [debug, otlphttp]
logs/host:
receivers: [filelog]
processors: [transform, resourcedetection, resourcedetection/cloud, resourcedetection/env, batch]
exporters: [debug, otlphttp]
traces:
receivers: [otlp]
processors: [transform, resourcedetection, resourcedetection/cloud, resourcedetection/env, batch]
exporters: [debug, otlphttp]
metrics:
receivers: [otlp]
processors: [transform, resourcedetection, resourcedetection/cloud, resourcedetection/env, batch]
exporters: [debug, otlphttp]
logs:
receivers: [otlp]
processors: [transform, resourcedetection, resourcedetection/cloud, resourcedetection/env, batch]
exporters: [debug, otlphttp]

extensions: [health_check]
Empty file added test/terraform/nightly/main.tf
Empty file.

0 comments on commit 44583f8

Please sign in to comment.