[otel] Add basic configuration samples (#5002)

dev-tools/packaging/packages.yml

@@ -87,6 +87,9 @@ shared:
     'data/{{.BeatName}}-{{ commit_short }}/otelcol':
       source: '{{ repo.RootDir }}/dev-tools/packaging/files/linux/otelcol.sh'
       mode: 0755
+    'otel_samples':
+      source: '{{ repo.RootDir }}/internal/pkg/otel/samples/linux/'
+      mode: 0755
 
   - &agent_binary_common_files
     LICENSE.txt:
@@ -160,6 +163,9 @@ shared:
     'data/{{.BeatName}}-{{ commit_short }}/otelcol':
       source: '{{ repo.RootDir }}/dev-tools/packaging/files/darwin/otelcol.sh'
       mode: 0755
+    'otel_samples':
+      source: '{{ repo.RootDir }}/internal/pkg/otel/samples/darwin/'
+      mode: 0755
     <<: *agent_darwin_app_bundle_files
     <<: *agent_binary_common_files
 
@@ -201,6 +207,10 @@ shared:
       'data/{{.BeatName}}-{{ commit_short }}/otelcol.ps1':
         source: '{{ repo.RootDir }}/dev-tools/packaging/files/windows/otelcol.ps1'
         mode: 0755
+      # windows not supported yet  
+      # 'otel_samples':
+      #   source: '{{ repo.RootDir }}/internal/pkg/otel/samples/windows/'
+      #   mode: 0755
 
   - &agent_docker_spec
     <<: *agent_binary_spec

internal/pkg/otel/samples/darwin/logs_hostmetrics.yml

@@ -0,0 +1,73 @@
+receivers:
+  # Receiver for platform specific log files
+  filelog/platformlogs:
+    include: [ /var/log/*.log ]
+#    start_at: beginning
+
+  # Receiver for CPU, Disk, Memory, and Filesystem metrics
+  hostmetrics/system:
+    collection_interval: 30s
+    scrapers:
+      disk:
+      filesystem:
+      cpu:
+      memory:
+
+processors:
+  elasticinframetrics:
+  resourcedetection:
+    detectors: ["system"]
+    system:
+      hostname_sources: ["os"]
+      resource_attributes:
+        host.name:
+          enabled: true
+        host.id:
+          enabled: false
+        host.arch:
+          enabled: true
+        host.ip:
+          enabled: true
+        host.mac:
+          enabled: true
+        host.cpu.vendor.id:
+          enabled: true
+        host.cpu.family:
+          enabled: true
+        host.cpu.model.id:
+          enabled: true
+        host.cpu.model.name:
+          enabled: true
+        host.cpu.stepping:
+          enabled: true
+        host.cpu.cache.l2.size:
+          enabled: true
+        os.description:
+          enabled: true
+        os.type:
+          enabled: true
+
+exporters:
+  elasticsearch:
+    endpoints: ["${env:ELASTIC_ENDPOINT}"]
+    api_key: ${env:ELASTIC_API_KEY}
+    mapping:
+      mode: ecs
+    logs_dynamic_index:
+      enabled: true
+    metrics_dynamic_index:
+      enabled: true
+    traces_dynamic_index:
+      enabled: true
+
+service:
+  pipelines:
+    metrics/hostmetrics:
+      receivers: [hostmetrics/system]
+      processors: [elasticinframetrics, resourcedetection]
+      exporters: [elasticsearch]
+
+    logs/platformlogs:
+      receivers: [filelog/platformlogs]
+      processors: [resourcedetection]
+      exporters: [elasticsearch]

internal/pkg/otel/samples/darwin/logs_metrics_traces.yml

@@ -0,0 +1,102 @@
+receivers:
+
+  # Receiver for platform specific log files
+  filelog/platformlogs:
+    include: [ /var/log/*.log ]
+#    start_at: beginning
+
+  # Receiver for CPU, Disk, Memory, and Filesystem metrics
+  hostmetrics/system:
+    collection_interval: 30s
+    scrapers:
+      disk:
+      filesystem:
+      cpu:
+      memory:
+
+  # Receiver for logs, traces, and metrics from SDKs
+  otlp/fromsdk:
+    protocols:
+      grpc:
+      http:
+
+processors:
+  elasticinframetrics:
+  resourcedetection:
+    detectors: ["system"]
+    system:
+      hostname_sources: ["os"]
+      resource_attributes:
+        host.name:
+          enabled: true
+        host.id:
+          enabled: false
+        host.arch:
+          enabled: true
+        host.ip:
+          enabled: true
+        host.mac:
+          enabled: true
+        host.cpu.vendor.id:
+          enabled: true
+        host.cpu.family:
+          enabled: true
+        host.cpu.model.id:
+          enabled: true
+        host.cpu.model.name:
+          enabled: true
+        host.cpu.stepping:
+          enabled: true
+        host.cpu.cache.l2.size:
+          enabled: true
+        os.description:
+          enabled: true
+        os.type:
+          enabled: true
+
+exporters:
+
+  otlp/apm:
+    endpoint: "${env:APM_ENDPOINT}"
+    headers:
+    # Elastic APM Server secret token or API key
+      Authorization: "Bearer ${env:APM_SECRET_KEY}"
+
+  elasticsearch:
+    endpoints: ["${env:ELASTIC_ENDPOINT}"]
+    api_key: ${env:ELASTIC_API_KEY}
+    mapping:
+      mode: ecs
+    logs_dynamic_index:
+      enabled: true
+    metrics_dynamic_index:
+      enabled: true
+    traces_dynamic_index:
+      enabled: true
+
+service:
+  pipelines:
+    traces/fromsdk:
+      receivers: [otlp/fromsdk]
+      processors: []
+      exporters: [otlp/apm]
+
+    metrics/fromsdk:
+      receivers: [otlp/fromsdk]
+      processors: []
+      exporters: [otlp/apm]
+
+    metrics/hostmetrics:
+      receivers: [hostmetrics/system]
+      processors: [elasticinframetrics, resourcedetection]
+      exporters: [elasticsearch]
+
+    logs/fromsdk:
+      receivers: [otlp/fromsdk]
+      processors: []
+      exporters: [otlp/apm]
+
+    logs/platformlogs:
+      receivers: [filelog/platformlogs]
+      processors: [resourcedetection]
+      exporters: [elasticsearch]

internal/pkg/otel/samples/darwin/platformlogs.yml

@@ -0,0 +1,66 @@
+receivers:
+  # Receiver for platform specific log files
+  filelog/platformlogs:
+    include: [ /var/log/*.log ]
+#   start_at: beginning
+
+
+processors:
+  resourcedetection:
+    detectors: ["system"]
+    system:
+      hostname_sources: ["os"]
+      resource_attributes:
+        host.name:
+          enabled: true
+        host.id:
+          enabled: false
+        host.arch:
+          enabled: true
+        host.ip:
+          enabled: true
+        host.mac:
+          enabled: true
+        host.cpu.vendor.id:
+          enabled: true
+        host.cpu.family:
+          enabled: true
+        host.cpu.model.id:
+          enabled: true
+        host.cpu.model.name:
+          enabled: true
+        host.cpu.stepping:
+          enabled: true
+        host.cpu.cache.l2.size:
+          enabled: true
+        os.description:
+          enabled: true
+        os.type:
+          enabled: true
+
+exporters:
+  # Exporter to print the first 5 logs/metrics and then every 1000th
+  debug:
+    verbosity: detailed
+    sampling_initial: 5
+    sampling_thereafter: 1000
+
+  # Exporter to send logs and metrics to Elasticsearch
+  elasticsearch:
+    endpoints: ["${env:ELASTIC_ENDPOINT}"]
+    api_key: ${env:ELASTIC_API_KEY}
+    mapping:
+      mode: ecs
+    logs_dynamic_index:
+      enabled: true
+    metrics_dynamic_index:
+      enabled: true
+    traces_dynamic_index:
+      enabled: true
+
+service:
+  pipelines:
+    logs/platformlogs:
+      receivers: [filelog/platformlogs]
+      processors: [resourcedetection]
+      exporters: [debug, elasticsearch]

internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml

@@ -0,0 +1,78 @@
+receivers:
+  # Receiver for platform specific log files
+  filelog/platformlogs:
+    include: [ /var/log/*.log ]
+#   start_at: beginning
+
+  # Receiver for CPU, Disk, Memory, and Filesystem metrics
+  hostmetrics/system:
+    collection_interval: 30s
+    scrapers:
+      disk:
+      filesystem:
+      cpu:
+      memory:
+
+processors:
+  resourcedetection:
+    detectors: ["system"]
+    system:
+      hostname_sources: ["os"]
+      resource_attributes:
+        host.name:
+          enabled: true
+        host.id:
+          enabled: false
+        host.arch:
+          enabled: true
+        host.ip:
+          enabled: true
+        host.mac:
+          enabled: true
+        host.cpu.vendor.id:
+          enabled: true
+        host.cpu.family:
+          enabled: true
+        host.cpu.model.id:
+          enabled: true
+        host.cpu.model.name:
+          enabled: true
+        host.cpu.stepping:
+          enabled: true
+        host.cpu.cache.l2.size:
+          enabled: true
+        os.description:
+          enabled: true
+        os.type:
+          enabled: true
+
+exporters:
+  # Exporter to print the first 5 logs/metrics and then every 1000th
+  debug:
+    verbosity: detailed
+    sampling_initial: 5
+    sampling_thereafter: 1000
+
+  # Exporter to send logs and metrics to Elasticsearch
+  elasticsearch:
+    endpoints: ["${env:ELASTIC_ENDPOINT}"]
+    api_key: ${env:ELASTIC_API_KEY}
+    mapping:
+      mode: ecs
+    logs_dynamic_index:
+      enabled: true
+    metrics_dynamic_index:
+      enabled: true
+    traces_dynamic_index:
+      enabled: true
+
+service:
+  pipelines:
+    metrics/hostmetrics:
+      receivers: [hostmetrics/system]
+      processors: [resourcedetection]
+      exporters: [debug, elasticsearch]
+    logs/platformlogs:
+      receivers: [filelog/platformlogs]
+      processors: [resourcedetection]
+      exporters: [debug, elasticsearch]