Skip to content

Commit

Permalink
Merge branch 'main' into dependabot/go_modules/github.com/docker/dock…
Browse files Browse the repository at this point in the history
…er-25.0.6incompatible
  • Loading branch information
jefchien authored Aug 9, 2024
2 parents 460223e + 630e834 commit ac29b92
Show file tree
Hide file tree
Showing 27 changed files with 863 additions and 276 deletions.
37 changes: 29 additions & 8 deletions .github/workflows/application-signals-e2e-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,26 +6,47 @@
# Read more about reusable workflows: https://docs.github.com/en/actions/using-workflows/reusing-workflows#overview
name: Application Signals E2E Test
on:
workflow_run:
workflows: [ Build Test Artifacts ]
types:
- completed
workflow_dispatch:
inputs:
build_run_id:
description: 'The ID of the build-test-artifacts workflow run'
type: number
required: true
build_sha:
description: 'The SHA of the build-test-artifacts workflow run'
type: string
required: true

permissions:
id-token: write
contents: read

concurrency:
group: '${{ github.workflow }} @ ${{ inputs.aws-region }}'
cancel-in-progress: false
group: ${{ github.workflow }}-${{ github.ref_name }}
cancel-in-progress: true


jobs:
CheckBuildTestArtifacts:
runs-on: ubuntu-latest
if: ${{ github.event.workflow_run.conclusion == 'success' }}
steps:
- run: echo 'The triggering build workflow succeeded'
- run: |
if [[ ${{ inputs.build_sha }} == ${{ github.sha }} ]]; then
echo "Build SHA matches test SHA"
else
echo "Build SHA does not match test SHA"
exit 1
fi
- run: |
conclusion=$(gh run view ${{ inputs.build_run_id }} --repo $GITHUB_REPOSITORY --json conclusion -q '.conclusion')
if [[ $conclusion == "success" ]]; then
echo "Run succeeded"
else
echo "Run failed"
exit 1
fi
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
java-eks-e2e-test:
needs: CheckBuildTestArtifacts
Expand Down
21 changes: 20 additions & 1 deletion .github/workflows/build-test-artifacts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,12 @@ on:
paths-ignore:
- '**/*.md'
- 'NOTICE'
- 'RELEASE_NOTES'
- 'THIRD-PARTY'
- 'LICENSE'
- '.github/**'
- '!.github/workflows/build-test-artifacts.yml'
- '!.github/workflows/integration-test.yml'
- '!.github/workflows/application-signals-e2e-test.yml'
workflow_dispatch:

concurrency:
Expand Down Expand Up @@ -83,3 +84,21 @@ jobs:
Region: "cn-north-1"
TerraformAWSAssumeRole: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_CN }}
Bucket: ${{ vars.S3_INTEGRATION_BUCKET_CN }}

StartIntegrationTests:
needs: [ BuildAndUploadPackages, BuildAndUploadITAR, BuildAndUploadCN, BuildDocker ]
runs-on: ubuntu-latest
steps:
- run: gh workflow run integration-test.yml --ref ${{ github.ref_name }} --repo $GITHUB_REPOSITORY -f build_run_id=${{ github.run_id }} -f build_sha=${{ github.sha }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

StartApplicationSignalsE2ETests:
needs: [ BuildAndUploadPackages, BuildAndUploadITAR, BuildAndUploadCN, BuildDocker ]
# Workflow only runs against main
if: ${{ contains(github.ref_name, 'main') }}
runs-on: ubuntu-latest
steps:
- run: gh workflow run application-signals-e2e-test.yml --ref ${{ github.ref_name }} --repo $GITHUB_REPOSITORY -f build_run_id=${{ github.run_id }} -f build_sha=${{ github.sha }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
33 changes: 27 additions & 6 deletions .github/workflows/integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,16 @@ env:
S3_INTEGRATION_BUCKET_CN: ${{ vars.S3_INTEGRATION_BUCKET_CN }}

on:
workflow_run:
workflows: [ Build Test Artifacts ]
types:
- completed
workflow_dispatch:
inputs:
build_run_id:
description: 'The ID of the build-test-artifacts workflow run'
type: number
required: true
build_sha:
description: 'The SHA of the build-test-artifacts workflow run'
type: string
required: true

concurrency:
group: ${{ github.workflow }}-${{ github.ref_name }}
Expand All @@ -32,9 +38,24 @@ concurrency:
jobs:
CheckBuildTestArtifacts:
runs-on: ubuntu-latest
if: ${{ github.event.workflow_run.conclusion == 'success' }}
steps:
- run: echo 'The triggering build workflow succeeded'
- run: |
if [[ ${{ inputs.build_sha }} == ${{ github.sha }} ]]; then
echo "Build SHA matches test SHA"
else
echo "Build SHA does not match test SHA"
exit 1
fi
- run: |
conclusion=$(gh run view ${{ inputs.build_run_id }} --repo $GITHUB_REPOSITORY --json conclusion -q '.conclusion')
if [[ $conclusion == "success" ]]; then
echo "Run succeeded"
else
echo "Run failed"
exit 1
fi
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GenerateTestMatrix:
needs: [ CheckBuildTestArtifacts ]
Expand Down
14 changes: 14 additions & 0 deletions RELEASE_NOTES
Original file line number Diff line number Diff line change
@@ -1,3 +1,17 @@
========================================================================
Amazon CloudWatch Agent 1.300043.0 (2024-08-08)
========================================================================
Bug Fixes:
* [Metrics/Plugin] Drop original metrics support added for statsd, collectd and ethtool metrics
* [Logs/Windows Event] Add windows event log service restart detection and resubscribe
* [Metrics/JMX, Metrics/Net, Metrics/DiskIO] Change cumulative to delta conversion to drop initial value
* [Metrics/JMX] Suppress sessions unit warning

Enhancements:
* [Metrics/JMX] Add cumulative to delta conversion for JMX metrics
* [ApplicationSignals] Rename and group application signals attributes
* Upgrade OTEL Contrib to v0.103.0

========================================================================
Amazon CloudWatch Agent 1.300042.1 (2024-07-16)
========================================================================
Expand Down
1 change: 1 addition & 0 deletions internal/cloudwatch/unit.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ var scaledBaseUnits = map[types.StandardUnit]map[unit.MetricPrefix]types.Standar

var knownNonConvertibleUnits = collections.NewSet(
// JMX/Tomcat units
"sessions",
"errors",
"threads",
"requests",
Expand Down
1 change: 1 addition & 0 deletions internal/containerinsightscommon/k8sconst.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ const (
PodOwnersKey = "PodOwners"
HostKey = "host"
K8sKey = "kubernetes"
K8sLabelsKey = "labels"

RunningPodCount = "number_of_running_pods"
RunningContainerCount = "number_of_running_containers"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: MIT

package internal

import (
"regexp"
)

const (
PROCESSED_NEURON_METRIC_PATTERN = "^(container|node|pod)_(neuroncore_|neurondevice_).*|^node_neuron_.*"
)

type AwsNeuronMetricChecker struct {
}

func NewAwsNeuronMetricChecker() *AwsNeuronMetricChecker {
return &AwsNeuronMetricChecker{}
}

func (md *AwsNeuronMetricChecker) IsProcessedNeuronMetric(name string) bool {
matched, err := regexp.MatchString(PROCESSED_NEURON_METRIC_PATTERN, name)
if err != nil {
print(err)
return false
}
return matched
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: MIT

package internal

import (
"testing"
)

func TestAwsNeuronMetricModifier_IsProcessedNeuronMetric(t *testing.T) {
tests := []struct {
name string
input string
expected bool
}{
{
name: "container_neuroncore_prefix",
input: "container_neuroncore_metric",
expected: true,
},
{
name: "pod_neuroncore_prefix",
input: "pod_neuroncore_metric",
expected: true,
},
{
name: "node_neuroncore_prefix",
input: "node_neuroncore_metric",
expected: true,
},
{
name: "container_neurondevice_prefix",
input: "container_neurondevice_metric",
expected: true,
},
{
name: "pod_neurondevice_prefix",
input: "pod_neurondevice_metric",
expected: true,
},
{
name: "node_neurondevice_prefix",
input: "node_neurondevice_metric",
expected: true,
},
{
name: "node_neuron_prefix",
input: "node_neuron_metric",
expected: true,
},
{
name: "container_neuron_prefix",
input: "container_neuron_metric",
expected: false,
},
{
name: "other_prefix",
input: "other_metric",
expected: false,
},
}

md := NewAwsNeuronMetricChecker()

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
result := md.IsProcessedNeuronMetric(test.input)
if result != test.expected {
t.Errorf("IsProcessedNeuronMetric(%q) = %v, expected %v", test.input, result, test.expected)
}
})
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ package internal
import (
"strings"

"go.opentelemetry.io/collector/pdata/pcommon"
"go.opentelemetry.io/collector/pdata/pmetric"
"go.uber.org/zap"

Expand Down Expand Up @@ -46,6 +45,7 @@ const (
Kubernetes = "kubernetes"
Region = "region"
SubnetId = "subnet_id"
RuntimeTagOverride = "DEFAULT"
NeuronExecutionErrorsAggregatedMetric = containerinsightscommon.NeuronExecutionErrors + "_total"
NeuronDeviceHardwareEccEventsAggregatedMetric = containerinsightscommon.NeuronDeviceHardwareEccEvents + "_total"
)
Expand Down Expand Up @@ -99,26 +99,6 @@ var (
"sram_ecc_corrected": NeuronDeviceHardwareEccEventsAggregatedMetric,
"sram_ecc_uncorrected": NeuronDeviceHardwareEccEventsAggregatedMetric},
}

MetricAttributesToKeep = map[string]struct{}{
ClusterName: {},
ContainerName: {},
FullPodName: {},
InstanceId: {},
InstanceType: {},
K8sPodName: {},
Namespace: {},
NeuronDevice: {},
NodeName: {},
PodName: {},
Service: {},
AvailabilityZone: {},
Kubernetes: {},
Region: {},
RuntimeTag: {},
SubnetId: {},
NeuronCore: {},
}
)

func NewMetricModifier(logger *zap.Logger) *AwsNeuronMetricModifier {
Expand All @@ -142,7 +122,7 @@ func (md *AwsNeuronMetricModifier) ModifyMetric(originalMetric pmetric.Metric, m
}
// Neuron metrics sent by the neuron monitor don't have any units so we add them in the agent.
addUnit(originalMetric)
prefixCoreAndDeviceLabels(originalMetric)
updateCoreDeviceRuntimeLabels(originalMetric)
resetStaleDatapoints(originalMetric)

originalMetricName := originalMetric.Name()
Expand All @@ -156,7 +136,6 @@ func (md *AwsNeuronMetricModifier) ModifyMetric(originalMetric pmetric.Metric, m
}

modifiedMetricSlice := md.extractDatapointsAsMetricsAndAggregate(originalMetric)
filterLabels(modifiedMetricSlice, originalMetricName)
md.duplicateMetrics(modifiedMetricSlice, originalMetricName, originalMetric.Sum().DataPoints(), metrics)
}

Expand Down Expand Up @@ -251,7 +230,6 @@ func (md *AwsNeuronMetricModifier) extractDatapointsAsMetricsAndAggregate(origin

// Creating body for the aggregated metric and add it to the new newMetricSlice for each runtime
for aggregatedMetricMetadata, value := range aggregatedValuesPerRuntimeTag {
// Aggregated metric for neuron device ecc events is not required
aggregatedMetric := setMetricMetadata(newMetricSlice.AppendEmpty(), aggregatedMetricMetadata.aggregatedMetricName, originalMetric.Unit())

originalMetricDatapoints.At(0).CopyTo(aggregatedMetric.SetEmptySum().DataPoints().AppendEmpty())
Expand All @@ -269,33 +247,9 @@ func (md *AwsNeuronMetricModifier) extractDatapointsAsMetricsAndAggregate(origin
return newMetricSlice
}

// This method removes the attribute keys which are not required. The removal is necessary so that the metrics are grouped together
func filterLabels(slice pmetric.MetricSlice, originalMetricName string) {
_, exists := metricModificationsMap[originalMetricName]
if !exists {
return
}

for i := 0; i < slice.Len(); i++ {
m := slice.At(i)

dps := m.Sum().DataPoints()
for j := 0; j < dps.Len(); j++ {
attributes := dps.At(j).Attributes()
attributes.RemoveIf(func(label string, value pcommon.Value) bool {
_, exists := MetricAttributesToKeep[label]
if !exists {
return true
}
return false
})
}
}
}

// This method prefixes NeuronCore and NeuronDevice values with `core` and `device` respectively
// to make the attribute values more verbose
func prefixCoreAndDeviceLabels(originalMetric pmetric.Metric) {
func updateCoreDeviceRuntimeLabels(originalMetric pmetric.Metric) {
dps := originalMetric.Sum().DataPoints()
for i := 0; i < dps.Len(); i++ {
dp := dps.At(i)
Expand All @@ -304,6 +258,7 @@ func prefixCoreAndDeviceLabels(originalMetric pmetric.Metric) {
dp.Attributes().PutStr(attributeKey, attributeValuePrefix+value.Str())
}
}
dp.Attributes().PutStr(RuntimeTag, RuntimeTagOverride)
}
}

Expand Down Expand Up @@ -360,7 +315,7 @@ func resetStaleDatapoints(originalMetric pmetric.Metric) {
dp := dps.At(i)
if dp.ValueType() == pmetric.NumberDataPointValueTypeEmpty || dp.Flags().NoRecordedValue() {
dp.SetDoubleValue(dp.DoubleValue())
dp.Attributes().PutStr(RuntimeTag, "default")
dp.Attributes().PutStr(RuntimeTag, RuntimeTagOverride)
dp.SetFlags(dp.Flags().WithNoRecordedValue(false))
}
}
Expand Down
Loading

0 comments on commit ac29b92

Please sign in to comment.