From 8784c3cdc833b99e8e6136781bbefa44b01e191a Mon Sep 17 00:00:00 2001 From: Pinglei Guo Date: Sun, 21 Mar 2021 21:07:40 -0700 Subject: [PATCH 1/8] cadvisor: Fix pod detection for containerd #188 --- internal/containerinsightscommon/const.go | 3 ++ .../cadvisor/container_info_processor.go | 43 ++++++++++--------- .../cadvisor/extractors/cpu_extractor.go | 6 ++- .../cadvisor/extractors/diskio_extractor.go | 2 +- .../inputs/cadvisor/extractors/extractor.go | 20 ++++++--- .../cadvisor/extractors/fs_extractor.go | 2 +- .../cadvisor/extractors/mem_extractor.go | 5 ++- .../cadvisor/extractors/net_extractor.go | 10 +++-- plugins/inputs/cadvisor/merger.go | 1 + 9 files changed, 57 insertions(+), 35 deletions(-) diff --git a/internal/containerinsightscommon/const.go b/internal/containerinsightscommon/const.go index db8084e33a..0b9dc01672 100644 --- a/internal/containerinsightscommon/const.go +++ b/internal/containerinsightscommon/const.go @@ -92,4 +92,7 @@ const ( TypeContainer = "Container" TypeContainerFS = "ContainerFS" TypeContainerDiskIO = "ContainerDiskIO" + // Special type for pause container, introduced in https://github.com/aws/amazon-cloudwatch-agent/issues/188 + // because containerd does not set container name pause container name to POD like docker does. + TypeInfraContainer = "InfraContainer" ) diff --git a/plugins/inputs/cadvisor/container_info_processor.go b/plugins/inputs/cadvisor/container_info_processor.go index 2e8d0741dc..aca152fc57 100644 --- a/plugins/inputs/cadvisor/container_info_processor.go +++ b/plugins/inputs/cadvisor/container_info_processor.go @@ -98,17 +98,23 @@ func processContainer(info *cinfo.ContainerInfo, detailMode bool, containerOrche if !detailMode { return result, pKey } + + if len(info.Spec.Labels) == 0 { + log.Printf("W! no label found from container spec, is containerd socket mounted? https://github.com/aws/amazon-cloudwatch-agent/issues/188") + } // Only a container has all these three labels set. containerName := info.Spec.Labels[containerNameLable] namespace := info.Spec.Labels[namespaceLable] podName := info.Spec.Labels[podNameLable] podId := info.Spec.Labels[podIdLable] - if containerName == "" || namespace == "" || podName == "" { + // NOTE: containerName can be empty for pause container on containerd + // https://github.com/containerd/cri/issues/922#issuecomment-423729537 + if namespace == "" || podName == "" { return result, pKey } // Pod's cgroup path is parent for a container. - // contianer name: /kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod04d39715_075e_4c7c_b128_67f7897c05b7.slice/docker-57b3dabd69b94beb462244a0c15c244b509adad0940cdcc67ca079b8208ec1f2.scope + // container name: /kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod04d39715_075e_4c7c_b128_67f7897c05b7.slice/docker-57b3dabd69b94beb462244a0c15c244b509adad0940cdcc67ca079b8208ec1f2.scope // pod name: /kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod04d39715_075e_4c7c_b128_67f7897c05b7.slice/ podPath := path.Dir(info.Name) pKey = &podKey{cgroupPath: podPath, podName: podName, podId: podId, namespace: namespace} @@ -116,14 +122,18 @@ func processContainer(info *cinfo.ContainerInfo, detailMode bool, containerOrche tags[PodIdKey] = podId tags[K8sPodNameKey] = podName tags[K8sNamespace] = namespace - if containerName != infraContainerName { + + switch containerName { + // For docker, pause container name is set to POD while containerd does not set it. + // See https://github.com/aws/amazon-cloudwatch-agent/issues/188 + case "", infraContainerName: + // NOTE: the pod here is only used by NetMetricExtractor, + // other pod info like CPU, Mem are dealt within in processPod. + containerType = TypeInfraContainer + default: tags[ContainerNamekey] = containerName tags[ContainerIdkey] = path.Base(info.Name) containerType = TypeContainer - } else { - // NOTE: the pod here is only used by NetMetricExtractor, - // other pod info like CPU, Mem are dealt within in processPod. - containerType = TypePod } } else { containerType = TypeNode @@ -146,15 +156,18 @@ func processContainer(info *cinfo.ContainerInfo, detailMode bool, containerOrche return result, pKey } +// processPod is almost identical as processContainer. We got this second loop because pod detection relies +// on inspecting labels from containers in processContainer. cgroup path for detected pods are saved in podKeys. +// We may not get container before pod when looping all returned cgroup paths so we use a two pass solution +// in processContainers. func processPod(info *cinfo.ContainerInfo, podKeys map[string]podKey) []*extractors.CAdvisorMetric { var result []*extractors.CAdvisorMetric if isContainerInContainer(info.Name) { - log.Printf("D! drop metric because it's nested container, name %s", info.Name) return result } - podKey := getPodKey(info, podKeys) - if podKey == nil { + podKey, ok := podKeys[info.Name] + if !ok { return result } @@ -177,16 +190,6 @@ func processPod(info *cinfo.ContainerInfo, podKeys map[string]podKey) []*extract return result } -func getPodKey(info *cinfo.ContainerInfo, podKeys map[string]podKey) *podKey { - key := info.Name - - if v, ok := podKeys[key]; ok { - return &v - } - - return nil -} - // Check if it's a container running inside container, caller will drop the metric when return value is true. // The validation is based on ContainerReference.Name, which is essentially cgroup path. // The first version is from https://github.com/aws/amazon-cloudwatch-agent/commit/e8daa5f5926c5a5f38e0ceb746c141be463e11e4#diff-599185154c116b295172b56311729990d20672f6659500870997c018ce072100 diff --git a/plugins/inputs/cadvisor/extractors/cpu_extractor.go b/plugins/inputs/cadvisor/extractors/cpu_extractor.go index 7c3d7cc685..f5c51b211f 100644 --- a/plugins/inputs/cadvisor/extractors/cpu_extractor.go +++ b/plugins/inputs/cadvisor/extractors/cpu_extractor.go @@ -29,7 +29,8 @@ func (c *CpuMetricExtractor) recordPreviousInfo(info *cInfo.ContainerInfo) { func (c *CpuMetricExtractor) GetValue(info *cInfo.ContainerInfo, containerType string) []*CAdvisorMetric { var metrics []*CAdvisorMetric - if info.Spec.Labels[containerNameLable] == infraContainerName { + // Skip infra container and handle node, pod, other containers in pod + if containerType == TypeInfraContainer { return metrics } @@ -41,6 +42,7 @@ func (c *CpuMetricExtractor) GetValue(info *cInfo.ContainerInfo, containerType s if deltaCTimeInNano > MinTimeDiff { metric := newCadvisorMetric(containerType) + metric.cgroupPath = info.Name metric.fields[MetricName(containerType, CpuTotal)] = float64(curStats.Cpu.Usage.Total-preStats.Cpu.Usage.Total) / float64(deltaCTimeInNano) * decimalToMillicores metric.fields[MetricName(containerType, CpuUser)] = float64(curStats.Cpu.Usage.User-preStats.Cpu.Usage.User) / float64(deltaCTimeInNano) * decimalToMillicores @@ -59,6 +61,6 @@ func (c *CpuMetricExtractor) CleanUp(now time.Time) { func NewCpuMetricExtractor() *CpuMetricExtractor { return &CpuMetricExtractor{ - preInfos: mapWithExpiry.NewMapWithExpiry(CleanInteval), + preInfos: mapWithExpiry.NewMapWithExpiry(CleanInterval), } } diff --git a/plugins/inputs/cadvisor/extractors/diskio_extractor.go b/plugins/inputs/cadvisor/extractors/diskio_extractor.go index 8a9529a9f6..887413a8d4 100644 --- a/plugins/inputs/cadvisor/extractors/diskio_extractor.go +++ b/plugins/inputs/cadvisor/extractors/diskio_extractor.go @@ -90,7 +90,7 @@ func (d *DiskIOMetricExtractor) CleanUp(now time.Time) { func NewDiskIOMetricExtractor() *DiskIOMetricExtractor { return &DiskIOMetricExtractor{ - preInfos: mapWithExpiry.NewMapWithExpiry(CleanInteval), + preInfos: mapWithExpiry.NewMapWithExpiry(CleanInterval), } } diff --git a/plugins/inputs/cadvisor/extractors/extractor.go b/plugins/inputs/cadvisor/extractors/extractor.go index 5e6b20f094..d076f18e11 100644 --- a/plugins/inputs/cadvisor/extractors/extractor.go +++ b/plugins/inputs/cadvisor/extractors/extractor.go @@ -12,21 +12,28 @@ import ( ) const ( - containerNameLable = "io.kubernetes.container.name" - // TODO: https://github.com/containerd/cri/issues/922#issuecomment-423729537 the container name can be empty on containerd - infraContainerName = "POD" + containerNameLabel = "io.kubernetes.container.name" Metrics = "Metrics" Dimensions = "Dimensions" - CleanInteval = 5 * time.Minute + CleanInterval = 5 * time.Minute ) type MetricExtractor interface { HasValue(*cinfo.ContainerInfo) bool - GetValue(*cinfo.ContainerInfo, string) []*CAdvisorMetric + // GetValue normally applies to the following types: + // containerinsightscommon.TypeContainer + // containerinsightscommon.TypePod + // containerinsightscommon.TypeNode + // and ignores: + // containerinsightscommon.TypeInfraContainer + // The only exception is NetMetricExtractor because pod network metrics comes from infra container (i.e. pause). + // See https://www.ianlewis.org/en/almighty-pause-container + GetValue(info *cinfo.ContainerInfo, containerType string) []*CAdvisorMetric CleanUp(time.Time) } type CAdvisorMetric struct { + cgroupPath string // source of the metric for debugging merge conflict fields map[string]interface{} tags map[string]string metricType string @@ -68,7 +75,8 @@ func (c *CAdvisorMetric) Merge(src *CAdvisorMetric) { // If there is any conflict, keep the fields with earlier timestamp for k, v := range src.fields { if _, ok := c.fields[k]; ok { - log.Printf("D! metric being merged has conflict in fields, src: %v, dest: %v \n", *src, *c) + log.Printf("D! metric being merged has conflict in fields, path src: %q, dest: %q", src.cgroupPath, c.cgroupPath) + log.Printf("D! metric being merged has conflict in fields, src: %v, dest: %v", *src, *c) if c.tags[containerinsightscommon.Timestamp] < src.tags[containerinsightscommon.Timestamp] { continue } diff --git a/plugins/inputs/cadvisor/extractors/fs_extractor.go b/plugins/inputs/cadvisor/extractors/fs_extractor.go index ab46604137..f241d0c97e 100644 --- a/plugins/inputs/cadvisor/extractors/fs_extractor.go +++ b/plugins/inputs/cadvisor/extractors/fs_extractor.go @@ -26,7 +26,7 @@ func (f *FileSystemMetricExtractor) HasValue(info *cinfo.ContainerInfo) bool { func (f *FileSystemMetricExtractor) GetValue(info *cinfo.ContainerInfo, containerType string) []*CAdvisorMetric { var metrics []*CAdvisorMetric - if containerType == TypePod || info.Spec.Labels[containerNameLable] == infraContainerName { + if containerType == TypePod || containerType == TypeInfraContainer { return metrics } diff --git a/plugins/inputs/cadvisor/extractors/mem_extractor.go b/plugins/inputs/cadvisor/extractors/mem_extractor.go index f8efdaf022..7e8be186fa 100644 --- a/plugins/inputs/cadvisor/extractors/mem_extractor.go +++ b/plugins/inputs/cadvisor/extractors/mem_extractor.go @@ -25,11 +25,12 @@ func (m *MemMetricExtractor) HasValue(info *cinfo.ContainerInfo) bool { func (m *MemMetricExtractor) GetValue(info *cinfo.ContainerInfo, containerType string) []*CAdvisorMetric { var metrics []*CAdvisorMetric - if info.Spec.Labels[containerNameLable] == infraContainerName { + if containerType == TypeInfraContainer { return metrics } metric := newCadvisorMetric(containerType) + metric.cgroupPath = info.Name curStats := GetStats(info) metric.fields[MetricName(containerType, MemUsage)] = curStats.Memory.Usage @@ -64,6 +65,6 @@ func (m *MemMetricExtractor) CleanUp(now time.Time) { func NewMemMetricExtractor() *MemMetricExtractor { return &MemMetricExtractor{ - preInfos: mapWithExpiry.NewMapWithExpiry(CleanInteval), + preInfos: mapWithExpiry.NewMapWithExpiry(CleanInterval), } } diff --git a/plugins/inputs/cadvisor/extractors/net_extractor.go b/plugins/inputs/cadvisor/extractors/net_extractor.go index 5691f311d5..1e5b956149 100644 --- a/plugins/inputs/cadvisor/extractors/net_extractor.go +++ b/plugins/inputs/cadvisor/extractors/net_extractor.go @@ -39,10 +39,14 @@ func (n *NetMetricExtractor) HasValue(info *cinfo.ContainerInfo) bool { func (n *NetMetricExtractor) GetValue(info *cinfo.ContainerInfo, containerType string) []*CAdvisorMetric { var metrics []*CAdvisorMetric - // Just a protection here, there is no Container level Net metrics - if (containerType == TypePod && info.Spec.Labels[containerNameLable] != infraContainerName) || containerType == TypeContainer { + // Ignore both pod and container because the network metrics comes from InfraContainer. + if containerType == TypePod || containerType == TypeContainer { return metrics } + // Rename type to pod so the metric name prefix is pod_ + if containerType == TypeInfraContainer { + containerType = TypePod + } if preInfo, ok := n.preInfos.Get(info.Name); ok { curStats := GetStats(info) @@ -112,7 +116,7 @@ func (n *NetMetricExtractor) CleanUp(now time.Time) { func NewNetMetricExtractor() *NetMetricExtractor { return &NetMetricExtractor{ - preInfos: mapWithExpiry.NewMapWithExpiry(CleanInteval), + preInfos: mapWithExpiry.NewMapWithExpiry(CleanInterval), } } diff --git a/plugins/inputs/cadvisor/merger.go b/plugins/inputs/cadvisor/merger.go index 8d0a1e8145..f2bf9b751e 100644 --- a/plugins/inputs/cadvisor/merger.go +++ b/plugins/inputs/cadvisor/merger.go @@ -5,6 +5,7 @@ package cadvisor import ( "fmt" + . "github.com/aws/amazon-cloudwatch-agent/internal/containerinsightscommon" "github.com/aws/amazon-cloudwatch-agent/plugins/inputs/cadvisor/extractors" ) From 8a2665675a04337203b1bb1d8299b44004d9fe18 Mon Sep 17 00:00:00 2001 From: Pinglei Guo Date: Sun, 21 Mar 2021 22:35:22 -0700 Subject: [PATCH 2/8] k8s: Add containerd sock mount for example manifest --- .../deployment-mode/daemonset/combination/combination.yaml | 6 ++++++ .../cwagent/cwagent-daemonset.yaml | 6 ++++++ .../quickstart/cwagent-fluent-bit-quickstart.yaml | 6 ++++++ .../quickstart/cwagent-fluentd-quickstart.yaml | 6 ++++++ 4 files changed, 24 insertions(+) diff --git a/amazon-cloudwatch-container-insights/k8s-deployment-manifest-templates/deployment-mode/daemonset/combination/combination.yaml b/amazon-cloudwatch-container-insights/k8s-deployment-manifest-templates/deployment-mode/daemonset/combination/combination.yaml index 0d598cd3fd..d6bd40bc88 100644 --- a/amazon-cloudwatch-container-insights/k8s-deployment-manifest-templates/deployment-mode/daemonset/combination/combination.yaml +++ b/amazon-cloudwatch-container-insights/k8s-deployment-manifest-templates/deployment-mode/daemonset/combination/combination.yaml @@ -158,6 +158,9 @@ spec: - name: varlibdocker mountPath: /var/lib/docker readOnly: true + - name: containerdsock + mountPath: /run/containerd/containerd.sock + readOnly: true - name: sys mountPath: /sys readOnly: true @@ -177,6 +180,9 @@ spec: - name: varlibdocker hostPath: path: /var/lib/docker + - name: containerdsock + hostPath: + path: /run/containerd/containerd.sock - name: sys hostPath: path: /sys diff --git a/amazon-cloudwatch-container-insights/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-daemonset.yaml b/amazon-cloudwatch-container-insights/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-daemonset.yaml index 43b89300da..0cef4e19cc 100644 --- a/amazon-cloudwatch-container-insights/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-daemonset.yaml +++ b/amazon-cloudwatch-container-insights/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-daemonset.yaml @@ -56,6 +56,9 @@ spec: - name: varlibdocker mountPath: /var/lib/docker readOnly: true + - name: containerdsock + mountPath: /run/containerd/containerd.sock + readOnly: true - name: sys mountPath: /sys readOnly: true @@ -75,6 +78,9 @@ spec: - name: varlibdocker hostPath: path: /var/lib/docker + - name: containerdsock + hostPath: + path: /run/containerd/containerd.sock - name: sys hostPath: path: /sys diff --git a/amazon-cloudwatch-container-insights/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/quickstart/cwagent-fluent-bit-quickstart.yaml b/amazon-cloudwatch-container-insights/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/quickstart/cwagent-fluent-bit-quickstart.yaml index 4c5059ceab..c1fe2b46d9 100644 --- a/amazon-cloudwatch-container-insights/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/quickstart/cwagent-fluent-bit-quickstart.yaml +++ b/amazon-cloudwatch-container-insights/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/quickstart/cwagent-fluent-bit-quickstart.yaml @@ -140,6 +140,9 @@ spec: - name: varlibdocker mountPath: /var/lib/docker readOnly: true + - name: containerdsock + mountPath: /run/containerd/containerd.sock + readOnly: true - name: sys mountPath: /sys readOnly: true @@ -159,6 +162,9 @@ spec: - name: varlibdocker hostPath: path: /var/lib/docker + - name: containerdsock + hostPath: + path: /run/containerd/containerd.sock - name: sys hostPath: path: /sys diff --git a/amazon-cloudwatch-container-insights/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/quickstart/cwagent-fluentd-quickstart.yaml b/amazon-cloudwatch-container-insights/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/quickstart/cwagent-fluentd-quickstart.yaml index 9702625ee4..8d9d23cb11 100644 --- a/amazon-cloudwatch-container-insights/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/quickstart/cwagent-fluentd-quickstart.yaml +++ b/amazon-cloudwatch-container-insights/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/quickstart/cwagent-fluentd-quickstart.yaml @@ -140,6 +140,9 @@ spec: - name: varlibdocker mountPath: /var/lib/docker readOnly: true + - name: containerdsock + mountPath: /run/containerd/containerd.sock + readOnly: true - name: sys mountPath: /sys readOnly: true @@ -159,6 +162,9 @@ spec: - name: varlibdocker hostPath: path: /var/lib/docker + - name: containerdsock + hostPath: + path: /run/containerd/containerd.sock - name: sys hostPath: path: /sys From 58ef03490f924fed237add5481ef633d8ea0214e Mon Sep 17 00:00:00 2001 From: Pinglei Guo Date: Sun, 21 Mar 2021 22:52:29 -0700 Subject: [PATCH 3/8] cadvisor: Move log location because empty label is expected With proper sock mounting, some path still got empty label and we should not generate warning error message. --- plugins/inputs/cadvisor/container_info_processor.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/plugins/inputs/cadvisor/container_info_processor.go b/plugins/inputs/cadvisor/container_info_processor.go index aca152fc57..3c16f1024a 100644 --- a/plugins/inputs/cadvisor/container_info_processor.go +++ b/plugins/inputs/cadvisor/container_info_processor.go @@ -67,9 +67,11 @@ func processContainers(cInfos []*cinfo.ContainerInfo, detailMode bool, container metrics = append(metrics, processPod(cInfo, podKeys)...) } } - // This happens when our cgroup path based pod detection logic is not working. + // This happens when our cgroup path and label based pod detection logic is not working. + // contained https://github.com/aws/amazon-cloudwatch-agent/issues/188 + // docker systemd https://github.com/aws/amazon-cloudwatch-agent/pull/171 if len(metrics) == beforePod { - log.Printf("W! No pod metric collected, metrics count is still %d", beforePod) + log.Printf("W! No pod metric collected, metrics count is still %d is containerd socket mounted? https://github.com/aws/amazon-cloudwatch-agent/issues/188", beforePod) } metrics = mergeMetrics(metrics) @@ -99,9 +101,6 @@ func processContainer(info *cinfo.ContainerInfo, detailMode bool, containerOrche return result, pKey } - if len(info.Spec.Labels) == 0 { - log.Printf("W! no label found from container spec, is containerd socket mounted? https://github.com/aws/amazon-cloudwatch-agent/issues/188") - } // Only a container has all these three labels set. containerName := info.Spec.Labels[containerNameLable] namespace := info.Spec.Labels[namespaceLable] From dff19cc8b6552dd08a427dec7a88457c04752ba9 Mon Sep 17 00:00:00 2001 From: Pinglei Guo Date: Mon, 22 Mar 2021 12:05:10 -0700 Subject: [PATCH 4/8] cadvisor: Add log for container filesystem metrics #192 It's a bit annoying ... but it helps when user turn on debug log --- plugins/inputs/cadvisor/container_info_processor.go | 5 +++++ plugins/inputs/cadvisor/extractors/fs_extractor.go | 2 ++ 2 files changed, 7 insertions(+) diff --git a/plugins/inputs/cadvisor/container_info_processor.go b/plugins/inputs/cadvisor/container_info_processor.go index 3c16f1024a..31705da211 100644 --- a/plugins/inputs/cadvisor/container_info_processor.go +++ b/plugins/inputs/cadvisor/container_info_processor.go @@ -133,6 +133,11 @@ func processContainer(info *cinfo.ContainerInfo, detailMode bool, containerOrche tags[ContainerNamekey] = containerName tags[ContainerIdkey] = path.Base(info.Name) containerType = TypeContainer + + // TODO(pingleig): wait for upstream fix https://github.com/aws/amazon-cloudwatch-agent/issues/192 + if !info.Spec.HasFilesystem { + log.Printf("D! containerd does not have container filesystem metrics from cadvisor, See https://github.com/aws/amazon-cloudwatch-agent/issues/192") + } } } else { containerType = TypeNode diff --git a/plugins/inputs/cadvisor/extractors/fs_extractor.go b/plugins/inputs/cadvisor/extractors/fs_extractor.go index f241d0c97e..e81d0e91c9 100644 --- a/plugins/inputs/cadvisor/extractors/fs_extractor.go +++ b/plugins/inputs/cadvisor/extractors/fs_extractor.go @@ -20,6 +20,7 @@ type FileSystemMetricExtractor struct { allowListRegexP *regexp.Regexp } +// TODO(pingleig): it is always false for container using containerd https://github.com/aws/amazon-cloudwatch-agent/issues/192 func (f *FileSystemMetricExtractor) HasValue(info *cinfo.ContainerInfo) bool { return info.Spec.HasFilesystem } @@ -27,6 +28,7 @@ func (f *FileSystemMetricExtractor) HasValue(info *cinfo.ContainerInfo) bool { func (f *FileSystemMetricExtractor) GetValue(info *cinfo.ContainerInfo, containerType string) []*CAdvisorMetric { var metrics []*CAdvisorMetric if containerType == TypePod || containerType == TypeInfraContainer { + log.Printf("D! fs ignore type %q path %s", containerType, info.Name) return metrics } From ec5e21cad677cc6bcfdabec1acce4576949fc218 Mon Sep 17 00:00:00 2001 From: Pinglei Guo Date: Wed, 24 Mar 2021 13:03:45 -0700 Subject: [PATCH 5/8] cadvisor: Save cgroup path for fs and net * Skip diskio because we only extract node disk io --- plugins/inputs/cadvisor/extractors/fs_extractor.go | 1 + plugins/inputs/cadvisor/extractors/net_extractor.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/plugins/inputs/cadvisor/extractors/fs_extractor.go b/plugins/inputs/cadvisor/extractors/fs_extractor.go index e81d0e91c9..6ef2e84555 100644 --- a/plugins/inputs/cadvisor/extractors/fs_extractor.go +++ b/plugins/inputs/cadvisor/extractors/fs_extractor.go @@ -60,6 +60,7 @@ func (f *FileSystemMetricExtractor) GetValue(info *cinfo.ContainerInfo, containe metric.fields[MetricName(containerType, FSInodesfree)] = v.InodesFree } + metric.cgroupPath = info.Name metrics = append(metrics, metric) } return metrics diff --git a/plugins/inputs/cadvisor/extractors/net_extractor.go b/plugins/inputs/cadvisor/extractors/net_extractor.go index 1e5b956149..4cae233111 100644 --- a/plugins/inputs/cadvisor/extractors/net_extractor.go +++ b/plugins/inputs/cadvisor/extractors/net_extractor.go @@ -90,6 +90,7 @@ func (n *NetMetricExtractor) GetValue(info *cinfo.ContainerInfo, containerType s metric.fields[MetricName(mType, k)] = v } + metric.cgroupPath = info.Name metrics = append(metrics, metric) break } @@ -101,6 +102,7 @@ func (n *NetMetricExtractor) GetValue(info *cinfo.ContainerInfo, containerType s for k, v := range aggregatedFields { metric.fields[MetricName(containerType, k)] = v } + metric.cgroupPath = info.Name metrics = append(metrics, metric) } } From 5e340c154e5e41ea19b08465ed481afe0b59c767 Mon Sep 17 00:00:00 2001 From: Pinglei Guo Date: Wed, 24 Mar 2021 13:16:23 -0700 Subject: [PATCH 6/8] build: Add new target to only build binary for docker image Existing targets build binary for all platforms, but for debugging container related problems, most time we only need some binaries for amd64. --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 87c19a4f6b..198f4fe5ab 100644 --- a/Makefile +++ b/Makefile @@ -95,6 +95,12 @@ config-downloader: copy-version-file $(WIN_BUILD)/config-downloader.exe github.com/aws/amazon-cloudwatch-agent/cmd/config-downloader $(DARWIN_BUILD)/config-downloader github.com/aws/amazon-cloudwatch-agent/cmd/config-downloader +# A fast build that only builds amd64, we don't need wizard and config downloader +build-for-docker: + $(LINUX_AMD64_BUILD)/amazon-cloudwatch-agent github.com/aws/amazon-cloudwatch-agent/cmd/amazon-cloudwatch-agent + $(LINUX_AMD64_BUILD)/start-amazon-cloudwatch-agent github.com/aws/amazon-cloudwatch-agent/cmd/start-amazon-cloudwatch-agent + $(LINUX_AMD64_BUILD)/config-translator github.com/aws/amazon-cloudwatch-agent/cmd/config-translator + fmt: go fmt ./... From 9a5d849aa50b93064477ba31c98e05b9bbe1a38b Mon Sep 17 00:00:00 2001 From: Pinglei Guo Date: Wed, 24 Mar 2021 13:23:30 -0700 Subject: [PATCH 7/8] cadvisor: Remove unused log in fs extractor --- plugins/inputs/cadvisor/extractors/fs_extractor.go | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/inputs/cadvisor/extractors/fs_extractor.go b/plugins/inputs/cadvisor/extractors/fs_extractor.go index 6ef2e84555..e8f70f8cb5 100644 --- a/plugins/inputs/cadvisor/extractors/fs_extractor.go +++ b/plugins/inputs/cadvisor/extractors/fs_extractor.go @@ -28,7 +28,6 @@ func (f *FileSystemMetricExtractor) HasValue(info *cinfo.ContainerInfo) bool { func (f *FileSystemMetricExtractor) GetValue(info *cinfo.ContainerInfo, containerType string) []*CAdvisorMetric { var metrics []*CAdvisorMetric if containerType == TypePod || containerType == TypeInfraContainer { - log.Printf("D! fs ignore type %q path %s", containerType, info.Name) return metrics } From 2134d0b849cf74d37ca42f1e7d7255af4fbf1d42 Mon Sep 17 00:00:00 2001 From: Pinglei Guo Date: Wed, 24 Mar 2021 13:27:20 -0700 Subject: [PATCH 8/8] cadvisor: Add back log for contianer in container Must have deleted it by accident when cleaning up the logs --- plugins/inputs/cadvisor/container_info_processor.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/inputs/cadvisor/container_info_processor.go b/plugins/inputs/cadvisor/container_info_processor.go index 31705da211..4b30a58909 100644 --- a/plugins/inputs/cadvisor/container_info_processor.go +++ b/plugins/inputs/cadvisor/container_info_processor.go @@ -167,6 +167,7 @@ func processContainer(info *cinfo.ContainerInfo, detailMode bool, containerOrche func processPod(info *cinfo.ContainerInfo, podKeys map[string]podKey) []*extractors.CAdvisorMetric { var result []*extractors.CAdvisorMetric if isContainerInContainer(info.Name) { + log.Printf("D! drop metric because it's nested container, name %s", info.Name) return result }