From c9246c33cb77036dcbcb914e70460654d0417250 Mon Sep 17 00:00:00 2001 From: Kulwant Singh Date: Thu, 29 Feb 2024 19:28:05 +0000 Subject: [PATCH] Merge Container Insights on Windows to aws-cwa-dev with cherry-picked commits (#168) Add support for Container Insights on Windows for EKS * Add kubelet summary API for Windows (#142) * CPU extractors with unit tests (#146) * Add memory extractors for pod and node level (#147) * Define structs for CPU and Memory stats (#149) * Add Container level metrics for CPU and memory resources. (#150) * Add storage metrics for container and node level (#151) * Add network metrics (#152) * Enable awscontainerinsights receiver to run inside Host Process container (#153) * Add HCS shim api as alternative source for metric provider (#154) * Add check for host process container before reading from hcshim (#156) * Fix CPU utilization percentage for Windows nodes (#161) * Add List of Metrics for Windows + Design (#166) * fix fstype (#164) --- cmd/configschema/go.mod | 2 +- cmd/configschema/go.sum | 4 +- cmd/otelcontribcol/go.mod | 13 +- cmd/otelcontribcol/go.sum | 22 +- go.mod | 7 +- go.sum | 10 +- internal/aws/containerinsight/const.go | 7 + internal/aws/containerinsight/utils.go | 11 + internal/aws/containerinsight/utils_test.go | 12 + internal/aws/k8s/go.mod | 3 + internal/aws/k8s/k8sclient/clientset.go | 46 +++- internal/kubelet/client.go | 18 +- internal/kubelet/client_test.go | 28 ++ internal/kubelet/go.mod | 6 + internal/kubelet/go.sum | 7 + .../awscontainerinsightreceiver/README.md | 3 + .../awscontainerinsightreceiver/design.md | 11 +- receiver/awscontainerinsightreceiver/go.mod | 17 +- receiver/awscontainerinsightreceiver/go.sum | 34 +-- .../images/eks-windows-design.png | Bin 0 -> 160177 bytes .../cadvisor/extractors/cpu_extractor.go | 10 +- .../cadvisor/extractors/diskio_extractor.go | 6 +- .../internal/cadvisor/extractors/extractor.go | 6 +- ...or_helpers_test.go => extractorhelpers.go} | 0 .../cadvisor/extractors/fs_extractor.go | 2 +- .../cadvisor/extractors/mem_extractor.go | 14 +- .../cadvisor/extractors/net_extractor.go | 22 +- .../internal/host/ebsvolume.go | 9 + .../internal/host/nodeCapacity.go | 31 ++- .../internal/host/nodeCapacity_test.go | 4 +- .../internal/host/utilconst_windows.go | 14 + .../internal/host/utils.go | 6 - .../internal/host/utilsconst.go | 13 + .../internal/k8swindows/README.md | 253 ++++++++++++++++++ .../k8swindows/extractors/cpu_extractor.go | 60 +++++ .../extractors/cpu_extractor_test.go | 58 ++++ .../k8swindows/extractors/extractor.go | 91 +++++++ .../k8swindows/extractors/extractorhelpers.go | 227 ++++++++++++++++ .../extractors/extractorhelpers_test.go | 75 ++++++ .../k8swindows/extractors/fs_extractor.go | 75 ++++++ .../extractors/fs_extractor_test.go | 89 ++++++ .../k8swindows/extractors/mem_extractor.go | 65 +++++ .../extractors/mem_extractor_test.go | 69 +++++ .../k8swindows/extractors/net_extractor.go | 100 +++++++ .../extractors/net_extractor_test.go | 214 +++++++++++++++ .../testdata/CurSingleKubeletSummary.json | 206 ++++++++++++++ .../testdata/PreSingleKubeletSummary.json | 213 +++++++++++++++ .../internal/k8swindows/hcsshim/client.go | 54 ++++ .../internal/k8swindows/hcsshim/hcsshim.go | 217 +++++++++++++++ .../k8swindows/hcsshim/hcsshim_test.go | 147 ++++++++++ .../internal/k8swindows/k8swindows.go | 147 ++++++++++ .../k8swindows/k8swindows_nowindows.go | 32 +++ .../internal/k8swindows/kubelet/client.go | 64 +++++ .../internal/k8swindows/kubelet/kubelet.go | 161 +++++++++++ .../k8swindows/kubelet/kubelet_test.go | 118 ++++++++ .../internal/k8swindows/testutils/helpers.go | 22 ++ .../stores/kubeletutil/kubeletclient.go | 20 ++ .../internal/stores/utils.go | 45 ++++ .../internal/stores/utils_test.go | 40 +++ .../awscontainerinsightreceiver/receiver.go | 70 ++--- .../receiver_test.go | 10 +- 61 files changed, 3216 insertions(+), 124 deletions(-) create mode 100644 receiver/awscontainerinsightreceiver/images/eks-windows-design.png rename receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/{extractor_helpers_test.go => extractorhelpers.go} (100%) create mode 100644 receiver/awscontainerinsightreceiver/internal/host/utilconst_windows.go create mode 100644 receiver/awscontainerinsightreceiver/internal/host/utilsconst.go create mode 100644 receiver/awscontainerinsightreceiver/internal/k8swindows/README.md create mode 100644 receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/cpu_extractor.go create mode 100644 receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/cpu_extractor_test.go create mode 100644 receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/extractor.go create mode 100644 receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/extractorhelpers.go create mode 100644 receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/extractorhelpers_test.go create mode 100644 receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/fs_extractor.go create mode 100644 receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/fs_extractor_test.go create mode 100644 receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/mem_extractor.go create mode 100644 receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/mem_extractor_test.go create mode 100644 receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/net_extractor.go create mode 100644 receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/net_extractor_test.go create mode 100644 receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/testdata/CurSingleKubeletSummary.json create mode 100644 receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/testdata/PreSingleKubeletSummary.json create mode 100644 receiver/awscontainerinsightreceiver/internal/k8swindows/hcsshim/client.go create mode 100644 receiver/awscontainerinsightreceiver/internal/k8swindows/hcsshim/hcsshim.go create mode 100644 receiver/awscontainerinsightreceiver/internal/k8swindows/hcsshim/hcsshim_test.go create mode 100644 receiver/awscontainerinsightreceiver/internal/k8swindows/k8swindows.go create mode 100644 receiver/awscontainerinsightreceiver/internal/k8swindows/k8swindows_nowindows.go create mode 100644 receiver/awscontainerinsightreceiver/internal/k8swindows/kubelet/client.go create mode 100644 receiver/awscontainerinsightreceiver/internal/k8swindows/kubelet/kubelet.go create mode 100644 receiver/awscontainerinsightreceiver/internal/k8swindows/kubelet/kubelet_test.go create mode 100644 receiver/awscontainerinsightreceiver/internal/k8swindows/testutils/helpers.go diff --git a/cmd/configschema/go.mod b/cmd/configschema/go.mod index 3ffd3b4df77b..3654d7ce6a59 100644 --- a/cmd/configschema/go.mod +++ b/cmd/configschema/go.mod @@ -697,7 +697,7 @@ require ( k8s.io/klog v1.0.0 // indirect k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect - k8s.io/kubelet v0.28.3 // indirect + k8s.io/kubelet v0.28.4 // indirect k8s.io/utils v0.0.0-20230711102312-30195339c3c7 // indirect sigs.k8s.io/controller-runtime v0.16.3 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/cmd/configschema/go.sum b/cmd/configschema/go.sum index 9e375d65861a..9b52be2509fa 100644 --- a/cmd/configschema/go.sum +++ b/cmd/configschema/go.sum @@ -2268,8 +2268,8 @@ k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= -k8s.io/kubelet v0.28.3 h1:bp/uIf1R5F61BlFvFtzc4PDEiK7TtFcw3wFJlc0V0LM= -k8s.io/kubelet v0.28.3/go.mod h1:E3NHYbp/v45Ao6AD0EOZnqO3L0R6Haks6Nm0+bnFwtU= +k8s.io/kubelet v0.28.4 h1:Ypxy1jaFlSXFXbg/yVtFOU2ZxErBVRJfLu8+t4s7Dtw= +k8s.io/kubelet v0.28.4/go.mod h1:w1wPI12liY/aeC70nqKYcNNkr6/nbyvdMB7P7wmww2o= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20230711102312-30195339c3c7 h1:ZgnF1KZsYxWIifwSNZFZgNtWE89WI5yiP5WwlfDoIyc= k8s.io/utils v0.0.0-20230711102312-30195339c3c7/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= diff --git a/cmd/otelcontribcol/go.mod b/cmd/otelcontribcol/go.mod index 68ae0822c429..259b39305a27 100644 --- a/cmd/otelcontribcol/go.mod +++ b/cmd/otelcontribcol/go.mod @@ -285,6 +285,7 @@ require ( github.com/IBM/sarama v1.42.1 // indirect github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/Microsoft/hcsshim v0.11.4 // indirect github.com/ReneKroon/ttlcache/v2 v2.11.0 // indirect github.com/SAP/go-hdb v1.6.2 // indirect github.com/SermoDigital/jose v0.9.2-0.20161205224733-f6df55f235c2 // indirect @@ -339,8 +340,10 @@ require ( github.com/cloudfoundry-incubator/uaago v0.0.0-20190307164349-8136b7bbe76e // indirect github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe // indirect github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 // indirect + github.com/containerd/cgroups v1.1.0 // indirect github.com/containerd/cgroups/v3 v3.0.2 // indirect github.com/containerd/console v1.0.3 // indirect + github.com/containerd/containerd v1.7.7 // indirect github.com/containerd/ttrpc v1.2.2 // indirect github.com/coreos/go-oidc v2.2.1+incompatible // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect @@ -394,7 +397,7 @@ require ( github.com/goccy/go-json v0.10.2 // indirect github.com/gocql/gocql v1.3.2 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect - github.com/godbus/dbus/v5 v5.0.6 // indirect + github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gofrs/uuid v4.2.0+incompatible // indirect github.com/gogo/googleapis v1.4.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -521,7 +524,7 @@ require ( github.com/nginxinc/nginx-prometheus-exporter v0.8.1-0.20201110005315-f5a5f8086c19 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer v0.89.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil v0.89.0 // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight v0.89.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight v0.92.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs v0.89.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/ecsutil v0.89.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s v0.89.0 // indirect @@ -558,7 +561,7 @@ require ( github.com/opencontainers/image-spec v1.1.0-rc5 // indirect github.com/opencontainers/runc v1.1.12 // indirect github.com/opencontainers/runtime-spec v1.1.0-rc.3 // indirect - github.com/opencontainers/selinux v1.10.0 // indirect + github.com/opencontainers/selinux v1.11.0 // indirect github.com/openshift/api v3.9.0+incompatible // indirect github.com/openshift/client-go v0.0.0-20210521082421-73d9475a9142 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect @@ -625,8 +628,8 @@ require ( github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect github.com/uber/jaeger-lib v2.4.1+incompatible // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect - github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 // indirect - github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect + github.com/vishvananda/netlink v1.2.1-beta.2 // indirect + github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect github.com/vmware/go-vmware-nsxt v0.0.0-20220328155605-f49a14c1ef5f // indirect github.com/vmware/govmomi v0.33.1 // indirect github.com/vultr/govultr/v2 v2.17.2 // indirect diff --git a/cmd/otelcontribcol/go.sum b/cmd/otelcontribcol/go.sum index efcdc217b93e..8446157298af 100644 --- a/cmd/otelcontribcol/go.sum +++ b/cmd/otelcontribcol/go.sum @@ -224,7 +224,8 @@ github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jB github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/Microsoft/hcsshim v0.11.1 h1:hJ3s7GbWlGK4YVV92sO88BQSyF4ZLVy7/awqOlPxFbA= +github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= +github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/Netflix/go-env v0.0.0-20220526054621-78278af1949d h1:wvStE9wLpws31NiWUx+38wny1msZ/tm+eL5xmm4Y7So= github.com/Netflix/go-env v0.0.0-20220526054621-78278af1949d/go.mod h1:9XMFaCeRyW7fC9XJOWQ+NdAv8VLG7ys7l3x4ozEGLUQ= @@ -454,11 +455,14 @@ github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+g github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= +github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/cgroups/v3 v3.0.2 h1:f5WFqIVSgo5IZmtTT3qVBo6TzI1ON6sycSBKkymb9L0= github.com/containerd/cgroups/v3 v3.0.2/go.mod h1:JUgITrzdFqp42uI2ryGA+ge0ap/nxzYgkGmIcetmErE= github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/containerd/containerd v1.7.7 h1:QOC2K4A42RQpcrZyptP6z9EJZnlHfHJUfZrAAHe15q4= +github.com/containerd/containerd v1.7.7/go.mod h1:3c4XZv6VeT9qgf9GMTxNTMFxGJrGpI2vz1yk4ye+YY8= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/ttrpc v1.2.2 h1:9vqZr0pxwOF5koz6N0N3kJ0zDHokrcPxIR/ZR2YFtOs= github.com/containerd/ttrpc v1.2.2/go.mod h1:sIT6l32Ph/H9cvnJsfXM5drIVzTr5A2flTf1G5tYZak= @@ -657,8 +661,8 @@ github.com/gocql/gocql v1.3.2/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJr github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.6 h1:mkgN1ofwASrYnJ5W6U/BxG15eXXXjirgZc7CLqkcaro= -github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0= @@ -1227,8 +1231,8 @@ github.com/opencontainers/runc v1.1.12 h1:BOIssBaW1La0/qbNZHXOOa71dZfZEQOzW7dqQf github.com/opencontainers/runc v1.1.12/go.mod h1:S+lQwSfncpBha7XTy/5lBwWgm5+y5Ma/O44Ekby9FK8= github.com/opencontainers/runtime-spec v1.1.0-rc.3 h1:l04uafi6kxByhbxev7OWiuUv0LZxEsYUfDWZ6bztAuU= github.com/opencontainers/runtime-spec v1.1.0-rc.3/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.10.0 h1:rAiKF8hTcgLI3w0DHm6i0ylVVcOrlgR1kK99DRLDhyU= -github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= +github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= github.com/openshift/api v0.0.0-20180801171038-322a19404e37 h1:05irGU4HK4IauGGDbsk+ZHrm1wOzMLYjMlfaiqMrBYc= github.com/openshift/api v0.0.0-20180801171038-322a19404e37/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY= github.com/openshift/api v0.0.0-20210521075222-e273a339932a/go.mod h1:izBmoXbUu3z5kUa4FjZhvekTsyzIWiOoaIgJiZBBMQs= @@ -1486,10 +1490,11 @@ github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVK github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= -github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 h1:cPXZWzzG0NllBLdjWoD1nDfaqu98YMv+OneaKc8sPOA= -github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns= +github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= +github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA= +github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vmihailenco/msgpack/v4 v4.3.12 h1:07s4sz9IReOgdikxLTKNbBdqDMLsjPKXwvCazn8G65U= github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= github.com/vmware/go-vmware-nsxt v0.0.0-20220328155605-f49a14c1ef5f h1:NbC9yOr5At92seXK+kOr2TzU3mIWzcJOVzZasGSuwoU= @@ -1879,7 +1884,6 @@ golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/go.mod b/go.mod index d8753e322d0b..0b02070d6d1c 100644 --- a/go.mod +++ b/go.mod @@ -259,6 +259,7 @@ require ( github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/Microsoft/hcsshim v0.11.4 // indirect github.com/ReneKroon/ttlcache/v2 v2.11.0 // indirect github.com/SAP/go-hdb v1.6.2 // indirect github.com/SermoDigital/jose v0.9.2-0.20161205224733-f6df55f235c2 // indirect @@ -313,8 +314,10 @@ require ( github.com/cloudfoundry-incubator/uaago v0.0.0-20190307164349-8136b7bbe76e // indirect github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe // indirect github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 // indirect + github.com/containerd/cgroups v1.1.0 // indirect github.com/containerd/cgroups/v3 v3.0.2 // indirect github.com/containerd/console v1.0.3 // indirect + github.com/containerd/containerd v1.7.7 // indirect github.com/containerd/ttrpc v1.2.2 // indirect github.com/coreos/go-oidc v2.2.1+incompatible // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect @@ -496,7 +499,7 @@ require ( github.com/nginxinc/nginx-prometheus-exporter v0.8.1-0.20201110005315-f5a5f8086c19 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer v0.89.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil v0.89.0 // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight v0.89.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight v0.92.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs v0.89.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/ecsutil v0.89.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s v0.89.0 // indirect @@ -696,7 +699,7 @@ require ( k8s.io/klog v1.0.0 // indirect k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect - k8s.io/kubelet v0.28.3 // indirect + k8s.io/kubelet v0.28.4 // indirect k8s.io/utils v0.0.0-20230711102312-30195339c3c7 // indirect sigs.k8s.io/controller-runtime v0.16.3 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/go.sum b/go.sum index c6b853149236..32d2b4db5c0a 100644 --- a/go.sum +++ b/go.sum @@ -229,7 +229,8 @@ github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYr github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/Microsoft/hcsshim v0.11.1 h1:hJ3s7GbWlGK4YVV92sO88BQSyF4ZLVy7/awqOlPxFbA= +github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= +github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/Netflix/go-env v0.0.0-20220526054621-78278af1949d h1:wvStE9wLpws31NiWUx+38wny1msZ/tm+eL5xmm4Y7So= github.com/Netflix/go-env v0.0.0-20220526054621-78278af1949d/go.mod h1:9XMFaCeRyW7fC9XJOWQ+NdAv8VLG7ys7l3x4ozEGLUQ= @@ -459,11 +460,14 @@ github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+g github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= +github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/cgroups/v3 v3.0.2 h1:f5WFqIVSgo5IZmtTT3qVBo6TzI1ON6sycSBKkymb9L0= github.com/containerd/cgroups/v3 v3.0.2/go.mod h1:JUgITrzdFqp42uI2ryGA+ge0ap/nxzYgkGmIcetmErE= github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/containerd/containerd v1.7.7 h1:QOC2K4A42RQpcrZyptP6z9EJZnlHfHJUfZrAAHe15q4= +github.com/containerd/containerd v1.7.7/go.mod h1:3c4XZv6VeT9qgf9GMTxNTMFxGJrGpI2vz1yk4ye+YY8= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/ttrpc v1.2.2 h1:9vqZr0pxwOF5koz6N0N3kJ0zDHokrcPxIR/ZR2YFtOs= github.com/containerd/ttrpc v1.2.2/go.mod h1:sIT6l32Ph/H9cvnJsfXM5drIVzTr5A2flTf1G5tYZak= @@ -2273,8 +2277,8 @@ k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= -k8s.io/kubelet v0.28.3 h1:bp/uIf1R5F61BlFvFtzc4PDEiK7TtFcw3wFJlc0V0LM= -k8s.io/kubelet v0.28.3/go.mod h1:E3NHYbp/v45Ao6AD0EOZnqO3L0R6Haks6Nm0+bnFwtU= +k8s.io/kubelet v0.28.4 h1:Ypxy1jaFlSXFXbg/yVtFOU2ZxErBVRJfLu8+t4s7Dtw= +k8s.io/kubelet v0.28.4/go.mod h1:w1wPI12liY/aeC70nqKYcNNkr6/nbyvdMB7P7wmww2o= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20230711102312-30195339c3c7 h1:ZgnF1KZsYxWIifwSNZFZgNtWE89WI5yiP5WwlfDoIyc= k8s.io/utils v0.0.0-20230711102312-30195339c3c7/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= diff --git a/internal/aws/containerinsight/const.go b/internal/aws/containerinsight/const.go index 38a94360bd8b..bf476aa266f9 100644 --- a/internal/aws/containerinsight/const.go +++ b/internal/aws/containerinsight/const.go @@ -12,6 +12,11 @@ const ( // We assume 50 micro-seconds is the minimal gap between two collected data sample to be valid to calculate delta MinTimeDiff = 50 * time.Microsecond + // Environment variables + RunInContainer = "RUN_IN_CONTAINER" + RunAsHostProcessContainer = "RUN_AS_HOST_PROCESS_CONTAINER" + TrueValue = "True" + // Attribute names InstanceID = "InstanceId" InstanceType = "InstanceType" @@ -26,6 +31,8 @@ const ( MetricType = "Type" SourcesKey = "Sources" Timestamp = "Timestamp" + OperatingSystem = "OperatingSystem" + OperatingSystemWindows = "windows" // The following constants are used for metric name construction CPUTotal = "cpu_usage_total" diff --git a/internal/aws/containerinsight/utils.go b/internal/aws/containerinsight/utils.go index 5f734597bfb1..ee84e4033a55 100644 --- a/internal/aws/containerinsight/utils.go +++ b/internal/aws/containerinsight/utils.go @@ -5,6 +5,8 @@ package containerinsight // import "github.com/open-telemetry/opentelemetry-coll import ( "fmt" "log" + "os" + "runtime" "strconv" "strings" "time" @@ -83,6 +85,15 @@ func IsPod(mType string) bool { return false } +func IsWindowsHostProcessContainer() bool { + // todo: Remove this workaround func when Windows AMIs has containerd 1.7 which solves upstream bug + // https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/#containerd-v1-6 + if runtime.GOOS == OperatingSystemWindows && os.Getenv(RunInContainer) == TrueValue && os.Getenv(RunAsHostProcessContainer) == TrueValue { + return true + } + return false +} + func getPrefixByMetricType(mType string) string { prefix := "" instancePrefix := "instance_" diff --git a/internal/aws/containerinsight/utils_test.go b/internal/aws/containerinsight/utils_test.go index 124aa1d22c70..e1c782dbcf70 100644 --- a/internal/aws/containerinsight/utils_test.go +++ b/internal/aws/containerinsight/utils_test.go @@ -5,6 +5,7 @@ package containerinsight import ( "fmt" "log" + "os" "strconv" "strings" "testing" @@ -866,3 +867,14 @@ func TestConvertToOTLPMetricsForPodContainerStatusMetrics(t *testing.T) { md = ConvertToOTLPMetrics(fields, tags, zap.NewNop()) checkMetricsAreExpected(t, md, fields, tags, expectedUnits) } + +func TestHostProcessContainer(t *testing.T) { + os.Setenv(RunInContainer, "True") + assert.Equal(t, IsWindowsHostProcessContainer(), false) + + os.Setenv(RunAsHostProcessContainer, "True") + assert.Equal(t, IsWindowsHostProcessContainer(), true) + + os.Unsetenv(RunInContainer) + os.Unsetenv(RunAsHostProcessContainer) +} diff --git a/internal/aws/k8s/go.mod b/internal/aws/k8s/go.mod index b596cebea0e8..9171fdcf1d32 100644 --- a/internal/aws/k8s/go.mod +++ b/internal/aws/k8s/go.mod @@ -9,6 +9,7 @@ require ( k8s.io/api v0.28.3 k8s.io/apimachinery v0.28.3 k8s.io/client-go v0.28.3 + github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight v0.89.0 ) require ( @@ -56,6 +57,8 @@ require ( sigs.k8s.io/yaml v1.3.0 // indirect ) +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => ../../aws/containerinsight + retract ( v0.76.2 v0.76.1 diff --git a/internal/aws/k8s/k8sclient/clientset.go b/internal/aws/k8s/k8sclient/clientset.go index 0a115915b520..294e760b0678 100644 --- a/internal/aws/k8s/k8sclient/clientset.go +++ b/internal/aws/k8s/k8sclient/clientset.go @@ -5,6 +5,8 @@ package k8sclient // import "github.com/open-telemetry/opentelemetry-collector-c import ( "context" + "fmt" + "net" "os" "path/filepath" "reflect" @@ -20,6 +22,9 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + certutil "k8s.io/client-go/util/cert" + + "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight" ) const ( @@ -251,7 +256,7 @@ func (c *K8sClient) init(logger *zap.Logger, options ...Option) error { opt.set(c) } - config, err := rest.InClusterConfig() + config, err := c.inClusterConfig() if err != nil { c.logger.Warn("cannot find in cluster config", zap.Error(err)) config, err = clientcmd.BuildConfigFromFlags("", c.kubeConfigPath) @@ -459,3 +464,42 @@ func (c *K8sClient) Shutdown() { } } } + +// inClusterConfig is copy of rest.InClusterConfig. +// There is known bug in rest.InClusterConfig on Windows when running it as host process container. +// https://github.com/kubernetes/kubernetes/issues/104562 +// This copy fixes that bug by appending `CONTAINER_SANDBOX_MOUNT_POINT` in k8s token and cert file paths. +// todo: Remove this workaround func when Windows AMIs has containerd 1.7 which solves upstream bug. +func (c *K8sClient) inClusterConfig() (*rest.Config, error) { + if !containerinsight.IsWindowsHostProcessContainer() { + return rest.InClusterConfig() + } + var ( + tokenFile = filepath.Join(os.Getenv("CONTAINER_SANDBOX_MOUNT_POINT"), "/var/run/secrets/kubernetes.io/serviceaccount/token") + rootCAFile = filepath.Join(os.Getenv("CONTAINER_SANDBOX_MOUNT_POINT"), "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt") + ) + host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT") + if len(host) == 0 || len(port) == 0 { + return nil, rest.ErrNotInCluster + } + + token, err := os.ReadFile(tokenFile) + if err != nil { + return nil, err + } + + tlsClientConfig := rest.TLSClientConfig{} + + if _, err := certutil.NewPool(rootCAFile); err != nil { + c.logger.Error(fmt.Sprintf("Expected to load root CA config from %s, but got err: %v", rootCAFile, err)) + } else { + tlsClientConfig.CAFile = rootCAFile + } + + return &rest.Config{ + Host: "https://" + net.JoinHostPort(host, port), + TLSClientConfig: tlsClientConfig, + BearerToken: string(token), + BearerTokenFile: tokenFile, + }, nil +} diff --git a/internal/kubelet/client.go b/internal/kubelet/client.go index df16aa3b3c84..6d00e74ef320 100644 --- a/internal/kubelet/client.go +++ b/internal/kubelet/client.go @@ -11,23 +11,39 @@ import ( "net/http" "net/url" "os" + "path/filepath" "strings" "go.uber.org/zap" "k8s.io/client-go/rest" "k8s.io/client-go/transport" + "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight" "github.com/open-telemetry/opentelemetry-collector-contrib/internal/common/sanitize" "github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig" ) -const ( +var ( svcAcctCACertPath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" svcAcctTokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" // #nosec defaultSecurePort = "10250" defaultReadOnlyPort = "10255" ) +func init() { + updateSVCPath() +} + +func updateSVCPath() { + // This is known that k8s token and cert file as available with CONTAINER_SANDBOX_MOUNT_POINT in path. + // https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/#containerd-v1-6 + // todo: Remove this workaround func when Windows AMIs has containerd 1.7 which solves upstream bug + if containerinsight.IsWindowsHostProcessContainer() { + svcAcctCACertPath = filepath.Join(os.Getenv("CONTAINER_SANDBOX_MOUNT_POINT"), svcAcctCACertPath) + svcAcctTokenPath = filepath.Join(os.Getenv("CONTAINER_SANDBOX_MOUNT_POINT"), svcAcctTokenPath) + } +} + type Client interface { Get(path string) ([]byte, error) } diff --git a/internal/kubelet/client_test.go b/internal/kubelet/client_test.go index d9fa5f6547f6..5b960743a53b 100644 --- a/internal/kubelet/client_test.go +++ b/internal/kubelet/client_test.go @@ -15,6 +15,7 @@ import ( "io" "net/http" "net/http/httptest" + "os" "path/filepath" "regexp" "strings" @@ -26,6 +27,7 @@ import ( "go.uber.org/zap" "k8s.io/client-go/tools/clientcmd" + "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight" "github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig" ) @@ -71,6 +73,32 @@ func TestNewTLSClientProvider(t *testing.T) { require.NotNil(t, tcc.RootCAs) } +func TestSAPathInHostProcessContainer(t *testing.T) { + // todo: Remove this workaround func when Windows AMIs has containerd 1.7 which solves upstream bug. + + // Test default SA cert and token. + assert.Equal(t, svcAcctCACertPath, "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt") + assert.Equal(t, svcAcctTokenPath, "/var/run/secrets/kubernetes.io/serviceaccount/token") + + // Test SA cert and token when run inside container. + os.Setenv(containerinsight.RunInContainer, "True") + updateSVCPath() + assert.Equal(t, svcAcctCACertPath, "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt") + assert.Equal(t, svcAcctTokenPath, "/var/run/secrets/kubernetes.io/serviceaccount/token") + + // Test SA cert and token when run inside host process container. + os.Setenv(containerinsight.RunAsHostProcessContainer, "True") + os.Setenv("CONTAINER_SANDBOX_MOUNT_POINT", "test123456") + updateSVCPath() + assert.Equal(t, svcAcctCACertPath, "test123456/var/run/secrets/kubernetes.io/serviceaccount/ca.crt") + assert.Equal(t, svcAcctTokenPath, "test123456/var/run/secrets/kubernetes.io/serviceaccount/token") + + os.Unsetenv("CONTAINER_SANDBOX_MOUNT_POINT") + os.Unsetenv(containerinsight.RunInContainer) + os.Unsetenv(containerinsight.RunAsHostProcessContainer) + updateSVCPath() +} + func TestNewSAClientProvider(t *testing.T) { p, err := NewClientProvider("localhost:9876", &ClientConfig{ APIConfig: k8sconfig.APIConfig{ diff --git a/internal/kubelet/go.mod b/internal/kubelet/go.mod index 60c5eb829dcd..0c360e49dda1 100644 --- a/internal/kubelet/go.mod +++ b/internal/kubelet/go.mod @@ -3,6 +3,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/internal/kubele go 1.20 require ( + github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight v0.92.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.89.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig v0.89.0 github.com/stretchr/testify v1.8.4 @@ -37,6 +38,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect go.opentelemetry.io/collector/config/configopaque v0.89.0 // indirect + go.opentelemetry.io/collector/pdata v1.0.0-rcv0018 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.18.0 // indirect golang.org/x/oauth2 v0.14.0 // indirect @@ -45,6 +47,8 @@ require ( golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/grpc v1.59.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect @@ -63,6 +67,8 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/commo replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig => ../../internal/k8sconfig +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => ../../internal/aws/containerinsight + // openshift removed all tags from their repo, use the pseudoversion from the release-3.9 branch HEAD replace github.com/openshift/api v3.9.0+incompatible => github.com/openshift/api v0.0.0-20180801171038-322a19404e37 diff --git a/internal/kubelet/go.sum b/internal/kubelet/go.sum index 4285b59c7fe3..43e990b84bda 100644 --- a/internal/kubelet/go.sum +++ b/internal/kubelet/go.sum @@ -232,6 +232,8 @@ go.opentelemetry.io/collector/config/configopaque v0.89.0 h1:Ad6yGcGBHs+J9SNjked go.opentelemetry.io/collector/config/configopaque v0.89.0/go.mod h1:TPCHaU+QXiEV+JXbgyr6mSErTI9chwQyasDVMdJr3eY= go.opentelemetry.io/collector/config/configtls v0.89.0 h1:XDeUaTU7LYwnEXz/CSdjbCStJa7n0YR1q0QpK0Vtw9w= go.opentelemetry.io/collector/config/configtls v0.89.0/go.mod h1:NlE4elqXoyFfzQvYfzgH6uOU1zNVa+5tt6EIq52TJ9Y= +go.opentelemetry.io/collector/pdata v1.0.0-rcv0018 h1:a2IHOZKphRzPagcvOHQHHUE0DlITFSKlIBwaWhPZpl4= +go.opentelemetry.io/collector/pdata v1.0.0-rcv0018/go.mod h1:oNIcTRyEJYIfMcRYyyh5lquDU0Vl+ktTL6ka+p+dYvg= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= @@ -438,6 +440,9 @@ google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -445,6 +450,8 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/receiver/awscontainerinsightreceiver/README.md b/receiver/awscontainerinsightreceiver/README.md index b7d26825d01f..fd063ee7cc15 100644 --- a/receiver/awscontainerinsightreceiver/README.md +++ b/receiver/awscontainerinsightreceiver/README.md @@ -851,6 +851,9 @@ kubectl apply -f config.yaml The attribute `container_status_reason` is present only when `container_status` is in "Waiting" or "Terminated" State. The attribute `container_last_termination_reason` is present only when `container_status` is in "Terminated" State. +## Available Metrics and Resource Attributes on Windows +Refer [Metrics on Windows](./internal/k8swindows/README.md) + This is a sample configuration for AWS Container Insights using the `awscontainerinsightreceiver` and `awsemfexporter` for an ECS cluster to collect the instance level metrics: ``` receivers: diff --git a/receiver/awscontainerinsightreceiver/design.md b/receiver/awscontainerinsightreceiver/design.md index 1669740c36ba..a333fd1b4a46 100644 --- a/receiver/awscontainerinsightreceiver/design.md +++ b/receiver/awscontainerinsightreceiver/design.md @@ -3,6 +3,7 @@ ## Container Insights Architecture for EKS ![architecture](images/eks-design.png) +![architecture for Windows Nodes](images/eks-windows-design.png) ## Container Insights Architecture for ECS ![architecture](images/ecs-design.png) @@ -16,7 +17,15 @@ * Some pod/container related labels like podName, podId, namespace, containerName are extracted from the container spec provided by `cadvisor`. This labels will be added as resource attributes for the metrics and the AWS Container Insights processor needs those attributes to do further processing of the metrics. * `k8sapiserver` * Collects cluster-level metrics from k8s api server - * The receiver is designed to run as daemonset. This guarantees that only one receiver is running per cluster node. To make sure cluster-level metrics are not duplicated, the receiver integrate with K8s client which support leader election API. It leverages k8s configmap resource as some sort of LOCK primitive. The deployment will create a dedicate configmap as the lock resource. If one receiver is required to elect a leader, it will try to lock (via Create/Update) the configmap. The API will ensure one of the receivers hold the lock to be the leader. The leader continually “heartbeats” to claim its leaderships, and the other candidates periodically make new attempts to become the leader. This ensures that a new leader will be elected quickly, if the current leader fails for some reason. + * The receiver is designed to run as daemonset. This guarantees that only one receiver is running per cluster node. To make sure cluster-level metrics are not duplicated, the receiver integrate with K8s client which support leader election API. It leverages k8s configmap resource as some sort of LOCK primitive. The deployment will create a dedicate configmap as the lock resource. If one receiver is required to elect a leader, it will try to lock (via Create/Update) the configmap. The API will ensure one of the receivers hold the lock to be the leader. The leader continually “heartbeats” to claim its leaderships, and the other candidates periodically make new attempts to become the leader. This ensures that a new leader will be elected quickly, if the current leader fails for some reason. + +For Windows Worker Nodes, +`awscontainerinsightreceiver` collects data from 2 main sources: +* `kubelet` Summary API + * Kubelet on Windows node expose summary API which returns CPU, Memory, Network and storage metrics for container, pod and Node. + * The receiver generates Container Insights specific metrics from the raw metrics provided by `kubelet`. The metrics are categorized as different infrastructure layers like node, node filesystem, node network, pod, pod network, container, and container filesystem. +* HCS Shim API + * HCS Shim API provides Network metrics for containers. The following two packages are used to decorate metrics: diff --git a/receiver/awscontainerinsightreceiver/go.mod b/receiver/awscontainerinsightreceiver/go.mod index b597027e539c..2e978a4a6cd4 100644 --- a/receiver/awscontainerinsightreceiver/go.mod +++ b/receiver/awscontainerinsightreceiver/go.mod @@ -3,12 +3,13 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscon go 1.20 require ( + github.com/Microsoft/hcsshim v0.11.4 github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-00010101000000-000000000000 github.com/aws/aws-sdk-go v1.47.10 github.com/go-kit/log v0.2.1 github.com/google/cadvisor v0.48.1 github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil v0.89.0 - github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight v0.89.0 + github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight v0.92.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s v0.89.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/metrics v0.89.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig v0.89.0 @@ -31,6 +32,7 @@ require ( k8s.io/apimachinery v0.28.3 k8s.io/client-go v0.28.3 k8s.io/klog v1.0.0 + k8s.io/kubelet v0.28.3 k8s.io/utils v0.0.0-20230711102312-30195339c3c7 ) @@ -51,9 +53,11 @@ require ( github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/checkpoint-restore/go-criu/v5 v5.3.0 // indirect - github.com/cilium/ebpf v0.7.0 // indirect + github.com/cilium/ebpf v0.9.1 // indirect github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 // indirect + github.com/containerd/cgroups v1.1.0 // indirect github.com/containerd/console v1.0.3 // indirect + github.com/containerd/containerd v1.6.23 // indirect github.com/containerd/ttrpc v1.2.2 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect @@ -71,7 +75,6 @@ require ( github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/fatih/color v1.15.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/frankban/quicktest v1.14.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/logr v1.3.0 // indirect @@ -149,10 +152,10 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.89.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus v0.89.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.0.2 // indirect + github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b // indirect github.com/opencontainers/runc v1.1.12 // indirect github.com/opencontainers/runtime-spec v1.0.3-0.20220909204839-494a5a6aca78 // indirect - github.com/opencontainers/selinux v1.10.0 // indirect + github.com/opencontainers/selinux v1.10.1 // indirect github.com/openshift/api v3.9.0+incompatible // indirect github.com/openshift/client-go v0.0.0-20210521082421-73d9475a9142 // indirect github.com/ovh/go-ovh v1.4.3 // indirect @@ -174,8 +177,8 @@ require ( github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect - github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 // indirect - github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect + github.com/vishvananda/netlink v1.2.1-beta.2 // indirect + github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect github.com/vultr/govultr/v2 v2.17.2 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect go.opencensus.io v0.24.0 // indirect diff --git a/receiver/awscontainerinsightreceiver/go.sum b/receiver/awscontainerinsightreceiver/go.sum index 8c1b64cc5e36..7a603b22a981 100644 --- a/receiver/awscontainerinsightreceiver/go.sum +++ b/receiver/awscontainerinsightreceiver/go.sum @@ -66,6 +66,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= +github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= @@ -103,16 +105,20 @@ github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAc github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.7.0 h1:1k/q3ATgxSXRdrmPfH8d7YK0GfqVsEKZAX9dQZvs56k= -github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= +github.com/cilium/ebpf v0.9.1 h1:64sn2K3UKw8NbP/blsixRpF3nXuyhz/VjRlRzvlBRu4= +github.com/cilium/ebpf v0.9.1/go.mod h1:+OhNOIXx/Fnu1IE8bJz2dzOA+VSfyTfdNUVdlQnxUFY= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= +github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= +github.com/containerd/containerd v1.6.23 h1:KYJd6UJhKHzwMhiD70iTtSmU+k4565ac22GOTI3AuTA= +github.com/containerd/containerd v1.6.23/go.mod h1:UrQOiyzrLi3n4aezYJbQH6Il+YzTvnHFbEuO3yfDrM4= github.com/containerd/ttrpc v1.2.2 h1:9vqZr0pxwOF5koz6N0N3kJ0zDHokrcPxIR/ZR2YFtOs= github.com/containerd/ttrpc v1.2.2/go.mod h1:sIT6l32Ph/H9cvnJsfXM5drIVzTr5A2flTf1G5tYZak= github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY= @@ -165,9 +171,7 @@ github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBD github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= -github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= @@ -416,7 +420,6 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= @@ -506,14 +509,14 @@ github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= -github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b h1:YWuSjZCQAPM8UUBLkYUk1e+rZcvWHJmFb6i6rM44Xs8= +github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= github.com/opencontainers/runc v1.1.12 h1:BOIssBaW1La0/qbNZHXOOa71dZfZEQOzW7dqQf3phss= github.com/opencontainers/runc v1.1.12/go.mod h1:S+lQwSfncpBha7XTy/5lBwWgm5+y5Ma/O44Ekby9FK8= github.com/opencontainers/runtime-spec v1.0.3-0.20220909204839-494a5a6aca78 h1:R5M2qXZiK/mWPMT4VldCOiSL9HIAMuxQZWdG0CSM5+4= github.com/opencontainers/runtime-spec v1.0.3-0.20220909204839-494a5a6aca78/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.10.0 h1:rAiKF8hTcgLI3w0DHm6i0ylVVcOrlgR1kK99DRLDhyU= -github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/opencontainers/selinux v1.10.1 h1:09LIPVRP3uuZGQvgR+SgMSNBd1Eb3vlRbGqQpoHsF8w= +github.com/opencontainers/selinux v1.10.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/openshift/api v0.0.0-20180801171038-322a19404e37 h1:05irGU4HK4IauGGDbsk+ZHrm1wOzMLYjMlfaiqMrBYc= github.com/openshift/api v0.0.0-20180801171038-322a19404e37/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY= github.com/openshift/api v0.0.0-20210521075222-e273a339932a/go.mod h1:izBmoXbUu3z5kUa4FjZhvekTsyzIWiOoaIgJiZBBMQs= @@ -572,7 +575,6 @@ github.com/prometheus/prometheus v0.48.0 h1:yrBloImGQ7je4h8M10ujGh4R6oxYQJQKlMuE github.com/prometheus/prometheus v0.48.0/go.mod h1:SRw624aMAxTfryAcP8rOjg4S/sHHaetx2lyJJ2nM83g= github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo= github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= @@ -629,10 +631,11 @@ github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0h github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= -github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 h1:cPXZWzzG0NllBLdjWoD1nDfaqu98YMv+OneaKc8sPOA= -github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns= +github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= +github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA= +github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vultr/govultr/v2 v2.17.2 h1:gej/rwr91Puc/tgh+j33p/BLR16UrIPnSr+AIwYWZQs= github.com/vultr/govultr/v2 v2.17.2/go.mod h1:ZFOKGWmgjytfyjeyAdhQlSWwTjh2ig+X49cAp50dzXI= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -882,7 +885,6 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1085,7 +1087,7 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1115,6 +1117,8 @@ k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= +k8s.io/kubelet v0.28.3 h1:bp/uIf1R5F61BlFvFtzc4PDEiK7TtFcw3wFJlc0V0LM= +k8s.io/kubelet v0.28.3/go.mod h1:E3NHYbp/v45Ao6AD0EOZnqO3L0R6Haks6Nm0+bnFwtU= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20230711102312-30195339c3c7 h1:ZgnF1KZsYxWIifwSNZFZgNtWE89WI5yiP5WwlfDoIyc= k8s.io/utils v0.0.0-20230711102312-30195339c3c7/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= diff --git a/receiver/awscontainerinsightreceiver/images/eks-windows-design.png b/receiver/awscontainerinsightreceiver/images/eks-windows-design.png new file mode 100644 index 0000000000000000000000000000000000000000..950ed3636072e3826a5b8b4cc09d11ed228e1061 GIT binary patch literal 160177 zcmeEP2_Tet|L;DW6*-clZ7U`BeUnNh(&auHjN8m$n8_JJhojw!kZ2VZA#z_eijt#J z?jt1R7;=o;|M!@~ux!2WZu{=;e|z`Mc;bFWVuiLj1Ht zM9329h~6Q(e}{;OnF9=Cg%mJFI-mq#b}$zt0(`auUx2&pkY-lqR&X=oogzXaV*ElP z{361qgr#0vZ_{ z4YMGQ3WGMahnXEYh!8yphnd);_F=@7iKB6bJ2+Y)5v0Kj3rGrx5x+p6MZt+zO&pLI zN4NuYS@ggTA@Cs7BA941E0~1?3{0bt6>0WPY8WHnCQ`x@$M;+8lTZ_qJc2HtGosw2&2XU+^ud`MJ8NQ(F(v$LeWib{wRuUI&co1cBF zh!Aw)q?HS)Jz>yWXf`aItjyq!#D_^YqLD~jv=wUpMN=dK0XHSBFzH5^12hNoo;63> zk_Jt>2L;A8?+)m~NmH2ZudixbnW2G*K$0mXPU;?X_aV5Ig(bOVVIk5tCNNVQ3kReV zg47-Yi2%zF4ap8hehukXM@yI)5<`6R>?P_2BTg}Sq8*S(&=S>pQh!ap$quUmw}m8_ zJj0}RfS~>7&r;^efucaa=XEBca5UCI+R0hVTvF%??8z6VJxlC@RKN*DResq$R4<6iMF!^zX^knj%Wv@4O|6ji*%TsR#BnZ zKa_9CO9^xXXgstiDA?Ti$i0C(py4D*p1t(j^FBBGs&Js{Xos`lwWP=S#Yl5TQUfuP z5@6;;N?4pMIhIuNBq>FD5Jr+?3(C8pUVqDqf_q2;@!S1}Oz;A^S$wfbJ^JS^mNZc$ zNNXiYGGTwmwIUna|H)bb&u&Ylg66Y-{v?HiR0C2F(i#er-O1l_5y3#9v3%PiLgw#3 zwT=?Ae-^cl3l~w`9Q+U5X^uposTNms_RpWRxRMJNSDN%Y_2P;xW=+NZde&52@_R3? znS_aixWqRtuDQ90sp+4zh{6k0R77HtRRn&UI6c%pQIrh6{yIKU?7OQd1;7yJo*I#F8NY4BZGoF9O{0mEwFvoAqznCaV=KkXI{|)vH zjxbY(s2YIkrnWFgM=Qt%QK5M_)(WiyzJ!23^jR1BEC6{S=(qh4#wLHYADXe*Ul0I* zLk~gz5HJ^qvs6z(zs@~2`y2VWC70I`>EtlWHozT=;$l&fsY1gXEZ`(Mxaf_fb%UGD zW1hba&KD&YRN6>3iPXYCzUBb8g`usS=M&n+6!crVFxL@HBnXTn*t0iFzq-qv;fytX=#uD~UP)uO<3x3Dh016cB5tc$>cZiU`1fMLRo50W6 zx0sMGr~n)XvqC_DA`tj;gd!P^;NuCnDIAClxJtYqOb6u=s%sDmB)^HMIdlCI0puBK zl1h>1)AHGf2b zRA}g(e-OO~aV}yrB-s}w3nG<<3X}DLT1Cmvkp=dD&+diE!4K+C%y*GIH766e0|Jf) z(LE4R)jSOT2Jy<@DRXmc|KIQ+X$eVD7zCB6WpamzC|uY~94-m2{~5xxFjmAbya@Ud z7AK1swK$PQ?5`tEAS@1<`gwK?x;| zc6Kl*p-f@pzsbCRA2|3^L(QV(R5)2o#1~18FD55saU$EV|2=gU5nY7r{1Jxe>vDoZ zngKES#|eidIi2ti5Drp-);uQsM+gVX2`csZwx$1XAm{=pNePAg3QAIfCriS@e;A4r zmso(}$ki5Plk~@`2+8cv(~kv_qW?yG{~KO2%dBP-PDC>FU$U;>qbk%4 zYSARpw`ZG3Ug@8vd}N$Jat-3-l*#u~J~2>QOkGs;&!JEviv^tiZ07_C=uwynC?KAf zGy3*ikOLfKzhEY_1*1@S0tF=~XUitV_w5khzi7Z>QQe~0qJig18%nV=aJJe`=AWIGM4K1W$|%V5lfMIo3)|(HDY_ z|2_30*TQ@Ud4M@k2Yh~$cK^-Afl|Ch76-CR{rianDI`dpWdz>kBxyGR(gAIWv_K+Y zwg={}?wbpmK+a?Co})-4iWCaBhNIDENqbaaPH5zOPeo)z+N6^H-`ra8`@z_Eb`DTY z6>td)3jxKTeNK?0AUlu$I%+)U6c^MHNf2EsrF8pOP5#%c>VNIDz`Q|I_lMi4Ln5GP!{&B*ycZyj$Z|X7R5KD>aIUSFcwA* zMWsaqBuGq{S~AEkj#@IvkmoNX8Kj_wH4G#KhB{|lPw#q!Myru=Jl9p3^K7Y$92 z*ZcPaNGdBx=5y5ZPQLb?aVAM3$WEJVa{p_LwS5{(AOgNW|4g`&v&DCaLwZBwO$*Jc}2c0>`6jDQF~jQoqa6&5;X zU>~VJ`z_IE=&T7))_D;8XKIEJ6R-kdU~?;k8Qg(*Ajv^97#aq?gswY+p8#xw9Fi5x zmLG;fLE+qkP)X-OGbEbd4#XGv?cfL}eoJt$1;4O}lmH3=&inXvX&wba|JqqBAY+tK z{o2X@rW5%CcR+}d%n+4VAzNH(uR^{?w)6jc(k1@A^73RfP1!c1VvBS#(}tlQLRdHqS2b+LV=B4ip&EjN_V6lnnjkK!-?V^RBkd{Dwv3T|Q!(9Tzb4rVa8l=+`_Sd;|WP?6+n z@lElNL4+U$wO|2HOS<|O60OC84hyglIV|>{Y#06YqEWcvY@s1^gaI`{1od9Oy^nPE zW=()m!FiU@K^&BGcA)C$|AeD1h*(jChuCAP2pvTLi7UBC`M_7%DLJI`Rd!0&@P8UR zr5qSRQkFl1ol=fBA+`C>VW*O$f|5Vn51^dDiPLkI{x<;xB%*(f;%G^-?OGTY{NwY+ z#Nz1rMfaqrl3m5$u{K)rPXo)xP;dkY0KmzI9cz;7jD8n@`DmPc|$i@6#w{8xTqv4U`)n03!D5qM8bbdMCK_CkqZ8HAbc)b zJA3ROWULm3eWByl<{w`1o1xhMh(<0%_g_D5=Fc#uqGI#oUt(k=M`cVGjYa%V38N0& z(a!p$jg^(EIZQ)LQuvUhx*7jsp@{#~m_oTf3T>loX`S2QI+tAhgR+F=Y5ad`7{x_N zt46X?lz-ph`x!5|8{ID)d{B>R%NiZs_|HREV6mCi^{dDLGkd|3WG>TfM!+zo|v8?4J9F z&8REiX1^u>Q)m8uc`;eQa>LQSyU(SLCv{ zcXsaUeENFTs>7@95_aZozkL|FVm;@!bG#ew5Sq5}{PN?vL%W!*E)E54;#q$8{Ktur zmPi=F;p$O`xSOdR(l%o*_)_T{xfyr&>JntHd!6^?Wy@ErTDO5m>D;eBDABJTk+;!I za&U|(Bq&xV2Ztc7FA0s_Tgo3vCgQnLT-ws*+bMiaCe+-L|;LaXWF5( zitONDzmIt3MZps~(PNUf@9w=ZJ|=y-eky0E-fdFbT+83Jh1tkuNWD8S%;b`2T$u9%>DX|W5`o50yu|6AGeKvwj~-F%+p*Xn zphg`8`UloVnQxyQ%{!LQK2pkYa^_5aEple4dAo7#;5B~6im})9NrzXo(XQ=?a;^Gu z;ZuZK?5W@{xf5^VS|S-0IWoG0TQW7<5rym5_=awEeMMWTAv-LmmXQ0R$NQF3PpDUDCNP$Evz2EG;Z#)t=xjfjR3W^;6g0yh(eiPk_$u zHbHvr2$zBMM`z`h;kah^%AP^AZnJ%ibkc*Cjf)ESGOwNMbHBXnHqI68$&I3Tt_;+C1b|(WJ&NUo2-RQ=417q+TO1kl#cVq~BAk*i#&yV`bB*WA?2|A_n{#Li>W_C$ogdy=t+0f?96EO9 z!ww%@lZ9;roIY`A6)>@{qlI=3IrLcSrSaqU*wXr}_O*InWxJGBE?l+Bmp%q@L^X)h zj>c6*W~xw}(R^wh4F~Qa(-Jy#<<_}qp}cUd@u9=5 z>faR+AoDM@lP=t|Pq)zAX2kNd4%*0dxiYL*J?+pU9s?3iy(3DXkt~9|sb;}zmT?zF zShzR~bF8j*>JLlUXL_a$%n||FEu8CVI5e7GGjO|3KkTm=M7pTLRNgxsT%w&#n>H}- zJXQKA_6Odh8R_=s-nhrzHqJ{J;^@8=-neWg$%iYIVU?X6Zy!Y`Z!4AAwnT@z!1ya4 zK8zM>tSbszPg9wgRJ-I<(SoVcXn4%9%2z!Gd8pVM=O@&CehDSL!^Z>l)fG;74h{m^@PZP%r0EG25$VAjw!eE&}HKWKt~W@ZyLKWb@X(~Uw5UX5y<@YuAIrPqun10%%Yy-@!8WAxAm*JPxtmes|@z%cdn7jnHgd# zr_;AfSASpesjH~k$Wz>HcsLYRXB8W9XRO+RA?BbvlS@LjnN~d3F1zJZ3nA{7s0OQK zgNvcmQ0Ge79+x-VPK-9g=|D0(z)sexxj3#ateExoZ|A+ik z7IEiEB~7U?)N1TeO`UT;IGJ2% zrT4oGT~x(Y$s$A4Kfh}m_qR*0NgQBGP!8_4%{IqcbzZ(*k88w5dofl=KR=8#>GvV{ zM~+zI2{==g!lR;P4T+Dsk@-8fI`=$f>fu@AhiOD1Zy>U}>Q?)D<_{xb7yHyS#y`j@ zQzrbJQrYTNzT&&e)P1~h4cC4VSh{Ve6x;kf1ukr0p%7b;L>ESm7v$(gk zv*Oe(H zTxd?M@wNBYYrY!H5NhmrxT<|?>I_cjkB z)7a7CmYS{KQq34lflu$*(pcx@=u&=)5n&`1ok0z^n!$J;Z zK1*L_$4K&*OY!F)O4jeZiK?@&#YEU;p2gyQ2sn42K6+%qpq5StYqzgcwVvYMv}bQm z69@zSGv^g=&P<@%?5ab9EdmGY%%;>;D&V0ND>5$zN;;2>b!;zO9U93!J@Ul8WmwXf z@zvhi?Fr*oqx_<#9&2~jnr2-L!F+tN5!Q9RfY#a+r7z=w4vZS1(KhHejMmQvBQ<4E z6zIusyVZ^t$ohgqHQX-_=?$9cJ$#j!Sz+X3Ov|KNIlW-s@8VOD7EuuWNuzh0|xVR|yO(7ElG%c)o)d*H=BUIP9{vO4smsK?a^>~)V?7lDk zp%Zxe34*lHv&O3fb(VSRg>^QGC%I-G?V6H!7XGX|;p8-1Y?yiY>#+P*h7xv*8}`3U zBj!-gYdWCzqVG5=&uKW|}5Xe`TUW@C`Cp6R(|%BZN&;Ls7+5+$e*H8njNy$#lvl}~%TP5gGWP+gI5 zu}#dd#ClITccu5rEqR`KyL!eV3}v@=l<*~PNz5Ac6KcP9(r;UZ!e)tDXCDUq#Bidk z|8_# z-)kFgH*jikitOXdzbcx@#7c49UUC444C-933plK)!%>`@{sV8R4LpFt*`rmwPH0Mv zgzh59^V?>;XK0hF1m1tm5&N}Ad+0w~~>bMl}j$HsVIN_n6$LYp+f3!&B`~i8f{WIE3aEI0iV@nOuEXXnIU; zlZHau`JU7?mnjhgx4w2JgUI@xZ1`xgDI=U9HyK)780ncWf79jD;aYq#r;|y|*__6b z3hgz%!q@7Z{N?w_X^i-WMpZcL7x4%ht?hF>By|Sw6VE`ShPMiMTDRK5cl+*>PCL%JMiX!{Et8DNkbTDj$ zHoa#LK4u6nYfr_u-R#Q*YME5+=PNEbsXLaJ6eT(>9QVq5(ci>UHGaykIXZ$37?g zP(;|S)yEs4`l5*Q-XpC6nZcWtg|#coOFaa7GZS(Z`&PMiH)dGBtv(;%HV2WT!+J+fC)aL%tw`3FCqJ$S-bff50gnxPm6O%V@FZR^UxAizr269;-varM zHOU@>UjAz>3K;VzT9S)Ba;^n$43s4t(e8>w4-nY4lnK{wZI4byyP0=2 zfiY)r5?%y#1yo=7X**72h7eT1R$_=6*#UbPSat6~0d}vYsxmCCLNM={$n6emwZM~C zE$b}^?ka8-XTZ$3Rr>;Op&>VwLlEG4jK5tgyS6Q7qv_1uHklch;+5^mXcJ}F<6hT~ z1F*JXiL$x`3Elu%HnrlnAw^rN)#MVLt2vz+9m}LKIfmf_w}L)r?yJ~bSf<_hMu0bA ztEXF{$hyEm`JQtz`v~nU<=Kt4G5je6rXY9b&mTv`4}0!K?4(ef*Gj;iuYAPOa6LNt z=?D5g4b7S}s78uxqcx&McykF=6NMSg9{LQeCa;8Vj;lxEgWLkDN4;?+R}y0|%(qZO zFJZ-Y&bu>G>lI{u3mB#myBel2@@(l1_LW}-qT5`N4bpLpFPx=9nvUzpD5So#Y-ih3 zz#;9hU&kdO8kO}v^!|f_(ObDe-nheO6}S{#ilpPNH}mP}J{oPxk6}%CLs_8CpsBrVgp8p-TU=pTZ?+nd2f5*$yY)uknov9Njw!dlJ_23!gwI zitQMk$4*)g#<5-uJ|RF`)ycs8C0FglPCm>!2KVtmq29-p={A@te$TM8&rRXSx-fb* z2*SfuyqYW>G8a&$;rCeuJjdoDBKLv;RZ>ViYk6AphX0;-NIGxac&&99ncGD(thgI3 z$5s2KZPsPlaDFh9Y03Z8NPy0*cNiM%gt6`A$jHwp{td$q;{^1*=LYUE?__mHZ- zT=9F4E4YZ|`y;CJFr`8F0@_A9Vv?=}-`?t4GSGPLZgl^ZDF()g-eGqk@!iU>t~^hs zZo!e})Yj9#$ zd68+;Kk!r~#We~$J`g@Ee*BE&T&%~^8_eVSPWsO$G;4S|cRFq7wcV79phRqT9*d5N zMh$hoffXAgIY3-&rEZLHbQXSu?Oo}{X+rx9uJsj$&(q^bb0yB3R)Cp7&aGhYez3mE zut~W!t_22;k*Id3P+z{J9J!nfG&8*YO^E&4wj@l~RnCW5#U8T$)lFFb973&UE_0JR zJ~Dij@B5Kzc$EMMg!#D~o0%H3RW(Yh;KqCwZV8b^loWYt%nTN07Z@ngX76dv8y)hg z$dHXqJ@v9gA-&n9IC=bm&11eVhvT@789uk#-+Gh6rQ8DWJ&VgjLS+JeH1XwVP%-vh zr%Lc-UT&pemD@}n!(Q&xE!9Wm(w$pY=lI1Z=H$A$S?6|qm}0QWh)Ir#?r)wM<4s5| zO6mQSQ;nE74`GN2+tFQxHdWOhr`oZX$Rc-6$sG`b;bA-T?pZrn0PNsY;4nqSHC4QT zoYhSFbGvVFK_|=Kyy;;pFX<+9L~bmX$?zOG-jK(N8F;^)H{j`jpMFwhU+25b%a4YF zp4H5#5ENKgZg@ULwX}QE?7CHj%;g*ET00ZAx}2&0c+F1}jgpP5A$lfV4)tk0@qV5s z!snt!noPigsk1rOUWgV7h2O7@1|%uw-3GGxSK7X+2>AE>8HbsuLaqt-8Kb9do~|cq zAU+U*EZFGWWRvqnqkzqW6Z7t3JErJ*(2s$|wur2@owZ`E7M@u%O$RDWch6Orlmw6<)Wwdrg z1=Fcb-Ekcm8@EBSmCFw15PD8m8t3BiGgzJ;rzd^wuO9ae7~2>|S`i<<4|* zXp|tmRQ^o=wfs0>-ELXjJfR3k04zRIqD;XU^7gWc1r|45TV*F(b0%9i+w@Ov7xa(7 zX9{n18S<*Na1WMJ=*ceRVlp^*T{XA}b4sAe{UQV26XG~otT#CkdEFt-gS_t*T%Pge zw|)7{k+?4ImXkxyL`Y}8*{JEv@>;%|bzqttU26ze@={z|9X<2f zayS$3#)r78Z+IC>;mNDQd3bnjSGf3OGc!x9%Rbz=jwWJNl0JnL7SX!NO)A2k0TwjP zEwCZ>mdob{Yc^nRPZ0`?&-=0%U~dxMUf*&dhuzq7a9F5&Z|eC(+9_Vy&#AYx#iyPd zI@#wB6j(&wRm%}RJvbR^?7`mTL3n5zXYR(`k~>&h4O7yoI9v5guT=213i<1+y$43$ zWWIV7GwwG0Ub`ht(wIKVeZ;CWTQft5V9@4wJB{s_ZOl*Z@2yMg=Hh)q!poIgo}Irm zkr$nunHF#U(Hm!dtwDvtdl!5N&;~KhzSa%QvHJB=qROzkVR2dVaLM9FQc_I}5Yonr z5mjE%$yP)3T{@aI8_Xl;tRa~{&0g_B1AXqQ(gFcfySCC*zE;)MJrpoG|ASH^xYcPF zLT{}%F7}P&9aCkP$_=M|3pz?}=j~>wulmZc?6g2pCgxZ}n<$ zPOypxo38||_N9katzR%I@*{_!i9pz`2A!5DKdO1d*}W`?Vx8uHcxR}jse@^abpo=o z#a%^)g<0af?yfl@nm0TiLojREX`BC@;D0BWC~JRf2~GtmNca?jz0P!a2TP>sEdi^F zx}_9*nOFEax5=k*EwyWnx1_5Tlx0`nQ4ahFWl`cC-nsF*hkqv50toeb<1Po~cSXG# zHnIlDuq;x1sVvOT%pgm3MOkptF@q^XXXnfH>7#dmeGxnddUkgTF2F=^nl* z(rXp5RJMRelh*6@v4ZJ>48Q!DxJHed$tQY`;{x5*gRsQuV7=g2LrtjXS=K|W7q?_jIuo8G zZ$q9xrZCQD57xlS*Zt5xJg>pVdYui@&+GmGub*!YNMbE$peXpD%_w;rx*)1L+ z=@lwb9#gMcxILVUtNaskhCj;T!%Wt#qq%%M?jNG{|wq=CKrUFdLzs4ci2cf2yCzCiHA!>CHmb(NN;> z^Lz^Q@>4sYf(JiUJlCe!sI%Eln68Z4sCbtb>b}4)u%x3;)4Sy=!`#L%wIS;J2RF{- znf?T4oi=&Yo>Q)K`Z8h;a*^#msRbZa#ew+~b^hD|?4q&3SAjHXWb z`0x}y-#%7AR6w#9U5QT35$atHTPh`Y#?~wIa{pb`8S%+@!2%D@Y9!j``ugb+R`P*Ww;n-BGHp&~EfIq{{3pc#F7fQ@YbVvE8$DG$ zvMlTNFFCb(YG=m7;y5H+-gPGJ)2}x!m~O7lAaoU!eW=4F*gk@5=5 zfG1i028~H}NA;V9?M#E}<7S+vwp7L&&GUeeBYzK)iSZeGS|h!2KWLTRAW|eR9PP6! z$*+1+C(Y9WwL3*dV~d+c-9EC0($dlzYt7MB;u5_Cs(F>qDrfz?w52pN8|p<`UJ+O| z%l2UxbG&O1-W@yCx9#@BhYvGi3>0_-xT5}W&47o zB`~zH?H9+ z%#^r3UM^6Im%pK5WVLVEaJ4M4gRfVt5dP7Yqp}JEDk-$Z>)%e$)YsLy%u2!oE(fLs z!%zaz0&kp5^rw4=m0>}bV5EL3xvW^pkdlc!D&TvW!SJ*-;X5SJeu<4KtZ1U3!GZ%OxyarP?TFB1IL$xSi*b8=9>HX zorb%zOc}|zkRo!{0Kz9MHwlW^!2Bsw%8!&}XV(Al;3_`X_SS08M2;p#7^b6r>aK93LENNP%&^};q-{f+eeZoFQO z`Z*t@OHW{hs)v16vQWo!NxAtz{K^d=caOY-#5t*9di2v@O|+qP)kHD3+ZL`o*F#V? zv@vc*!pEj6*^e?uv zC+_Y-eh__NRuqk>MbR>vYUy0SbX(CuE7x`xbE|gh4!^4e$~t?2TCUPN1%LqvBv5e$ zuQltR5jHvNJ9tHgOuqbuwmRpPDxQ7Q^x)H7kXq#3O*^)LCP1!kiDjD`T5#x``>z2`ZzxkuF@UK2)>;#|*pl6p*+(zanu*gU&^x9Nt|T42VvaN)tIj z6qo1^D>^vc%5GTu;PNVJ(jR`P6nC31NJ>1*0Vi2;;N}zNc`LsaSXjqXbSiUXlBkFe zfUnkieyS#7m9H;08d1z8Z8VN-Yf4u@hK=!5g_}!-5Li%tTv=?u(r$^lz zhYKlxLUIbffUB3agxeO@Jc6z!fo{(cR324ec(!gX)FewYY0kYKfc8!cZywpH41=jz zd>~#{dIPQ=h`%vF$@7tV0feO*GU$}bk}KOmGjo=VVycu*gZ3WWpmDt#ot%7E6h)eD zOK|m<)1}x2#ue(S4J1bt9{J)Y(2VMHA?5|E5xoc8d-|wv9!Q1Nyk(an+73-IaP`d| z^V|iJ1a(y$a$uASn~#S;GchJ?Gz*N5=5-LA+!Kpy`jL09Z5Q25;y`%A!Bxx6qDBjP z0p2mP1z3YQKJD-$4{zylVLD&R3=luu0{~uU^GO(T69l-{Xb>ftSsh$`e2HCfK`%g8 zUFjj@L#Z^92IBPT9-H4nhJR`YFx;2!CrhrUq1BgJxr-=#mv-{-?$vu95W1ijkl|YM zulNGt`&WE{CjTqGL^b@^`V!6hzt(rwzW-}|iC*Mi@GuLK{sj+26!9;3AjVX_CwKrw z7d;UprzlyJ*9KmL4EeFm<2fU@I(gO@fv??*(x-O1#Bj_5@X_m56mN!IDLQnU3*ROLg2SL<3lRQTc9nO#=7HXKN*^-Yf@ncEl&pVX1>G>Xb{pXXa#4X= zg=CI0vch{GpEot8eYg#zlLBQjO7;Tcsjc@_=I{;*ny4D6OB#TJchna@XI@@g-i&hM zTzYA-Y+z1z7%q#PYr;zl0Ppk@X_Zm&fJ1DVAHP}zlY7a4z;XA5_%1&XCYs&801{~J zRqUimECB=4O3crqikiGW0$qFHJnjh6U4|f!_Z70HsR}p_r@eS13j8d5$x(MZI{9{k z3@Ol}rwayF@?H`{#Z>eTgKJ0LYj`FC(Y4w8RT}h}7jTO0iLo4D(}AtjSru83*S)51 zL863wU|{-Le1LxH->>Vc0JSmBf%9Sji}7P6PAGZ#>^-^={G{K7wDbr2 zBpkW}h|_Kf2Lt2NZf>BGFv}g#wbQawSzuQ~;%vB%(QL)96<`+&3e0sfVkP-wT@HN1}gyVQ27d{b7BUvxly~49^^|U zZsy+Rq@m#`c_>Pp_UP+iU=I!49Z7r)lCS9NfJ;9-)jQO&2dqF5SPIJi6s5(;esmC! z3z=5y>=y9T@2ccu6_Dc1BJY2BoB{@RJs#OdrOJ;%ygB*$gUD$EXa#isRfd0+Vb<*Y zUS%jhpvg9)T1nmla>(#Z;!ll*vo1+&EpSP0VuP}f439ZJ#;_3t$c^1GOWO3>TQSwDKZ+}KATN=Kn%dO=Pe^r5|SBHid)sH zm<_7L4|KKw>vYtW^Wz>~W|r8zno?rH!QA!2JE??bQ zRI2vIt>Nh=d%_LC*ipGqY%ExA2==d?OQ0n$ zE-@K~7*s!NvXvXWlu1tjJ% z!+gBd1P1Ivn_)BoxkQ1ly2MQ&5WQ2cW~YXlX3hS|HE4cb+hr*N`kBLZ&^kE&s|^1t!|zdsT(}QzR53(=to++of7((jpHN_fzvwL8|Fd#SOqjpM zkOEc3FeI8@k5;c?j67sYGAD!9W?HQ3?f3H)X*Ug4 z(B#tTDGv?E;`8w`gK#{K+}xrGiFZeV9TzPzkob|e6s~J6!K)0=zWL?DD#Xz|ISZjjvt@838h}NHF*Jk@g>y|EuP=H z7Y5v``L!%tutEP-$~#+4sAFX;S4qieul2zCKin|SBOd}z=!VJZf~c#I1E8S+ekTb< zqTY)qkHkSPvMN9%lzu&pqi#qz{SnZygj$ogqzY;w81Mv}=jWSt12=mz;WNPzfh(T!0t~=R_tjPMyI1)N=8u}_Sl_>!viVVOdc9wKQhs)>Op`mI zG5JdgpFgJQ?5+vh=bN?ZeAY5Dh?WYejJ94D#BtAz7WcnaVe}<5dn)dI{CXcOZ@7w9 z7N6^Q!}`H}nKT@iYtN&EW~>vwNj?|6r+vh^*`0tIc+P53K9a1Pvg*@TBfMNFejoS~J-G+7Llz7`_w)nNA*CS|89u)B!alR^y^5~R~&&socrHKZM^oGxJl6sMNUPJlVUZ-ud+Je&nn{M2` z9PWG{uZ``?>PWf1N#&8(%C&xA^28_dJZGlcQMSEl)ePNV~UaPo`gn> zMU>O0Iz&z{PQ2RSs3jA{2BtNF?&3x-igyN_rLBiGSX zD9%jwzcQW{dDbT0wpj(2c~z+IEuZ0lR0UsBuE>OVUqggldM&PyQE{u5zFk^RD(J@c zt?1XvwG~NuBYILT=b2KT^nrnt)m^^bc#y-6Ior8crs+EN{b_AqPUo6!6Vn&A2d#^{ z>2O3M_U%tmch{O0#3yE3JFn_enCQ9czO^`gYNX!3U}g&6(%@K8rXV}|!Kd&D-@|L1 zkuE?GLKH7M=72j2#VU_7r8hXdN*?j}dgYEWNJ*>qe@y1hoWPcVn@|>t|2F3fESC$1zHc16=$;(98SnX;* zw|r#^9tku9)#y|WS1dNX%$JaiGjold+J&4g(R+Oq%$w-&u=+gWG7LgqRcaF4y)-m~ z$NgOb_kkVs$}l+z^v4%J>E2*T!VXzBa7v8`xuP8CobDN^5m_3+tdBn@*CTS^y+!?^gR|I%I3IBqdwoKaO$GNbqe-R#+ZM=Kz6+e)WaMI%uI5+^4=wB4 z-Rm}4T`;|w#Ss1Gi{s2#g(q&SW2tcY&DNljm%|Cw+!4;Rt{wQoEo7`bD`*RSAxgvt z_bBv$8|#NnYI}vttuLxxEo$_+i&^d5UBzGD1?K3ZWy5hvuz$@WXS7^yu(W58?aOn_ zwjw2AKc@ki?u1m_^0ixqn%-wlet8wDlFjz>bJsS+6JYfu-9~CHbT{bG)PK6=+P9$@ z&KAt={`r1fio`A!u#q)L?#qk*_u9tSi`f@Q?YMLim_&6VskGO&4= zw$Dmq!2L?F;xsl+zH&O1=;tj%fQ!=oGCr^31RMYWqM!HcC(}W3T#~Z9D)7h0HQ6xC zO`bDTum|^Jt+kVq3Z`XfJ56}N-tOS7fnb*kA6FcE@PeEXu8VpqQI`e55zRofB-#Tn=mai52tddE~hXe1WI)QOkzq?Ddzl3od0H+siB0 zlU#_ydk}JZvPv|CxTe^PIfC~Dg6hv!&*B)xw$o$N&%my;91Q-8MyXVlC2JBk!K&-V z+ss!Yed&|KNuGuZ9z%_o<}&94Aw?Ubq`U51#c9a)o>EvL&eiBL^hOm^mwL++T?g%= z^r`}sFdflbMk)~@%EWHqm`^4m))Dbt&wkCC7}emt0eptS_b#{Sw{OP0dX8Z)>Siq% zJuuRgZC%>vp6S|`Q7~n9!Ck%cf!i+FBzOA>%h8_H3O``D z(J``~v zxAx@@mOG4Y1`>ezG&RvYQ;ti<@&nritiI6;kD#2+Sg@7V8Kz6eX{rx7zYDjUflI!Y z7mRDo%d8jTlHAff2G%NP^oX*4!)Df% zscd_TIGyZOO>&Ex?pMEgn>kWvle#P!eL7z|`OCU@qwaKTfEY}C$%~|?;cGs?=2K}J zB)t>ZPSM(pw}qcw*51vVK*v~Zh-tKCh0`4aPUnRK4;yA(NI+|a>Q?*xXA|r#S^xwM z=RCYcT}``v;9gpP!c0^Y7lhBF-$Q|UE&D-gQ>J7yTX~*J1k*Mfn2UC7!c06rmyEvd zoyZpS{EUs8c7r%8rsBD4Yu;VHzWoVu-eP{|3dY|j?abVw2yCcOh1}pZd{&De2QEY( z5jN!k5T-vcE_suUuJ}CGPm!3WFL*E>e|nA(-!P`^Bg!onQSKp)^=~%D4~${a)^ho1 zKX&sM;T$9gRH7cb;wiu#d0TfD3AV{Hp##++yrAiS)`Zo;xSQ6p3CMs`Kn&S)PTfNZA|Yp)(tU~d(U*cIjG&{>~> z%+V_nvVC%HIn0M%;)O!6>}QiqECf-S1$A6LoLkAT&f=#1!E_aSlWKrpm~R0`)058D z#GmrwhuVtkFnzj4u-EUtv$~j1Olr1F zJfxFT4d#kmeeSMvvrG5Gd=&8CJwzvpum>w6b~Kfc#6?aR#@8N>?iCwIVt?mF-) zZyuc0WClWge2@6qHfCmxTl&XuZDQ8gr&$yBi^q6ZT#NC{xHg-D8&=2iw(q_SL?iCq z$z$>wSB04`FE`BVjU&=fhP_AdF5*Wyh~VWtuSzDjS(404-DL z6+27>V$4uPAlaK;fZFJQ9Uj2blFILO z{Hj0lCfqfju7jBDV8)(6BJU@KBxA{8|Jf)E3sCpiyk-9|9t#6IEQC#qu4!uk9-DLe z)HtH2qD`G?JW*H8-o!M#hETR8!(&#fl$yYtosGXRFn=cb!PPX)QY42rZn7$tWjhmd z)i&O(;?}2b0QrA4d=?N%9I*WnQyvtOkfoo2XaQ)AzZnBjFF-fsCZ|ijilv(k%3*>9m|_xk6XX%ylJq! z#Kcrq$1PgGw@ZIaQ_PPzrZ_c(s)6~#jTM1|g#oIZa5!fRF_KdT0v{45jhpAIgCJ0c zJhQYh`Lw6yTZ3Jt?oM?(_jNr7PiwJ>7>+h#hDPR#X`3tDfH&@6BlTt2K|{hmMu1 zA0tdp;4|v8s>9$qucC#5`1`zXTE&K?Y+Fwwdy>5Zs(V(7hqBubTR%}3ZF^Ngy8sL= zh@q*f-Mns7!_}!V2f)cJWdX$&LGC|VP4-<3Y>{k)DV-)T z(nYp}_RcpkuT*b=V^-V-%Oq1>+YI~}xw)X|Me>`29BK9KH%6ioh~VxgV3`gi-g-n; zrJ!`~$tVzVt0a2Lhyz}kKYGqWtK!d4rlr}in@uGVL`h^O?`EAT=$4kl%h~(LYc0g}A`gIyDtyoWO~W zrcL81o6+-bARdfBeT~uw`PU%)xjnEKWY=2UI{RsF>E)bh6BGc^Ck41vvVkc30=%_r zVy~a)L|+aP+w-Wnw3C}U$w1hlvXfQt1{ zfb8s1dj`PW_vrwcNR12~_*@xxOb7kufM`^;kt_d14+I^9oSkOltX{c*pAG7nYS$TIL@I7Wa>ZcTrF#SVn7!aj_%%dQK+pOX7{MJ2_;WR$%yE8l&>yr9@r z!*rcqZzcxt^w)RQLVIf2Y(0QMtWAZ&A%!BZ_MT)|W~#~7VI&joRBx9-uWjt^be0Z~ zEAtUYw_P8hzLihPV?$;$!|zT5|EX;hI8U~ft^(Os_eN@FfW)e@07M(zc;YI~xSU zZha}wmmjIwE}Q;Xgf2k!vH$@3?Gt^V1ehj7OE}hfo{L9UG5>+n;1{Gat0Wp z>nlr2gah&vOtHG`O3sfxWM%ZwAD(zYaFMNJgpIif^|%h0t#f%U!zFbWb9g{#RxzZ$ zr#i{_!c4^uqc`2{p~ejMYfZW?-7+;EuzZ_&nN(%)1>{5LoozM?eq^@u@b&~4O-F5J z{#0-f9I#BjMTM-^i> zrh|jn3Zk6rtZEYrvCn6gD~!~b*7x#Geu_Y*M7rON7ug@6dc8;~k@;wobG;p`El`pR zzvVM8Z-8VmW^++w;B{>}8djY%UJr^nnvoYM!X^d4ifi#e{tRo(BkW3=K^z>>^2=p`Ju^n^A)LU0)F$R+Rtn{B?+1M1 zWglO!MgrZ+UChiu?4mmxAahwrFIjpk%5!?G5QH~Rc+9lablbTpx2Or`z3anPjv4?% zcl?+4L9V`=wZQ4klEx#5A}r_Gk3Ye3dedJ8@dQqVBuW(@VA1`t6a*D&RfVMj@kaup z=-Gs#C(EA!78cv+=rmpbhPZj}h8d8+AFrOB7c*Q%12z}#Rr*Orc)v1iS3gbvJ!sir z1}g^)eFh?Z1}@3Av<78jpPxejh9oF>l>?b8YzsU~dvn{PJHlv==Ct=(`rx3-0)+{e z6W%!NOt?)VdtbrKbWbg?4}REp7OK#ZR9Gb4-U_LBb z6%xyTHSocQkqe(EjN0paarJ1u=zeG^ER})Pr)6opM$MVPW=?=8F3%Rq0ei5@SC!N0 zloUAKFXcMA>u#tYm}%*WF%U7a!|d-Y=JqsDNKbv-!v$MsH^62|l*zv5 z(?*&)J>%-OL$jl2;=?`5=LW}%VgUjyOokOgfzgdD`nHb-xHii?O5ZXzbtgbJKVtwG zNJhocr;I5CmXN06nm5=25Wy5ecfJ;K8|~kq@|+t4cLQ&Ro|u79*{+CkpLt&R ziYZu+pvxzHZm*}YId`LN=PKXTV1He%2VEesuII)oNW#u!U8gSdSw#a3!``F!ciKdQ zh>Hndsfstqm%&)#y=M2VdHn*Kx@@G}Mpcam0(el-omJBRvG&oe5LKMacpA?_|1Rq9yRJ zYu0$U$1tUCXuh9o+`>ilkvH?Nv7e`H`YjX+QJriDu4F~srfJ^$cIt2WKh#;$++M^y zW~rGrvs2G{$!A?}gVq?K_WE0)L@85$N8g`pjWwQCz=={Gk}`sy_NqM}r$fUypY-Uf z*JrN{lz(*J;i$Ap$q2G*Mw}arj*FjhfR2Hmkzc5)>RmUVN!xZ6QHZ@kz0%h|%zn+z zFqL!3*BHFVnVH3@lnTkC%i^DF2pEL((Y+C~uW!)BlkUKr4^Ouc$H`&(d%=^s zb?Sj1&raxHR9Ug>9|K8vW3J7O1xS4A&DDx^*DNu+$UqTGBByL(yEOjZ5{kchi_1}A zl)87pO7x`iRRNj!D5WNybv-YYITm$po;e)Fh5<+u*TpVK9orPIT67;j|MP?_(%l{s z*9HAh#gobQaB?+q2qfitgKW?-&@sk1@jjxTLq9v##S=$>91!Jm(D=7LrI{Sp`Di|m z@k!Hp$3i+dJg!69-(&qP((R1p)tKW2WMWZ4)vG8HSg6rFm%A;07#`|-YdlJZ0DPg0 zo5v<}Y6OEt`yI!Skt`3lp=sAQdqHvi`J6FpI&j)D>lc9)gX(*g$Rg{sMBRs|0$aSp zIW!}tYYyklljB=gjn~n^v9_Fw>i2f$KD0LEtDj>*_5B?h-0F-hD@CAdvOSS~WwO<_ zeH7J)g_-z{1IFRjJbfaTEa7|KqZgs$2$R4V1CvVUc5gNu= z7R8<|J^>zq%e5wM%AO=;UKELTbw;$qBNK-m!cR@fq&p$EcPZ`fHOxya`Jsp>J4RfK z026PJ@1#bL92jYEdQTCWR_Q}VR?XrD>vDoW=Kc!$4Qhrse%ti#01YkIoo8>r;{8sU zm1rAWcY~O!=i>3rdWb9Sj(r{FW84O&xPBX!1sh!6$>Bz4==z6_Q6&7Fn=+2qio6J| zh_%7%P$E3=o=Ph78GOMkpNP=9aZsjH=`iVeJrSQ^+Z45v@VLA^EfF)hC#@dEhG^#>PULjo1-!5&%L*^!5a+dlmfrq zbO^WAu(o6nDdXoCmw4rEr`p0Z5#7FX>MhW zuJO+54D~klZLCWgENaj$@_)9nz5x3oY>8P(PN z9;=sYPY_1Oc*4nGLVsNNXi{s(8^er1ai%X46PlScM3jT0StEr8e|931V8mwJ1OXbu ztWK83Bf~)X7$sSJ6q1~V56%h<0&qiHV)m6&v{<9y-YIeYzv|B^NFGxpUZ3#pF?Ka< zzewWX%)c=h3!Hlwmcng3L_P%W!_~K1EJ@XBj$m!Q6!-8@_|Par?ox6yxXtC3jfcvy zSvv~2Z5E6-M=B|bY#H{yv!)iOI(>MX{uxF3uTyn(zXYEpakypQ_zq+`Tvygz0pt!o z;L}7y|9C+8_RpG=#gm64f5q9h$#{h_tFiF|Pt0hlEuQGCZ=3Liaw)1Y^yBvir{MbI z0*?0{Ag=9IFFnx2I7`NN3rn+YP2sQV=%Ht&0=Q65v{5EE2Gn%V}E1?7B|4hx)A z`YI}+EO2MB4gt3E?b+<1y}b9!ve8_|LFI}1pz(~Ln-fq0%AyV|`to#Qrp{rLr=I@D z&p5gruUW*@qj)&P^?N0sy+0?ee`Q0r*Pb5240$eDiR%(o6#9@oDib1eLie*7h_Ut5pY zdfXtL+CfD-&$Wc_EwS?@Cz9t1!>USmbs=&SR0CusBz8nndyX{PRS-If+~2LdBM%oB zVBMO?g&<&5xdHSbk}Tz9?}y(C#1;7`2)YXZV^y-#=*m*fXu=lVy{Bs+segZ!zvRY0 z=LmkoJ@o0C2|A^Ohr~k__U5Kv-`pwqi_?@9gwY?A#mkd3nN&MsjK=3Ym>`ZQ0> zL>Y;5n<3LTk)Rw&_Ep0tkeJ6t&lPeKsgR^twAFcY+n9ikp1!$qshYVph9?XTA*S{> zpwuBGmhY%qZ6nA7EfZA)j^7N*5KTD!y{qC7Gqr<4`ZIXIBMAvl&kj?v!#lT%Z1+TS zq+G6MiQz<3xXpL#`bHw-II1%k1k$R{0BCOnXqf&^_TBK;H(y9<%Xrs4ZR9()Vpu@K zTd`@*)GF7F2=KGU-?iLeM=DcXIaano;Z}MgR?=v1dyUibJ0DjUSOU`s8H2->!+tfO zs+{w+pX&|GdHir*K{*PG`@+d+QwWEWMgAF>v=2#z@6ZriSpVb=qMccH8ZvP>FB^955Y8^ARA8Da^Lk5oZU`OiF-NU< zvaO!P-jyQrzPx?%nZE<$^QZ}A0%~lPxiS$v&fu(?ioRBe*m-;U`0s)i+&zPK{wA1V z(F-XW&>+cV_Djt}@4ClleHEoMuRp5uLH|whtmo(ey#7MNAvl{9W_soLZ=foQ855YP zXKKy$=BB~Hd2vy~!Xy7)I5B~Z9;~_GNB|zkgA?Yke!fKF#F^SK!M@GnrB)D_hFn9| zcJ0sF9*ZY)#fI$QZSQoRf7n%$CC+4N6r~fe>k7?I zO`slX<=AeI%Pi1TWf0eY>|LkM z!e0S{n0_nJuQ>hQZlWpt0t2_z%tAKXQe`?nTKa4hzliB#d3b zBlz-#U)Pw50aIG}ayG4$T8=suD1|0FA@h5D?%Q@Uva-+n-UT;1Z!C?lg9;5(h`nay zw2uI;3GV9}jq!qx-|W8^83#v)XAVLtffieMRryvUHhn7yB=|o*^J7%sx9T>x&pk3x&g438_5QacSDxqg488Ng-iFakSK1;U zRJge&MGW>lDh8=f-)oKKjRd5$Nrpb2CR;5(0`!)a!@_{kuxiG6_V*m~HRn)$^;O>@ zv%zr$8F=S->f&Lv_RpXDR|+1DN`O)J;#b3o08t8PEO-?31%tT{;y-)?;5@>uYtszBvK8)$miN&qfulZuU&* zLQZfQm}jYck!=2NHnzHVPMb+PftS!=o?jKQn4$MXVoFVnxQjno*JY|JL|NfE0ppdi zFM(mn*ZpHSfdY0#Mp5w^m5~?9nJ!@^zJ;%p-Ta;=ROY=oGPniXJMEVbttOrn*cj{W z=P_4KC}#ePXgc0KOGRLEYy~vuMG}%a7WW%X7tk==$qByLTuZ+2CC9rC?nX?9QrVk3 z*)c>u$scrg))&$=iXJ?mOXOPlHR?|h0f(xy4R<<>PJy%D?3)uH@G!vB(ouHCf}NWL-S)5$;5i-njb`b_4jQN*Rt?fa$F4*4_BSetl@ zwmAOu3*f<3_hl$iY}Fj>h(_~S-;Z&(e|D=%N`F>i?)8R!Pk#`k&i!^AVBthT0+lMp zRVfEH1$$PRH^` z^uO00v<7SM*qNDOsfO#>44pU)!AW^A)A8ZSxn@1jJtGPJs&~`rqp>1vs_6mIWSR9d z`PR#mt<11LVapTEv3p(T$tgT{t+@o1KUU%^@_pFjo89}%em%+WNzh9+(>K9j+-LlP zD_&)?S-gq$%YzhBymJm!8k-I=F#+%mRN=K8qhU|Tkljq2TW_EamWD{C#Q^K_66D|Q zjFo10Do{TF!`fh$(9*SBDtW@wD2p7?5LW<&e)NxKI9#$fEzT>AL*%&+;O9j>ZwZQX zzHbv0GAY7gmEh65V8eei2p5q31tBYEFi(gp9AZmL4XDYq%^p+bZ@F5$St@DIcPjjR z+tQQurpMrZ4d}RVuj)SLG3(*jkPw3Giy=60i_Z3h$(B%aa&^iYMZjI*I$zaDl4S)! zrfrS*-Ut+cWuRfKRX?rB$+C1A2G)I(kOPWzMi8;jtULCI=YhLMq2V(PO&+`MelO53 z=PRo;b$u}|%Hw7&ou63})jM<#q=n$>ewyPl3;4$zAaXr^yEgg`aEGlrml*cm!QDLu zg-?$qar73Uucd_EQRcH4;GP+-s@%Fg^yniNr~T}0XGlQ~rE#s<#}hdL*QZg-s>*1+ zJ4xC243ZjO%(xYas$kVUakE5dirn}7B{uF2o}LAu0&!Crf_teTY~TI33mt>P?U%T1 zjc8S3l4V$JIJD3D7<@$-JTyH89l<0g zvkM|v81el+0&drrK3-ZFjnh!x1VUmXT)qXi%B?2m{3s(42*2j4OTkR@L+Vnsr2!Ti zNnJZM2MVNTumckZH_c64(@Ez7r!8_segh-=! zB;JAYDe07}%N2M8l9mzDQjovkW(cT%C_(?1c~9ocElh3TN!S3Pq#2@HW{CX*%S985 zX06irGp;v2He49;T zn@T!AJcR|I0k+Vu$NXgp29Pd#KzGl^rZ6kf(uG~3>YP4}xV}j_*uHS^nDm(LyfKX6 zju&ySfIK07`a_FgHUnZy*XvItux_o9`@0-SUONoiAbP@Bn8P>oIU$apBx(;;tHIUh zKLL)b+w8CZ2~P%yKt31D$hn8p0d)aq3)phAihegmvRgS=N1%SKC#IZTPEL+(Tkh`n zfR{UbX=rJUHp@F4<^pQw&-x;YAL^u3*Ox}DBK9z=HuR@u*7kjf)d`tyGrUexJ}s_u zmS;`)vMR6iQg)XK^$^*(VAotA6L?3Jytp}& zkthr80>1B@o$ZQ+fUm%Awx{t`O1tOg=s8C1cLnbc_IDom>6=7lF3qtB=nSZq$%N6# z0F$b93_>O~zNwc~R$H_YU#5To33J4dIobmuM@hIo3#IEK^bF4zTY z6+H;Noqd-z-Mzi>P%RdMS!fw}Dtyu`)L?8ngDah^C$-S&0BQsY%G@s#2z0iM~@Px1fg z4Y5tQgQHr1BO`}O1ORraQ#EX&vdYu-oh~^Fe=I}dS>NC44m~3j7d8D41zM~P{9jv3 zY4kte`WurxziGd93%U}Jhiml3Cvljk+!%h6)B%LM*KC3@4HQ>qgO0xb4jEeZEyaoJ z|3m4|(Cxoxi=Q8*7p7B^LX4#L_Vm;24~Kfj4%qpoyY%@J=i?QJv1yGA92bO984OWK zrKgRiAXv1szAS<<-N9$-sSjs}Lk3o2`ke~XiE%;gC@QO`NMg+ZJMGFH;g6ttwqKKQ zJp>RQiTOLE#FLBWO#q=ub6K0+ColgKkb0pKK-F%oVoKAAU3grN;_+Sygr)w;r^26y zfOv2=+?ueYM}G(LLUi&`KYnv0^BqAvIKLzw-x2xx&l>?r9p`U0owU|?dbL)~!FU98 z`%x`9XZ}7GRNuBebPeQS)Q=7(>jM;K^$vab_T<)XGCq0l3Xbqqm>-pXTcmCUOt>7_(8qO#hxoA1nbYd=ix+(i}W%to|p*^ z_9@Y!3_vo;sTmxgXgDC=Ua0Mc^7NCWNBg^5oL0j^ru~I!#h}Z`$DZ2$afrn(!&d53 zzCC~Pr*L2~kL`-FV`Sj_iY=+=2pIT(o4x;}aSxo&!h0W3vTmjLtu77;wv{yY&qUvf z;u0MUrei3s^0&Dw3IdDjTQQ%>Cig{oq3Bn&7tV#=43+S0R5$SW*V*ma%TRRK_*fU1Qd zUn1Z2^}tI!)C{W!ue@Y)i;>*cMV08z*@XB!fEA zvM?M#n7&y2vc7*bv^O0LT}+-)b#mxK&O*oy9Joh5x=kMNdN{riLys%|tT4vfu#Zcy z#E~^c#VJq&2_CLaesJ>pLa3|czApK7K&y#$y+}9vr7PvxfbHm4K$(RyZVbIZ;OkrU zY->k=H03DF4nMCsljFqXaqht6GnZA~Nx=@1i)(6PLKZ1G?RuEm5jOur6jDeBYmBV_ z1L6UvItpJnC=ij}oYXN}awJ`|86cDxzt%BqZDSH0xcD-m;;4g29+JE!zQ+d(Ug z<%J6ZUJYJt9{L}N)`keN{ref|e0gHOWBQr=pV$C-B+B->?9*x(VKs2f`K)&xDmKQ{ zQMxQFf>lcveDk0D5h&J{^f*0~$`lzF&`x`aQ6C04g{%ded!_p4d}S`xBrL2vqCBza z5VBKPnDHG2>J!)c8NCgzz{Lr`2^a#Bsq*DZ>rwHqkga?0gcVn59Mx;5w564sfaP|M z%Ja#KY&MXj`5o1*D_by#|s$wGX=~|CCN_5wGP^wT8bSH`OeiRJEae#M}7QJ z{X28D<~*VFA8@UcUj|c_OvLRSCdoDV_S6F_Crrgg#`};Wy?lM1FYY-qjSC;! z_EpWqqHghJCMTWs+OF|MLtR*sH+u^DLY?N<=C3tarB0~1`fb~jtQm2fZ&mhLrs-#& z&G`EJ_zed`FM%Zm#!go_&?N;6rkC*R9qGR>_f;4YTri_luTXJU<@NsX&H z!5bQ_3^{wu)5hUDMKAPyP4DUSUj+GGy+VfR`fZS{g?tg8z%`Jcx7I;@8pxd)<5|z) z&u=6BUdcbf+ny~PEpC?qa-B5@EeND#kpGr+nZcBn5rkXYY#sq1fTa%dFME}~(=GD zG-QFJ_Bsu=UJ~OiY72kq3Dxk4a3uVS;O;2MB?$>ik3O!9?x9{-+__bnnhPsOEVFjPJ!?vKSn&e$<_>1ahZeQ?*pdgAAM z=6N3`C%U+PlB=jSv66BWpL=+G%waaq_`ABJ{A@@_nq=npXsVA3LR<;__-}t> zZG(Gjz4(|RF#)-2P1ji}K3i=SJmEh@rEeF{NQUfZEeWU$D^kWY$2WJE8V7P06JOdl zGxYUKeTaPZCEjXj!!i7Ey&-~i54kyqDySBW8-GhZQ=N7+!KYItUHfn%?C`;=o>Z3O zv-jujb3JaW*&K;@oy$IfNgQZZ zahn#9j@%VF^h{BG(Ru62+4bw}i7p;$lTD?^Ng>Hc5yIaNo*_$t>zqn|iM?r39dL3) zaGz0vsY9!JwI^^WN}j8qMjPU@DMGHh)`WG1&&p-Vu8k?5Ry^Y5h^@oc6vT zJ^GDx1rO%m*bFdE3~gZVT1)G?n)~j&m;J2LwOOaCN-DZiVq`PkAUaucjdfP~_iNvl zF`XLnkc;NL&65<4&83O!(q6mHY4S7oRk9zPDNj=5X$)sBT-@4b;d_;~T0{qGHN_Fds(ixM*-@amZ77l5LlB9$))*SxgUK`9t-cCBa`A@2l^n^_ST- zUa>LlE~*m7!zuS9yAx&(5Z2ZNY*c;7Q z!d50@-1kTZQUVG1tHmNL_5p%6S`^f)7}a(;_loh(?TV4!7K^@!g&NuAZ~Ph&3-gcM zW6!4=spJ-{k5ufb5pC=By9{65&?V`$o<~&SrVco;OTzLy$B(*J4oNMbd{zqQEIH?- zQOsk6OQ6|(@iq-UXd6##Y%m%U*-}m{5AfNv-ioKU2obI_N`E|WCS5;AQhtrXvs5=B zVeO@}?&_vDr}enPqHD0|hL_v=1U4gSJyw;jY{H}W&I=!vYQl_lx4#LoX;nlCaZD3$ z*|#rsxqiwas2_(cmx76URns%h&4lKn`IMQW$cCRrbA{iP94`+~VPv_f$Up+^#f!t^ zQi8i6Jdw(ATqmGCemr7)5s%>PJ)E7SXApTzf0B<8193N5LknJiD1jfR5XBr@x+sf3 zFq2d3wzVA7=Xl^CwjJQsIApx+&MP-6vDLc5b~-Uk6njta`Iw;gpct{9ogF`FyhUcw z1$*<}Uv{RBM#7zG@_J5tswz?uSmD&3Rv{2WD;B_@OgcDRszW%l}xkT(DC55n-g57 zb*#~KESA2Sy}T*0r4yWU7K;m`2=AvmDh}Eka0irK6C94>)FT}x=5S_L1abr%mDEqh z?lXcbAh+->@i5d_Pho4}5-36!*0U*Yc)NqWI8ddvec|%Y-f8He686y_J9dg}pe*bQ zRxqo*i*X;h&2)ZEi>vTrwOpG>cEs*`+YHQij_NUl@mgg;c5^oV&Q*>zt`mJD9$djo ziw+rawhk7~%QG{}jfL$k=JstOa*H#)pIOI(pPWlONJ^e`-gnzdlbZ@ssTyb`dLzuY zh`Tj8m{NrPf)El!6vV8~E96*w@e7q}Bh7o*L5_(d&N~)1qi3el7>mZpbcT{1ZFY#O zl&gH^-eK8UwcT>t!sxHCkA20WA)%P3d6TWO*q4BRe;QMEsC56Gg5QOjb|E&~`=&%( zEQiL$E5}8hk_n|Pr|##S0ok`r7O!oBoGwD5mfg9HR4qE3hNpd=R)j!QL@5R~;neKI znCE{RJH(wzKnHZbq_!zA@pq0k3C0zM;^a%2*YCndHPC6-cM#2XH|U=6HT~rK;i)#c zb5*%xo}dHubSl>5ugVp;l^1qd{pjp>s|X~%rp|PmuTkg%)|u>h?K^&$v-H@3o;RxZ zC#N4)@4LG+^_aF-6rHOV;gGfa@-yl!cI<{+1OwF=xsX8f;*SraX(LHgmbI4##IT6$ zC4bVk#u^b3@2~agb@9_Yb^EoCYjc05YsRHDX73G))0FUvZlLO$=A`|q?)gYa9PGcw zHu|zsznGnAw7a{a*zft{F=NIjbo!`uBt1{$I~=4g3m!5B0up3$nPMCZO3*&tuFc<85lV5uDg@n*YGh|#Bv~CE=VzYw_v<2Uh6D2 z2`yw0c=LH5Dk}*LyA(O+&Rle7@xTq4Zq!exV0G&;9_htW7ua;#f zWLg{NtXh;*BdQDB>y?(j;VVTSw3^I$^pUA-#0d~p*BCY5l_7Vo*{SZO0;qLnkwW0b ztw@G=zR?fdJ4S>Aer#1PJ0J-jRp(pXFZg5Dw6EFBHU~&Bi{8DBaSD|M+{)r4{7jVN ziv%{^Ti1)duQ*F~=nYA_vVjh1RngUlMa8gczkz{>csTWU4Tsg@vwQFJM?A&#rytg) zN@B?uWW=6zZ*Oz!6zUcG^;-s?Mz7k9j$p2*JeCA=xVJi9-qj(!UulVo&*`3C5laT! ztv6ajwrvpz=v+3}8#|@5IL&Y=&eJkxJx61wgk@*mR-?ovcNfxkro$A`Zuz%?{h;Gd zCg>3=eqHL_lCZ!i6{E`dw#lt=GqP)Gx6`(V`|_oAoDKE|mj~voeC{|*&sPQnk-UUs*|zFU zWc$1|7Cfl(imak1qpfXjI3<`wGX2t`g6E>M#%?E`>X(^}udh6+mKqE495?2t&PEu2 z6}av)WXr;5ccDwH$av}0BL84lC!ZHaViRs zl)S=#X>N#cMNw}77Vxx6G4?q3wU+sm=ppj|bvh}|1fniURo;CgghctplOX+#_-z1w zZ7*gqCy#3WzW+H@y3>Gb&n~hC`+!w+Z?{OR=;g<=a-T8i8~p-B8!~q&WN#T%)r^J( z3ts__5odmlz!X;W#^fO5y7hwre)jcgVX^haLF0tGxhfid!-FHj`%v%rQrG=l`4VsC zyJy9fqXU>bGl5%Hjpq67wj(vaE6h{34q__Wd7|PttP9<0^iy^Q-kn(gS$Ws_EY*o|gE1Q!l1J3kSG3^m!jvn^LrOO%_RrNSjc(er4)s& zG1L0=@XfY7cS7;xJFsqu>#l)b9_S(IIT6L;-TISS_{o5&8;=gmmZ?K?6!B_lqC)Z>PA*gOQXDOO;%- zjmLG`i62t^KJ{q#NlMy{XZYXUxxbSLJbd+Va4kGkbB ziwcdv+IkG@e*hmnaXrxavPwzBSS}p{!MELdP1*d%n`s&bgyMzwKpk zBuf}HGUWPfP24-=v{&+A-Qa;1u*f;Lbh9=`+%Lr&c_Er*i+DMI|QDwI(4t2F%Ly{kVWaLCAdhv10nOXM!psXlHqDVSCjYAs-rqDU?>>8=qY*Asu>L=`esm_FWAI2^R)Xhjdy*utg)yZ z7k6(iPYAZLT%Y|ediDEQ(SVZijm2-j!wA^4rDfwDJxn)DufLkY!F+dE>aTw%Qr^(1 zRoY+#ow)sI6cySw5x6`yvBo{L0+>ou`#!$U?}Rp$bFA$%wJJA4mR{gfjQ zWjD8MUz5TPO#a1w>H;ki!9R%lGQRKATO9HH5N`Wdl}t&D(&moA%eb2aDIy5pmmmM0D+8)Lq$wkw=wgQWiE$0J*4&Bl%L8&V8TR z(3FZ*lRy4vZYF;C5olLy4zXzdO-AhF*pfF>fzbjB|2wN}4VT-IlnOn8M81%C`S6Ap zNtwnXu=;acMP-wlyvpcExah?Ip}r(h)X(F$l9B^v>tN5rg|RS;oo@~)(vnP_351%g zdG!k1&FQmDhn#jCBS1(QPOAzX_cKr>K`-a!fxu*ozR$xFPwso|5H5vi>CaD#85zD8 zY+cf*T$JkH9kcA;8(dN;wY!?-q=Ig$vc%1JH}zi7``&|JMJ!rxJ73+JE5@*Ciwig!b1w=kN;fKhD9{9D{l>38uQmkmGIg6G)4adktetU(6-4@#^Wc3oU+gjUnBdqFC8{c5Xn=hWNU;>uCE!{d~2)c=K$3>aWXp7nMPg-%VegkAR}yS8+RB}9pw#Jy-7_$v1WwY=-9Q*=WC zvjW%a-8p`Qd))vKD3XRDKC#qCU=H93AP}@Q|GpAX{hB>_%ZEzv@=I$hj2AuOY(uV% z;Cq;|5c8T*l3A8B`ZgCTiAFN2CSJ`CXSpS@y|F+=AHlY;SY*0Fs8O`etg-Wz*XBp< zsVwdC(6~Eu@}|m5bp#XdOMZ|=?=_LG4187~Zf@n(u_AYTpECD0O^RNW1;Rwf;K9w0 zbk0jpJ{OvH6Y{f-SlifqW$Mb2G00MXD-lyYSm~J9;^~?UWf^gm;%cKW3t{Q#P1(*ZM}ro9|># zWarm}Q+|iAz^rKwO-q!G=1fcLzj-nBZfmi9 z@%`Fc$gQPRg!^gjJym?%!2A#dbx>IWwlr`5SH(3{U)>i2cgPVXe36M*8oW;rmHD(^RrLdc>n`y2zc4sN2?2NcQm$r|Bze5hXRN=jV>782Bn6OPy}dlqPOr1H7;ote(fx;F zh3oCNQYhrh;$Gi;F1Ek19)#)H*7 zW`&iNuA;6T&r5jSxC8-R_Blv*|L5HuBO=Hfymf=z$;8{bAxyALXm5PHKq10yIZHl% zJ*Tp!?HY^9xKsaoqwJl5e&es9ZxKJZ8(*^GDoV%oKk*JiU6g1bCtRa0(c@-$jh&x1(VCUbsZ zFPjFtrd?R_^p~)vNQAInnLo2XMnP)vfY9%(Z&oKcxkCy8`(1{Iat=r~$s6$ye00+T zj~^h;5go(d6ZmJpa`CK{fUYxN@@oz;beH!{p8ZVt4INW*g&5k~oIVuiIlaBV{OjGh zI!n|1>L=r6!mcg*0+mX}TQlpeWpm>ntb^Z8V3L1Rd!r`E4{S5zSyP;hShl(Ta@BJy zo1Y_fUqZ(uD{^`sGs?1Io3Qs+4=kJ))_7aYCz>L{$Ei-YV2Z|*@wxSMuA6l5xfm8g zC7KifW>+d5magRoOINEg?Ue=0baxv!)XRxn%{Lc|5^{Qsj%qK{U47dBow(s(vzE+Y zGrJr$QD;Y#wfS`N;ZN&=^Ikh z$-35aF=(%^;co0dA0Rz(|5X_RgC+Ugh_r)7HIgE!?SM71d}sIl&(Fg-w|-}|*cwk7O?RbSWQwCn@xu~v&;=9krow;+S$9Cn zq53Ig`#9+O=f-~ObUEa}>Gt_0b(r6(Y@$9iR5!kGG3+vrM0b^&WA}}&4-Uxfxm-$r zw6#zs+?72-2}IBoE9C=`P$+Vp;eC`@nb>xthv-?`HJjBZg}?n3CbIyM(C~fvJ>}Or zn`5P%`)pyF%fZJUt>il27Ml!%FNz^neYw0=b1JD7G+9M60d-Kym7b|3$%RpAw?qql zdNTNnmD($BE}2h1w4H2=EMFPsR*H`fNET1-y%ni$57kY~B3id0-#-s2a`0F(v5xyP zGc zxD~v+5*)q8cf~p6726%A&%o0W@wtE6cuOplXff!PU{VM*wA;n1lSRIPH?> zZg1Vva@5P4+}&3j3>o$E9>OI?Omxlg1}TZs|EnEHzT&q386$eRr&xaJywjH%p4QMf z1YNNduzl2X+F)qsbhR+2)xWhAI?tSo#?7dc;IODd9|sF_O?Ca zogs-xa$1JspD8zJAtMfzXA;eVPiIfWRCS2|`}NMhx?A8ZVEhJ~)~Kqa6uDG05_N^6 zhsfQ6tww&|(HftNi^HxvO(vt=N&IfE?%eQ*(n!8ad8pmYcNy*u+tLmV?@PRF#= z{O<1-nn&*L?@)l=M`zS5Hc?ls@R{*CT@xGaoOSwh0Y<|;daP6S4@O5ndd3zO4S4KW z1kd+Zs>z14Fn)fXrndo#M#^J*Yg+hRb2uwC5RDa8=Mz22IX7fjw9*3zm~L}qcc;50 zrb$t++_wVc(gz%3%8F>o6VWLq*bRfnq#qa|IIfkg)dBNY(0wJ76ET0NA8B1G2QCvx z=9ipej0SE6nF1GNH1Ht)k}m#sA{NJKP)A()PVs+JS~5uF=cI)WlnG%*;AXl}i78LN zuuR20?ugv;0HolTB3J*>U^JFW6b0_+4~(`e$8 z|1A6V6V-hE4yx}6tQil(fm&h^R`Uw)qgeY7zeUU&Jb{6)UjHhra8K<{ABDhA3t=B$ zc`=^g)S^%$hpmSHZ-)&5Y$q0??ZJm`LL&0B=_iSC0f!N}!hhYm{|7-c7>vD9dkdhD z5SC;MA&FvUNg8m%hs5A+g6{`qIonKExc=r>wI6zzP2NA~`68?9nmejoLRzMffRP{2la>B8u*#q$r(CAj#A5Cg#S*MhVKx zSEA4en&hF_u3(^qb zQ+a*;wqp#4M$kT$XEiGO*;q91$#3haoM)`HBP9lV;QaNe|0|jW4dwDk2-$V0KQ4L7 z()3xQ`V7B4ES&oB>~X7ja(AtPo{-(k@l#~Nro6tX7k{|4W+IEb&s@94S~{!|A2@dp z;ns`wqU`G6Qvt0h?vo8RF!Ign_n}ziEpPAy5yAyC;2-7`x+I${qp)xM0JreakNCzI0?(zT@VcoNVv>kR z5LWTr2_=rRpf^ML`h3nF#h&mElqTt&f8_o`X~TvrAXv~C%TG-CXsGJ{GE~tBR-3}e zC05N(EOql-HZ;h5@x5-G$GG@;tA6;8Y~24jULW?TYTY2eiw?M_hOnNu3&MF%-mXnP z{6K*AbA$)1)E9w&SG)S_75bY|Gp}o!t9l|99dIqTU2Ffcu4i1t}pYo;z#Y9{~wg-on%Uhh$R9Jd|hT1WXBAUCAC6GUU*Y zej`qlXrJ5bojg{;(v>R7&e30LN8Sr#^Nly8)`~$o35fEAa%)4r5K`|IK@Gvk%g=T%4~Rp|8>ELkouJqz*sbr@D~5g~n36 z6<6RtINWlyZ^FX{=pvRMtyoR zY*8wm69mR)bMX?RSj8bA8^JO1TL6%ten1%A{}n-mGPvXx7O&|5{b%#szKUh!2Z zt(@eLMST!ip?M#hZ34VM@)6bJ7deVs*VhKL3m0Ex0;Yq95;@_Y68aU_bzex;=H^n`Syv&KIy&|+Qbr|gmFyk(1F zISJLfE(dpQ|3YY46b&{-{94<*m)KbpcoQoTn$qxHPsF&$)KKSpUg57xt+`5xB1oqb z+vfnvYH)8pH*7XlCTuLf(%*-Uu@+@Ae4RUE!7(Ubucn$sqmZVXZn8N4%J1`a)gBla zB{i$n^o^%2!&0}6&m`~EnNj!66-0eQdioS-KAzKBuk@e0x4yuK=VgMixrUq$x0J#- z7?wYkrgSP+H+9tztJol=Kmh1T3vqawR^he#!7XR*=IgS;TOXyB-Ox&w5niFng0g-J zPLExuEogJl;wmE}Gf5uR`Q3(CqzYR7*!=n-UNk=qy^_gW-$>qefST&K(!?Z=*OqH` z#x-YrJ@lFw1moM+1O&}U5ytH|TF?VKdmrpMb{t5biE*FV-F5_WZA=Q?GFU=tUb$4T zi}+b(SMm?}`{OBe7+}pG1PQpca9wd$3K`d@PDxUFDUNFb<+$bfQI=pACv_v+9Hu*{ z0H`k1Xj3hNb_BX_)bcIKtpTIkiX3V*>~tq$!Uj1xqm2kzuv ziFu#w@1LL3;Kd)6d|lykXp!IYyTX29;Eu!6Fe@~9c!d-a_tgp4sE!7UY&Wo!Y{Ndq zq$!bbn^2MoI&e?fT+1DV$uR=N84f*h8hWx+Z!RrlHj8AGrClPBRAH}L`U&OtEzZye zj2c?B@LV(NNLL`)81|nj(VJU=<|fV1lfqj%4ShDu8wh6SIWG1W=&b zbJ;9JyVAj;9EwIKpf~Z=9iv{g(Ow>g#QaBtGi6E2gF`e~Z)vdS8D!Sl&>}yS?O0c> zTl94S`4KJYufDPd!T7m*;}1x^#8}SkZXzcYLCr8{i%amp;#Gqyq)lv=jC33kNy?Fu z!CVJPBGvE=Cu%jqH0byoccl#lA+V4_r(;W2?hZkCo=&Al_PQ|ERLi9Ug_s7UH==j7 z99#LP?~d$i=^w({gmSdX@|>H|6Ia((*Aux-L(w#DOmbm@KPlFymxnIsw>?o!+Jk&5n}s|-C9 zl+z;to*V3CZGycQf4Y#FTMM>rKt8GgdV`pF7^ZBDe5G)n4z>Wi%1G+Wbu-*WPbGoA znjGREuP!a_YlPSDpnanO!CxN!A#k~zNQx3i?v&`h4mCr-1X|NJxF;GUw9DlESQwsL z&i=cFEA&~n(_#o%?+WRlkR0gXZd+ya91J(+R9`Eljk89wf0x_%o^6Yu`V!&Cx(bo1 zdH4I=qVU}p@oK~9DUd6JX8%%mbJSlNJ>k*rA$ri(0+sng@B#rNO!<9|c6k?Relpa> zyFWlF=}?x@tATfbT5=?e^`_nnkSxKQOkwak!{DoAt{>29#NYu_1e?x)Pk?5h$KH1I zBUQk}(@&+gO`aE4a}iHg%@1g+fu1^CgKdnek>bM8!VDfbqPDEv0?_w7}?JCGc(!rurz-NyH;0?Im)#Gi~Kk0@jl zLr032$mvxq0lwSP;xV)uBmAHxs-P)oO%Xck$4@t*L#HbaQSbZL0Ve>@*4&y5bL6=& z?$eCNBm3Y2ezE0F6Na?#VYmuuUHgk~qPyvuKN^Hi1djwOh-9j78J6$9AxYE9{}4)V zrK|cm(Q|+F(~9E~UF0q`RcyybJy9v-dvx zzfXMM4HsOjIp-Tsj%SQ_zeTcN8!{=EX=?s>EQGs0^;9orKiv6zWKih z4W~b{A4+GlS2r*@P3!s)!K$Tf7Xf3ZiE)wEaXEnubKJ9Mg3iD>`2f|A*CC*7v_s~^ zt>M{wZYBM)Y85YD{uHpp#H91tW1rz^I|FHy7yabtLCJu%paU?&M`r5)DF7pOLBgkz z{3G%$-v?4*eD635&7nG3&}5!%PoT}&Jy=k){!TWYI6Id8vCH+@YU~ey z4eRA9O)gb9z7!Oat~@$t{#s-7xF4y)2aw8$UJE`ROzt=qR4#6vn+)v z$MM}9^E3NU1~wP{Y?gu2iLAr?PXCqmM^%%<}Bz!Kx_9dq17K@KCkt&4Zi^e4>oL?@k?9WD8Y&~?(7k@gO&wFiv7;?#8yV>)RCN+F(v0KFCkkoW87K0 zA^CMb$C?JefL>ro&fQ%W!gg|#qtM<9e%}VVUC&B=O$7i5l(prkw@@zX$>rVvc8#Ag zn2+QMjlbaDSb{Ul0Q}-i5+~2wu5LCUZI7T3Z`+MRf^FnWYH*M_9*TY}CKTpcxP&MqT^jP+D&}h}NM8e;w-0W0>-sW+kMdU~kyDZd2V(@MuXhFKik` zgOwU9d~0&w;jzcXV7KpXlXSorJdZ_jMIj zwkfK-S4|(U_gbsf*YR`@eswG*z%z&UgU=pT_t~o_U%z?_q%c-I8qXzhahHyMP@T^m zo>!iO^uf_{ZQ)*)oYw}PhMixPUzkgLEue|(36+VH_hd~Ods$a-vN0l;p%Bjulu0wr z_WgUM#+I{&zR#|gdjN*Z46>Il+Q!8~vDZa1g;y>&IxqFqJj+s6M{n4zC%*xy(cWU0 zmlRxY10KfX?N6pc;bRN87sHRQC9sTwGrrnX6r_<5Bba(h~1k z&lA9s6%L5CVlL=6Gfr4XqYus{A6TSbS3T$FRgp8k0OAqd@7HUx8w&axNpKMB@T{PP zRtvv5$C|9u$MFz_P%e)^`#?q_<#bdm?h~OArO0ePW^h zNqO1x9E*fVHVttSwmFE9=WN2lr)I@S_6TxC#joZ1?EG@!3AP?0*<*XJt5aP?nV|D) zcsKkOmkQM6wnyI&<=7jfwaaP-g|D~AUUI#jpINa<4btiSHb(W5UA0m}&rN#PW6AK$ zV}}2IuQO@K_RK{G@r}^#TK!EAWV*sWj7Q{#=ukVqwC|e+sT;G*^|esj*R*rJUDw^P z-kFOo$GN<3qw;lr z#NzNF8dJctm-TGT2i3?k-ojZu{!spNPApoeqC7QAqUS-f7Puy7VQG4)M(* zON>#*e#UN0K_L2q~M@1562NtAO3dyf?M<$8K-qwfH)bLhs6xkRy;tSG zYCZ(yYrLV7kr>BZkWXm^S}CaS&_&Te|EVr(XnSTyE5>z ztf_SbJZ3|*CRuzS{X48VWC#IVFK}wnSh^^^n>fQNpy8%MsGDB7&gxnBFi9^{0$KD* zXIHl?eyN~p5o=&mY6F<^y-D*zUOK>AwmXtLCW$M;jfK_nSUr+$?1FqVLNDI_vBhcw zx5VZ=t7Ac-p*=OOC7^JN&cHpNqdadRyFv`HoFrNG=&ljYO&E(d%`OyT;qtxiHvxoO z88{WMFi_6UNi6YfscWy`E z?&`qqrG!ttj-Y*HQz~QBm<;X?Q=fd#sVg>sqPoY9iN55s5_Z9BR--)W{B5DLT+%Mn zAoI>uUWo9(^zO;{(O9<#(y^X*jfkLq1tzPcF;<4%oQ`msm3;S0nemoq`hD4wWf^m?&GOI zVwUbu%_D!V-YYMlK}rvl&I8xVHI0|nSGtjKa4&g&4w(m_nI((Qfiy53uy!%!L@7Z9 z0BYJ{%H3Vp{K{)F-PTu4iP8(2D(*Q`eB{W#_>d*~IKlvAm@r{+sBIcunuIWuqr^FLl z{ki6Ci@)K*Z;aDrIPJhTxr#QRAiix6`VtcLx-UDgPi?Pti^R8$8t<*K=6;zK!K$>H zqL=T@U!k5l@1#s~2_E`YogFrvE@w}Y*t-%0W~SqhGhvQv(~`e;&g56RJ$gkn0O;Cb znAZ9IpwCL(s-C-|2S_iE)?I>Bv(+Vc)Q;>6q{D@v5c&8&83cz(rm*%P@RU4Zcl;-V$S4A3rhF{p6BYjc zGRqll-)2>Bnv~#q?fWB3cMOCj_jD-j!Qs{{RnDUZ_ zIwSxk$(3z|Z(ZpWkSg|vGi7tEjNkKpLf>SIKZnbCg7}rb$<F^E>Gz`S~J9BNbZS%}Ry8Qx`Zg{bslJ z-1N;(-tq5qH-A*G=dyH|U8)jZ+a}^1ri2*Zhry!x@*? zWLUyU(ZT(E073Yks5`YcUUV-JzM2Q9@(th)&O92o1GgV&Uv_tr8!cKb1jSz^g42L7nCG91c?52_g)N3H69I!zG(10uVk8hP8C2LNh&_)Ddzz_AE>C z5xxN#_YB?Y_a8-{K_PVyQ*f!Pt>2+8+fBmSWSuN!ImEPR&OesYB0l<(cOKgCriskG z`B)PH=AOjjf&Qv>B3yRiouR#S^0)j~3^NtdL#%b~JC4u*QpxjG+GCF=yq~OHWQFxJ z(OLX|EA?8Oq8OuhI!pFE2HcTID3XA@E^^!;--QUyv+Qk)i;sk3Q2PaectMS$?tG75 zxW(`*rR=YE65`W^#~g=9Ms&hVU)8d@t&{E=Q+7K4%+U+?-;2Z8B*($BFYEzys|ZBT zt>nUkb}3J1y-{BI2g~h^gSu?yDWyb8Uo^K&Ttn87^coc|(d!-mlgt=ZA-ZOy6=h6x zM0T5u`@iB)$po_o_qLu7Kv~1E|mBykSQ>Rk})p@`OMLGI+6QG$Viis667IB zm^4w&g9Mof0Z-@{F|;`yC~XE&UAg|=!VmhON!#_API9`L{)kkf0I_I+W)Sd=vN`N5 zy`as7ja9LzVIgo%ml(=zu*aheXAU`(!Lc- z2Zm@PlKOtRcXlAn8uY3Vg4+2aIvFI0{UO>D4Uy3z=L}FQ@)dmBtV(k&^Lyi@-3p$L|~%ld3Cs{F zDX)oAV|2b+Df85$?oYpz0^0Ikg}S}-`%q9)I4#e=;mQ=zZfW z!_bQzzKYSSR~GR+l1)a_)xF-mi)R&YQMWmW+c?8&G>>Rm3JGQwrRs#ytiq|QEWXg5 zsMy~|npsbHXKU!GlNh~t?&5>zFg71?Ivr_8qLdHg`p+YS-SwLwg7C8PPv(PyW61w8dT z+YAY%0vvdldnRB{KocTEO!~O`lc?ofeZ2Kb!bWz63w_SAfj4Tx)}EA1v$3M^zLaKi zJQ|Ga^5F{MyR4=OZ}pCiPZBfcBivYT5MedPNize&SR?0#q61ddTA28E5q!QN;2?58 zP8UUO{oo=PJ7<^wzg%qI0&u$x@iKK%k+JYy4p+F&kkmXmJ&C`uMN%XXnzmHxtF*XU zl87P~(8XnZlUe9;PsEFx7`qwaU-xd$Rti!Dbe(q~73LR;s+s4wi?+T5Fh?2!F$OP& zuP87973&qDFy)3R#dEKn+SMofhn>rbDnaPx0f|213l~EVl}|!U8S?Yzoa2D-$=CKb zcw_WZYk|0~Z9>DNG<-_i;qOykm~igTC_-d`?vWRIV7GLZJtH9gi5m`jrm9&XxdYfa)D zHe)JJ%$Vi4^9s7whS+0ag!(l}3@}ieoFA=6@Gj|Q0Ah0NJ}HF`HClJCX3Mg-JDVNE zg5KqaZt-HpxH(t}(+E&5U%qCq`n_gscxM=YMFn>sH|g|zGyB9B3u|*ER|ezz$4iV{ zHpxH_K*HQSb!T=#cJB#F7LWqYZS@Q7jaQMJJOVt;y1P6ORS|rAF5=@R^DB5I{pe(G zp~Y^o_2b=8u(ys|$(AA*HaUJHevz+1Z1GGawI7Q}Vh)I42LKt;I-b;>NPUlw zWZq3lbe-}xn%={x$49&k>(77wXEXlqcU$RUFQioqCcD=x3F$TOnw1r*xRT|t_vD-Z zHda3N%4O@LLPQc2)$wUyWZJ6Bos`_?5h?qr{)4V6KY~iEyCYY(4MSc!Sn)N_Pp zNIVTxS200S1b!_Vg01IG!+x$E9<-N* z5ynjD#n{L=7%4}p)FpiqkVxZMf^EP47#dgXB$NIX5w~42k)`q1BFr}1qV(eF>@2bj zG)zW?KajrgNcA3W1X>!TKvtpM@GL&U=1R6jh?PJke2>IDRj$8il2Ad<)+eapeo zeCPdZRB%hhWDtGH+oF{~$NRxTIv#ef<~=rzf_oxBq1?N_K+w>J`h;jt2;%!*VCrys zzkKJTiDrkeKBduT6h33XBCDD`cvi}lm*YoD*Zjn|D?;84gipDbdkP?QNUsXDw&X;3 zUFEwf{3>NA>27j{&23od^H`ff#>LHriE- zR=wj(sy#Id+3B?yaiiQmNoXkF-s<6B> z<*So3^K2=zCq0PkZ4nBr3W_Ugrq=%>s3AR9E|Qq0DHSRqo?A;ml3~9oYdtXvaIb}fKFgM!1P~+Po#==m~P_y9zMal*p zIb1XKJ3KNjI=xqWFE_>uveQGniuBzsxDCJa2>CQSp~3$Q+g@vf1TN+PJMwVpliXYQ z;?quGaY9~HpvDy@u9~m%9xHzoz1%}CHyx6W(= ze4{af63DdG8^i9dHZImaabmvSm)X!FlNGQfF-8GRHSNm~Zd@BG0%0n+R`x!30VNN_ zK9lLyg94Q>EHWVEAP(jgFx@^Kx-JlZ#D$n`%r%I@)mInq={k|qck>clGhU?FZUae+ zEVr}ayHb!+ol46^)L*?>bmYCyB#e^iu_cbU z`u2n4lFpJIsYS7%j?J1EH6q88IN|_qdc-S7o>w@qSQnSPYor;2gmf+SvsG7 zaD1EVK$R`CTCwP;McT`~)t~Xm{e1t$zN3@0N4aUksdM|(QkHRK*$)~7!TQI?49rjn z?|aQLW9>ozU3q*_tI4Z}08cW>$m_fLfp|`3U{ND;K=}&R2l>T~)p939%A*lDP79BP z=f5z1)!aHw*-uC~ScIvz^yUj^f?v#Hb;hh3uA~Jb@6tU+?7Pdz14I|-aMi`>5XONN zkO@f3RMTX?AkgB}V>DvXDcEGtNjzhyHBA`=ZEe7)?2!Ug&H`)jD^&Q~Z1XLterBrR z>D3}q?|a6HWKD&JhZ8NNhoZcKh)8aig&v;EAd0ePeK){OK%aqr>w1eykm^N~U=H4g zTiwpw0GgE>&VM+82rKd5o1ZFw6T3Ix<@)pqvv!`Zmy^SwzGp%b)l}u+eyU8?n%aZ) zKo;x(3|4ha(4R^D>yL<4 zoN=%u4VOp@qWlQ-Td|6e^1KGXwc?o+e#9HHx*7FWu96hhr`e9hcYzB1o^PaKD7A=` z9H2yD>Gl5=UjOHGlKlE_PNzO=Qh%(7d}6*4{E-1)`asm2jF0?35BL*Ao;Z*Pl z@}gUFARi{4T%bJL{R2G6VaS3Pp83_hX&Z6F!6GNB5p1loR%Akmbju}T&l4*Vgo6Z7 zdf7S?M1%~xqaJC=FSv-LA^1q(k;|24*NWvZ5g8L=0mABUaDYd%es2tKklG_vZy+@i z1=p2gh9-Fl$d!oup!rpV$9|Lj^Im$wcs0q)ksVm~r4dEp2@a!C-r8t$5>$i9YUK@d zP5%3OMGQRNn<-v2djc%k0kCAbk|Egqs)%tuzTBUiAFz&(6zV6QA8*P;>X~t5R1dxM z0K!s0496%%llAv#u14QFbCoa+b7NA!r}XmKq$l4pbum&N0?Vs(FB+2JnIhIcAIq#x z!xE&AN{x7ZylAkG)-%{O_o%SUl~YgfFBE@Mv-VbLfcI@ZWl z4+dbZ@UjZ)wpk@FAXoS7nqBnrm28o_suQuv#{gEG~Wzv-8_h3QCofo9~ZZ=7uN{#ylt8A9mKy5Vx0OnM7 zrmM3hU|2^$oZ)UQk>Mbd)!D%^kJ&K9<{41WkP<;g!@1j#ng`g~(K{)H9P&&k39du> z>^Dx2h7?Ufn{2TL*r6eSHj;`O0=OqA&<#pf|7G-48GQXsJILa$OPNjrh+hG3$nxjg z4gW9ihvf1yDZ7UMJyOdoqyaIyq>0o}V39=MjdPmK)r(L)e(wv^02GBBwlLP!fKm~v zX4e-6bWjus+4;Wv1Q9cT5D2kkDW^Z5Tt@Y$t^*K|!MIqbjbWxvQ2ENraoS(>{*8Ed zkvpC*^xk7-1f{gWbh#nWuxRT;2;emz1ghB9)6vXjD{jiVn7q#LM*B=sxTuX#Jag*Kb#k+Tn)fN@=N? z7?`xD;JZlCC`e6fk(eO>U&Cd;HLg@fo-BiigpNxg<$1a9e@dC81!P3BKrGM#p(sE`E2*6~b)3`w|^Fh{)vfKJJhy?XuBuhy$4l_w7VT2s! z6*Y)#4e~8jV6Budj>g}F4xH{Um2X1DlXw~u4LOh`pj*hblNKShL;gf-Z~VecKuPts zg!cj+7CTy)dd|!I(Q+*Nh7*^{;;MeRjNOCj_Hr|{bNMtwx%aRm@g>73zq~?K%_7O( z0I4|ku2|aEL@BmY6u*_;WTjQ2rYaChf&h4yoSPm%pxV?M07W_9htz`rSht=LYe}Hz zhhXXT8GIz6ft>A1ceYvrG30Wqv=1WzChHaE70$dRd8*`#Gh$Kd!q&(n##qbn*jWP+ z+_dtSRS}17+oG|IDKeqL2N?44g1oQQ076Tk4rthaG`TYdcLEexrv~CLQ)4Si_k>M! zxaz%#IsaNw3hp>5$-nvJ8$rnQ>5R4+w6tW6&qv~7)EMqhuCr_wuz)>H0rnIdIXH+% z5Ah`wLaK;IEi4Oi80RE?FBsPXcl6zz(svUj-g}7xx&6#I-th-dPUn$7$Y2mO#{$zD zdVO`G-Xntadc+8qdEfvgdTXrla_ZtxTZ6w#XBdjRGY$CtY8u|JSY;hXiFZv1(> zpBM|b=T<6I6&euy07EIbXjtIp_l)BlE)U`%#tNYVFifA_{#?$$|92H<$VS!+I{4HK++v*d3jISX7T{r~ewUhw z!0n4Ey-V7M=E9?ozhC`>1!%a|j?-Zo0yjgE#{KBaDRFWcjnh1$*+2@a0eh=3wB+qG z{PuHzPT+Ce(fwit>Q=**R{GjN?gFF7ea3zaUn1&tF&d@jFbc;7*di^bu{Uq9J0+=r zuu&$cETWenX@`(K`T#^VbQbbGffOf!gh_64AwI=DTjI%50H0>oE`!Y2{Gg&g;uF4E z&bT+@=5S8%j8N$KLG!GxzbMgd#2hjI-^3(|b7FKUpL(2J@3BSk2i5CTYLhLuWAP8w zorp>u)0ic+%RW@RewwPkt;o`F8W%!+7N3(boFtxL+O`mkts`#^6{Ph!-Pb~ z{w$4)A%63Q6wwf>bV?TW;Q$J zy`z@JHIT}eca>s^1RjhQ>MMdfOK7+>E$Ku^k4T{hFq3xYN2c3?!~LlT-7P4wzPYxV z;2Pe+HWRY}Y3C~xJU%)CNGlq?@XZ&vNFr@AiX}#7u=Rv2pwaPRQO~<;eALP9ruMkV z+o7yTsjmf8?#o?FpV$L%$ z&3{uheV5-OmT}kcrc2J>Q!dzO)aoZ0BDUl1q{7LJR>HOLzKm@3Pf^!_F}t&Gru^?? zxG2bgbLV-@$_2sUMQ*Srf5cObkEu>e0eLr)Lo6TlSx{+7Sj>s-)&60uRXx5ZvQ+D9 zrX&hlDuUkM$g1zg3aEm~xuq}Rn5J|9iW~$!3pprWvn-y=FwnsnB-WkicYS62U+~X5 zF#u!0zW3cSmYj&u34t+55DkWT_i>Zjo?HB@CU@m{EG)zyoE(?nsvNbE>O(21>+;nuJ(_C@!=p6#73WhsuF7pOS=Km>k%Up`<}{J*j7W=L4{qsz_c^Ej zN>^kk7AZSxIGV}UbamBB{uATg2OCP6JazvZ+n(DZtS+-B_uml3|t-qIcgSB$&-RFsCH z!WOpiER-`t+AZ`q0xV zRbRHJASXb;CF2zHefZSL%6CE>-A?|#26fqDjGnN?~V zb5%mLx51+(-V1<+QDET)eZL3?I{WKztad`-J=rn|FssztAw8h6p}+)H99sG^tGx7&bTU~>?%HoiZo z02zCog=p^RKY;>>Lwm{Mgq^wIf@mp^2?7mz1p(M4=_7(OxF7fpEvbbdLh}a2B-=;E zFM%Z25f~&1>MLH)aChB|&)dkjH~g@VO?6eld8c55j)MhlUTOxQ`Ehk~!}=8TV|Yvn z)CtQZGwY?sqc``!2#F@tv4#GBW1eaka&*1-lIC{Lg1C%ch>Wg{7c;0XYzQx|vJ|k2 zbv4X9<>IKtY?v$)eN}ssDW9|xuRPawAEI0D^rRd#CR-cJH=c-XCu~b=M6#Rb?ZdMV zp4Zm@-at_+BhZaF>`Xrc8f+w=3rq%5!a>OYnYuCcHIV;lZYcu7q^c=Wn8i*Map2&~ zq(}|pD_)#xuM&frb8SWk`pin$GJ&G02=$6z4`S^0DEYgR;{sH=b^SGAfj)5 z*!-n4&80{&PT&i*`_Vw7v7{dkrkA9cz=#So!QCXCx)UKy3JFs7d^~Qb!P67}fC#z= za+y0F8GbXP9mbg)d+%{FF)31168UBBR0>AJ7p-Tn2Ne1Bx;AR>-u*cx%7Mkcdc3y)zC6Vd-- zY<7}MkM%%gkNPP)a!(rsSOqe?Ii^*jXpxxwy(+G{(dl^DLG2ebDpu zL9W2L!AuOt`p{!;)9sDno`g!(?_rERQM+ZP_x>aUGHA-RFir@L+GqW$MJhEn5s=0g zA+0chP)|6vb&5Wra7m5RU2m5Na=BeO2T@EI?iY|1m?Gh{1}I8ID+wo5`uo4Zlp0Ez zWb+)TlRl4p=|B#KZ9<*^xY_;*P?l73njXthle_c)I;Z_01A?u(1Bhqags-f?jU$*1 z83xVFeWBjv?V(IO!>)*6my;bTug@6UK=4&5->q`SA=W{E-kJPPLbHx?=1Y)k$|d@Q zx>A+G?`LTBAwK(mS7S`c3U^IJrzB?*~XG4q)_z?XM&9Pm-2onJQ!|f)F z*iNlu&~7nw$?na^X?m+7jg)5$-nVY6o#EfPtv|)ARFD(csVXpFEuBqba61*?D|k!T zxvljl4+9rnL;*5+M(Yr|UT!Hev96ntSR>4A0LT}KwXh!c>73$pQF#G=DE!i^+w@8nX$=e z)oZ9pgh-raaHN<)TK!U7(~D^KjPJSOB{+D2<~C0jK75O1vRz@N?~DIdcqh)(M6Ch^ zR#Q+`PnR0qk41i3yhviX90_u;accsFZtQw!do zOkM@FEJL-{K-4=9(>*BV<>@LS$1 zx116DwTtmI@N*g$yqPT42BruyJK%6x0^$9Vw;lA?cR0kH&_Mi8FJBKNM?ad|E_Xh- z270^i7rq2=BaWbOV$fn{nLMO9*!cb6E4XlhWA$z~ZMjKTn=`c>spai>*c$`qWL>Kr z3FvQ3x%FGXG0*-}GVjkD=uYZqbrqPQ7!qFq1)406;O9!;>s$^w!dXsk)*_Kzv)@c* zAmT|lFIX@+4gqWllj1B3k|EW2ga7&QnixxDW_{3%Rw+9%lllk0hH070C-dbt!+Q%d z>i~6tY1MX;p>TIOH~Z=4?<2QFjWsZ0@oBRCEkHi~OyVS8Voy^5iK@{&DLaE7Bmaln zvW*PfdS}H+6Gg0@QM`H$Qf&Hn_pcw=4oqVpSB+y}df!zh?C(B;hxH3h{-2Y8-e(3F z>Q(W>A!m&Z=1c=KGWfF|;6CAcHOzM-`2BQT=_UtCk>Q3<+$sP|2RyyslV%0+2{D+# z^kq>CvwWiaCX-`f>viUnms1B%I%MT`C^D)S?G!FSj8{YUJOL zB*Uj!^Wx%SL9*=46iE$`;>_0kn9F2b&^1V8zZZFB2U_QgRvIfDvZOD=#cMYkZSPY1 z%D&0y&e(7qyQAv}oB&dlu*)yU8dgO;W@r}1^3N|d6NAI8N&O5Q*9&O<0J-;6GYQyd zWuTS)g34HFstr{4={*(#1>XrE-`cAL6;SwEeL+&*^f9KeSRjPd3gAo%$Oxo)Lm9|_ zeel=tN@j>(kr;$Do+LUW)03a1-xfySrmTqbPxgY!==#e))!7){K=MCb>Lm@pT9vsv zB`RHNRQsT0#I=Zg=0;2fU!lx6(!tw0t*d_GnZ^nWSw19H^tRv-fY|vj3fPGOsLEo2 zpegVg`V9q{MU4^p#r?`3x;X5D+0>Jf(*CZGe@qa^zcxX(uJsXm zzFXoZzSZOu(FT6L>lg=1!@)Q6)%n}h z->nLbg)h!Bd~xP$|5W&S%%!*?JYCcZm5d?)FHz1@Ec_>@j)dDLibr>=@o%frRe7L@ zABn0Uv2V%txifMLh+#fR*3eB4LDJMkZe1>=w;w)Gr>?)LHRLESx15cv`qs%3FLfa{D1F7@I)zU2&9j(ptBcHyXBbgd($ zNb9#HwDHvbyoJucz!~71w3SLb-`w_L@H_s+G2U%Q%GNAf_?-HzDC3HGq0Sc-|B1|C zy+ejx(@A1>#DG<#&g~Z+s}1+9w+mu`D3FbAB*7C3`m0(&@0rue=}0Ce@`tAR=oOf-fcp|^I8i2IFnYf zU(%qh1^NRe;BhZW1Tfo#>clz?@WNiT3gp^#xia+{?So_7c%qM!LhkaB^3Q&8Mh?OE z>(}i)6;EEj-LDym=_e3yuIHBiWUkTz|LV{WVW(L-HQg7Fuv1H?6r+EMX#zDqyXO8$ zX2aJOd;x-3vi{F?!v0+$wzUV#pe=Q~dLme1ye@g0hM|87ydbR2&(gZRVJ58sZ*RBF z`{&=N*APU#HLTVIfsPbJ2WKYFIA6!9)gazqd`F;9or(k&4HD-aS^am>RQ`Wjv?O{= zX}hH76|ubdJZ&fDF<_+v%?i|=sp9$trS-Wzd71~nxBf=~&L9fEjXUrK+{`fIg|!Rh8C8R@R~6HlJ_{Y-T;4vpZW z8UI+AI$}j?gS79^?;|)Ify4nGC+?U z`GV#jHDtQhncE$Va{`F8;5q0VnG9vfhd;J8LOcY+0gUt1Nz||OL3JDD5*Y;r2Zadb zUN~m|-Mzba@j3CK`+!Igi3{lolHksGv832fFjlS{&1IMcATnUWkzWbuv1i$>4@!S} z^Y;@5KtqyDo^IED=Lji#2zzQ)WjL12=)3cfWmCS9(0NNp38}(V82kb@Qp2umY^wOz zpTna9xT&P;`WL3M-wj&je&GO^)9~3q_Ybh&DuBv*8hCvyO>`5Dig#Lw$TJFF0hIif zMAihK--Aks9Y}0)w9A>*3iaZ5YBt%K^=hl8dgc#qgXe%}!0JEhFgkh)s>6JW%@nIV zpJg>$s#AEdim0x5Y2zv~Z$p=&|7nD1an_7OCEoa^!4l)vd4nC~M~l;=wRbU?@(hffV5o?Vb|{`=u1I~^*4?{XCPGf+hlmTd z{daI<{u@;K+wPPrQ4DYEMCzXi`#-zR7-X!4cz(l9q8L643P7~*310mMEF3fdPty@l zoH7&uZ#~PLqT#HR70Qi3ZzQ+bCC#MyIKPr0FU#*TeHcs z>{IkU@xq5r6bkWzYM17qr%`S(6;X$C!@cp9jb@;C&LhoT#=&W(d)h-DP%AS_19765 z9DUn{72>!aQ?Mn1`6dTlwtsBdFP88pU;zHmKYk+Qx+rxE_Fr5dfVFVPv0hUeVr$*@ zxxG4f{_X0#poRdZ2sT_Q!Ep6_?Wdrxp2}x2iNPV<@W4qXgi0u4)@cbh2j1M=82$(v zwY!y;W+%prflYSj-KL=A^0d>|)MI;7AhnWM;ONH_!ShuPyVF)gm$FXV*O?-jtpH2F zS?uyKg)fs-HePRB!52-}|1d!=&-6V!Fne6{T`hO$H}KeKJm3W@{Wv4Lu&N5+n3gT2 zwR6s5HAGjU$**0u(d0xfy>BZhy5SK;KH`$P5x^dx!$2V`qu%`@(Dn-6AqQwvU!4`+ zf@zTP4FTwLt8-AD%W(>5kCTr{^HcJewsnOw$WbF=K2BFe^%!r63As6@6mjXYU03nC z|2sz9fS}DW+YE$Un486~!PZl6)?3AZtq*2;D)V35mPYM=BK?}bk$y{3zvR=>Ayxu!TAb|6WCNI%^e6APJg*zQ5QjnStC^I%aq#GKZ@iGq)k=9f8gari zyqV=E($H7SqoHdBgSgpyh7YtdQQ%AH{rX`R^WP=>jI&-(Q4uTZhcGoj;}BkFJ(+MD zXf`VVbVk`1c%TVxenu$b9TBq{Sd}{z#DWb!5O2eXPu5`F5Ml5RopFM*9{>PI0+pQ% z`lo?p1~+f{;B~-ZFxMn#w{|xeG|1pR_U@G`y_1rBnjtGs>J;TO=4w4sjaL;&U6tJ_ zt_AJ^;szgj06ewn!^0%Ozc1*8q6@=LkYhFb|GvjNKFu9r4~J9~MV4Uzi!-U zu6pYbt7zu(5yklb(;c?MBdRbR^1B22(Yy5*>@VH++oZ>*eo4m(+6MxD87UyKkq+mi znI4Xs1h78F2{&ha@Cs4i#0}otIhLyb**1sLU0u-hZU{!KzqvG2%G~Zq;>0ll@zwva z1EYeTo-TzBfekYS6e7~&6LP^OlVwbXy>ZciXKyv#NXEiEMQouE-6Ne!bUW!&G?#Ky zGYqmz?2N%dr@jsV19q?G5GN~LXCAFh;$Dl`2OtTIo>wLcAekS% zYs<>*Bk?1RCxaFZ)*eNljcY?8TQw4GZCk8!-0{9IUTD>UI3#?RSV(Yi!VVhn*CrY{ z{?Cv?ljScqRhs`V*5N;g!enNCPWn|f4%^SCDy=f3xs53d-f-}xfI%zOS|=L{VW%R} zSgQ#mD(HB+1?l27XuvzqIS&1{vz5Z}#|2SSFbwK)VYzwBbg zlwPI=w9)GqbiE(=FmjlxMl-f5RJofnS(%oAFJSVEHfM9){f3%Uph40;< z7X`1?eOi{mL73cnhmqodWDiZ6o{uZcg2t!0>ILax8xGc+h~9DRrT9W0p_QZNisI(8 zJOjPrSS&1ngn9wSNF3V;03S#O@a(9aM$omuZe0^1)VO*MxC3lYviZMVNxdp-l7*;8 zZ%jZx8FU@c7~XBQNy90QRRQOV*;GZo!1sQf>_-f=L-~QpT!Q=2@*;7W^ODj2$q8wlOnO%YiD=7G zQLlx4TVId1C_U|>zUTGRxx8!X;ZPp;Sso-FoXD++WNfV6y7USJn96GHcFu%8&81+l zGaGNK#Wyqlp(^rZvAq$G!U5keglm3*{Ltduz`$DL-ZVXo4Bxv29IUX^@3H0WKn0_`YaM36f6;50sL{ycfc+a% z3%Hp~K?gh}hU>ut7^rv0WV}Fwg*@+unKI8Gxx@1v`So{(wl36VYLAVLK>AztH~rO> zY}^k5=`VZsHqwLcRNJG)p9)M1q_B69-Nr=iw9%RD2Df<}Vi%Yiz$vb`z5-Y;RS-XF z*SYY%^mKP>eC@j+(jNeakEK8VW&&Eq@U9PDnZSF(86?E##%UzyNz;BRXicAh?9(Qmd5>stG8@Dol$JKrM*4<(B9&8mLHIqG zrh*rh)Zm5FfMXc;0@niSSn(SfhIgC((zSr>tXXL2h_gWR$Q9s3vOq^>_`9Q(+&WmP zy~e3>cT(I$fw-MKdl}go0m`BgpZvA{LjSex&zBRq@b|{`?UudE1@= z2ATC}lI?8kVu_&~((Ft_b$v8&IO{0`*eBqvc_c2`z~Gl^oe@^~dcPlg z?Ot<0%S~6CzH)54#nU>_lO{=k#k<;P$RwJ4S*ejtx=T)zGZDb&UimS@UHDt48U%n_ z_x$hpNVMP*NglqAFKkg;K5BNrdATU}8VU}rq|n9Dvl&g+o6y8~FIdxOXuiL=0gg`~ z$44E+%w0KvC;P#c^Az!4??D;>CO;;H1`-y5p|*qVO(X2vuCT3}X=D$m!{%t}Rly6D z03{^s4sfZHleW?#sFqa=XcRmqcj17z-48(VyFUCQYNOdm&c`;<=xGGlk5H=2*unz( z#>tIDbtP-fQfuWS*Ls>d?z5N7dL;t5WCMxM^)&>Ub+OsIEXfgI8)qN)tWoBrB-Rvs z_oT3%q;j4L9uY6k-(V=!oUGP?)=5Z^V#MT$QwVL#c7$5C1t&P zV4CN0nKnH0P9KADLL^vx{$%X58QRUXY1=3E^U?sM6V^k zoW~hY#O&)$=rn0YJ-x2drj$*vPsV)- z>430C3BZe)KxZ;!cwRbM5I2^~5DQRBxbiq!nvK5*Q{l|=@izz}Ve^elrh-OYcvdsH zhHTNjL?x4odJ(4|V2fjOJ&)`t0_ml0CHJjaU!@WqLUt4-LvZx6tW<0GW|`wgs8*Mn zt1?jBPmUdj(-c%{9xRi%9L5iqqUR5OnG5aDPq&goMoZVoa0y{3_?EPn z&8`^8M<4&#){oC>{DrM(l+31jA>m}7=3W6pKG4?+j``&QMrf+fzbm=29@?il2!H~K zbYgWk6t(s>KQU1XsX}A3^=BwsqBz|gZwIvT@A8i~{mbPKeHHa^A=E@G?!P*&{jZyt zT%Aiwvq~(c_-eK*rq&w0ktb`u$J_jYB8Zz!S3LrdfSxZK(-4(P13*yszkmO9DaEo}BF1#$wS=EFx*=@U{) zz*nVj9_T;ZXd?Fp(>e<=7P&A|y1{V%xs0YdG(rTu?+V+>4YnZv`1V|tq{hA2)LD~! z{u!p#Pxd+?l^}=gWa%HlWHK#iIGHOJvFH*Bi@Sj6odsu`ntL0US*s@P=WBE#zS{jq zHo+VAIoGV4zQ#E()Dt%+V-N8kL3ligSg+nw&3J6ePvY9&W~&z4A=d4{jRnqp5T_HQ z!ejk}f;m#M?Uotbb=O93i&&_AqQ}m*`fs?_{$GV_GAF1tbeY{dewo_FsE2VM0Q60S zm*^dT1U%=O@@WqSHpMT;>;#7W$^=v)jsS^Hl|f2WrycS}wNi%VD!`k{Xz081!sn-I zA_Zdb?m`(G#f}}%N__=3j1MS$>E)eI83m74#o3@39FH3t=gXEa*i>&q&=8?jp1U43 zxtBAl8M@>#k8Wn7E-o(?7|IN0TyDcZt5IR4ke3{7CWZ(I+Tyhv{L|tOuLZ;ryk*ss zI8STvGWt^Vs9cq97OvK={de$=D97kKO7c*|qXzP0E0t%^H~NXs3hrt->`u%757yo? ztjewH`=+})q-zNX2+|FUR#+e)-3THeAxKLtq)S2s>28n`$wdlC2@*;y8l*!~edgNk z>pt%1!}GrH{T%PdeeHc*oa;R2oMX%}{{P=N*#z@fJ=RJ2Aj}>!z$8!u!C9-ghB6nH{WPkMI;*n(?MP4J#A2cY%u%PtZVd-= zzic|ZAib{=UIPlR}5rMvJ zjK4v(xg^)ZwPaJV-9(ZW)JP=q{+>|p=HPf`brTg| zc>dbR5wsSmPzA87^8>-(IY2*Lu3ttG$1MAnfC{3}d2>wIeljW@E<){|%`U@sbbx%t zLgSm{#i0lx) zzGX3ipuol%&K$VcID+Q`(^iy{!v;q+awF=~@^rMY%mUA;9@hYDhc1344E2(gk(`$~myo z46ZiO*Mbs&w-wEO{|)ORp__M$w#Pq3g_()ZuFT-;e?0RHLivUN?QQnE{(+i->mfOB zNfigfNaook&iwU{x~Uj}+!XNjNBy~=Ph1=7DVI#ECp1983P`^^Wcti#z#`b*?Qlzl z=qhF9;3hOf3xUpGug<1#Yf?*)I>s3I(`ENS305Cepk0AMg7R!@Lb+{(V+_Cnc@PO3 zV>y-z-QY$N)wu+Ct{C?a56OIiMpbag2c!#q#xbSJX)*ARKtB&pF>Y}~LZ2J^IIr9q z%n7>EsOV5fBg4k>M_+kHfzwa9a|FOhMc}3yBPIaah7XK1#1+_fo&-FjH0cD;$-qWI zQAaa*XiodbbKSUT(cY!bOJ63YU%%)KI`dj3Ub%JeT&DSnRbAr+e%vs1-)EB{4Es`p zOENFeg9VdlIUl}fze}}qw^n+()Kug_GfhC7LO5rY*B{x_p*N3u9bCf@Gp*X{l*Kf* zWVV#!<0L{;t+zvki_``|@gglL`QmBeqdXH&2`k4J9r0foOK6)4rU!CL46H5=k?bB_ zK+CJiB=vkeZ9wBAat^NxzIOUvf&u~!l&Xc%Qk1WA{pUlIGLJ$Qv3U*eFGxf@2)h`K z`|RqJoJ$5H&!qH9{(H*CyLz9@zQa8SMca#4tG7b;_}Nzk|849vA!vFKP!)hfCS>S9qXB$o;;sEjND8JzbjAig)nW*z9fD0(2QluSi*nf#8?N z4v&>+6I>O;U$D@`9J;O0RP7(zILDo=fE)gF{c$}YX=oR*kmigDdI;kC1aZEcjgH91dL!0t%W z(apcBDQKJoeJjVaqfMy9CcVXdlv`=3A}JZ4K(~{L&S4+|2pWf?u?Y8?w9<;+M?Nh9 z43YUKB1x&=4Q5kJPs-It*TG)FXJ7)0%0~}sOn(}3Ftcjf#L{VDQ+(1<_rh$ zTck4ku8|*!T)hvxzVw^{MrC_sk?RW!$)FnvYg++|{2oxH>oH^jSV3hJLa${5pqh(g z1W9CH$mxZ6d~t2iBxl?ia2|2@52{(#*f@kon08@nElI7z z81pc{_cw+roliG179R&De`<=d6h(`^9q@Rv{FdU81=Y-%w%N^0ovc3pQqgmb(WhAu zfqmg<^NgRemL**---gO(e10ilv zig}cp-}sbiH~KILy@s1+Y(`z8LD?Lo1fT&)_uKAK1c!y4KziwG zF+Dnt5X39d$tec5+sJoa7xh-yXNQRQHhlz01Bfg-`ZD;Da>zR^z9)~tMDHMECvY?2 zx#k5)3@xIa7{6n{OKla-L+-gC?&&g(e)(8DB8q_}U{M?X z^LJZ%bUMHGLC_V6{inAoYnRT84pYDDrcRiV@W_*MflpZ(O)hh)2lxHdy6#c)n>=~a z9BUVb7?Zm+Q~1K*8^1(`5v`+%?$nxBqa=v-THwW;i3>_92+G|)j zI&^$}^6(xkM3Rzl1Lb3OuIw&O1!fQ!Q0n2wcnY(z1iw7l_>6sl(7FLaoZf+8n=l;W z{OO8^V~VEPt>s^JU%OtGfa{vBLpIZ9VKr;N-adTfg1Ut8Uz~_0;#- zAwb|nUjeMBvXt9$a4D{*{CwvX)F39Q4C_9=XsonYDSCdu)pm(ZUK^0p+F&y6-}ODnH3yBQ{Bz(f z0Ax&caOlVaY+Vc*;gn2LJaJ;+mrn&>eyRtOcHnYBpnp2AcKD7+Le8;eHm5e}l@dV5 zdGtPLu$PSOU~~3<$10@~u;#eE&oyEksm-xO3Gil0!7fqSypT4qLAL;9H4hZga^$Wc zXBQYF_7_a}=lt=$iQ9nXOqr>bL>7lLhRYeS)ZbqA`Y-^rAgTwWZ3PcUYG0FO;2wNY zZ?WX?y*5gf23@nG1vV299F|x@Ig+eg6cCnBO2j#lD6qC)Ha#rSCa*wTAwZ_)lri9H zT5xtdBWK|0xzYSS79_UFqNCp!+v33O<9lqUv|>zxoS?nV(lpvcZy&ACJGR5#X=#1l zd4!M5x01OXw!K?}{T}g}zTr*1Lsf_Lwg&kjZGVWO%3mdg8|Vc&;d~@ai24cXMX=Fe z*kg?MxN6<>RI3%b!ALES2jUocEYfO)Fj^oFg&^TG>@<>gV6t+rmSMHohb@3oXJeC% zYe~AtkcUa|v0+asXXJMZ_t;w~b$MYnZ9y?A)o$N8E1RrJ#oMGCdcLtFa{7BJMp%{JGZTB>0QJj98(q0Cd`i!A2n?QCNa$6)N+E z?nF%PwJl?&HyO<>{7rn|WiYxSl8BYO0=}+%xh}jL76)QX=6s@?_&#&~D~>h^nzlpw z({6U5AskjHoreM`XZU3=Muj%>+Kh{TONSccL*QvE9ANh6-)4q8I@I!H&3iMcJ4`<^ zhebXOv$p0x%S0pCbEMz<{hC4M&&|n`wk)@oKrpTS9hXZhjcUmq{cDxd_G$-)Rm*ys zDO4g5%_}&l%O_w=#W>h&HY?(~84QkG`6w*ogofXFu|haOq|gtnt}!`xAK&#(f(dk3 zu3BLGC~v*=QBsi>L(9B*evB5m(;#cM6qYA5&#%aK72-gdO*>BhD=izgt4p`Hy}rAt zojn}`5k{eFazi9mnK$q6&o^pyam5;Nc5(X@We^Y?{hn|2dOBf9ZMEq4qYA@$m~&GX zO|MOQ=#o6st}xx^-qSw9SitD?(Sa>pPd8ycW@kvGhNQ5PUEUqTEG_>;K(+FFAYx@X zMf9m5`G-N?vRqh6)kK5N#^%wyY+92RtQ3<9k=l8%{L#4Xcbgn#P0W z2_wO{?bhnJv(f#yvO_%DISjb=qtSI01BLSrZA?PJ+#(L?n-@5tD#V~DGPJ)<^fzt= z{}ZP-^6nqr>qUf9PhEp+6b3Vn5v*5987M^Iv>4vZpi;#z ze9vY69?V-fnIUV1c5iNKOwKiorPfNhA z*uwj#qf_|rh)_V#hFQO1}u zk1CJ+vWkx1yboS?n!Fc2ye_X9`x7gV<9|4JX7^l=k($}4o8TuTR%g#KR|y*)Obv+L zRMdzHvQ>;=61E9|$>mmHqp^C{cBJb5j6+{m>?0c8%E1o%BH-Pt&Yoh@h15R5OEm+H znv^$X?G`YfpmaUV~@{vS*)vgAF`Hs@E( z$?(s~myw`#j_{*e`9)|8%XUxj#ac&Zr{vu^8jZLn#$PT2{DBpR5m6o4vy=;Zi_n%o zb_m&5FN}N^Z7sr@Yh?%@`cY{n#y=F4`%(p)#u5F7eFJW;sUI0esI4=H^~0`G z-2%M8f%fRDWtUgy4#6tlrDtKR;#14N`{}cT-PO?&4(of{62uh6oezBha|j?Qs>ys& zK3CZmn#ESU^yp3eE7~X8`bft4@=%M4iGolinapujuO2}eb|n0^yfVFEE==w^KVket za<_qc>5))SA{q-h3GEU+SvMYIRpQ|S?yv60M2@(DNb{&Ma&77y+n%uRDO{$7^r@lAcS$Wl|;e3oH ztNwWg9vD>fb+nsaY9SbNeeHcbtKtvdZEf;XN3qj(oy2>?7u=6QBfWLPcTiY*t z5F7oqBAW#>c{UN!(vFU=rW*kbR%#w#PCtmnprA6mV{F>iYon+eL1bgJ729uE)XU&^ z&nLt*JAwB{trU+z&<3YninW?SjiOJ!P2Gcd7WCHm8ZVt*tkhWFSzfN|SS7Cs(TneF z^fb3(LV|Gst5Af5=3!DT9sHV!5WvjGw7;lqcSpeQcFC^wV5AUPMvNGx?0&7vJQRRf zcC-a22Gbb5F`%SztwVSG@wEy=dkAh7F4Y&xLqXF{mQTE>AIvgw%nB{g{-&F4u;dk? z-3IcKXA~K3FIN}85YxKQQknRQMo8n%l`kCyBkUZb9I`;~d5EtD2Asf{=IP^@gx7R6 zNTqE{;8(>ELJ*RlC*G&HSiP3%@Ei&N^^yXKF;i}SoK+Kk4-TT+H0el}7|=~v_B8?y zdv|XJnn|yC;_*)f-+sCUHZ$;a=aNmR)sExMvR;D{bCOQlRBM(*#Uom!~vN@huqUQqCF1BB!+X-ZbM1pAi3V!;xUo9LCmWT|Xc5GsepkQMi83hcZ@ zY|ln866r|%`zyfM6q37}U_GjcXpe9g+{ z9OO}cwz7!!d>(YDB1zC52B+ismxn4*kA8|l>)_VOzU2)oJ0xagnZlDlW)D`S=QhTM zZ^?H&9XueFV8%|;%1yr^vTs2rC}^a%qP;?9goWSCvW9FDe-%g+iYFsR&nrYSF$h@m zYmSFLFqkFc8c;nHoh8Hg$gbfSorBJ1-yWQR*IB{+BiR=AHQSeF>Yh#!en329AF*Gp~>|A;sVSMtOo;e zaWv83lfM`hZL&L@f2jThZ^|&&I~m;gZbxB#b5#LBhm~#!kalM<_=ZUV8RP zDEG}d(!G?M-`fUBREzO(ar&gJ_6>ewXeaWIku#B8&k!#-t?S61OrJZ>ub%jTuw{_% zTeaPwxDmBx;6O`;)6X|bHXZOMit0+AmZqP8U}DkkpA1P#GeIfK6ep6EGN025X%=yr zT6_PjvOE97&3MVsuCW?$zlF#a{xW47FQVHg!ln?|*ULfVH9L64=}8r;^71c+E8`rJ=R zv%}DY(L{JFl{13h9HL?JLos@LuZ1>TeW`GNCnWZu#k_fGpILi}c|{Z0~_urE!Nll6c*vD%KTvt zbDWHo5r6O1CT|C>HRY@OptKxw;hK+-35cjCH)P0;KQtn0l6EFKFqBl~oP`EX?q5{H z$y$&@{W!wT$4#0bzmiRAJNiICP@r-Pv3SQ$KABJ?;5jK{fj|UGE z;?}YjP6uo-oli6?!XRTd?@eSl?9K&+06NdkDFZ(+QbZ=L#7XJH=&JiBM)OZpbv`@U z3oBU55Ev<{DsX0XQs4GnD1e=D6XD|7|Jehrf?aZAef0z-z66DE+@6(`Q+`A!RW3Hd ze+VlvQbi;DZm+SNQ?dp1o3Og^XzYb$vSp{BF46#&E<`p_4bAsXfR~9`_=LueR|4-- z<7XWVA4B6g3}iE5PAU50&41tMO!n>13mFSo>BCp9%oQhcw%FkzN9FhgVQhJ3vj*)q z&#kOw0ON?~T|lde>c>BJ<`gto{#YxXDQvWZT>K9YI6l;k4pGuBD&aBV9Y6v$dqWv% z-c9>Uv~+qTkto1IRYXY@@`)O0_h!+|Xm67)O6poRG-0BFznfK632-!=%Xk}X3d%nD z8E6Dv{ic2+l%tfiNJjU4*5qj%YQBH|o3}(f<=JCVZuu=sc8Fq1AIoh&DilT$-RHJ4 zP60o32t7+xFXRoumkE+CF|$|aWte%bBL);&?M zcAG;s*u%CGc$)sRu$_#y+$I!2Qg%)mAg_=-#(Vh-(6Gb@j3kTz9VPSU)c=OVQeKa< zN0M*mcYP6%u;^gySN5UX#nXRbn$O&ZX0=%w{=&yPIfio9JQkajpLZyo%`7aPqk$gT zTk8|rhp{WuyZloBvTad`y05qq!1zT~sW<(5C1d0{Fn zu;hl_#B8MXp?LXFw+Fv(kl_D%?;Rq>?&4n&Ccx#V#faxRf}k#cr9 z47efvjhFyxLCP?YDD?R6IO3h3!z!Z_!Ns-BKHt`~lIY2jFvxo~osbqL7EjKO&_4uZ)NpxJj2 zCd^+zRC8xI(UUeP-Qm_ws8XA>ttT->a4bG8Z*E7)f+{Lv1406BGzq# zyoG;A<|#}ewyjP>rh^1*owl#J;cFU8&G9Al?T9(7w8pnYAs^l>HWRG zMF*{1Mz*4*+Ur^4{VoPH!*s^GYvi1RVZ>i9|Bg#-o?!Y7i3NM4NHg`D0Lm^@bqReBf6A(ac`nZ(^mP3k{nGJgPO5OK<= zA<72-&z} z2;RqgU+DMklbDq|=EwWUq>|snAL)iu`Dka|U(w#VWs!i1;9#nb7p@$7{(elNhgi@T zDf6NogYczXTPlrfwnlSG+t`B-EzYEn!|oZ5Qir`dlRNfCrChHKxdp+cI~Ga+hx7>f z%br3Mte-IwcGr!}B;jZ^z+JqKC}5-~loW^<8+39;*F#c5C)E12mgpqQB}X=|A!xgp ztuSbEYt>`=`1m{6S%T6|glVLazet}cIg=oi(2`rjn61CKF}2;QRS16nwN?RCC?)C= z0n?PSpDGx`TV8D#bE^~_%u{|$AZd~)LCc$xnWu**vF7pj5VRqGH*0bSD@p%z60?Bv zrQ;oE&e!;hTws1DEgQDGfQf_=ce|h4ouNbxzcY4W+*}E<31p=py*Axt;SAd{{l33h z7}@hu#zkG>PbU*rnEWF@ZZYP*R%_nV4pur&3m!Umz`AXjg?%Y##wStTf~nlg*>dW7H!v1-5g)u#B^Qf zL~**`>U0I!zj%;bN-mgk$abR$c#5ZJn;mklB>Or>CD5m!%BW81IZv`$-i~pE$MS7D zfwEjO&-(DpU=!hnVwvNfaw6&u^iUgHokA)i0`$l_V{ne+O%UBVl%>cFXAe>GIqSI| z*RKC&XxtRgF|>qKEQ;Vv&4u|z4hPhhoGynm<;KbQ8)xJ5oeUkAU6}h}mJ8`L5+56d zD22xuh}KCxCEM$6rzzWYdn*iS5SbtO%%_;A-Am2~gDeTI(Zjc%G^n%83Pm0e`_OiM z41EhIRnmsLP5gK?#pA}U?*wueQ-^4ZHSHgY*SH-g{M(^;i>maneT5H15Hm(IHGAL| zkmLOH;W!kAlR3}MG2QmXrqMPz5?&wilQa*s_f0U3$YiO&kl(jd9{4S#p0JqQ`ccv( zS=yofIkQ899M=y|!4RccCS<|k%NxvE1q9-rjQ&(@;m+)#6r@2~45bpArXavGK1ZiM$wfZ&<5%m45t-OGuk_(G8A6l_U&Pi z5E+`0^+1OgW~vO9$Wh2tNO--OxzQVEFmis_@jPd@(@#6&w5dBIR3Zm=_0~DX{+L^> zGw54|c*@qjiLWEB;WSH7dE$eH&TMs(Lu4O$P=KfN>f{reCMQ}-B8v+qe=oO83aNC# zokNDu&4kf9bW}@Wp9&I=OWPJc;lK%;C$hxs>8n%5?T0>y6C>_e7Jg_&=dg(D`dqdl zGu){~O#zaH9a-pifqW568%w>N8c~}_5sQA$hC~t6*w+U|HkUtO`-Ec&N@QI}fyISn zDslBwWd2~RdN$j$nCL)) zNryhl#YnY6Pj`zN??e{i_7;tRyyKzlB9_#OF<~u_xM~?Qe8)FlctpCo+mmjS5)nVS zh&JjGMj&RW@l5Y>#L(K}P_hw31ncx8Cc@hc4RQM3Ez;SF);8H?t~2catj5@}<$&Ys z2rPIcV6aT$W&;!GLX)${utNf3Z6_yo9#y2c-`_%51`JQ7^fu-`y$B|y;$>UxCjD(9)dM(v|z&x3)&N{!*(Rg{7n5 zqm8a#j&Gr0H2e9@-GU1dL^5V+DLxq{VfAEz=HH5%>{~CP>^$ON?uPLP6T8KxB*$e| zi=$przCFHF^I*|2GyPq3TISM2@W5?sK2dZk1-c#pEG(u!H_SUF_anh8<;m;7!7SRN9`<&ypv)0GwfMRZ9a7&WEz{ZXzyeoS|xkOK}|dUhep^p zJu8JopJ{JCw8C7y1ngSsYM$!6^T+j_{BpCgX55Y^87+y9KNjdiW>bl2gE{-DF75$0 zP*l)Z>Iok^9WqwEOtMX9{l#r(R|Gc?CjhGSN?k>gjm^e#DFTuVikr_h{-gyMTVH_T3 zia5mM%@H5q^D}5j6ie&X`M}NpNN^CxJeH6j=Vgm!RBEygA&6-fHYH2rtG-ArkK&{GBeG-7%;I*^XyXF{24ZZEm)hf6IRxs?`gj9 zd=CO(#I#~t*19CHTh>49BIVz;B7cQ@qPzcDW)7##$2!|NlimQjJekg|7Jbqlxa_#IxN0mV#Fmo!qt$q zCF`^$A^Hl<_9qzR>@fzM^d=~sjWa>5<{O>6;x@=d4H(__?_gTtwk$BuAX{G2V-2rM zVtZ~|3=;d*a2DT+GCNZ}xH0(YPK1?Yygtm)KmaV7cI+|i@JwRdhwgX8{8@>l^*G#wh2;F6dm0sU!7<#mPeCx<;f41@RWaD{dr4ONI1GhL;E97v2)ufl) zSr1X%3Fe)|Y*Na!li$VC0Li2ys}BokQy!?_J-IT@d6wJW$*kq_UDD6pt|UdSz9>L7 zFGI|;+yK>ikrGW?;RQ13(tslY8TQ^jz@9CL7>*rwlk23jd|UgCK#+IVMh+KW;6VD^X!OT)#}C?Zm6LX@aIHOfhe9aMjF} zkLr9PlkPsvmmtshYqg$yakY%#5fPi4#*q~KxIUwqx!c2DS9ZkWLOh$(@4F1&{ zZJ%mf9v?Sk&Z#r{ZZe}~;U-hHBK-wZwxJUS6f^v`Rh9oG3sat;iK z_A)jNbnc@?Ty+jg8ET-^5$MXCo{M&@Ib8|kk)l~lLOYv|YdxGywZ45MDf63*#>g+D zTbjbc8qn)#W`iEWe-es z*s`HU1WCO2vo22(78P;^IhK)AEJUiYnZlEYsU;$Ew}U7j|vla}-C2_LIE&=T|& z(TPhuBF>c7_h*;km%&9=;UCDxGId#bJwQ98%T@8a%9HWoLOh;qst-JjR_P?i#=y5F z&R$jT&Qk+?gH-~?Zsc~kUwT1R@IdX$hWeguXX8OML7_b9;F$NnC6_!9?|$>Y4(1Gj<^C6=bVP|bTX#10v!7%3;4jiiU?Ofsr0 z0Y8_xGYRn`{N3$drRZb+P6t_>``E{6KokXm6igr~&-f^!Gk>cyy*&Z0FPM2XMsiM zG}5cM&C%PlIPF497id@#VNH-*pyET<`MD*0mfG#A<30+{f9$=X=yTXDO|!^*7=R5S z{K=nSD2Q0^_9R{0M>9vtxa*S0R0%ypFj|B%R2;i&?*oHkUxrz*Am|Ime2sl0uVx-O zi1sXq-bU2S`mi3C4r4?wk81HKkK^jaXVDovVhPEL;8RHNMffBB(H<-;6-UQu8mB6> zz{6W`7c0xyF}Dc@_C~R+1@u7lPVxw*bCUf$pVyRQjX5M)QO)%3=6}BJQ=iV%Ej*Wa z=+(-(De9iOKGpZ_JcpF8+H~Wm%QvXu>{nN(pchUJw9@2iWT{wVYWQ!fcBbM^$;;X) zsOI-5Hxn#Fa9&ubwRVl~$S76NfX%Taz@6Up7w`2?p0eBKrXp+ue2~3_Sfg5r()e*g zt>DxrMDxb5nc*JwjTXNPi+ZHkv8a#j<2>*<0~(Al>WS+A$`NzO#{?GmStBskw+kbV_B-@X>BN1s2f?am{LS4SW9w2u%@4)1O^JiM z3v!xEvyaHj*e%Z$X7{Ya<%&zhcIF>GqU52O^7?VCCkHf_f@kl(?O0-mD&|@G?|?Z9 z3dp<#K8Qoo^=&oewan9UaD|QZG(^89bDGl@_T@lg|LIeFxzoti{B8U7Q{S;<`aK0) z!9u6%(-Si%7QWOw_rYZ$+C-)01g}Z}ED8f{9R~mg>e6x-RzpK*E$99KavU6ze(lHp zHdUG6#PZPYa6{=Y1CU+$INs_W^@eEDSnf1z`#j;e?!0$l?wT3`1NX3(<3d%IeQ47k z{7BE-bH}8TqSl@&3%a)M?vAI?9-S$j0bU5{KxA7T-!k>bymyBYv%p`P`(9f?$=FDU zKgH{kn5th+?_vmway?}%Zit(k0+Phe*I@uH6i1aT|gL(t^KGG zM<*X7gHnFmIJC&vB0l}R3rfrn1co`W>1gKPM^FNU5)!(1D%^gjP)gQPy1mwAz&lIa z8#oA>|A=|vO6_oRroeG}ly2TBTJPG%)Wn(t$w-fHII8RC|57$>zKamTbpaM?+K-fX z7tEe<+5nmce9jMTZ&!N&Is4FNKjBRAgUqcAhivk@XYO?3-o7R0;Y?>f_!6KiEwS8j zKSQ`$15rCJ`sBkhCtp`BYzeI{6k+#@a{2)IlG**1-&DNalViY)VDePHP%=enzClxK z)8a#tTYNIOSGMK}r_(?9*Ttjug+Fa_E9o#l z)60GCD*-gjH#nRnpo#5lh0t+(kmUAlP^0BDZN4??Q2g(8RQMm^!DrfPKpKpMg@8Ii ziR(fu{au{oiAZ!z9N3@Vzwj;XfO|VDkz0uWe&mRmMkqB}F9yF^y8<%IVP9pz)lLJT6%A2aK0b<(_nypo27AC3`9A7V z^*^HvBJ03lmbzTB#=XV%%&VV!S~0YOG>N-WJlR4v{G9pZKz+vDH3@Qqhcy=MfOoO= z>RM_}AOfyk{>eTgAFW48Rx2D6UeRH3o506q3WgZtB1|g|=sSsiz*HR8y_^m9h~n7W zvJd}45VS2Np6vn~u+7Y9A>Uj)w=u`>CJC1@)ao+n+V+JtTZ|3>-oVjc<0|4}AmlX{ z!0qULo*g2%Ez z3WpjAM?d^^To-Z_r3?a~*)U#U{Cp?nl@C-8(#G(rdax&wV?_L+=n?P<;-XL`G{s;= z&Kt82O4K`tc05Pd$heZyg~V3{w2oFJ^86CBNz>bx^Z&KmyS!t}rC(RuNx!v=yZE;9 z?{`4|fBp_X%4X8|^E{vXhe_8I-1vIJDmTeZra{wHEYoM5_b-#F0>wE_+@h@ps=O~K zM=7ulOf@`tcvJ(XIjF`a_%PG~W6fJBukwo-U=0V}(mUn=-Fif|J=;h`$n-jyP*@$E zc?KDWo+wp#tu7UCrGQ6=#i{?LElP`yis4>0hiKp5ACfS*b@{k{LFSp#1VEF>??U|l z-vAIomp@^OY$gTSYLT8Ce-he7&};IgSk4`$2?i`0>aSAwxENnTk}-6;&8E zcX%}ooV!J#eDJ2yBKDQgV8^QHXsFmr(cpZ=u3%V-Y-I!&LmCE6)t^7~{$CQq?KT!g z&a)*?Mo(J0jXO?)mf8nKhBJJRtC?f+<0y3L?tHL1Ri%A9_hGoNT)pO2}FSH>%E`^p4;~Oo- ziHSs9H9j32mBsB~S*0>QT{D;UgNjRqkihA2 z8w_4)qN1Ww2kggp=T~K~BLKHtG~ZjF3#up`d?R}1~0jAQ#mWQl&B1$5TDMPYn^`Eo?A*_~M@&DJekx${jjdFA zA?MPPx33lfUPVyv;2W}Fg*hze3yW1~p7C@Xq{Bag*UJS(dg~KaQiAO-8UCE zSM=!lWV%}pfghL|b&VJd$X+h%>FIoG1NR;&4-}X_e)n7F%TA6zDOmc#Fbzd6Tn_m* zz9*$?dC8HQ+`MHzqmr8v$#p>a9{=3=drKV$TWUbVr4r42*1}=+ovr#qy97!v4jiq= zOuCLsn+oSaRoDGZfxVy2$bCgb1T6~K6D*=K7A*dK_WtMpd3Ln?@l%FnXx=tGyPOJ| zuHy?PRswRks=h@2IEn674Lbr*U~HrPT#Vc#$srWawEon?E)xL1FC!$pC44c8j6M$s@4^8AgWFE*5E}J z)u2L!!=V$TRyOpoQ1U-o08{k~Cc)Jl zs85y$0iq{=W&GEN24qAl%l33T=@mJLtr8l>`mR6jBP<5e{PD_DHiFMXul60u;H^<_ zMVY}jbA+m}qh1MBzO^DNA+Z)7viE)$r*Y|td%i^%>;I>pkI6%@cePym7Dir+^e5JJ zH2lQ=fBq{WsK0WbfUOV8dK{+Q#-h8vxe9GZ)6K6{aayHpO*y06ApT&w{#D*`)85f` ze%`I8oA5p~kJjX8)ASwAT`v2dLxTl#mq!fm_5&hZUV09jp37}{Tg)q>VboFn^Y`ie znYM1B>z2MQ_O%5+cQ|S^k_;ZH^zMx6%n0mL>#07<(D8!3rJ%RP*Oo zBT8(zg*6PuSP3}X91?*^C%})ct2|of8=vJlDydhZ%~WRD*>TT#76#C){2&>A4UT|I zxWI_J>JT+|7ZxZo!!AUFrq-hLq|iF>l(mXZljJ}{Td7Na>FX# zR9*uLuvycSBY<<^b|8FRT#!?~{4T zxZ6`@ud`k(`l?Q)5R2bx3!ZAX{LxXjXMd+5E*s4^Ci|U+?56^axky=>g-fDuzc<7% zThA4nPbNt)$Rghb6E8iDM9ltB*u(=L?%rocLS07-?dPiDAb#vrmXr=Cf#6;skWJvy>P4;Tz z_x(w)HT8mSjt2CMj3DM8y;Fw;KHeAjH<&D0r$~1E^VDAg!}LM;6MUNIa*8!*db)dm zM}zsnkm_#T)~LhmPw{5@@pjVIcj?~XnA}Ajlgd>67PwMs3RF9VPZq&)JMqXDVTZEi z!yIH)WNE;Dqw_inujHr~vco6s_Gq7ap3ltF>UZYkd-7R)9RQ)zLPIgO=iZlatb&il zjSA%4S|E(4^~KOiQ=m{{|2>RPR-1op>3h;cj4N3Ce2H~B(DvJ~i6f&d`Z0~H^lQIl zwb%9J{`{>c^i97$yWoIB(RH(k35%6X`fsGJn+X{uCxc5pc`HtnKi-zL6$`rVkSSvw z-2^yz(W>S$Y`a@3mEmNhS|qF?$4*k@36utS)7x$IB98~P1lu_QWj+zyUev+ur3gSk zL)hB>dm1H8`towvawu0ZEbC!nkcHTHw?l;5Ik6rHyzQ0nrU^>cXA|#5%6NS5-Da#C z$d}Hs{F_fbQg}jhyo?`;zpq_DwT@D6&HX<~9O z6>a%Ua%x?uGI-2;QzN^p1DnFm*O}i&b{26LMzy5z6j;46!~$yA2yAOG-*K=Kcun4W z#t+T&m7Jg_2YaXVeRe)GA;>S7&nm;N7xZZrLI50g9gEt%8JTn;N?(mFcMG5*lx}ymlw(ANF(n@h+G}3E@w5I<6Qz zmO~Q#I2QTuQ{DfcJr&3wTYa{+$P{Sw@VLn%&)oBOUJ;5$SSB2qe)O)2rMs(yI$Ku< z64C*<`1K(dXIfjE485pR%I>#j9d7L`2pG8#B}KyMD&b??#sU(K&PT)lJRjQS@#x4J z^V*lS-Ij#U2{Ndtr}ZrXM1=4ELgmIk zODOTS65*!jF9l-6+V1?h9D8Z^0%?ec`$oa!OSVUUnS=Ta&9=L7-zR1shCL4GuVgr& zN4`c4?a207!M~iY38JPIQ+&JjVKe)+^g-K0)GRrAVLNsGyl)^%4cW)L`H=?8YK{OZsad|JHs2k*k&3BMzXt7}uR?#63mf9CHR zHM^(-2Yn{$pm+ZFpf|*9_2(e)q7;291Cry*aVBEa8vR(D&p-;+C?i;-2Aa>KkM9=7 zh>s+3G1lMGh-7UjiT%km&!r`o!xbjLa62y9w6~$why}dxwYtHB6;BdIvHL&w7x-%0 zj-#{O`{^elDBlKIzWmi_1~lAy6^3_!AwRmr>z@}M`fI-Cmz2Q|dzXegIlTJkJcn;#qRr$9*r!i!;S1=FK8@cQ|VQj0?=QIi!A{U4N1NsQB7FKO6Jn8E(y) znMXFbgc`>)%kqM;g0W)Z<3B(CzWPxFEJ?>;a#`zRD-fS%VyMBC(Zqg}Ym2Xnrq8gR z=lJ{-ldf6?d(Kh@^5b34!I(WNoX#D2vOH(R++#%x*}|JWY>$A%3Y-U>fw5XDUt`_L z;g8q4Yapm>uG4AwOAEM1wp@e^Kg6GCA;Q>sc{R{-6YtOY^bOJOJ&br73Rnq&`1(tB zadY8T#B|@Pdp`nugGK;WOOtYhgq_nU+xE;^pi->huhfAQCD z2;}s&>&k#C6}KF2kZLgGxi96y7-z4rx1nJgh&2Pg2{2G7v!UnB3k3K0qZ7`7O*W{_)$km+usC!>&|9Svntiq1Btp-4g$*IrGq(g)GL2FGJOGF z`5W-c4YtQiF)-FYDV>$9y?_7y9Y`a*)AihS9fww-1#J5&G_K!UbWvA$AlH8lQfU=lkD{QS&~*-Cr<04s{rF%3EA}t-W$RdnG>wFFj5!nw?ke5TBM`xIt#l zJ}zjE!n(N0NKb|S`_}#1ekwK}je|VVO$T+^g63TZ3>0wX3o)lqisR-Bx$myt6+tCh z2~rhqBv9)o(3>s;>$&kRu8?{=>rVqBYy{>ABFFdm4-ZoB5Ryif!&i8$Lw$sAABVk4 z{=VKDzB$|2E36*D^uC`mMb#!v%Yz}Vb0+gi%qS!?e z3ye6~y>VTeYP@=(?Wrdmjh55dI~{`m_XWRaIsh-&4E2I}|5kizS7Zj>W9JKHH>}Eo z%F*wURY?xi6OZ{xZ=0cZBRSmBos7%gT4T;86LRx2^VvF>^WmXdBm(_)NcmGM3c)DQ z`t7Y6jo~SCK4^dpf}1{JM48YDjYgMWB=lH-0vfB(V^6;d*h(ETu1+d_~a{`XSOcxAv=JkEdO3hM66 z2_DrE3Z%xAFFTbqGYc5q> zw=y4qL7)k{pO^SjEA-PCG(^qLzbqWu9YCgJbz}zkbZCl8N`}XReu~iFbA)|Y()+O3 z_%uDscO%0CP%#Hw7JYQ?+4d9po=rbeY3I1K`zQ9ht>cu@nl!-CKq730@lVbf9XNjH z_Pk=3%-`hWb2;CC&U2W!?)(0~l|*$^CGik5oxJFYIT^X}o8)A6UtizJjzIs-DT90{ za^B$S`eu7gDj$v++sgo_0P+J{wS-Ua#22^@I?p?%p6Oc_w;zROnI4$UT>9*uCk&tH z$O(EbzBGM%@7YT0(kF(FU-P#%Ue%#gcB4^X+%*{2YXUIbDyxJbqJf(4?z&YRxW-Ar}s;|93H#pBM zJbHrm;5065Xy|xY=?bW$Hp7PHxmL6{nNY$%w=JdjI|ShBF85q53^Mg#s`3bR%5v~h z=o(28-*nkSpO}zC*O;_kEO@0?nlx%ACnp!y){YxzUY!G}@WVy<8*%#CC(!RRrfCo` zd!m$x<@rbcYpaX~i5j@f`MCqvhsX6Lvt)3dm2!2e@;@1m(*1BBjaM_v`ehw1_VHx< zD4_@@06881Ll418fEpeCWUjvGI{=(SBK`Dmh8DWGBCgM^3JOMbbI^hz=H zE*lxk->#x*5TA1H2E4GXu)eRTFGwSYRS)`nvhudK3Ti# zaD(sJP-rvlP{f#+dId{kiq^|XgCOsov9+m(|BJOZkEe3|`p3~$+eU5kkl1FKLxU;v zJkPU;GL(`?$e1k^$vi~nF(jmjl2Yan(PUPcB{K6{x9S|7&gXeOzt8ji<2e4veckuE z*0t7qz1ML050p#3y;?W7vcE%o^r{2XU_Nt-_XhFjW`$vvn0KGw-0)dX(WItbO})4b z0~3ar>?$-ffbPtJWXIDVtL`HK zFamldHVcjJ#34raw)gmcgoa>}_BQm}T-_SVjz6zwzwd*!*();c z@`md%1Jw{lk3e62q*^prBYT7t8;mG&_1lE`7dLyXeY6l04ind+ftqY8v3&-gqqE-c zke0mU?xt&(uL9S6&iKcJ`o)TuZvk$*C*S3-S6~BxBGbSjm|qj%6sM9TI$?`3gY)Kr z|D7;H%plEG9{!Y_egh&IVnw9#>pCzy<8|`O8_I$UOC5v}mImQCd*Y}Z7_kor|6ajEXu$V*>8%Xy#ITsqM*CpOG(`;IbHK3uI= z$`I~M(})l(9s8!0!AxAB_)$`$EO~Kf!Nk)?%`4SAY&tC`E z%PHFrB4Os@=X|QORmDIlsl#Z5EJ)O}jn(^ad`pGKU0R2wsrL_>p>%yTa2K;HN2>Gr zjpfgh5iCVkk1Am7dj0YO41CgoaX{<{1ju`R#hHnd#eeoPOT%@c$6u;-^jawx7SH;x z-a?R~8w_l3ged?im0hOlhoBd{Co8j;3dbyM{eglG)W8ZI$JX2z6YVFy0C!UhWaoc$l?pAHyr=Z->Ihoyd^~JRG zqj*aEnbpMaH|mChN{)QMrE-N8vd?X_s>kv+A~P{!`TT`~JS@lcZt2LZm!F^0AyKP8 zwK6~EHXMGq2`czUZ%9tNjvQmLh~bl@Ujg%P-D+B0k5$>#dm%DE=K6HkMP zZx9&TDJrI4TS&jc90l0+t#~+$QIa3)z7wC7oh`1fd-XZO&l{Oojn4bS$ffLKPyB|s zj_zY(v&%DxYHVzDgGnzUSCHs}&U&)P+OdO{2mrV%NigDO_XB*%0?_#F2fh`%Qw%wi zQ3X#i2-Xx^A#^}eJZwmcr*@B}a?DA`po<)*Oy!kqQSR!msTcoFt0gd)}<}I;;y^- z^hTj!XX!|$%E#}bO4Qm3-gh-?&T8|Fo2{3urf#mOOqh3PYIYAFI$JqP%K*eCLImDm zkr4#yB>!8U9w)k)5-+pT5n+!cIesDM_Kb`1)&Fmlks%4E ziN)_Ieq{b)%_;1|fsx&2YyKKpuj^ArmV0gsdF!O9hnVOk+!xbeJgl|9DO2Kp!kam^ z&m#jtC%%nd6CAsd02c@-%{>!s<8fY`x%(t;51l{9c-C-NPpS|nW=n9&%+ls>uGT>) z|G6W!!V+e@L;`bH#Ek(oJX+c;$S0o_d|RS1cSC=z3(Ntp8H3lAw5V&jDHqK3{5^*B zQW^{**o`6%gony|IP2B;2)pIK%tda!>$)_aWj~^yi|O}WwBqJkSkgcIb*x2*8>5re z=vP!$MNbO46i|se4N00^R^R1h@*w#(q_BmAZ7(_3Pw|J!KgbGtyIYUGrMfbl5!A`u zg70yK0(IPr<~^2=n=+qBlw6GGAsX4Bt@7n5{dRlHwJEm0IXxi7B`qlM_C= zac?np>x6P)*h%^Klf^pfbALGfFzN8w;`M#_YuCMn`hkkY!L`=sv`Lf&J)(4Kgi5k_ zq@Z@={9Fl)E7R^D>Va!uv*1F2wntwSqc6n@W& zxtw_TfgU?>imW@8%ElE?4et}1?YbXBuUtsVyVh%4~!U;1$V)zu+7aT?P%nJVlVXMu2SBdKdR zc(zY!?EQngd8Pog7oxAhHE7kkaASrb6#7I#)J3zf2>~M0azacD2i)^V1)6kx@DkTC z-)=Aa@j|n*CZ;zCHoSQ6m`}Qr1%j=fa$*s#IHz~l$Eb{bsFokxQhQ+)BpG7&eCz;9 zZj0^!q9F8|B6Xbh4jESly`*#!n9Y<|+wb$T=0=*qjKHP!k3Cs#u}!5mZA6u;pCU!p zZoeO?@HumSuI$uH!OE(a1fT6~)buWz%{1E&k^1lVj(8XCpTrT(tr@46_#|5H+0Mqy z#qe0T1&-gdZ*gy>*Yj4TWxuKV9{ZNFqNCsYcgI_(^qH$uS547T;A`$qa#Gz+yjkLX zP0D-To-l7SaR!>~X>u{CKwGI~Xx)}F+k2LC4N@KaK&1;xyrmOAkk_Y|r;S!tv>mCr z?Ci$5BbGWWSt1NIf2)5@8OXRV&-40x=)LA08>#S~AI@EoS{{i;fL0~Is_SpQwTQkt z62W8SFqpi2 zMy+Ev(WhUjGONG1y5ACV8o7Zi55ss`+flX|JW>}2(!T9}Dzxaozi0PyrPPsWxwnr! z9?e(g-Hnd#|FF7Oe3Uw}=U5V_r0kqsBKC1I=Z=b9vR!jJ`MO~FBye6P|ztJkW?>hu|){` z|2u#IpFzoq*{obioh!4JiTCTtYJSN5GInUaBPwFVL}qm(>zE?Zbuxx}eH3ucvh6H0T>d!ICvQKiO8Dt}}(;MVfc-dk|TFLPyXzF9|OIA_1odEb@B z+`0D_{w5rIyU&WbbQm(KSEJ^w7#kaF%T(D#&X`5)Atx6InSsv8%k_8Ru3!0+ZOLO&M21%E z%3(a-4g(N5>wLbK1+!p6*vY#|o;P?rC8j$PF=ZHW$ta$2IArgcn}C|kiXTl^QGGJ4 zIGxDO1BpES`3dI_kN`9me7to0bI*qtxjVp9izZV=nVFe!?`uQsh3WqTd%>ocA#7fY z@B zL8TGYEU1)1ye99x z9C@}!CI+UZ)mYLa*`6qu|9MG_@cY$YwEzlGcku7aZ*fZWLpY%6xMjjJTp{jjYm{90 z(gt4QZOXur!ZY9>+E!_MelT+?)U@oig{j^$@|`{IW{w$&v-FL$|~WEoVNgu z%$`g9ep2RpZ2rrKaLbS3iwKq(XJJUZJT(8}zxUkbE>eMaP&R=@3kD*23(@+cU*f2z za5l_83!J)XppxQ0uwo%cLh0f*^pd*HMDUcC>^%CuM$_3)Ju0=F*=BJ3gTNU0<2cBa zH$gb~(Id@>A`s>nSR)qUiYebtOocgz}f? z?f#W>u(HU~ctE?(Z=SfAP4GcT_BZk`XU0hH6$zaB@->xriya0~xk z{EifSg|hBp^I1Q=Y%^>re%IOoG79G#oKQZ?>evDewK~Dpm4*{V#hC9wVoIB}JN7gH zF_Gd)cN^xICMn-Pn*a*a-Rchg*yK>iLs~%MAGp4@gt>x=rSpl|TU!`k3!Ia+YBEoq=-^IsP1-Z&sEQ#v)^_udgp(GCmid5FgJCoh4D=G#eq* z&4$`mJ<)qedaM5Y|4E-IgJV%KJWN3C;ZNPVe~B%lL|*;E!KH7hoBRw63~oR&HUm@c z&@&`ec=A`CbtQfJw$Jp}_Oi*Gt$cOh-mL#J1_lNa6NxM2nB8`qM~i1N!_SX!7wuBb z9HBLKv(Y<2)%)1@5dtyh`g~@@qb4ifpCKhLKoZk2{56xM%_B%4A42{_#%u4y2?Ust zJ^XK#=NB7F8BC1I@m=C}vb{N^{uxRO7!`q9V@M4~YrzPp)@ z;Ql4GfAA!d_$93SZ&}(ON?oT8Ws7{T$KPid-?bJ%EUo6PLt4d8HoQOIg?jVlL&vIj zT0^Mbk}Upu`4rF}ZeVv+p&0s7Vjtq!YhC}(%7Vok=X5VB(a_R10d7DjX-W^wS-9(~ z6YlyN`%qanbABSp2AXI@U}&rtP&3J|BcH5|3Y}C24 zVIK2TRsLi;2Di&oh_j*5s91sw;XIhCe8!_`w)Jp-3=9Ufz+f<0a7RR&&w7{IGbp!V zr1}C(nNkXq;f#glL)UZ;Ztmt-6YKpjDF6Q1++7-ie$Z?gm3S`I-eJ9)UeIqB?ABr1 z7gr#YaPuOO?T7vk`VK!i_3HrwP{H({IrTW6YA8(E3_z-RN!S`tSiD@@ZLfQ0At~7h zalgt70khKQ;whekcslhfyZ3DsRl#IkacJ7`-Lr<2UmZO4rU5!^?cpLJ1SVgXAbtzf zipp_ozva>lB zwcx#%iM2m#&2AJLbe;jDrC;fG!s$(6B7iYPxb1-l;6}8B$G9pK!|l5Yc|nhS^ZNyc z1%_(cFf=S9Gy)i>Z3SmajkRFe)cpJ&L)EWF#o2yAxQ;s#j9eq!afcYY8_9-Ir7)h= z4Wh4luZ8Sf0~z|JuSAXX2agQid(IcYJlXSAzEuk0pDC4bVesX>|M+rit8fNUW#a6O zI45IwMJhf*McYDD>vY5xeoGIn6FF0H;}-ASGxMm(NL$4lXd)sx&Hx@V=TAX;ctRV^a=GIY%+)jo{b`S``h&tn?>rbtb8BKayC%0S`i zvuBpeep2U3wQ)%P)5#-tKj<1%ngQt4PWlv%RBtir#Uh~7bpEM&BhzM-y^#6o?QdO2 zv3L!C>cqF)_oEq)>0&AelD#TBvs`wJ^_FSj0_<;i4fyC?oVmM$;_eL`Ff6+EWC%TQ zk_tTxQ7rpEWM^u%zRBH!gQCx?ZBSbJqo52)5*o2#nzf(p@0OIv)qIHL^ht1-Oa~Pm zf0CBlYkTs|*EOAPC+SZs%0=O!>i#m{8nMH`m&u-;T=<(f$W%FUVBf5_3dsLtOwXfx z=b$1$EH}rHqZ;lI{S$(fOp48`K2Z=6c{z7mp1mLj7o<+0N8i=*cHaXFwLBi`5jyp@ z50+#86A{^GUYoxNGAnnCv_JWQ`>mP(!9&VVh#*UGoqrg?eyUP|!ZT;jjQtX|Tx|{_ zYKDU}SBd^TO?H7%aDveZI}~jeKh;3$kVe7h;SC9NX}i~}#x13NzEE}Aslm@hgUf6e!X9*@>Z+eq z7VY{e2vKzJJR*q99E3W4dKYBw7kSXm5~<$^RR>xoC1j?5aW=zV$h$vOiUNfSQkA`0 zho7*H78EEh(4~gbSMw0dwf9>up)6Z-%e6UaU(_4!ZjX_@=f922KJ^#pCe^C;c7I|h zqWlr}QE1EtH1WlAzbQWJom=Kkm&mrS|3lC(b9XsF3o&=1&PWQ8dF|Rxy)SfTqA!%T zS}+u$qu_N~-{Q@bl-6MtXvqjdv%%@YmeJlwh~<|mnQ_I>@bGItvo~{W_R-ELI{JOW zv9V(hLUX=F)jft7Yxj>BOLmB4>$^2CNz+6Gs6_l5BXwJfJaIUuDI@z7r%%Umb~h#1 z21Zz|U4*vyVomiOVRtZpd_p%b-Pur0dwj9`VjcHR~M0w9auAGZt(f6GlYJrLl z*;;?1$+Oc8u>TH98fcEF02ibrRngdBtP;vl{)Mj}%?*@EaZ*gARFnlR-;wkXi+8!q zUp}V8`BL9|vgclVrVwl;(EcS)yqZZ?(+*pghsd&_d%wd@YAVvCT+;scr1P?qY59C= zl=!rlHVva*5sE^LD6}r^1plS4NCQd?zvjd(-bRloOG2Iop_t^#%C=R^#G#_mf{0_` z$uhBPttL?i#+9Gspj#eJ_5`ZY1R~mm_zGuCz^c+>BSZZWLbWpwhQiBrd2Ucz7C%=#o%8>M~z?RIHIaf!7 z^USyLt4f0?n&K8sks_EGqhW@;&8?JkGXe}vi-jII(aGFX(!Ggt#nlw7f7fg1iJwIZyzp~bD#UNOCK^1Ne28hLX52) zRbeMR|7lb`x`O;tJRYviih>lr>h}+yvsO8$cBmdc%35~p+9wF*zs0&(?Z%n+!w|;j0UYCox#%dDS&n|JLgagi@-|xBXySe`EO-rdD zSQaj@EE^}HecNR!V9?kcI(Q3uz4;9NDgRP`L2m_)5K0J^wK`BV_!ZQlcHRBBe2Eoa4zWzkQrXs*wE1Y$Czxtyck?l2WTS zmyU>%qNoE)d|^LY3w-|F^vpXoFf9g!F-WStKvl;hgf(DM!?BRjK zI=flc;?I`~zu}uZL&bcUaj)9Tb1K(M7HBN>Zmh%#|I6@Nqbqiw_vH#v4 zB0S*6$;YShsU?veZoG+uJF*jvte7+8sJ#P>lwT{cjHpLEKHQuTb_EXG$%rx_4cS4^@f;g7~?TIIR44G%$w5i^o#4*AW8 z0E9}r4l)po>U&|!GXLWG8| z5jHACGmkyX_E~IJ^m7fdx3_=zMUDy`d>Kv!lUIdrENuTz0QIU=c&>qMdHSKfgI4K2 zmk&Kqi03(?iAG!s9L1B$tR;JrBEh6xmhDAM?#b>0|H>xm9)oR{7P5Px9e03XkLrsA zm3i_ORV-SV@Ii)aM@^jYj>Px(J2qfB z(W1)z{2ztH`xm6d=0OVY_6nIRfF^|E!T$5luZ=;fY>$aD`5wZ@J&#`SEM4N<8AQMDAM2Z<9P1l?*;!=ai3EK9R3aiM zG7gBXg!czi9}J`+?P;3BpBlNAE^Qg17su~@M#TQ)wM&cP{N`(czih@rf>@VEz`D7R zjxyBg)xoa<36%~kWGDMVC61z&tR=-A6d}FZ{-4vUE7y7$E_LkUJMp#iVCk3j*iy$v z6%npszeodZPrtiRD%d7N5xVg?kbzQxKrexQx@DY&kdyR5;agTGUwtc`(w}EZ>g(ba z2tgBvv{`-@FJHPqexuDbbj z!D=zx_Bct&G442-GSID0l)C-{N+XhyGaKw5g{F8u3Oc=Hxpd&Lm7dCrqx1(QBzE2; z=s(pL4IZ|whGV?~;)E2UoQR`ETptqc!LQQb zJ-m;MGZO3NUC&A!hA^e&dT}07-ez^5`t^~}x`f#d2EarCwH0+j_I+`dWAB`0D+Lyz zOxxngl4?k~&XxTrA_W37`UFD?p5Gpx772a3?3&~XteAtcVi(bh=}#Lwu#89jRa`W z`1mhzQ+ucw7KJK=fVTY5ef~oFK$`2M_RTA2Ig4}(P6=+s1SURNn*%GwtL@lda{cjQ z0VE(^F8e|M1IXk5Y%L5|-61DiYYyP(ltFWyI?^6N`TL8RZoQb;|K`QWh(KmB1rFVD zhz>62A>fB8=Elb;P(s$&jVyIGoMFKowdU_8pSpd9LAQaXK-( z9<5i^T@VWhXhda1;>EwYWvewCMB=%)<9dHnsz5gQpt5U(|LUz`~ohjn80;u!{Qvc@V;9T)*qd#5=`Ar-^P{@IOK#y~$QvGk9zBAC#-d4^peWayT ziB@0lw`T=>g}phMjs9Oe$CHwXkCv_>ZlPb@Jk_u#tZNl>c;rm1g{b+%X$6H}I`~rt zU%3K!!VZ6(`wR)Ef2j&!jNBO%5YP>48|TK?$132=%`Pt8k?!<&_3cHXj~~Z!>zTq) zps|JJ|C8!C6Im+`rd?Ri((9}={$_J-kWJWUMLXZy*af(QA@9L)d)sivUDIIS@D({u( zLoy%%_;0c6&%;2dnBF~h;v6%&(9{xV@y4GI@l1>Iv{lMVzpDUF8{~S9Xvz%;Ck-<=+s)B7n-a-N4@I5c~nb3`=?%DbxUV!1IIh14%Qzev;XNowjJ6_ zptSQ0B`*gl@XUe4h2j~)HYDHZ1;JK-xo5xTqm$G>`&3 z7|8kFxask5uG&OZO7rOK+vyYf-{#L|oyy#u{;7WGncTm-A*P$Azq=tDjzE`F){*SH zDm11pWM=8>^Re5xQ*{LqliF{rEqC5MCZndKj3&$hK0Vs#nToVfm;OQcWWCgm)4gTGJ zEIK)vp!DYWw$l#31}P&^&Wk_QFo*4dz~hdq*$bRcM7g+#0trx)tl*Bg;|rg&{AQY{ zPPep1KRY%Rxp8^o(Vw0p*09yPj%zJpaD1Tg4f>X zWl!^!6&udqk{HjYWq$h_f0)j-1mf*>Ii0J-e^vkO1K)g(Y)j6Aj0=0jNEL0fs@sc~kdaJXgZOf!_x@cwr>c==qg@dOe+$wyx7Z?bBq{ z9&)4=VVkYK$XVjj%I7vxdqFYX#Pw(e)VCk4J80vmm`a>?ky7?h)j5Js3Clb&sP4xGN zTG!xsy(sGapjqznH5^K}48DZWx`53Pf}H2tlWLuj4B8#dQx!AN`Zlk0<3ra5M(EjV zTF@HPdkW&aKCU{#1B7paWZojLEn55G3tvY}ajRo?%hNP#urlo~0my~rUIE8z0d zN!J_0(W6aOKyR$n&2*E|Ew>*V6x=qlNXpH5cv~-pzff;D&3!WNU$)3l5yys!)-R4? z3|}$n`modw$)Hp=B5Df7^p7V35KCn=Se@IR=Z+;q6|%$O70l-6x$zs2I@dyFfKU;? zj6yB+`$}Ih90lIkr~ci&qlk7&z2!x~`7m$t%LvAJN}`~EgP>GzvZC-D0smAgl>mPC z@N2&#C|P3EX`e|RIB9z@!2Ifp&{!oS(sqv>wi9BO8E3Z)36X686Qy8Hz%O%hJ9V?` z@Xf0Iv355C3AZ^8Fle)sB~$k@L+fVWxV0PK;2|@AQC)u>KG^$C!jb!Z6-&wvId9_E z1hqv)$f2ufC(*0B-YQ8cx45?0qDmH*1Wp;XN?AtEl)XFo3n^fee{--3e>&JSCZYS*b2z0trn)S7k&WHySU!2M*y_X%M zV9Cp0LxgoG_bRpwJ%-=hswV?A$tV{gc29}QgRN$*AE*y_vH+J&Fs6_ z!3ZsOpeJ8$P!-_@LIR@jyLKDK1MohRBhdnmvoXY)SEWTYW*Dk{b;w#%`r zpYdoiPdQhtu@DM@K*Wd&R@AXau())6m&n)847_)ak{;v*%qd(wdrkH7e{iPs*E!U; zn+gOku;r2%;litGZq8nG-wBRKE&ARX1W5DO3mJxyc_t?fk2B~o^&YaoeCvziJ;3M#4NJhT?}{?mi<-kXTjS zVj(m4OH2=GE+$-?>Zu$~7_<^0)OUvdQ!ACMZyw_jKYu=Zb6}_b;q1~oN5z03-lkG@ zYPFw0`gN{v+_5I%JY>h;e0UiK`c!o{4ply4g(nGUi@}y8)P{ANb>_7x>6e+;xS1?( z!ydG+ZJe;}DmXUMyY%m#Eu=Nb_vHNsegk!>mL@ODNK8DK`WPIU<(47bHM%dMo*lCz zx*(4T+2~B(dVhKCp@LseCOH(R<7jm;tcGM8xflp~U~7IDqwwmS!Vi+#XEs6nOg_sdW6yT!*7J z78^R^0D?G%v)>Y@<_0E(R4+vpfgnl<1PLsd9g4t>`r-rC_rYRl@I44_Wo0WlelGB| z*=AAJf|^Io_%^fjhrjHV!?BTcI|F)qdPB*m1fyun)5)|1BZ_vU*16FkPG)Cuo!2qj z0e$6KtZ)e6;RHjhYAE1gg_zTEm~F2cNRJL4s|l_@0GSmY#;Ox1xYchJ!rVB6&w!D)8-VPxbCHXALB`^7Mb87ocv?82x5XxkQhOWv}?<_s7KnLv1bPSRT3@n}?yFM!OQVd!rz^e&^>7e<=jASITbDDq`^LBKO!lJ0l# z-uvI`?z#pdTSquX8X9X$+BtfCKK&*(+`wKq-e(;d=!uF+evebFUj2jmsMRD=t=k+W z)^j`=PT74GHt>Q83iQcQ3m+eYAJ@w=auzE&>C2CTa@&9bgdA8 z#fpUl*kJ4TPs-MXCS7C5i_Kl5hxd| z;YSl40Pf*M*iAxQT6pc`m}K_2*?sB8A8Clu{LXt8r4>i$(SJiNIL5Q7vR@Y|3Op%z z)f(zERq_LEfEIv}Vjap?;b19SNGdN%mbg?DAlsMLyjNr}Tbv%I*TyeSW< zYh86xWgO^>oWd?fabiao!LpgSmR||bL0$N~FBZ<~FFM6J?pN~dr-58Z-%7Xsb&jz9 zL{iS>$3KPEU1R8ALuj$9uL4sk5ToVyCZ56$WUGCHrr7iNpk9`8Rxl2CQy0v4@Q{NW zI!5iMmp#b(2df|}k4|371CyB9VU)ov-sHg+h_uT>leRsy0Cr|5U9TN#Y-x!(VD}<# zsk>if$0IC}P&62=={i}k&r`*-o5jT=WAedZ<=WU-b-5sz!4ndSl5EF^ySCD7{`XZ@ z_7u=ucU|ynIx4<8A)8w zxbRPkGk}>Wo0v6%VL?8Q3J;Z`hxCyBU}H)3qQDc<;R`h3&X!Mx&BGBg%&7(U?Bk;9 zkRa3TWcW_Bg2r642+mc6NTZ@C#uw^s-grpHla7^rY|i?T9oQED+|u(zf>PnO+C-QU z@hi|EAkxF~2%s%g!<9|W2eq>|L)%01*a5ikJ3_kExlQs9?%G(13zrmCr;ATxRx1$H zX5hlHYLXD9LfEiH8Me#Xc=qWfO8Z{PTF{Cekx(mE{)d?}Vqx4K=@1b0p<`k$NA^`; zY2W*DsE7jJQI_ej2Hf}XO9=`XNpn% zSg|Gu6ESphs|o(shsCJL!aUNc_m2&4F!lP$@9mSz%Y||v?;|4Do`6(0=CtO&gvygD z4hEA@*&~PT(vmA2HnWUVP1lxV#iWdvD!-XD^38?nFH9sKDQEl0&t?yQ$EtCpnOtmp z8eqcXf8t#Re2CP^cv+Y3Zf=xu(2b<^slP3Eiq7+-u5j#-SH8iN?>T!U;zg#)Fqu@_ zYgWw9{|QjyeMf#l}e9aI2cklBUR_&i(m0 ziKMK&=-}Ymxq4!^FOKGhr6n2>F0AR#MYae01K|Obf%SKza*3~RS}X1@IA*ZfmK{<1 zJKNTC*p*0Go|=omn<*DC8^r~p9C>OebJ?ObbW^wtb1{``oLDc#u|{pC6-miV$Zp-E z>|OlmR65;a+by(+uV!*I}Cf=`8P#eyJI9uj<2N8y0_@@p}XAc4m{!5wr4 zjVdvIzBaxCZ)9t+Xi`E>j7-ZhPAmWU>u&Lrp>^0kgc;xj4i-O&YJ zQCs;7^HCw58r`_F1ww(vuG>VN)Smc1b@brFHF2jg@MiDGVTR+L?>aW(}9PSSF zLg9}n#*I`loj606W&pbUlO-;}#JtOv5zlu6(5m$$lEfTt9MU>J5*23>D+%q{x4xYp zeMdSyE6v3P2alKQX6l1uw3=4DKR~R0{l1O&@{M&ni^f$SG52p?b%ag+3tb|==3pOA z6hBJV&CyIeuCA*0G7vQ13MT>RFES_ZvLdxP8>Tx`rJG&#$Ir?wyOnkt#KpS<>0SIA zYn*bV%a3-`cBRu9`E^vbmI^RJb*agFWay@F7;wn;H0;Fna-cP=s1ZJS#=h&9d)KE;9 z(8|-MDR>R@09fg^ENu*=1XNGR;+7C<1WB&h46=@S*u@X*up$2KWn~Az+yL;^6+j?% z6+&G;;!EyE%nHxt+Tg|MD5UzS;pCgSt0HqEee33|JdCn9LR1XDl>X9vF&(XQ(vp8b zI|8DkRH4hqgTn(IF0-_jW4XL7&>8Q>vsq@YyL_~T8cvbm;0BO|Rx9`nC`5K%*D}?Ts{KN${Ar~Sg_J4l(C#PM7 zhPk|IUHNq|dsfZcyQfi)7 zdDM~geK{*OMNc`?><1(3?n>9(#hTH7xI`tseT^X1+7~Zx03J8J+IJiZ^oD85--OCH zE2i_xGRrWQN3ebc{^UnS@Z6`%vTJDmi?;cOIt8E9Xg{?y$No6%0>gAXqQAStbPk(g zI~5B2_jnOFc1*CP)OabsuIW7jcObq#&JA8Kg_T{l)}E*VYT^`5|E6R@gcitWmp{ot z2pW1l$8?^NubCJ89nF5sw?q;Y41`2nR1L4*3u(2TqF=;DFo{zxiYNZgS)q3aHaPv< zdT&=EPYS1#Cv%{0LHWL+f3tfBo{JDjB{JR83^Wb0hM`AoejU$Ia(75i$OFKqPckfP z$5Q3#XRCiOiQoy7bmQ8-L9#YzMQy5rfLzT)%7lj>cV6Rh*WgWt6^j-F1~cuo3Pf3dZ+yv(fL5TtN`VxOog^WZ z-?Cll*>G|NaxqgIv!xP{Qu8Kaak0YTH#gMHd!{U>aj8?*Txv2>8iLLr_*NRB&JoQK z3=uJFZz7No+aAtyIbbT61lbnppW8xNZqLXMH$X!I ze~*=CO3d0>W@*Nlah&#c!~Q&8R+0MM(wRE6nl|-|2RTF1dozUlo)1g1WV9dG%Tssi zEwR;BB4;^jnRIZt!|Ut%#RGeR;o$`!o1e%ki2lMpaE0g6*LpgnMSdgXCsAQ_mbQDJ!c?S$+P$&6+Q|0pjTR+Z-@{oyoZQF^L0vbBETI#$oO zV77w?EQ-kXh*l_2uADDuAxb&qm$Rr$Z7zs0B6KTgp9h5pMc$_HBs_Y^rRzuhZJc53 zZ`+e6FsH?BviATo##L&rfgQS9Mo@>4ZWkEnG71x?qsz<7Gi>_3Cj=5PoSt`6uKF%aPK-aS}$c#ARX|0CdDx@2~iEo zH<6+0NNSX^MkCV8SL9(FqSZYJGL2xpF9@ z1V-SLp7|(&v_M>f3AjF1UNN`>aevc}?;16wwk=S@nK*tnvEuIG(-R7y zROgUI(|PSUVLpXnly0N^7Lc(;8bVHeGV@G8Q`6yLAa3kY-xX?sO_<#__l#vj7n%PK z?WZ89d|^|*zn6qfl*MUAFoh2Q3{5YgJVKly95_2Hd@w^;F+;c8V1yzonK$fN?)Clf zmUiGK!nMc)wvTKh3%gzK7%-B*#Qnzb!c7qvBH>R?n~tStOS7p z6g+|$=+*XMIO1y4fq`+@pH=IiRZ8Q+g$t29m!|Ibj3k{Z-tTF zf%T=YN-&u1=C@bS^R0@OCZS=?EsMMJ~pPzJz*`L^>-yR*V7{ z#%ORBt#8G`@&Z&iNYkdy$Fs3ko1NFPu5gs<#K?)Yr`IrY9Rx%n$h-NkyiRSpbN8`1 z^cVbZ+f|nVBLGtuMoh>_n9f7rtP=`Ob zUIu;FKxSeg^Ic&Z8ln9u3sk7o={1*AUv(uFjN&@VJAN+UjmZre7QsWDR%Sub_+mgk#!`VnL&+2h1qoKkc?Bkh? zuw8^@h`PbKEt7)wFeSo*mp2Lwf?=k$z{-83vkqxirCopE4dbX_<^??%pht7jscVYJ zya4peUo!yoH(N7RS_=p>;0ymg3{oV)3z3lcx@c@R^a*068@^H}L4OUY3x@GrWcPRs>o4<%EwC&4gV)OgyeikqbWWjmUtm;52^hI)i5KnTnVxRhRuORr!eswZ zGU13~i(qo;$iTxO1rf)=Q`hPA^2oCzcd%l@HLq8sK_Co#1hkIs{X}gD4ag~;i>_O? zWd}Ix*=OWuo3pj<-rT9yS{HynC|}D+bQOz_o>{&%&-hMhJS*9bT!ofs8zoH$B@wc{ z{TimpWZ+wMFz{myNmiBGKHK@(p0%`ndXFRmezZAeo4EaBb3F93p?`feW7%fiqO-_M zMv;{m1&0yL`FL0U-1wclTRq6FNy|d3=lfSJ-9CL{Bi&)l7mTp>&%WK4Bp|>Ms42=B z{)({z9nsYB!Q&k15_3Gy%6gh_ zy!r-ZfX+%<*ZusF6cE1@N6neZp??xlfi`duNuI3%X>6uIz4J~;jq-J#d3Ao%DTD>j zxYpX#6eWKC(~(W-OG~p4*BV!Cd@Vy^E>_7^pu=)bm(rq_R0zzzuvjP~Y5l*fF_r{J z(%c4@q=a@)9z6-&5XN$+5wh~O?t_LpoL|^gsN4*NG=MOp^L!vsjz7UC7ZQ8#(B&q* z!;!)|;Ebm zjJ^m(gXJ?FKp-d4|HgDfDhvi1p3}zu4A*UMmYJNJ1e$wL?GDP!^y{urTA(FdNiLOp z1X6FT@*(Z_mDiU}FN&zG44S#=*=d{XI{aR7Hx$bLRYPIoO+Lluq2Ja)h0ffHoB!IA zbL;^e?pGKgD{P<(^>de##68a;iGfjLOlkYaU001^tyvJ#<#(H&kHrn2#rswUWoixizD%rrV4`XvOa(4accD zf{q`jAx;;3m|p+n_{UiD_-i~Ik?JlVY3hCZizI2DLXD|1VP+7U+)X86PQ|zPX+~xw z=dGi$QHeed(uF4wgtmR)LV9P#=TFRGwRUvE~{?+_#(R3~9VYHORKNG(ech{j;4 zm0df(h!C+-sB-XIT9#@aDtyoLGWoR(M`qeyG%kRE=;sgJTeZ@QAJ|u{<)w?O=hPdU znurQ%34`Rz;&;iVzx}46tE%gG3LhUA*93#zonxnZ?`2xLHF8$8Wn$qGSYyJVx{@}s zb$J{kkpWAJfL14?M($)qg=C?v?d9@gUmwfAKTfSkK#0Q1ZvCkBCnVqH!OhB=F6t8` z?_jORr=?4OpW*zQWAF*`DtyY%PAWGwo3T(h| zP}0)s?2l{_6hM=*OHQ~m{rznFt!Jk*S6$Ziq;%`0!ts=Urta29;q_9)eb%{nIC#n* z=<~)7>1Ew$JVHo_lIJOL`xq}W+a1l-J`kiJN9a3VD8&burd)_$rA*o5i_$a zCXJA_#m&0S`>MwEidM2t z8#tYRZIGf)+MJ;L)H+NRIRTyS9eY)4rMY;>E)q$dxt-Q-HxO`#8@@}^nWXfQ@cyIP z$^?M~#AwWp=ebTOGO8u}6}IJzD|Buj8l>K{v-6j0DarlX3D^!Pio6X$`HxC@tmzx6 z&ngMz($AOiq{!*gXTNppGeI`w^PQY`43mPINjEWASukfobp-8Bl!TXw#rQ3kl@dqD znkuB5>N%V3pH+pswTQ2~q$+G*0IOnai_9^5DYV+%M0 zbYBa~laq)?(cySB`-EE#VyQ#CE=0}jm1C8&ox78#PkwL;o=rtx)ued>jT;CG*oiNT zTT@s$v0m}!UDM&>sQUhvXwlspmr0BCiHXn&g2D?OZ#y}P?q!ygTvWemf5C~hW=`T( z+q0#gT8yYNS~1OAHZR47F8z^#obH!mhhArP9V*z9?CyvZK2`}gc3q~rF5+%7N>CqnnqfU@%`iKDQtY`PVvi!^=h zD}HIMcG2$|b~;6Yb(I3Q$t7f0rseakslb?0~g~d_?$;S~Ap)c;AZJbvsox3j5$PnZ5DDgilmcCJ1 zj;pQ!8ohpu0)^kWwfdG#EmLTCBCuth_}efM5PA|HkD%!`x0ACiBqoygEK2E8*8QT$=Y4_%lX=)d+glTP zw0+G*{R<}P>4$P=M7u5zv{QNWD&a|NhV0?X1{aE}nNha<@r!^vl zCLu-wtg5WdcxG9)nz`Lr%*Qw1-&U2C$i2IpWn%5{{HLR=&*y~5nrFo6mWO57&T>Ttfz{a zE8?7qNH-*{b}~t2wWZBYMWn`RFYJliG*om6o~I*3)v@V*?>3L@4(D)it;l+yTK2KX zUrNMEVz(pft%{;D-`&I%zdj$1)g4)_r^mEXji&uaZFFTK)+Gz=G`syMsFS3>q3|bP z2_c6%%lP5zSz-OlnPqUCAIVOz`=5~Hu@uQnPHSIe$dE&|P&yh4X-OP9(ynJ}s#l?X zt)D+q=)omTQ^z-MHe!FDwS#CZM%mL0oTcaVH(M zLnqV7;0Rb!nLc_Y@6=Omk|K$5h=0txX;$^1T2bY>Nw!y-9RFS3s4A|3kTL8Iym<*`P@Y7xurr1@ean;ov3J z2%|Brx3`>XnaL5z`4FO53tFxR+KZhQ+))}ba23G7L$=v>S)N&X z(IK7t9~DilxI5Us4pZ**vGM$ef}*gq3IS>DdqxM|;dsc}`XWG0NigfC4&{OHm|%UI zlVkS#`>U0!Dkd#XZGzht_jShvHjw^w+ux-{NJ!abR>`)XyfSd_tx2!OGg*1WeEg8- zwOS8`?S^BKrxcKOa}=H|N2*yhZq~?>3@8X>p^;==Y{$w5BP*Yu>=mbeUTyp-}a&1S2~zPRZZ?c?u%s2&&pD!y!Y&HvZl zTZUz|b#23fpoBZHjSXPw4`){poBZ&R4Rdv+gQK`*g^~ z5258C5nQKS5(%jSa#f=JiU(2JAvAmd5d!b=L^?50G+`L{CBlgF8(DsBhJ5M(IMI

m z?i)&wf#37eg&%s~O;Up0*7D5qC%OL{n}RaS1BiRp_l#$TO(gbGW^xDu;-2-rN~l1b zxjDQ?hWsr;P+x{D=X&ziEcTxsM#=VCFE68e&USOE9adjqGl5-izd;pmdKS4vFZECO$h(>(7Dy0L6Hz^BC~~7flQA`#h{h>-ha&0h!YU zb%_@Q6GMuI2eTI!4S90gTX-Y2U)hE}qpOPC5c5H}kq+)-rgMSh@t zG<*y!Tmj*P8wXXxS_`z!zD-UE2^v$@WtR}wlEKErgkbU4=Y$cA_c{|RKGCy0GwY|- zTX=q9Jl|F>p14BUGq}d{r$S&50m8Vcc!7L^kodZO6w%yy#1X$XfA|fFT|Z=U{fL+6 z5=g8@_Tu3Kn4h{0_9-B$oR?2lPL*Ch9(UivI-|8aAt>?J_ga5OGm~9NmM578s#WKO6)ga@ySyL z>{usR{Uv@GIt>e_vmyJH15&yVfbp37@zPl|k=W4UcisDHxy=-hpfSMe_!TZec6aXj z8*o}GnX?&z$3I*WdDfyW2Yr@R8nTi%xSt~Pgy5%zutLI0SRCf-=YZ|bIjCcaf@fD*-DNFsM3KvLCY zds^4?fXaSnhP}uv7)~c7-ejG(azPfin{fqtv?0f$16|9;j^%^LY3Qy7a=q5mYx;ag z(kj4ci~pZ~2Lb(;W6!ETi(!9#!Z@cNdDP`loDYbST>_uf7@=|%l}oIesXa65^Of$imck)jo1(24t{q zb@lb4Qic4V`7{~|clj^kKcPJ9DSQL_;NJR$|1;kBfNb#MvB+o`o-K~{PV#~|#6C9^ zB?SGd)P!BKop=#I<9`GF@dMOTsTcK$c;V@hC;D*6F-9x_8WX+xDrz{tPt{fCPD@&y#7`akr`F! zX=xfnX5fjB4M()lVh~8kcxJKl2sK?l7{n*wvzi)T+Z_?n3pQ$S(m~=}WF~nNY#hch zB2{g9CIwP{w!_5{iZjVb&Xw8ade(Toq%WrR5;44?{GYmE{)ohICjNUP(Tp34m5;N) z{y6Hd*M#r9!a)4J<`7SQ@?k+?AK(fB2x-2(rKcOjMX& zjbY)$@mM*yAdbYuF9cPF*=yT{_>2;{9H=n)9;bvN}W!Jjer?^J*QBI*eM*?;a><-Q@Edjje@ib;fx%lxrR zaw+`$15`k?+mQoXMAyKKPohL&(z9k5vfa%4|#qUb4T7*-x|afTp&>SJ`V4Y7M_2KZsjs43(;_la88NiLhU$squ1 z5OYwihchnVKg3n94gzWby-@&p|3vF|un36zgkNK!W1XZ&V%U!zKgYsp$;D26z+?l# zEN?ck%^IH|_HpwGoS{O(vJLa293-+F{uNn5kr*~(av+8sJIN&mW~4~=XUrY{M{@H!;jB`#Nvg|>goLeTvvT7Cb`?P#cXM{P7;GOe{bLwcu%`=lDsxx z3*5ha ztxMViRDKG>0ZZ|0g zeF(kv#662PzP4mZF>yV;rC=h{VOj-e{8`E}w08Xi+elyynNAByO^dRq+{wO_y9xTp z!!uaUF~N#3UbC?0*kuP>Y13+HtfkwZ#8vU2K7|+F;F(4vPkF%&u>doYFuTs2z>q=T zEPwkyAap(~*+Su3HnoV}xuiyiB*ZaWu#xWj!9*li zAO8%)ZvWzdZy_LNw&=g;sDB_jIcWyl@aUu)3Lt%v;z`cm+9~`55b3M3L;uK_?_jZH zGK82$S1|14$Ry|4ZPX|&|4gI$lS4rBJv~pMZ{LhOQc5dd}6FcxK60$mxkqFb9W*Z=f z{Fog9_$#N3T(PMA(<-OWn_sQ|y>s^eb?2yGfeN=qaF>+P0cA+iXoF~4t7g#(IXu={ z9v*w`<{MEsq6cE~d8GOgp966MLGqF(e2UTUSINoBwnfq+fW?_m9>_at0(*GROXemMw`Ufjha+ z)fo|u4r3HnH5o7EVEQc~G>kw(aDUg?wkN;~d-w)U(ID=RW9FeP;{Nn8YGliWeO6B? zP2BriG8V=Xv%q|(BR&>^R4!q_$C+NIMIh0fZEj3{r?D>nknNjZ1Bmp z$FUkn)J8J)pP5v8mfY~;E5?1Ut$Q>2QW_2>lOcWflS1y~HMvcKH6L1+oaD#fGXAa+t=vn&Yyn+d{4|6=8CX(IN8i74@( zu~MOOF2))tj9kg)>|S03rF%`{m}C|FWWihr-exVn*6>l`p;uN(nsr#xdV+z8@du|yK!}HXx?46BQXl8}izO>c zQ>9jHlswZ#oOpo2R59taR?`d&3v6?9vhv@;QHp)y6r92Qc4id-OkS0d|3hPU>Dk?v zG27CWbY!bV-9}BWSC2n{i=4>4eEKj?(GV~9n>d6s0}i60eEMv|3x@zm1_}f^CBcn%U9vsfU!dl} zv9@6BHWXu;*gK4d@Rql3=x!8SjXstgT0F<%YTBe_qnsJrMrTnG%wd=XqO8oo7;j4U zPk5Ry*%UJy@1I72lIE6Jplu6!^DyWyvm=mD)@|4f;hRseRAn4%7!G;1R{MCiynu7Y z&MaF=eAg985cSdz|K4IlrzO>ZIiVZZjgG&I=^b7`-sP&n__7m+elI_qJ_Vao=8%=0 zi(EIQIox+s}3>Xj{zN#Z!U3G2vh2&mS zbs*(4L7QCDG7B5U67Rr_r*N2f2<-%>w#&A8Z-vIhgJBiOm%hBwz4=ScJk`+6RZ8i{ zn1@0xyVXj1yB%jYx4tYcCTDVj>GcOJ99rJHmrJDO!d{ES`aEb5I>Tn+e9V|Ew$Y08 z35_TGvE>8Q+jSyefiJE()+g~mdp`#X&o0G9X7>{k7aQE*`t>=G))Kc2qQ;nwq{78& zZg{WEkIcqDEIZ4vLcooo1dv*nh<_GcMDNKF6mC)k-KnkhP@}E&b!#ViE_f(`B^n-X zgU5gvklo>O;!bROqIV)dqKs!N`EV2rUZ0}-Z8yT2!ozy1yQ^nRgKYIbrXZMU49U~j zKz>J1=2Fqby~Rn&LO&hD%FvZMQY;(=RtC+L2X$Tc{5G6* z3%q@BBrS*YEPBb)&b+grDH7Jh9^5XT1Pj7ICnqXj`wm41mJ=Uks(Snpmbl0cPUe3s zNw_kg(pTKSDdaMbhl8DVFzyNB>rQLa^N;}!UYEZhA{TaBB(^*}YevuG$?~g&1=v+j zdh~{FH(FWOWuIe{95*@03;GOCg9J~wGhI96^$2%z3u}T{N#X|sH9l3-#u&`+r-RUQYomtP^*}*_D5))@=ZAaAknLBaoWewZr z`a)DCO=e~pEqg!{#a|4%?}e>6B$*u9V_d zM0r;y6z^9}eo{Isa2_V;vhq$45GTsX6_(yhGsqQGCFg8>)`XLggsiO@{ZeRi(5Z&; zx!fH8b)rpvLQY6M0#3AOp`6py*UHkg^FM656qAte_HDYnM!O9uncC9ES)ScJb@8SL z$N!?_<1;FKYyI5MS=d-gU}HcOO!^rGZ>Gd7I@g;F!g<2%f+>>NT)={K=z9qV52tyU z(&>fsc4#3?`?_dXFfu*HCt9uQImyWnD6l@EI_B3R;k+ll&jp+~ZH0k31;W%oAxW61 z2+aaL=T`X3R?xB%z~dJ-^KWiCQqpZ|SJC!SvLG(WlC}eJ9`P}r?Th{~?3iTOV!v6Y zV}z>T=ZE8GVkYoTNzYXyUhmaE^5#I83W)*-Cn6E{fHnz%JoeZf#FW!K#anWIIy=)9 z{LZ--u9m~5{qz9@yhuhbjZJYw&NGHxr0`M1bp(P0B;4R8gpA(nwWCl#RX8OXyD{TQ+sC7Rg7g7zt{~?JKkS(LXCX)12|z%ys$vR^2x>MVrKJ!#2J->|Bc8%7Ux^$iQas*<;+ur2HYm6y@HAAPTSP)WqjbaW|c{KIlQWi zi>U3ee?*}=&N*3EV~$m|3Uj(>j;Zg4V1>6tI#do3Q^FH$Hc*{d3>jz zdc)%^%d?fMZ5t9;vxTHotQ(lzS6^4ywF!Au-G6&lgPVwSQ(Q*sm-`m30`Q34e)o1A za=>(%=7QrxB&i=ef6X4evmu(>K=SNMx(!SZ-(~j6l85L{MTgH7c;MGl3A9x!CzE@1 zkTR4`5e5jfo!gtJ-L;{X3li3zC)_I`{dV$}GEgr+;vj($Xb5}raW~tuz$yQz2!KUea`r((PpX{JfLzYs_t z(L&Ps{cSLV%*DVd3?6Cn@n#qQaRjCRJgV(SM{x_VWE=DPoylW;QH=R*+2;J-tG%Fs zT>%9$uK?Q5;1|oNwX{EA{K~bcf=uSBjx!e4zY7uFTbayP7U| zQ6UBO0PtC4{UsnE3iC0@(oxr;^ly0^E7*#wtV zcs3pk_gL!BbXjCi2)d;&)5=h>$nslz&)J$Y_ry*tW!v>K@g%E|z2u%@6t~aVP1O9@ z?QR@tE8KlNyOwhGOr)$H#%qY(Z(nmLj7%o&e!*N|JXWjOf}4X;NE*%+t?lPy{YQBuuDIMBM%Oc6p^5!rX2Kj-<2VGmffgfPD4Fqr5Q5z=H_8ZPU>H$*1d;c-A6^-e z!s4{ZND?k-!_>sH$K47$cjGoKD|@=}C)>1Zo(r;RuLUyiF_|-WNzS}z9P7gN7+_@J z*OSQL->D*2E1YEJ$uc%Bnj8o^zrN_5y5clG>o=OQ6%`aHORDCFDKOh|cx)g?#^2q| zrKVxdOk7WE!guG}Ij4=)spSnasr2F|LEGeQx&puL`P_HDgICbJvT&_!)eRb6up-yJ zn!?@EoOgQIXB9|I_6m6@7=)gmW~s~zGk+U2)|l7xm~{VOZeJbufP~rMPP%3VqARXejQ7)dU4|6xKfIOchz9{+FQYPf2&QQc|Q*U($9wW z13IK`D>fl9)+!XKr5bNHzsfL2e_N;DqpS86YTHasgafytEr*i~<{vB4+eq+msUm|H zdY+#kz$)?7;Z7>B9Att9mWc~Ia0@W4_fh;8a9c8VEn3mJsSsgdP1x?;#RX&ab z74_4S;tk*Yb_|m?1F>P`Y{%y%?;_lIlP><$`SDX*bt<^p1rBHPr!K$WOuHL*#HKTu zaQSOjy3Isc^T!%#jkIR&kVdFxpI!hc_PzL1S5dohQ-}+dt9)YVPL$H{j-zkDthQEs zv&i6?lDp{WdSqB%##f3aqBKy3a8b)GMa6UEFIE?Zb&NiR(ZCHZwjuM-*^ytBd#UM) zSd{ywX{u$J43|Y;hneF*;oaq};k_Y;&xx@*EZbXy3+^iBIu+1+z%(*C+nbXBcP1+} zsa3d_!IjEt(MoH*S$f`|!ntK%To`;??X>jeLmQO3%`Ci)v4*>X+7hKttCTq?o#IY% z(@P8>CjPAcDX*tKdqREo>eK9B#_G8T#=0Bc{+b)U4i)+&C?lbv*8HxDNUSpFYUg8) z#kvS4dpdGrv+pYE%TIE9xV7>v2kDBun?rj)M624*WV~-Cy$~Rkxf#B?aCE*{ripLs z>dXSl!$}y&eK)T5lHUMBwXC2~QcTveWmR!g*>o}AXje&e_B8XB&8yk|?NAB{-z6{^ z0^ILzw#Crp@OngL^=rRX1<9aNvR98dmD@QOsh{ztlbCp#R^0w!?0i=^k8NVyS_B3P zU+{Yj(wY(&5^_EvA;G5g8d-hW`ffI}#%W(n)aG{uw_|2wVwp+EHD8ZU(5vrtYxz4v zZQ&~p99jZ1@GhW7EOPfn>D7a;{R)*o2(1XxG6`~)|u z-iTM1v5l9VV_k1})E#ajcV%+YeR*8U@x#o<%$#4tFLw!QIhsW_ewqLB*iz0XBzm)0z6ZI)86`K~)@F+;nmK z-5H&|z|&Txt(z5sH91zp<;eX$YJTXQ&RnHucR}OA5L>oa?4r~_{>l&`m2i6EZj+}6!$lKTEBex!sOT8QEP1`ob=FLi>fTek4a!G zSRqaP(X7nG`N+-Pa!g#(I=D~WYEH3(L%?M%xZy&{P0O-X zW#-DKZVvqx1&;+uSDr9u_^W6;U6Z*o^5A=#S#N4aG1D%kScypN8VtUlS0>oloXKpc zr;lb~D_aR%orn*TdD5+;BA3jEy->bod5Xs-p2T1pilAGqc#EG4;{iq~BIVntf#a*! z*VKMNXsBZcQnk^v^)!67(yYG43=lf@Yo00(!?tVF)3!WYz8Q-9JmOIlnNp5RQUBlh!!weNopK^A!737jfh6|Vz8z^07?+Nuv8f&v=mR=m(-0II9 zWTtA;pza>b`WoOCUAmsOGCO1%s-HqhEK@i}*bkI9ulCsp0(rCwD;ehYl_|dRq}3Kr zx$0c*p?hLO?pqc-<6-)vP`004p2#<`GF+S^D<*<>*yH=iQRd2U=-DJGH9eH%w_FPA zPNVR4a&qcoW8&Za{?(?5XQEEaanKNTKTp7!lk|&eI9xPVk}5JX*%~*N>9Lc#y|dsE zs#oo!h8F9Cs-^NMoxJd&hY%OXzE=Ov_9oY$Q}TjU2k(kp5{;NcX;KIja}3+ro%SLJ z9L4=?PZYEJOL4_DINZ4W8sj(0j)#1~e=~P35%R_GS0kxU_aVxgG)9Jw8{#y#3T0m$Y+p5fl?T9Z(b`3 z@vibS5@78ueDRW7(Z9Q%=?8r=?iCI==xU?`=rPtE}QpS9f#ezH*F@GA`^0RJK$P$fF$>*QYed4Fxeh?>4c5H8NGYLYV&CGMj68EE25~PQY0Y|bOvC4ArYYX>9Gd`?!)13cB zIiO(-KydMZwGk7q}WG3t`guEZ_8)9V9CR@(!sr!Uu67=3!&?XgPL z6j(v5N$#;1D_G9CSF<$+&@`397eGD3`p{){_sFjIo26pSU(+2KC_7T0ehOa=UVS-e ztKzCfC%(6Kn{memWgj(f<7#)SGko2bkOyKd4#Zk@?<~C2muJt#MkDZJa~nc#h|b<- zcUMy-z%zB4nE5jq?k4`p3v=yA9rkOqc)?fSzBLQ^k#$D;YzkHzcvtn%UR@1HI(+mv zG2a5iv$;nDow-zYEg~Fn$NEfzJq(~pt8v4 zsoBqRTGJX^@NdnJ4_kO5*4T|%=+v;DecSM*LH>{mIoBQCR13V7l?Y!7js%e%9S0&M zLA3@J^-|4zi!I4w2g}p=SaY1Z>jMj27XFRl9{zOFX{?!He+rCk{P& z59W3W5;iJMhC1H4!T}3 z&rbjFZdV^+@`x^Smuq&zg&@E&A7L=jXrH-<)5^kOe12*&O=~X_4Byk-O?^(P74B|o z&?f3S5j%giqnRtYz2yQaag_6P%JTNNW(|cCE;=!Slq(R|3IyB3+td?d+L|95HoTB* zjuJ>8eD2^~#kWwZaG_$`<`kfV0qAPrV@363guT_T{fNgsQ0ZI6;F{ASj=>LbQj?2e zR9$oHHk>Ifh6lX)Z5(9aGqi-`pUA=Cjn8dz{uvBpRmM-BJ^_wP9a&ZS!cx1qGh_n| zHeVjotG|n$KiMHxT?LDg6Wm@DV}9Q*9_9|Hr!Cu1s%Y}{(@|ss00Nj{ERtzi^&(u} zSRh(~%T_8!C;f^TA9Ix-Z@fux>rStEW?8&m^&9S_3dl63I#Pl<6OXt$Us86Yzo1pz zvH8hT?p`I@QB^q}xlUFv^=FBl)@;1SZh-!T2R^5D{(?2)w(Y=*qRyrIS z0_>kCQSpC(5`7F=+wkG6QvqA3BLUy>iJdViJ%V1N7QmCa0v^rfZ?6T3;aFZlP@b1@yd2!G)BLObW`%bmVPw z5Z3b?BpuIByAbr}nHyu?JHv-j$kTX3xbidznX3dWq!(rWmV5YN2MoafVIzg-fdzYU zZWV(S&b(KS-Rl=bbg6*(Ss&;U2f93@)2BWWPcm>PDd+~Cx=oFcJbHe2;(+Agv;W<$ zck3!pCfDeAF?}R~nmUJc2mP$T>LL9+o(pw)*>V0{o}R|==K^tD*|+bD(k6w#`NaNf zc#43P9+5VBf{%1X!NK$ae#w&AR&&azqu6qv*zS?v%?)A zDEgqQ0UJl{Fwlw+{4c4~GDA&^jryryrBo-Oa@(ZEAo05#oWE)xIzSxfV;1sQKyM-j zDh(R@PJC9MJ(V6nvQMBL!}*z&A0qv^P*be={I1WVQLv!GL(JmO?jSaM} z@0!rwhTrx|yX>tB_nlFuCw09TD8iQv#YndZLJ2ZL&0|=$g4IAJHqP_#Hm1X84~p^g z%J>S*4UCrxy|4`Ny&lQFjroS%)+6uq>KbNhA{U{a)qBa$%fUpNwZbqr$8m7ft~G$a6BO{ z4QGgHyBu_XtV_m9W*Dvqwpr~|X@KkKS<~Ty)||+v7m6idOo-qHcwmI%uz!)MN&V6` zk21;4zN3hk^;6>%apXOpxgyVY^S2WeH(q;DXOuzd%gvMfM+!d;;Q|-}FC)njq@13h zRJL)yQa`uDZc+|2wV!Cz2tQs#=r?%IqI zq5V`~<`FK*(Ytf!8SM1F0B`y`3?uca-3%!Vy0A!3`pU24$kDif_H9MrAms_IZET(4 z=G^SP-K_?|`AjOBcTs5?1u~Y!Q*=&aAu6NcR^jb zvVkEvN-xWUWOyZc9cCFkw)%(Dnw$B*-RuQOIF{Qklj9(@3wLV=mzMGuegE<_0R|Y@ zz_=TY=&8aEvAr?~%k*v;6x^)e5N^e6Yj1a9Zxn934uad`Z-Vn@ZU|)yMS@;mo;exS z@Dmc|(KaczyqYA#82~RcL|7Sws9Fd9H`? z768*%;QIDpB)07?hgK*;&Bb6e$!>DtFi!XFb?FqsU%O+i0H!Q0J&l9#)jy6vp?KFl zq!k@bzeOv7lZDSxxllApqz0E&lRPrbS_j3~;J&+hF`j&P>h8u^XajiQ3*=d*y7_m% zUQd0t%O@l>T&(5_xLC~)crhfm=-S&KS;A{)@_YgSMMl$Q{YjavHxpCOyWuWuwRh@c zVnPe;Nq(+C*>%-Qzv6sbL(E6cZFy%$V=p=&fCeluBrT1Fk(D*?iAiO$pSkp!CZA=W zk5AaJ$6i@y)LJUsNvxcl!ri7%W0kf5J6Q0Hoa+WK7uVcocnf*7LJ4%lsV}0@cYk@W zzBrNgv0J;0a+;0x{qZtVE~_EUCr_WI`%!vqFMV&mPp0E4FC(z3IAgnWDyBQ*_3N`U z!<)9_$#K5fySoFn&Go=PrdQ0^t9eLXqe7)W)b?JbK=l_`73}0xQIV``jHu6+Gjbt^ zcKa4)2}MxFDUw;kF&W zX`WW>9p|yP`g)JM-)d)m+EGz*%~;qdHje?!Ks z6bPoeXAE9m9k{wV&B@TQJ2f9Fu+}36mpI!YcYCLWe_km4=9{N~-LW?L;gL61Wvef^ z5>@NM>WUl&;rE?*Cu6+R4lVZR`{fN0_NJ<2GI=GfiBolR!?>gk$Z6tXijIATnx#y) zPcpW%#vbM9&bj`L&&v}0$y8=CG+P1k8_A&}41fcYbm zu18~cU)Q&aQ?aIN6y&>)-x(}cQqMKfLp^dHY>RvXs3gao$MDPh$4fCS_02fZs+v}i zna;#3{w}mCMA~JT=tD4})b=EGIleR!a5u9)nSif(O$`_L<_=YM9x~hhzSI@R$u&CD z8uyUvs@u(o@}L{|Zlz;ji0Ls%h6!lSNZ^Tr>ahwjf{*w!d>8Lrx(c?n!>r8yKKhBa}mV%S+#OLujlRu!6><%J`G@4Mjle7 zMPt7ebL6RjG`nlda&_}w6*j%9pP4)O`P3CvZ}CC3D;~S9Gi2m4JY&J%Ggeha%A%(8 zF6YO!%s1munQB@+`9BoU3dAeAi3nR6*Mf)hUs~3sq$7Dd$rI;~v_{%)zTDMu?46Qt z_Z8jI9A-_he-Rv92P0`tdDv-M{FF^lOqR{}q_0k3rsY0vXdg}tt0Jp5i$v9drHh^_dRk`E^!fjyO$qJGg8E%eh(p!=f&*yb(chTxHQx*5;SoG&F)qbdPn1-U| z7pxk6|00#{=2s3lL$=KVHd6_qPs+m=!O3S+N^LGTur><JkbstAQ2<(y8Th# z@;&puT+{qeChjB)1$hEAadd}_557z*Mb^?n^tYpxDl*KvGYY@pm>(Sh0*~dxhYvO* zC^GdD9IK+S7fb@(Gr>n5Z_bPEoQf-n7Jp8D9*&lbOCxsAqRLAaw-zc2?u+sXF|SN0 z1mkCXkdG4B{B;zV0<9QfI5`pI`Ut9q#)bm`w+CD{pWuFkFtQvjcS|p(a-O8&0p~jU z?kH6*Wzi`<92}gq^_33}T_5g?x)}!_@ha9gfWdlQ8GNh?JEgDgSM?M*^mRDup6}~E zPQhE(mLO%=o+#aADIu^@lQ*x6&C6i-MOj1c*SNl!)R*VRU_keTB5ELUhgqx zHJ_eH?TyYc+vJhopg@tY85$dFM7GwRU8vaG&4w5}<2T>br9<616S&(Z0q0(6Hk%S{ zKs`u)8*fZ^Or2)CGI_zyts`$&RD71Od+&R#&{fAvHAC_y6UNuZB8PXeiuwU1I0~fT z{YD8IfD$a^r*WYvk)6TGqi?@6**KvU2im2zE1Mdp+GY3}ok#n@;7@o=L>@6CPn&w) z%Qs&r#_KqLM_^5wis}uw`|3<_|6;SpZq73bzAoqJj*W?EHjBR8x0Xj{V*P>|>gou} z>bE)+WYwZqXM5Y7hjoCQaO737I^Nwm4y^5o`P6a^BuNjY_2a)KBe-F(u~?0tzRoaB zXeXCk!ZugjNlH8|6K>S_>O*(Je0eKRt_!3i8N_1*)^xgXK$UjgL#lhl_O*>%PqDrE z3>n*6uQAn3r1$1GQAhk~V`r0=Bq3a%;}m9P6ZUY(+Vxd}>q^X)2wWv~6d zbewK?HepZkt1X#I48Lk+16r}wHL*GLZhGOA>u7-ZD1ewe^yW*={Ff|W$7;>qy;mP) z-YvbAD8A}QT*I5-Hvs0|ohH5mIIO;v9O4)41sqykjhGDcw{uU>846RQPlOCFy;+cs z6mW4Ub)*&d&2=Yz?||vPHCOODdMYnvIabJVMF8bI*DB`GE0XyZ&T*fUwVP+o-|DDk zs6`ZQLgA2N23LJv57!MQr1E%QYW&F27|a?u4DCe64h`1QAyRYGwB$rVtuU#?8pQXA z9Y7*!EwW;70XKFQip?R1o(_SNTgO&=@-4zYy}VT0C!bt`ppfp}J9Qi_d3g$X=8&_?9x{nF^*a^k_iogHiL!Z2 z<|%oY#>1erE;fx0yUA5;4>ei=0j*+(Ir6=i@&XYj9v1rV9dQlJh<~P_(>k zV}8_lOgx2miikUD|Ey~Zolg5{>QU4h7lT^pAvkV$@SXSPL#YX%tyM2;H+Mb?5_y*w zdTEb=+hl(>6DSBHdJdt}4MiOHytpE535n|F`U0Dwp&^9F`XKs-yR{S(g~M?2eX|A| ztA~hvNgZ-~^XU^z@AIzET$cI08LrUUvhw_u4?n2#%E>FAr;*J#tbLyok~Td;_N8$< z)jQQ4eyke#b%10ptj=_Yj#NKvI3KDRD7Rk76G?EIo<=Uo^Ktj>viV6qGg}**_?Va& znZr0eWS`aV3_l6{AswjJCIA{lB)L7248Ce7`y|Og?pw8(cHJ3_#R~@txo$4w(o0xc zTE^_eEG^;^laNU2?VfxoxG}%z_=55*!A0Ol2*a}ul{DmC5q5p`yVUN^kM;xANjPH= z6S^j8?~jNm1@nD9w0rpq5BW=-t5}3Y+cvQyCt$XaM?gJMw!Ajya`H(pMUv0z5pOOn zxDLQcZuniKk#Qf-e=Y!N4}Zy>2!XFE*(1qF{}uZg<^XM2$LAllbK#UhQ`at`FW^#O z6$tCSQw^Z12-$9>tsazzSpyiDPmG#Fp%iaS{mi?fZzT;V%2}IrpW^8W zbbx5M95HSgZ1Q9hL&|YPXXx@gube`Ixgaf$`qI&Yky

C~CHkx12ypj$NPN^mZf zLp_US&6*!~_0+HAn_3l;ZrvTzL8NnWouyj!1Q1<%C%ezPi2~V%fwQ+6MqIY__$J${ zQ*!twmuJyY8{LoZXt6;_AhwIj6cu^^Necz4xFOv9u0c}EH(!nJFBYfI?Y+touzGsr zClv+uON5Lq8t}bT4ZPaPj1B7&ZiLpc%w(Jv`%$s_*5=!h8}9G$2&u3K9W`RIbR+7N z%PtC3&Amy1w(mFCNWGrTH{WsfPak_Ep&aAsZ~owsF9^YI{7F<7P!&)BM)Ma%BOy^) zEE4GwKT0+f*0K>{@IqK#K49|s)Ijy|v*$R?%}z?G05j%>{@9}d+Vt45 z@$pxjS8HxP^&Il=ip!$bd1CY_`2NMv#X@=IV=!QFQeOg>Zh#hZqq~8ysA5F@OywDj zUW09QO;ze^*Dl5De)w(9R4NnxyV_u8v(t;~y` z%`PEsbRC|3$5C7;Jnt!XZIr&gVokJuZul_lHkPWehg00-pc4?4QAaDn_r}y+y++p- zrX0WJBCUNt^w%FbpA7Vjpghn43hBOqo;>5PU-^}{{`!lL@Ny~qTSm_#V_MQS(gev5 zFjIdm;lOw3``$Z}PGH}g?{}B(mjZuBYH~H86!=&pdOYnf>;L^~KR(yf6kVe$Gk7_3 z;zoFcQ26Cs@>&H|RR&B<%uTKxS-l&_cu_|XuGQcKaiXH!6=e=}DsW$G>zw=3VgAy` zU*7?r?0|v#7BAiO-P7fZEKJ|oy>he@M?_C)$f=njq)AMk7NRj7RAwb*4`K)o$G&F z+WtSGTrdTO(;GrVzxCSs)GWQP<862L-1Vl2 zfU5r~mp|VS<1CwQS-7=dVjGM3)zR8720-r~92vofiIS0VC`pTs)nYn|;$``B5h8x?)35_)R2_nvp)z?+lt&!9GCpOqH z|E^hk&@J-}ahFzK3DJg-%8sp!D)q+3rpkh!;;?*jl?D@O-7xOv>TH1Z{u{symz8HKA+a3yG@H9F)eV`8nX++yRDdS_$n9k~U%b=e$AbPiY zzA8(GXy$M9{q4!*4@9*g%oe5 za}_u&;hq)Rp&pakqd8@9M8cacJ+0a4VeIg5-~o5=hkED{x9H0GEt4!fo>gw%3~HTH zgyx7#&jwfXWMPV$&DPLKk=TXN^2-u%PEpu?U!&wZPp{@u5tNW?lxxh++>gI_zzH0D zYwBm{cdhKayPah?_F%B!I>5ZR4)j_+MtaL!kZi>N<vS{5c!u&uAY3i4`x&Q)Q?7RVIz0 z?L+&OtXh|X|7uJA^Y@|XWFNZU{@vsGYc$^9e{WwJl?osOS*OF95i#tGjElL45kww! zft~4JpU;laQ^-V?zjX=!@8bbA{68U_vX!-|qHL|if}w(xh?VdtBmB#l;^Kfr!tOUj z(frjE_&bjOQ2ZZd9tKxtXc&%7loocTh7NBimhY}R_co^E&??IAHoLKY-|*jd-C_Q7 ze%&9)2@|`t5_` znyw6Nzf0`F!p7;954$ZDHS(_@rx1G0CKHXW%oM|-+YL)bkMq4r$~7C}jEIa^@x{Eo zKp#99|A+$RyMfGqK8Upba3euP3h*2lezEW(n`av zzl1npLY<53WwGB$_dnjbk{L`s)1X?(3L2dIm>o!b%q=6W9AG4*yF z%yk@hZ8QJpi~QONbkLk*!9xBuY-JXk4i<%0ewW({YInU>ysk&$z8{uiIC~24$9+O!V{>@M(JsG@R$RY~ zp5a*yf1A|5KR^^AY2HjNreAuyPEtdD#z9fmw6iqDYOdX1z)Q)ztKsOk&^;oec{-;P zdtoYyJ9qAjxT56?GZgG@flHtS3)5P;5Pj`2| zaTMh)OEmXy$t{q3fx;ppUdP4v6inI*Pd;@+n<+yHNoLH_ZFDx9m02m6cJep68wTgH zW|=h>-|PL(hov7qOSk!`wdxfmmwK63FdmMDl)}M@#($Tk8IYp}ZmQ^A$Eyd&uzD{P k`G05mOnoM8AZPDr!R#5f3Yu(!L-3#EHJPg!qPqA0KXf|uF#rGn literal 0 HcmV?d00001 diff --git a/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/cpu_extractor.go b/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/cpu_extractor.go index e2b8851d6740..4d4ba6115734 100644 --- a/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/cpu_extractor.go +++ b/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/cpu_extractor.go @@ -33,12 +33,12 @@ func (c *CPUMetricExtractor) GetValue(info *cInfo.ContainerInfo, mInfo CPUMemInf // When there is more than one stats point, always use the last one curStats := GetStats(info) - metric := newCadvisorMetric(containerType, c.logger) + metric := NewCadvisorMetric(containerType, c.logger) metric.cgroupPath = info.Name multiplier := float64(decimalToMillicores) - assignRateValueToField(&c.rateCalculator, metric.fields, ci.MetricName(containerType, ci.CPUTotal), info.Name, float64(curStats.Cpu.Usage.Total), curStats.Timestamp, multiplier) - assignRateValueToField(&c.rateCalculator, metric.fields, ci.MetricName(containerType, ci.CPUUser), info.Name, float64(curStats.Cpu.Usage.User), curStats.Timestamp, multiplier) - assignRateValueToField(&c.rateCalculator, metric.fields, ci.MetricName(containerType, ci.CPUSystem), info.Name, float64(curStats.Cpu.Usage.System), curStats.Timestamp, multiplier) + AssignRateValueToField(&c.rateCalculator, metric.fields, ci.MetricName(containerType, ci.CPUTotal), info.Name, float64(curStats.Cpu.Usage.Total), curStats.Timestamp, multiplier) + AssignRateValueToField(&c.rateCalculator, metric.fields, ci.MetricName(containerType, ci.CPUUser), info.Name, float64(curStats.Cpu.Usage.User), curStats.Timestamp, multiplier) + AssignRateValueToField(&c.rateCalculator, metric.fields, ci.MetricName(containerType, ci.CPUSystem), info.Name, float64(curStats.Cpu.Usage.System), curStats.Timestamp, multiplier) numCores := mInfo.GetNumCores() if metric.fields[ci.MetricName(containerType, ci.CPUTotal)] != nil && numCores != 0 { @@ -60,6 +60,6 @@ func (c *CPUMetricExtractor) Shutdown() error { func NewCPUMetricExtractor(logger *zap.Logger) *CPUMetricExtractor { return &CPUMetricExtractor{ logger: logger, - rateCalculator: newFloat64RateCalculator(), + rateCalculator: NewFloat64RateCalculator(), } } diff --git a/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/diskio_extractor.go b/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/diskio_extractor.go index 384acf0e5e4f..c7a6210acec2 100644 --- a/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/diskio_extractor.go +++ b/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/diskio_extractor.go @@ -41,12 +41,12 @@ func (d *DiskIOMetricExtractor) extractIoMetrics(curStatsSet []cInfo.PerDiskStat expectedKey := []string{ci.DiskIOAsync, ci.DiskIOSync, ci.DiskIORead, ci.DiskIOWrite, ci.DiskIOTotal} for _, cur := range curStatsSet { curDevName := devName(cur) - metric := newCadvisorMetric(getDiskIOMetricType(containerType, d.logger), d.logger) + metric := NewCadvisorMetric(getDiskIOMetricType(containerType, d.logger), d.logger) metric.tags[ci.DiskDev] = curDevName for _, key := range expectedKey { if curVal, curOk := cur.Stats[key]; curOk { mname := ci.MetricName(containerType, ioMetricName(namePrefix, key)) - assignRateValueToField(&d.rateCalculator, metric.fields, mname, infoName, float64(curVal), curTime, float64(time.Second)) + AssignRateValueToField(&d.rateCalculator, metric.fields, mname, infoName, float64(curVal), curTime, float64(time.Second)) } } if len(metric.fields) > 0 { @@ -75,7 +75,7 @@ func devName(dStats cInfo.PerDiskStats) string { func NewDiskIOMetricExtractor(logger *zap.Logger) *DiskIOMetricExtractor { return &DiskIOMetricExtractor{ logger: logger, - rateCalculator: newFloat64RateCalculator(), + rateCalculator: NewFloat64RateCalculator(), } } diff --git a/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/extractor.go b/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/extractor.go index 398ad4805a59..8e0e47d2b575 100644 --- a/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/extractor.go +++ b/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/extractor.go @@ -44,7 +44,7 @@ type CAdvisorMetric struct { logger *zap.Logger } -func newCadvisorMetric(mType string, logger *zap.Logger) *CAdvisorMetric { +func NewCadvisorMetric(mType string, logger *zap.Logger) *CAdvisorMetric { metric := &CAdvisorMetric{ fields: make(map[string]any), tags: make(map[string]string), @@ -114,7 +114,7 @@ func (c *CAdvisorMetric) Merge(src *CAdvisorMetric) { } } -func newFloat64RateCalculator() awsmetrics.MetricCalculator { +func NewFloat64RateCalculator() awsmetrics.MetricCalculator { return awsmetrics.NewMetricCalculator(func(prev *awsmetrics.MetricValue, val any, timestamp time.Time) (any, bool) { if prev != nil { deltaNs := timestamp.Sub(prev.Timestamp) @@ -127,7 +127,7 @@ func newFloat64RateCalculator() awsmetrics.MetricCalculator { }) } -func assignRateValueToField(rateCalculator *awsmetrics.MetricCalculator, fields map[string]any, metricName string, +func AssignRateValueToField(rateCalculator *awsmetrics.MetricCalculator, fields map[string]any, metricName string, cinfoName string, curVal any, curTime time.Time, multiplier float64) { mKey := awsmetrics.NewKey(cinfoName+metricName, nil) if val, ok := rateCalculator.Calculate(mKey, curVal, curTime); ok { diff --git a/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/extractor_helpers_test.go b/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/extractorhelpers.go similarity index 100% rename from receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/extractor_helpers_test.go rename to receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/extractorhelpers.go diff --git a/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/fs_extractor.go b/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/fs_extractor.go index 6e2f888b461b..093ff0d4b1cf 100644 --- a/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/fs_extractor.go +++ b/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/fs_extractor.go @@ -33,7 +33,7 @@ func (f *FileSystemMetricExtractor) GetValue(info *cinfo.ContainerInfo, _ CPUMem metrics := make([]*CAdvisorMetric, 0, len(stats.Filesystem)) for _, v := range stats.Filesystem { - metric := newCadvisorMetric(containerType, f.logger) + metric := NewCadvisorMetric(containerType, f.logger) if v.Device == "" { continue } diff --git a/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/mem_extractor.go b/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/mem_extractor.go index fe7750c39093..bda4e8dce905 100644 --- a/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/mem_extractor.go +++ b/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/mem_extractor.go @@ -28,7 +28,7 @@ func (m *MemMetricExtractor) GetValue(info *cinfo.ContainerInfo, mInfo CPUMemInf return metrics } - metric := newCadvisorMetric(containerType, m.logger) + metric := NewCadvisorMetric(containerType, m.logger) metric.cgroupPath = info.Name curStats := GetStats(info) @@ -42,16 +42,16 @@ func (m *MemMetricExtractor) GetValue(info *cinfo.ContainerInfo, mInfo CPUMemInf metric.fields[ci.MetricName(containerType, ci.MemWorkingset)] = curStats.Memory.WorkingSet multiplier := float64(time.Second) - assignRateValueToField(&m.rateCalculator, metric.fields, ci.MetricName(containerType, ci.MemPgfault), info.Name, + AssignRateValueToField(&m.rateCalculator, metric.fields, ci.MetricName(containerType, ci.MemPgfault), info.Name, float64(curStats.Memory.ContainerData.Pgfault), curStats.Timestamp, multiplier) - assignRateValueToField(&m.rateCalculator, metric.fields, ci.MetricName(containerType, ci.MemPgmajfault), info.Name, + AssignRateValueToField(&m.rateCalculator, metric.fields, ci.MetricName(containerType, ci.MemPgmajfault), info.Name, float64(curStats.Memory.ContainerData.Pgmajfault), curStats.Timestamp, multiplier) - assignRateValueToField(&m.rateCalculator, metric.fields, ci.MetricName(containerType, ci.MemHierarchicalPgfault), info.Name, + AssignRateValueToField(&m.rateCalculator, metric.fields, ci.MetricName(containerType, ci.MemHierarchicalPgfault), info.Name, float64(curStats.Memory.HierarchicalData.Pgfault), curStats.Timestamp, multiplier) - assignRateValueToField(&m.rateCalculator, metric.fields, ci.MetricName(containerType, ci.MemHierarchicalPgmajfault), info.Name, + AssignRateValueToField(&m.rateCalculator, metric.fields, ci.MetricName(containerType, ci.MemHierarchicalPgmajfault), info.Name, float64(curStats.Memory.HierarchicalData.Pgmajfault), curStats.Timestamp, multiplier) memoryFailuresTotal := curStats.Memory.ContainerData.Pgfault + curStats.Memory.ContainerData.Pgmajfault - assignRateValueToField(&m.rateCalculator, metric.fields, ci.MetricName(containerType, ci.MemFailuresTotal), info.Name, + AssignRateValueToField(&m.rateCalculator, metric.fields, ci.MetricName(containerType, ci.MemFailuresTotal), info.Name, float64(memoryFailuresTotal), curStats.Timestamp, multiplier) memoryCapacity := mInfo.GetMemoryCapacity() @@ -74,6 +74,6 @@ func (m *MemMetricExtractor) Shutdown() error { func NewMemMetricExtractor(logger *zap.Logger) *MemMetricExtractor { return &MemMetricExtractor{ logger: logger, - rateCalculator: newFloat64RateCalculator(), + rateCalculator: NewFloat64RateCalculator(), } } diff --git a/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/net_extractor.go b/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/net_extractor.go index 3affa24971fd..b2211cfe8a24 100644 --- a/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/net_extractor.go +++ b/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors/net_extractor.go @@ -55,14 +55,14 @@ func (n *NetMetricExtractor) GetValue(info *cinfo.ContainerInfo, _ CPUMemInfoPro infoName := info.Name + containerType + cur.Name // used to identify the network interface multiplier := float64(time.Second) - assignRateValueToField(&n.rateCalculator, netIfceMetric, ci.NetRxBytes, infoName, float64(cur.RxBytes), curStats.Timestamp, multiplier) - assignRateValueToField(&n.rateCalculator, netIfceMetric, ci.NetRxPackets, infoName, float64(cur.RxPackets), curStats.Timestamp, multiplier) - assignRateValueToField(&n.rateCalculator, netIfceMetric, ci.NetRxDropped, infoName, float64(cur.RxDropped), curStats.Timestamp, multiplier) - assignRateValueToField(&n.rateCalculator, netIfceMetric, ci.NetRxErrors, infoName, float64(cur.RxErrors), curStats.Timestamp, multiplier) - assignRateValueToField(&n.rateCalculator, netIfceMetric, ci.NetTxBytes, infoName, float64(cur.TxBytes), curStats.Timestamp, multiplier) - assignRateValueToField(&n.rateCalculator, netIfceMetric, ci.NetTxPackets, infoName, float64(cur.TxPackets), curStats.Timestamp, multiplier) - assignRateValueToField(&n.rateCalculator, netIfceMetric, ci.NetTxDropped, infoName, float64(cur.TxDropped), curStats.Timestamp, multiplier) - assignRateValueToField(&n.rateCalculator, netIfceMetric, ci.NetTxErrors, infoName, float64(cur.TxErrors), curStats.Timestamp, multiplier) + AssignRateValueToField(&n.rateCalculator, netIfceMetric, ci.NetRxBytes, infoName, float64(cur.RxBytes), curStats.Timestamp, multiplier) + AssignRateValueToField(&n.rateCalculator, netIfceMetric, ci.NetRxPackets, infoName, float64(cur.RxPackets), curStats.Timestamp, multiplier) + AssignRateValueToField(&n.rateCalculator, netIfceMetric, ci.NetRxDropped, infoName, float64(cur.RxDropped), curStats.Timestamp, multiplier) + AssignRateValueToField(&n.rateCalculator, netIfceMetric, ci.NetRxErrors, infoName, float64(cur.RxErrors), curStats.Timestamp, multiplier) + AssignRateValueToField(&n.rateCalculator, netIfceMetric, ci.NetTxBytes, infoName, float64(cur.TxBytes), curStats.Timestamp, multiplier) + AssignRateValueToField(&n.rateCalculator, netIfceMetric, ci.NetTxPackets, infoName, float64(cur.TxPackets), curStats.Timestamp, multiplier) + AssignRateValueToField(&n.rateCalculator, netIfceMetric, ci.NetTxDropped, infoName, float64(cur.TxDropped), curStats.Timestamp, multiplier) + AssignRateValueToField(&n.rateCalculator, netIfceMetric, ci.NetTxErrors, infoName, float64(cur.TxErrors), curStats.Timestamp, multiplier) if netIfceMetric[ci.NetRxBytes] != nil && netIfceMetric[ci.NetTxBytes] != nil { netIfceMetric[ci.NetTotalBytes] = netIfceMetric[ci.NetRxBytes].(float64) + netIfceMetric[ci.NetTxBytes].(float64) @@ -70,7 +70,7 @@ func (n *NetMetricExtractor) GetValue(info *cinfo.ContainerInfo, _ CPUMemInfoPro netIfceMetrics[i] = netIfceMetric - metric := newCadvisorMetric(mType, n.logger) + metric := NewCadvisorMetric(mType, n.logger) metric.tags[ci.NetIfce] = cur.Name for k, v := range netIfceMetric { metric.fields[ci.MetricName(mType, k)] = v @@ -81,7 +81,7 @@ func (n *NetMetricExtractor) GetValue(info *cinfo.ContainerInfo, _ CPUMemInfoPro aggregatedFields := ci.SumFields(netIfceMetrics) if len(aggregatedFields) > 0 { - metric := newCadvisorMetric(containerType, n.logger) + metric := NewCadvisorMetric(containerType, n.logger) for k, v := range aggregatedFields { metric.fields[ci.MetricName(containerType, k)] = v } @@ -98,7 +98,7 @@ func (n *NetMetricExtractor) Shutdown() error { func NewNetMetricExtractor(logger *zap.Logger) *NetMetricExtractor { return &NetMetricExtractor{ logger: logger, - rateCalculator: newFloat64RateCalculator(), + rateCalculator: NewFloat64RateCalculator(), } } diff --git a/receiver/awscontainerinsightreceiver/internal/host/ebsvolume.go b/receiver/awscontainerinsightreceiver/internal/host/ebsvolume.go index 23690675c287..a40f14be95a0 100644 --- a/receiver/awscontainerinsightreceiver/internal/host/ebsvolume.go +++ b/receiver/awscontainerinsightreceiver/internal/host/ebsvolume.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "regexp" + "runtime" "strings" "sync" "time" @@ -19,6 +20,8 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" "go.uber.org/zap" + + ci "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight" ) var ebsMountPointRegex = regexp.MustCompile(`kubernetes\.io/aws-ebs/mounts/aws/(.+)/(vol-\w+)$`) @@ -141,6 +144,12 @@ func (e *ebsVolume) addEBSVolumeMapping(zone *string, attachement *ec2.VolumeAtt func (e *ebsVolume) findNvmeBlockNameIfPresent(devName string) string { // for nvme(ssd), there is a symlink from devName to nvme block name, i.e. /dev/xvda -> /dev/nvme0n1 // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nvme-ebs-volumes.html + + // Windows does not support file system eg: /rootfs to get Nvme Block name. + // todo: Implement logic to identify Nvme devices on Windows. Refer https://docs.aws.amazon.com/AWSEC2/latest/WindowsGuide/nvme-ebs-volumes.html#identify-nvme-ebs-device + if runtime.GOOS == ci.OperatingSystemWindows { + return "" + } hasRootFs := true if _, err := e.osLstat(hostProc); os.IsNotExist(err) { hasRootFs = false diff --git a/receiver/awscontainerinsightreceiver/internal/host/nodeCapacity.go b/receiver/awscontainerinsightreceiver/internal/host/nodeCapacity.go index 8c77d4634831..60e3f4771316 100644 --- a/receiver/awscontainerinsightreceiver/internal/host/nodeCapacity.go +++ b/receiver/awscontainerinsightreceiver/internal/host/nodeCapacity.go @@ -6,11 +6,14 @@ package host // import "github.com/open-telemetry/opentelemetry-collector-contri import ( "context" "os" + "runtime" "github.com/shirou/gopsutil/v3/common" "github.com/shirou/gopsutil/v3/cpu" "github.com/shirou/gopsutil/v3/mem" "go.uber.org/zap" + + ci "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight" ) type nodeCapacityProvider interface { @@ -46,11 +49,14 @@ func newNodeCapacity(logger *zap.Logger, options ...nodeCapacityOption) (nodeCap opt(nc) } - if _, err := nc.osLstat(hostProc); os.IsNotExist(err) { - return nil, err + ctx := context.Background() + if runtime.GOOS != ci.OperatingSystemWindows { + if _, err := nc.osLstat(hostProc); os.IsNotExist(err) { + return nil, err + } + envMap := common.EnvMap{common.HostProcEnvKey: hostProc} + ctx = context.WithValue(ctx, common.EnvKey, envMap) } - envMap := common.EnvMap{common.HostProcEnvKey: hostProc} - ctx := context.WithValue(context.Background(), common.EnvKey, envMap) nc.parseCPU(ctx) nc.parseMemory(ctx) @@ -67,6 +73,10 @@ func (nc *nodeCapacity) parseMemory(ctx context.Context) { } func (nc *nodeCapacity) parseCPU(ctx context.Context) { + if runtime.GOOS == ci.OperatingSystemWindows { + nc.parseCPUWindows(ctx) + return + } if cpuInfos, err := nc.cpuInfo(ctx); err == nil { numCores := len(cpuInfos) nc.cpuCapacity = int64(numCores) @@ -76,6 +86,19 @@ func (nc *nodeCapacity) parseCPU(ctx context.Context) { } } +func (nc *nodeCapacity) parseCPUWindows(ctx context.Context) { + if cpuInfos, err := nc.cpuInfo(ctx); err == nil { + var coreCount int32 + for _, cpuInfo := range cpuInfos { + coreCount += cpuInfo.Cores + } + nc.cpuCapacity = int64(coreCount) + } else { + // If any error happen, then there will be no cpu utilization metrics + nc.logger.Error("NodeCapacity cannot get cpuInfo from psUtil", zap.Error(err)) + } +} + func (nc *nodeCapacity) getNumCores() int64 { return nc.cpuCapacity } diff --git a/receiver/awscontainerinsightreceiver/internal/host/nodeCapacity_test.go b/receiver/awscontainerinsightreceiver/internal/host/nodeCapacity_test.go index fc903775f0f1..3f4abc99906b 100644 --- a/receiver/awscontainerinsightreceiver/internal/host/nodeCapacity_test.go +++ b/receiver/awscontainerinsightreceiver/internal/host/nodeCapacity_test.go @@ -66,8 +66,8 @@ func TestNodeCapacity(t *testing.T) { cpuInfoOption = func(nc *nodeCapacity) { nc.cpuInfo = func(ctx context.Context) ([]cpu.InfoStat, error) { return []cpu.InfoStat{ - {}, - {}, + {Cores: 1}, + {Cores: 1}, }, nil } } diff --git a/receiver/awscontainerinsightreceiver/internal/host/utilconst_windows.go b/receiver/awscontainerinsightreceiver/internal/host/utilconst_windows.go new file mode 100644 index 000000000000..2bd7285f21ea --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/host/utilconst_windows.go @@ -0,0 +1,14 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//go:build !linux +// +build !linux + +package host // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/host" + +// These variables are invalid for Windows +const ( + rootfs = "" + hostProc = rootfs + "" + hostMounts = hostProc + "" +) diff --git a/receiver/awscontainerinsightreceiver/internal/host/utils.go b/receiver/awscontainerinsightreceiver/internal/host/utils.go index edd4d00833cb..b9e2fed4d854 100644 --- a/receiver/awscontainerinsightreceiver/internal/host/utils.go +++ b/receiver/awscontainerinsightreceiver/internal/host/utils.go @@ -10,12 +10,6 @@ import ( "time" ) -const ( - rootfs = "/rootfs" // the root directory "/" is mounted as "/rootfs" in container - hostProc = rootfs + "/proc" // "/rootfs/proc" in container refers to the host proc directory "/proc" - hostMounts = hostProc + "/mounts" // "/rootfs/proc/mounts" in container refers to "/proc/mounts" in the host -) - func hostJitter(max time.Duration) time.Duration { hostName, err := os.Hostname() if err != nil { diff --git a/receiver/awscontainerinsightreceiver/internal/host/utilsconst.go b/receiver/awscontainerinsightreceiver/internal/host/utilsconst.go new file mode 100644 index 000000000000..adb162d5d1ff --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/host/utilsconst.go @@ -0,0 +1,13 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//go:build !windows +// +build !windows + +package host // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/host" + +const ( + rootfs = "/rootfs" // the root directory "/" is mounted as "/rootfs" in container + hostProc = rootfs + "/proc" // "/rootfs/proc" in container refers to the host proc directory "/proc" + hostMounts = hostProc + "/mounts" // "/rootfs/proc/mounts" in container refers to "/proc/mounts" in the host +) diff --git a/receiver/awscontainerinsightreceiver/internal/k8swindows/README.md b/receiver/awscontainerinsightreceiver/internal/k8swindows/README.md new file mode 100644 index 000000000000..9d6169c96952 --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/k8swindows/README.md @@ -0,0 +1,253 @@ +## Available Metrics and Resource Attributes for Windows worker Nodes + +### Node +| Metric | Unit | +|-------------------------------------------|--------------| +| node_cpu_limit | Millicore | +| node_cpu_request | Millicore | +| node_cpu_reserved_capacity | Percent | +| node_cpu_usage_total | Millicore | +| node_cpu_utilization | Percent | +| node_memory_limit | Bytes | +| node_memory_pgfault | Count/Second | +| node_memory_pgmajfault | Count/Second | +| node_memory_request | Bytes | +| node_memory_reserved_capacity | Percent | +| node_memory_rss | Bytes | +| node_memory_usage | Bytes | +| node_memory_utilization | Percent | +| node_memory_working_set | Bytes | +| node_network_rx_bytes | Bytes/Second | +| node_network_rx_dropped | Count/Second | +| node_network_rx_errors | Count/Second | +| node_network_total_bytes | Bytes/Second | +| node_network_tx_bytes | Bytes/Second | +| node_network_tx_dropped | Count/Second | +| node_network_tx_errors | Count/Second | +| node_number_of_running_containers | Count | +| node_number_of_running_pods | Count | +| node_status_condition_ready | Count | +| node_status_condition_pid_pressure | Count | +| node_status_condition_memory_pressure | Count | +| node_status_condition_disk_pressure | Count | +| node_status_condition_network_unavailable | Count | +| node_status_condition_unknown | Count | +| node_status_capacity_pods | Count | +| node_status_allocatable_pods | Count | + +

+| Resource Attribute | +|-----------------------| +| ClusterName | +| InstanceType | +| NodeName | +| Timestamp | +| Type | +| Version | +| Sources | +| kubernetes | +| OperatingSystem | + +

+

+ +### Node Filesystem +| Metric | Unit | +|------------------------------|---------| +| node_filesystem_available | Bytes | +| node_filesystem_capacity | Bytes | +| node_filesystem_usage | Bytes | +| node_filesystem_utilization | Percent | + +

+| Resource Attribute | +|---------------------- | +| AutoScalingGroupName | +| ClusterName | +| InstanceId | +| InstanceType | +| NodeName | +| Timestamp | +| Type | +| Version | +| Sources | +| kubernete | +| OperatingSystem | +

+

+ +### Node Network +| Metric | Unit | +|------------------------------------|--------------| +| node_interface_network_rx_bytes | Bytes/Second | +| node_interface_network_rx_dropped | Count/Second | +| node_interface_network_rx_errors | Count/Second | +| node_interface_network_total_bytes | Bytes/Second | +| node_interface_network_tx_bytes | Bytes/Second | +| node_interface_network_tx_dropped | Count/Second | +| node_interface_network_tx_errors | Count/Second | + +

+| Resource Attribute | +|-----------------------| +| AutoScalingGroupName | +| ClusterName | +| InstanceId | +| InstanceType | +| NodeName | +| Timestamp | +| Type | +| Version | +| interface | +| Sources | +| kubernete | +| OperatingSystem | +

+

+ +### Pod +| Metric | Unit | +|-------------------------------------------------------------------|--------------| +| pod_cpu_limit | Millicore | +| pod_cpu_request | Millicore | +| pod_cpu_reserved_capacity | Percent | +| pod_cpu_usage_total | Millicore | +| pod_cpu_utilization | Percent | +| pod_cpu_utilization_over_pod_limit | Percent | +| pod_memory_limit | Bytes | +| pod_memory_max_usage | Bytes | +| pod_memory_pgfault | Count/Second | +| pod_memory_pgmajfault | Count/Second | +| pod_memory_request | Bytes | +| pod_memory_reserved_capacity | Percent | +| pod_memory_rss | Bytes | +| pod_memory_usage | Bytes | +| pod_memory_utilization | Percent | +| pod_memory_utilization_over_pod_limit | Percent | +| pod_memory_working_set | Bytes | +| pod_network_rx_bytes | Bytes/Second | +| pod_network_rx_dropped | Count/Second | +| pod_network_rx_errors | Count/Second | +| pod_network_total_bytes | Bytes/Second | +| pod_network_tx_bytes | Bytes/Second | +| pod_network_tx_dropped | Count/Second | +| pod_network_tx_errors | Count/Second | +| pod_number_of_container_restarts | Count | +| pod_number_of_containers | Count | +| pod_number_of_running_containers | Count | +| pod_status_ready | Count | +| pod_status_scheduled | Count | +| pod_status_unknown | Count | +| pod_status_failed | Count | +| pod_status_pending | Count | +| pod_status_running | Count | +| pod_status_succeeded | Count | +| pod_container_status_running | Count | +| pod_container_status_terminated | Count | +| pod_container_status_waiting | Count | +| pod_container_status_waiting_reason_crash_loop_back_off | Count | +| pod_container_status_waiting_reason_image_pull_error | Count | +| pod_container_status_waiting_reason_start_error | Count | +| pod_container_status_waiting_reason_create_container_error | Count | +| pod_container_status_waiting_reason_create_container_config_error | Count | +| pod_container_status_terminated_reason_oom_killed | Count | + +| Resource Attribute | +|------------------------| +| AutoScalingGroupName | +| ClusterName | +| InstanceId | +| InstanceType | +| K8sPodName | +| Namespace | +| NodeName | +| PodId | +| Timestamp | +| Type | +| Version | +| Sources | +| kubernete | +| pod_status | +| OperatingSystem | + +

+ +### Pod Network +| Metric | Unit | +|-----------------------------------|--------------| +| pod_interface_network_rx_bytes | Bytes/Second | +| pod_interface_network_rx_dropped | Count/Second | +| pod_interface_network_rx_errors | Count/Second | +| pod_interface_network_total_bytes | Bytes/Second | +| pod_interface_network_tx_bytes | Bytes/Second | +| pod_interface_network_tx_dropped | Count/Second | +| pod_interface_network_tx_errors | Count/Second | + + +

+| Resource Attribute | +|----------------------| +| AutoScalingGroupName | +| ClusterName | +| InstanceId | +| InstanceType | +| K8sPodName | +| Namespace | +| NodeName | +| PodId | +| Timestamp | +| Type | +| Version | +| interface | +| Sources | +| kubernete | +| pod_status | +| OperatingSystem | + +

+

+ + +### Container +| Metric | Unit | +|---------------------------------------------------|--------------| +| container_cpu_limit | Millicore | +| container_cpu_request | Millicore | +| container_cpu_usage_total | Millicore | +| container_cpu_utilization | Percent | +| container_cpu_utilization_over_container_limit | Percent | +| container_memory_limit | Bytes | +| container_memory_mapped_file | Bytes | +| container_memory_pgfault | Count/Second | +| container_memory_pgmajfault | Count/Second | +| container_memory_request | Bytes | +| container_memory_rss | Bytes | +| container_memory_usage | Bytes | +| container_memory_utilization | Percent | +| container_memory_utilization_over_container_limit | Percent | +| container_memory_working_set | Bytes | +| number_of_container_restarts | Count | + +

+ +| Resource Attribute | +|-----------------------------------| +| AutoScalingGroupName | +| ClusterName | +| ContainerId | +| ContainerName | +| InstanceId | +| InstanceType | +| K8sPodName | +| Namespace | +| NodeName | +| PodId | +| Timestamp | +| Type | +| Version | +| Sources | +| kubernetes | +| container_status | +| container_status_reason | +| container_last_termination_reason | +| OperatingSystem | diff --git a/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/cpu_extractor.go b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/cpu_extractor.go new file mode 100644 index 000000000000..09b00c540b5f --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/cpu_extractor.go @@ -0,0 +1,60 @@ +package extractors + +import ( + ci "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight" + awsmetrics "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/metrics" + cExtractor "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors" + + "go.uber.org/zap" +) + +const ( + decimalToMillicores = 1000 +) + +type CPUMetricExtractor struct { + logger *zap.Logger + rateCalculator awsmetrics.MetricCalculator +} + +func (c *CPUMetricExtractor) HasValue(rawMetric RawMetric) bool { + if !rawMetric.Time.IsZero() { + return true + } + return false +} + +func (c *CPUMetricExtractor) GetValue(rawMetric RawMetric, mInfo cExtractor.CPUMemInfoProvider, containerType string) []*cExtractor.CAdvisorMetric { + var metrics []*cExtractor.CAdvisorMetric + + metric := cExtractor.NewCadvisorMetric(containerType, c.logger) + + multiplier := float64(decimalToMillicores) + identifier := rawMetric.Id + cExtractor.AssignRateValueToField(&c.rateCalculator, metric.GetFields(), ci.MetricName(containerType, ci.CPUTotal), identifier, float64(rawMetric.CPUStats.UsageCoreNanoSeconds), rawMetric.Time, multiplier) + cExtractor.AssignRateValueToField(&c.rateCalculator, metric.GetFields(), ci.MetricName(containerType, ci.CPUUser), identifier, float64(rawMetric.CPUStats.UserCPUUsage), rawMetric.Time, multiplier) + cExtractor.AssignRateValueToField(&c.rateCalculator, metric.GetFields(), ci.MetricName(containerType, ci.CPUSystem), identifier, float64(rawMetric.CPUStats.SystemCPUUsage), rawMetric.Time, multiplier) + + numCores := mInfo.GetNumCores() + if metric.GetField(ci.MetricName(containerType, ci.CPUTotal)) != nil && numCores != 0 { + metric.AddField(ci.MetricName(containerType, ci.CPUUtilization), metric.GetField(ci.MetricName(containerType, ci.CPUTotal)).(float64)/float64(numCores*decimalToMillicores)*100) + } + + if containerType == ci.TypeNode { + metric.AddField(ci.MetricName(containerType, ci.CPULimit), numCores*decimalToMillicores) + } + + metrics = append(metrics, metric) + return metrics +} + +func (c *CPUMetricExtractor) Shutdown() error { + return c.rateCalculator.Shutdown() +} + +func NewCPUMetricExtractor(logger *zap.Logger) *CPUMetricExtractor { + return &CPUMetricExtractor{ + logger: logger, + rateCalculator: cExtractor.NewFloat64RateCalculator(), + } +} diff --git a/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/cpu_extractor_test.go b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/cpu_extractor_test.go new file mode 100644 index 000000000000..b172348ac9ab --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/cpu_extractor_test.go @@ -0,0 +1,58 @@ +package extractors + +import ( + "testing" + + "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight" + cExtractor "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors" + cTestUtils "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/cadvisor/testutils" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/k8swindows/testutils" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +func TestCPUStats(t *testing.T) { + MockCPUMemInfo := cTestUtils.MockCPUMemInfo{} + + result := testutils.LoadKubeletSummary(t, "./testdata/PreSingleKubeletSummary.json") + result2 := testutils.LoadKubeletSummary(t, "./testdata/CurSingleKubeletSummary.json") + + podRawMetric := ConvertPodToRaw(result.Pods[0]) + podRawMetric2 := ConvertPodToRaw(result2.Pods[0]) + + // test container type + containerType := containerinsight.TypePod + extractor := NewCPUMetricExtractor(&zap.Logger{}) + + var cMetrics []*cExtractor.CAdvisorMetric + if extractor.HasValue(podRawMetric) { + cMetrics = extractor.GetValue(podRawMetric, MockCPUMemInfo, containerType) + } + if extractor.HasValue(podRawMetric2) { + cMetrics = extractor.GetValue(podRawMetric2, MockCPUMemInfo, containerType) + } + + cExtractor.AssertContainsTaggedFloat(t, cMetrics[0], "pod_cpu_usage_total", 3.125000, 0) + cExtractor.AssertContainsTaggedFloat(t, cMetrics[0], "pod_cpu_utilization", 0.156250, 0) + require.NoError(t, extractor.Shutdown()) + + // test node type + containerType = containerinsight.TypeNode + extractor = NewCPUMetricExtractor(nil) + + nodeRawMetric := ConvertNodeToRaw(result.Node) + nodeRawMetric2 := ConvertNodeToRaw(result2.Node) + if extractor.HasValue(nodeRawMetric) { + cMetrics = extractor.GetValue(nodeRawMetric, MockCPUMemInfo, containerType) + } + if extractor.HasValue(nodeRawMetric2) { + cMetrics = extractor.GetValue(nodeRawMetric2, MockCPUMemInfo, containerType) + } + + cExtractor.AssertContainsTaggedFloat(t, cMetrics[0], "node_cpu_usage_total", 51.5, 0.5) + cExtractor.AssertContainsTaggedFloat(t, cMetrics[0], "node_cpu_utilization", 2.5, 0.5) + cExtractor.AssertContainsTaggedInt(t, cMetrics[0], "node_cpu_limit", 2000) + + require.NoError(t, extractor.Shutdown()) +} diff --git a/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/extractor.go b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/extractor.go new file mode 100644 index 000000000000..4ebfa6cf7b1c --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/extractor.go @@ -0,0 +1,91 @@ +package extractors + +import ( + "time" + + cExtractor "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors" + + "github.com/Microsoft/hcsshim" +) + +// CPUStat for Pod, Container and Node. +type CPUStat struct { + Time time.Time + UsageNanoCores uint64 + UsageCoreNanoSeconds uint64 + UserCPUUsage uint64 + SystemCPUUsage uint64 +} + +// MemoryStat for Pod, Container and Node +type MemoryStat struct { + Time time.Time + AvailableBytes uint64 + UsageBytes uint64 + WorkingSetBytes uint64 + RSSBytes uint64 + PageFaults uint64 + MajorPageFaults uint64 +} + +// FileSystemStat for Container and Node. +type FileSystemStat struct { + Time time.Time + Name string + Device string + Type string + AvailableBytes uint64 + CapacityBytes uint64 + UsedBytes uint64 +} + +// NetworkStat for Pod and Node. +type NetworkStat struct { + Time time.Time + Name string + RxBytes uint64 + RxErrors uint64 + TxBytes uint64 + TxErrors uint64 + DroppedIncoming uint64 + DroppedOutgoing uint64 +} + +// HCSNetworkStat Network Stat from HCS. +type HCSNetworkStat struct { + Name string + BytesReceived uint64 + BytesSent uint64 + DroppedPacketsIncoming uint64 + DroppedPacketsOutgoing uint64 +} + +// HCSStat Stats from HCS. +type HCSStat struct { + Time time.Time + Id string + Name string + + CPU *hcsshim.ProcessorStats + + Network *[]HCSNetworkStat +} + +// RawMetric Represent Container, Pod, Node Metric Extractors. +// Kubelet summary and HNS stats will be converted to Raw Metric for parsing by Extractors. +type RawMetric struct { + Id string + Name string + Namespace string + Time time.Time + CPUStats CPUStat + MemoryStats MemoryStat + FileSystemStats []FileSystemStat + NetworkStats []NetworkStat +} + +type MetricExtractor interface { + HasValue(summary RawMetric) bool + GetValue(summary RawMetric, mInfo cExtractor.CPUMemInfoProvider, containerType string) []*cExtractor.CAdvisorMetric + Shutdown() error +} diff --git a/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/extractorhelpers.go b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/extractorhelpers.go new file mode 100644 index 000000000000..df888ca3e97c --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/extractorhelpers.go @@ -0,0 +1,227 @@ +package extractors + +import ( + "fmt" + + stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1" +) + +// convertCPUStats Convert kubelet CPU stats to Raw CPU stats +func convertCPUStats(kubeletCPUStat stats.CPUStats) CPUStat { + var cpuStat CPUStat + + cpuStat.Time = kubeletCPUStat.Time.Time + + if kubeletCPUStat.UsageCoreNanoSeconds != nil { + cpuStat.UsageCoreNanoSeconds = *kubeletCPUStat.UsageCoreNanoSeconds + } + if kubeletCPUStat.UsageNanoCores != nil { + cpuStat.UsageNanoCores = *kubeletCPUStat.UsageNanoCores + } + + return cpuStat +} + +// convertFileSystemStats Convert kubelet file system stats to Raw memory stats +func convertFileSystemStats(kubeletFSstat stats.FsStats) FileSystemStat { + var fsstat FileSystemStat + + fsstat.Time = kubeletFSstat.Time.Time + + if kubeletFSstat.UsedBytes != nil { + fsstat.UsedBytes = *kubeletFSstat.UsedBytes + } + + if kubeletFSstat.AvailableBytes != nil { + fsstat.AvailableBytes = *kubeletFSstat.AvailableBytes + } + + if kubeletFSstat.CapacityBytes != nil { + fsstat.CapacityBytes = *kubeletFSstat.CapacityBytes + } + + return fsstat +} + +// convertMemoryStats Convert kubelet memory stats to Raw memory stats +func convertMemoryStats(kubeletMemoryStat stats.MemoryStats) MemoryStat { + var memoryStat MemoryStat + + memoryStat.Time = kubeletMemoryStat.Time.Time + + if kubeletMemoryStat.UsageBytes != nil { + memoryStat.UsageBytes = *kubeletMemoryStat.UsageBytes + } + if kubeletMemoryStat.AvailableBytes != nil { + memoryStat.AvailableBytes = *kubeletMemoryStat.AvailableBytes + } + if kubeletMemoryStat.WorkingSetBytes != nil { + memoryStat.WorkingSetBytes = *kubeletMemoryStat.WorkingSetBytes + } + if kubeletMemoryStat.RSSBytes != nil { + memoryStat.RSSBytes = *kubeletMemoryStat.RSSBytes + } + if kubeletMemoryStat.PageFaults != nil { + memoryStat.PageFaults = *kubeletMemoryStat.PageFaults + } + if kubeletMemoryStat.MajorPageFaults != nil { + memoryStat.MajorPageFaults = *kubeletMemoryStat.MajorPageFaults + } + return memoryStat +} + +// convertNetworkStats Convert kubelet network system stats to Raw memory stats +func convertNetworkStats(kubeletNetworkStat stats.NetworkStats, kubeletIntfStat stats.InterfaceStats) NetworkStat { + var networkstat NetworkStat + + networkstat.Time = kubeletNetworkStat.Time.Time + + networkstat.Name = kubeletIntfStat.Name + + if kubeletIntfStat.TxBytes != nil { + networkstat.TxBytes = *kubeletIntfStat.TxBytes + } + + if kubeletIntfStat.TxErrors != nil { + networkstat.TxErrors = *kubeletIntfStat.TxErrors + } + + if kubeletIntfStat.RxBytes != nil { + networkstat.RxBytes = *kubeletIntfStat.RxBytes + } + + if kubeletIntfStat.RxErrors != nil { + networkstat.RxErrors = *kubeletIntfStat.RxErrors + } + + return networkstat +} + +// convertHCSNetworkStats Convert HCS network system stats to Raw network stats +func convertHCSNetworkStats(stat HCSStat, networkStat HCSNetworkStat) NetworkStat { + var networkstat NetworkStat + + networkstat.Time = stat.Time + + networkstat.Name = networkStat.Name + networkstat.TxBytes = networkStat.BytesSent + networkstat.RxBytes = networkStat.BytesReceived + networkstat.DroppedIncoming = networkStat.DroppedPacketsIncoming + networkstat.DroppedOutgoing = networkStat.DroppedPacketsOutgoing + + return networkstat +} + +// ConvertPodToRaw Converts Kubelet Pod stats to RawMetric. +func ConvertPodToRaw(podStat stats.PodStats) RawMetric { + var rawMetic RawMetric + + rawMetic.Id = podStat.PodRef.UID + rawMetic.Name = podStat.PodRef.Name + rawMetic.Namespace = podStat.PodRef.Namespace + + if podStat.CPU != nil { + rawMetic.Time = podStat.CPU.Time.Time + rawMetic.CPUStats = convertCPUStats(*podStat.CPU) + } + + if podStat.Memory != nil { + if rawMetic.Time.IsZero() { + rawMetic.Time = podStat.Memory.Time.Time + } + rawMetic.MemoryStats = convertMemoryStats(*podStat.Memory) + } + + if podStat.Network != nil { + for _, intfStats := range podStat.Network.Interfaces { + rawMetic.NetworkStats = append(rawMetic.NetworkStats, convertNetworkStats(*podStat.Network, intfStats)) + } + } + + return rawMetic +} + +// ConvertContainerToRaw Converts Kubelet Container stats per Pod to RawMetric. +func ConvertContainerToRaw(containerStat stats.ContainerStats, podStat stats.PodStats) RawMetric { + var rawMetic RawMetric + + rawMetic.Id = fmt.Sprintf("%s-%s", podStat.PodRef.UID, containerStat.Name) + rawMetic.Name = containerStat.Name + rawMetic.Namespace = podStat.PodRef.Namespace + + if containerStat.CPU != nil { + rawMetic.Time = containerStat.CPU.Time.Time + rawMetic.CPUStats = convertCPUStats(*containerStat.CPU) + } + + if containerStat.Memory != nil { + if rawMetic.Time.IsZero() { + rawMetic.Time = containerStat.Memory.Time.Time + } + rawMetic.MemoryStats = convertMemoryStats(*containerStat.Memory) + } + + rawMetic.FileSystemStats = []FileSystemStat{} + if containerStat.Rootfs != nil { + fStat := convertFileSystemStats(*containerStat.Rootfs) + fStat.Name = "rootfs" + rawMetic.FileSystemStats = append(rawMetic.FileSystemStats, fStat) + } + if containerStat.Logs != nil { + fStat := convertFileSystemStats(*containerStat.Logs) + fStat.Name = "logfs" + rawMetic.FileSystemStats = append(rawMetic.FileSystemStats, fStat) + } + + return rawMetic +} + +// ConvertNodeToRaw Converts Kubelet Node stats to RawMetric. +func ConvertNodeToRaw(nodeStat stats.NodeStats) RawMetric { + var rawMetic RawMetric + + rawMetic.Id = nodeStat.NodeName + rawMetic.Name = nodeStat.NodeName + + if nodeStat.CPU != nil { + rawMetic.Time = nodeStat.CPU.Time.Time + rawMetic.CPUStats = convertCPUStats(*nodeStat.CPU) + } + + if nodeStat.Memory != nil { + if rawMetic.Time.IsZero() { + rawMetic.Time = nodeStat.Memory.Time.Time + } + rawMetic.MemoryStats = convertMemoryStats(*nodeStat.Memory) + } + + rawMetic.FileSystemStats = []FileSystemStat{} + if nodeStat.Fs != nil { + rawMetic.FileSystemStats = append(rawMetic.FileSystemStats, convertFileSystemStats(*nodeStat.Fs)) + } + + if nodeStat.Network != nil { + for _, intfStats := range nodeStat.Network.Interfaces { + rawMetic.NetworkStats = append(rawMetic.NetworkStats, convertNetworkStats(*nodeStat.Network, intfStats)) + } + } + + return rawMetic +} + +// ConvertHCSContainerToRaw Converts HCS Container stats to RawMetric. +func ConvertHCSContainerToRaw(containerStat HCSStat) RawMetric { + var rawMetic RawMetric + + rawMetic.Id = containerStat.Id + rawMetic.Name = containerStat.Name + rawMetic.Time = containerStat.Time + + if containerStat.Network != nil { + for _, val := range *containerStat.Network { + rawMetic.NetworkStats = append(rawMetic.NetworkStats, convertHCSNetworkStats(containerStat, val)) + } + } + + return rawMetic +} diff --git a/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/extractorhelpers_test.go b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/extractorhelpers_test.go new file mode 100644 index 000000000000..c86a2d1c9116 --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/extractorhelpers_test.go @@ -0,0 +1,75 @@ +package extractors + +import ( + "testing" + "time" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/k8swindows/testutils" + + "github.com/stretchr/testify/assert" +) + +func TestConvertPodToRaw(t *testing.T) { + + result := testutils.LoadKubeletSummary(t, "./testdata/PreSingleKubeletSummary.json") + + podRawMetric := ConvertPodToRaw(result.Pods[0]) + + assert.Equal(t, podRawMetric.Id, "01bfbe59-2925-4ad5-a8d3-a1b23e3ddd74") + assert.Equal(t, podRawMetric.Name, "windows-server-iis-ltsc2019-58d94b5844-6v2pg") + assert.Equal(t, podRawMetric.Namespace, "amazon-cloudwatch") + parsedtime, _ := time.Parse(time.RFC3339, "2023-12-21T15:19:59Z") + assert.Equal(t, podRawMetric.Time, parsedtime.Local()) + assert.Equal(t, podRawMetric.CPUStats.UsageCoreNanoSeconds, uint64(289625000000)) + assert.Equal(t, podRawMetric.CPUStats.UsageNanoCores, uint64(0)) + + assert.Equal(t, podRawMetric.MemoryStats.UsageBytes, uint64(0)) + assert.Equal(t, podRawMetric.MemoryStats.AvailableBytes, uint64(0)) + assert.Equal(t, podRawMetric.MemoryStats.WorkingSetBytes, uint64(208949248)) + assert.Equal(t, podRawMetric.MemoryStats.RSSBytes, uint64(0)) + assert.Equal(t, podRawMetric.MemoryStats.PageFaults, uint64(0)) + assert.Equal(t, podRawMetric.MemoryStats.MajorPageFaults, uint64(0)) +} + +func TestConvertContainerToRaw(t *testing.T) { + result := testutils.LoadKubeletSummary(t, "./testdata/PreSingleKubeletSummary.json") + + containerRawMetric := ConvertContainerToRaw(result.Pods[0].Containers[0], result.Pods[0]) + + assert.Equal(t, containerRawMetric.Id, "01bfbe59-2925-4ad5-a8d3-a1b23e3ddd74-windows-server-iis-ltsc2019") + assert.Equal(t, containerRawMetric.Name, "windows-server-iis-ltsc2019") + assert.Equal(t, containerRawMetric.Namespace, "amazon-cloudwatch") + parsedtime, _ := time.Parse(time.RFC3339, "2023-12-21T15:19:59Z") + assert.Equal(t, containerRawMetric.Time, parsedtime.Local()) + assert.Equal(t, containerRawMetric.CPUStats.UsageCoreNanoSeconds, uint64(289625000000)) + assert.Equal(t, containerRawMetric.CPUStats.UsageNanoCores, uint64(0)) + + assert.Equal(t, containerRawMetric.MemoryStats.UsageBytes, uint64(0)) + assert.Equal(t, containerRawMetric.MemoryStats.AvailableBytes, uint64(0)) + assert.Equal(t, containerRawMetric.MemoryStats.WorkingSetBytes, uint64(208949248)) + assert.Equal(t, containerRawMetric.MemoryStats.RSSBytes, uint64(0)) + assert.Equal(t, containerRawMetric.MemoryStats.PageFaults, uint64(0)) + assert.Equal(t, containerRawMetric.MemoryStats.MajorPageFaults, uint64(0)) +} + +func TestConvertNodeToRaw(t *testing.T) { + + result := testutils.LoadKubeletSummary(t, "./testdata/PreSingleKubeletSummary.json") + + nodeRawMetric := ConvertNodeToRaw(result.Node) + + assert.Equal(t, nodeRawMetric.Id, "ip-192-168-44-84.us-west-2.compute.internal") + assert.Equal(t, nodeRawMetric.Name, "ip-192-168-44-84.us-west-2.compute.internal") + assert.Equal(t, nodeRawMetric.Namespace, "") + parsedtime, _ := time.Parse(time.RFC3339, "2023-12-21T15:19:58Z") + assert.Equal(t, nodeRawMetric.Time, parsedtime.Local()) + assert.Equal(t, nodeRawMetric.CPUStats.UsageCoreNanoSeconds, uint64(38907680000000)) + assert.Equal(t, nodeRawMetric.CPUStats.UsageNanoCores, uint64(20000000)) + + assert.Equal(t, nodeRawMetric.MemoryStats.UsageBytes, uint64(3583389696)) + assert.Equal(t, nodeRawMetric.MemoryStats.AvailableBytes, uint64(7234662400)) + assert.Equal(t, nodeRawMetric.MemoryStats.WorkingSetBytes, uint64(1040203776)) + assert.Equal(t, nodeRawMetric.MemoryStats.RSSBytes, uint64(0)) + assert.Equal(t, nodeRawMetric.MemoryStats.PageFaults, uint64(0)) + assert.Equal(t, nodeRawMetric.MemoryStats.MajorPageFaults, uint64(0)) +} diff --git a/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/fs_extractor.go b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/fs_extractor.go new file mode 100644 index 000000000000..b77c90ff6db6 --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/fs_extractor.go @@ -0,0 +1,75 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package extractors // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors" + +import ( + ci "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight" + awsmetrics "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/metrics" + cExtractor "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors" + + "go.uber.org/zap" +) + +type FileSystemMetricExtractor struct { + logger *zap.Logger + rateCalculator awsmetrics.MetricCalculator +} + +func (f *FileSystemMetricExtractor) HasValue(rawMetric RawMetric) bool { + if !rawMetric.Time.IsZero() { + return true + } + return false +} + +func (f *FileSystemMetricExtractor) GetValue(rawMetric RawMetric, _ cExtractor.CPUMemInfoProvider, containerType string) []*cExtractor.CAdvisorMetric { + if containerType == ci.TypePod { + return nil + } + + containerType = getFSMetricType(containerType, f.logger) + metrics := make([]*cExtractor.CAdvisorMetric, 0, len(rawMetric.FileSystemStats)) + + for _, v := range rawMetric.FileSystemStats { + metric := cExtractor.NewCadvisorMetric(containerType, f.logger) + + metric.AddTag(ci.DiskDev, v.Device) + metric.AddTag(ci.FSType, v.Type) + + metric.AddField(ci.MetricName(containerType, ci.FSUsage), v.UsedBytes) + metric.AddField(ci.MetricName(containerType, ci.FSCapacity), v.CapacityBytes) + metric.AddField(ci.MetricName(containerType, ci.FSAvailable), v.AvailableBytes) + + if v.CapacityBytes != 0 { + metric.AddField(ci.MetricName(containerType, ci.FSUtilization), float64(v.UsedBytes)/float64(v.CapacityBytes)*100) + } + + metrics = append(metrics, metric) + } + return metrics +} + +func (f *FileSystemMetricExtractor) Shutdown() error { + return f.rateCalculator.Shutdown() +} + +func NewFileSystemMetricExtractor(logger *zap.Logger) *FileSystemMetricExtractor { + return &FileSystemMetricExtractor{ + logger: logger, + rateCalculator: cExtractor.NewFloat64RateCalculator(), + } +} + +func getFSMetricType(containerType string, logger *zap.Logger) string { + metricType := "" + switch containerType { + case ci.TypeNode: + metricType = ci.TypeNodeFS + case ci.TypeContainer: + metricType = ci.TypeContainerFS + default: + logger.Warn("fs_extractor: fs metric extractor is parsing unexpected containerType", zap.String("containerType", containerType)) + } + return metricType +} diff --git a/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/fs_extractor_test.go b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/fs_extractor_test.go new file mode 100644 index 000000000000..8ea57e92d8c0 --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/fs_extractor_test.go @@ -0,0 +1,89 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package extractors + +import ( + "fmt" + "testing" + + "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight" + cExtractor "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/k8swindows/testutils" + + "github.com/stretchr/testify/assert" +) + +func TestFSStats(t *testing.T) { + result := testutils.LoadKubeletSummary(t, "./testdata/PreSingleKubeletSummary.json") + + nodeRawMetric := ConvertNodeToRaw(result.Node) + + // node type + containerType := containerinsight.TypeNode + extractor := NewFileSystemMetricExtractor(nil) + + var cMetrics []*cExtractor.CAdvisorMetric + if extractor.HasValue(nodeRawMetric) { + cMetrics = extractor.GetValue(nodeRawMetric, nil, containerType) + } + fmt.Println(len(cMetrics)) + expectedFields := map[string]any{ + "node_filesystem_usage": uint64(34667089920), + "node_filesystem_capacity": uint64(85897244672), + "node_filesystem_available": uint64(51230154752), + "node_filesystem_utilization": float64(40.358791544917224), + } + expectedTags := map[string]string{ + "fstype": "", + "device": "", + "Type": "NodeFS", + } + cExtractor.AssertContainsTaggedField(t, cMetrics[0], expectedFields, expectedTags) + + // pod type + containerType = containerinsight.TypePod + extractor = NewFileSystemMetricExtractor(nil) + podRawMetric := ConvertPodToRaw(result.Pods[0]) + + if extractor.HasValue(podRawMetric) { + cMetrics = extractor.GetValue(podRawMetric, nil, containerType) + } + + assert.Equal(t, len(cMetrics), 0) + + // container type for eks + containerType = containerinsight.TypeContainer + extractor = NewFileSystemMetricExtractor(nil) + containerRawMetric := ConvertContainerToRaw(result.Pods[0].Containers[0], result.Pods[0]) + + if extractor.HasValue(containerRawMetric) { + cMetrics = extractor.GetValue(containerRawMetric, nil, containerType) + } + + expectedFields = map[string]any{ + "container_filesystem_available": uint64(51230154752), + "container_filesystem_capacity": uint64(85897244672), + "container_filesystem_usage": uint64(339738624), + "container_filesystem_utilization": float64(0.3955174875484043), + } + expectedTags = map[string]string{ + "fstype": "", + "device": "", + "Type": "ContainerFS", + } + cExtractor.AssertContainsTaggedField(t, cMetrics[0], expectedFields, expectedTags) + + expectedFields = map[string]any{ + "container_filesystem_available": uint64(51230154752), + "container_filesystem_capacity": uint64(85897244672), + "container_filesystem_usage": uint64(919463), + "container_filesystem_utilization": float64(0.0010704219949207732), + } + expectedTags = map[string]string{ + "fstype": "", + "device": "", + "Type": "ContainerFS", + } + cExtractor.AssertContainsTaggedField(t, cMetrics[1], expectedFields, expectedTags) +} diff --git a/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/mem_extractor.go b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/mem_extractor.go new file mode 100644 index 000000000000..fb8786e4526b --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/mem_extractor.go @@ -0,0 +1,65 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package extractors // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors" + +import ( + "time" + + ci "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight" + awsmetrics "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/metrics" + cExtractor "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors" + + "go.uber.org/zap" +) + +type MemMetricExtractor struct { + logger *zap.Logger + rateCalculator awsmetrics.MetricCalculator +} + +func (m *MemMetricExtractor) HasValue(rawMetric RawMetric) bool { + if !rawMetric.Time.IsZero() { + return true + } + return false +} + +func (m *MemMetricExtractor) GetValue(rawMetric RawMetric, mInfo cExtractor.CPUMemInfoProvider, containerType string) []*cExtractor.CAdvisorMetric { + var metrics []*cExtractor.CAdvisorMetric + metric := cExtractor.NewCadvisorMetric(containerType, m.logger) + identifier := rawMetric.Id + + metric.AddField(ci.MetricName(containerType, ci.MemUsage), rawMetric.MemoryStats.UsageBytes) + metric.AddField(ci.MetricName(containerType, ci.MemRss), rawMetric.MemoryStats.RSSBytes) + metric.AddField(ci.MetricName(containerType, ci.MemWorkingset), rawMetric.MemoryStats.WorkingSetBytes) + + multiplier := float64(time.Second) + cExtractor.AssignRateValueToField(&m.rateCalculator, metric.GetFields(), ci.MetricName(containerType, ci.MemPgfault), identifier, + float64(rawMetric.MemoryStats.PageFaults), rawMetric.Time, multiplier) + cExtractor.AssignRateValueToField(&m.rateCalculator, metric.GetFields(), ci.MetricName(containerType, ci.MemPgmajfault), identifier, + float64(rawMetric.MemoryStats.MajorPageFaults), rawMetric.Time, multiplier) + + memoryCapacity := mInfo.GetMemoryCapacity() + if metric.GetField(ci.MetricName(containerType, ci.MemWorkingset)) != nil && memoryCapacity != 0 { + metric.AddField(ci.MetricName(containerType, ci.MemUtilization), float64(metric.GetField(ci.MetricName(containerType, ci.MemWorkingset)).(uint64))/float64(memoryCapacity)*100) + } + + if containerType == ci.TypeNode { + metric.AddField(ci.MetricName(containerType, ci.MemLimit), memoryCapacity) + } + + metrics = append(metrics, metric) + return metrics +} + +func (m *MemMetricExtractor) Shutdown() error { + return m.rateCalculator.Shutdown() +} + +func NewMemMetricExtractor(logger *zap.Logger) *MemMetricExtractor { + return &MemMetricExtractor{ + logger: logger, + rateCalculator: cExtractor.NewFloat64RateCalculator(), + } +} diff --git a/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/mem_extractor_test.go b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/mem_extractor_test.go new file mode 100644 index 000000000000..abf06ae4b438 --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/mem_extractor_test.go @@ -0,0 +1,69 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package extractors + +import ( + "testing" + + "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight" + cExtractor "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors" + cTestUtils "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/cadvisor/testutils" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/k8swindows/testutils" + + "github.com/stretchr/testify/require" +) + +func TestMemStats(t *testing.T) { + MockCPUMemInfo := cTestUtils.MockCPUMemInfo{} + result := testutils.LoadKubeletSummary(t, "./testdata/PreSingleKubeletSummary.json") + result2 := testutils.LoadKubeletSummary(t, "./testdata/CurSingleKubeletSummary.json") + + podRawMetric := ConvertPodToRaw(result.Pods[0]) + podRawMetric2 := ConvertPodToRaw(result2.Pods[0]) + + containerType := containerinsight.TypePod + extractor := NewMemMetricExtractor(nil) + + var cMetrics []*cExtractor.CAdvisorMetric + if extractor.HasValue(podRawMetric) { + cMetrics = extractor.GetValue(podRawMetric, MockCPUMemInfo, containerType) + } + if extractor.HasValue(podRawMetric2) { + cMetrics = extractor.GetValue(podRawMetric2, MockCPUMemInfo, containerType) + } + + cExtractor.AssertContainsTaggedUint(t, cMetrics[0], "pod_memory_rss", 0) + cExtractor.AssertContainsTaggedUint(t, cMetrics[0], "pod_memory_usage", 0) + cExtractor.AssertContainsTaggedUint(t, cMetrics[0], "pod_memory_working_set", 209088512) + + cExtractor.AssertContainsTaggedFloat(t, cMetrics[0], "pod_memory_pgfault", 0, 0) + cExtractor.AssertContainsTaggedFloat(t, cMetrics[0], "pod_memory_pgmajfault", 0, 0) + require.NoError(t, extractor.Shutdown()) + + // for node type + containerType = containerinsight.TypeNode + extractor = NewMemMetricExtractor(nil) + + nodeRawMetric := ConvertNodeToRaw(result.Node) + nodeRawMetric2 := ConvertNodeToRaw(result2.Node) + + if extractor.HasValue(nodeRawMetric) { + cMetrics = extractor.GetValue(nodeRawMetric, MockCPUMemInfo, containerType) + } + + if extractor.HasValue(nodeRawMetric2) { + cMetrics = extractor.GetValue(nodeRawMetric2, MockCPUMemInfo, containerType) + } + + cExtractor.AssertContainsTaggedUint(t, cMetrics[0], "node_memory_rss", 0) + cExtractor.AssertContainsTaggedUint(t, cMetrics[0], "node_memory_usage", 3572293632) + cExtractor.AssertContainsTaggedUint(t, cMetrics[0], "node_memory_working_set", 1026678784) + cExtractor.AssertContainsTaggedInt(t, cMetrics[0], "node_memory_limit", 1073741824) + + cExtractor.AssertContainsTaggedFloat(t, cMetrics[0], "node_memory_pgfault", 0, 0) + cExtractor.AssertContainsTaggedFloat(t, cMetrics[0], "node_memory_pgmajfault", 0, 0) + cExtractor.AssertContainsTaggedFloat(t, cMetrics[0], "node_memory_utilization", 95, 0.7) + + require.NoError(t, extractor.Shutdown()) +} diff --git a/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/net_extractor.go b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/net_extractor.go new file mode 100644 index 000000000000..30ec0fa94c30 --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/net_extractor.go @@ -0,0 +1,100 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package extractors // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors" + +import ( + "time" + + "go.uber.org/zap" + + ci "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight" + awsmetrics "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/metrics" + cExtractor "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors" +) + +type NetMetricExtractor struct { + logger *zap.Logger + rateCalculator awsmetrics.MetricCalculator +} + +func (n *NetMetricExtractor) HasValue(rawMetric RawMetric) bool { + if !rawMetric.Time.IsZero() { + return true + } + return false +} + +func (n *NetMetricExtractor) GetValue(rawMetric RawMetric, mInfo cExtractor.CPUMemInfoProvider, containerType string) []*cExtractor.CAdvisorMetric { + var metrics []*cExtractor.CAdvisorMetric + + if containerType == ci.TypeContainer { + return nil + } + + netIfceMetrics := make([]map[string]any, len(rawMetric.NetworkStats)) + + for i, intf := range rawMetric.NetworkStats { + mType := getNetMetricType(containerType, n.logger) + netIfceMetric := make(map[string]any) + + identifier := rawMetric.Id + containerType + intf.Name + multiplier := float64(time.Second) + + cExtractor.AssignRateValueToField(&n.rateCalculator, netIfceMetric, ci.NetRxBytes, identifier, float64(intf.RxBytes), rawMetric.Time, multiplier) + cExtractor.AssignRateValueToField(&n.rateCalculator, netIfceMetric, ci.NetRxErrors, identifier, float64(intf.RxErrors), rawMetric.Time, multiplier) + cExtractor.AssignRateValueToField(&n.rateCalculator, netIfceMetric, ci.NetTxBytes, identifier, float64(intf.TxBytes), rawMetric.Time, multiplier) + cExtractor.AssignRateValueToField(&n.rateCalculator, netIfceMetric, ci.NetTxErrors, identifier, float64(intf.TxErrors), rawMetric.Time, multiplier) + cExtractor.AssignRateValueToField(&n.rateCalculator, netIfceMetric, ci.NetRxDropped, identifier, float64(intf.DroppedIncoming), rawMetric.Time, multiplier) + cExtractor.AssignRateValueToField(&n.rateCalculator, netIfceMetric, ci.NetTxDropped, identifier, float64(intf.DroppedOutgoing), rawMetric.Time, multiplier) + + if netIfceMetric[ci.NetRxBytes] != nil && netIfceMetric[ci.NetTxBytes] != nil { + netIfceMetric[ci.NetTotalBytes] = netIfceMetric[ci.NetRxBytes].(float64) + netIfceMetric[ci.NetTxBytes].(float64) + } + + netIfceMetrics[i] = netIfceMetric + + metric := cExtractor.NewCadvisorMetric(mType, n.logger) + metric.AddTag(ci.NetIfce, intf.Name) + for k, v := range netIfceMetric { + metric.AddField(ci.MetricName(mType, k), v) + } + + metrics = append(metrics, metric) + } + + aggregatedFields := ci.SumFields(netIfceMetrics) + if len(aggregatedFields) > 0 { + metric := cExtractor.NewCadvisorMetric(containerType, n.logger) + for k, v := range aggregatedFields { + metric.AddField(ci.MetricName(containerType, k), v) + } + metrics = append(metrics, metric) + } + + return metrics +} + +func (n *NetMetricExtractor) Shutdown() error { + return n.rateCalculator.Shutdown() +} + +func NewNetMetricExtractor(logger *zap.Logger) *NetMetricExtractor { + return &NetMetricExtractor{ + logger: logger, + rateCalculator: cExtractor.NewFloat64RateCalculator(), + } +} + +func getNetMetricType(containerType string, logger *zap.Logger) string { + metricType := "" + switch containerType { + case ci.TypeNode: + metricType = ci.TypeNodeNet + case ci.TypePod: + metricType = ci.TypePodNet + default: + logger.Warn("net_extractor: net metric extractor is parsing unexpected containerType", zap.String("containerType", containerType)) + } + return metricType +} diff --git a/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/net_extractor_test.go b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/net_extractor_test.go new file mode 100644 index 000000000000..2ad55976c3f5 --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/net_extractor_test.go @@ -0,0 +1,214 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package extractors + +import ( + "testing" + + ci "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight" + cExtractor "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/k8swindows/testutils" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNetStats(t *testing.T) { + + result := testutils.LoadKubeletSummary(t, "./testdata/PreSingleKubeletSummary.json") + result2 := testutils.LoadKubeletSummary(t, "./testdata/CurSingleKubeletSummary.json") + + nodeRawMetric := ConvertNodeToRaw(result.Node) + nodeRawMetric2 := ConvertNodeToRaw(result2.Node) + + containerType := ci.TypeNode + extractor := NewNetMetricExtractor(nil) + var cMetrics []*cExtractor.CAdvisorMetric + if extractor.HasValue(nodeRawMetric) { + cMetrics = extractor.GetValue(nodeRawMetric, nil, containerType) + } + if extractor.HasValue(nodeRawMetric2) { + cMetrics = extractor.GetValue(nodeRawMetric2, nil, containerType) + } + + expectedFields := []map[string]any{ + { + "node_interface_network_rx_bytes": float64(3474.633333333333), + "node_interface_network_rx_dropped": float64(0), + "node_interface_network_rx_errors": float64(0), + "node_interface_network_total_bytes": float64(5477.549999999999), + "node_interface_network_tx_bytes": float64(2002.9166666666665), + "node_interface_network_tx_dropped": float64(0), + "node_interface_network_tx_errors": float64(0), + }, + { + "node_interface_network_rx_bytes": float64(2293.733333333333), + "node_interface_network_rx_dropped": float64(0), + "node_interface_network_rx_errors": float64(0), + "node_interface_network_total_bytes": float64(4781.45), + "node_interface_network_tx_bytes": float64(2487.7166666666667), + "node_interface_network_tx_dropped": float64(0), + "node_interface_network_tx_errors": float64(0), + }, + { + "node_interface_network_rx_bytes": float64(0), + "node_interface_network_rx_dropped": float64(0), + "node_interface_network_rx_errors": float64(0), + "node_interface_network_total_bytes": float64(0), + "node_interface_network_tx_bytes": float64(0), + "node_interface_network_tx_dropped": float64(0), + "node_interface_network_tx_errors": float64(0), + }, + { + "node_interface_network_rx_bytes": float64(0), + "node_interface_network_rx_dropped": float64(0), + "node_interface_network_rx_errors": float64(0), + "node_interface_network_total_bytes": float64(0), + "node_interface_network_tx_bytes": float64(0), + "node_interface_network_tx_dropped": float64(0), + "node_interface_network_tx_errors": float64(0), + }, + { + "node_interface_network_rx_bytes": float64(0), + "node_interface_network_rx_dropped": float64(0), + "node_interface_network_rx_errors": float64(0), + "node_interface_network_total_bytes": float64(0), + "node_interface_network_tx_bytes": float64(0), + "node_interface_network_tx_dropped": float64(0), + "node_interface_network_tx_errors": float64(0), + }, + { + "node_interface_network_rx_bytes": float64(0), + "node_interface_network_rx_dropped": float64(0), + "node_interface_network_rx_errors": float64(0), + "node_interface_network_total_bytes": float64(0), + "node_interface_network_tx_bytes": float64(0), + "node_interface_network_tx_dropped": float64(0), + "node_interface_network_tx_errors": float64(0), + }, + { + "node_interface_network_rx_bytes": float64(0), + "node_interface_network_rx_dropped": float64(0), + "node_interface_network_rx_errors": float64(0), + "node_interface_network_total_bytes": float64(0), + "node_interface_network_tx_bytes": float64(0), + "node_interface_network_tx_dropped": float64(0), + "node_interface_network_tx_errors": float64(0), + }, + { + "node_interface_network_rx_bytes": float64(0), + "node_interface_network_rx_dropped": float64(0), + "node_interface_network_rx_errors": float64(0), + "node_interface_network_total_bytes": float64(0), + "node_interface_network_tx_bytes": float64(0), + "node_interface_network_tx_dropped": float64(0), + "node_interface_network_tx_errors": float64(0), + }, + { + "node_network_rx_bytes": float64(5768.366666666667), + "node_network_rx_dropped": float64(0), + "node_network_rx_errors": float64(0), + "node_network_total_bytes": float64(10259), + "node_network_tx_bytes": float64(4490.633333333333), + "node_network_tx_dropped": float64(0), + "node_network_tx_errors": float64(0), + }, + } + + expectedTags := []map[string]string{ + { + "Type": "NodeNet", + "interface": "Amazon Elastic Network Adapter", + }, + { + "Type": "NodeNet", + "interface": "Hyper-V Virtual Ethernet Adapter", + }, + { + "Type": "NodeNet", + "interface": "Teredo Tunneling Pseudo-Interface", + }, + { + "Type": "NodeNet", + "interface": "AWS PV Network Device", + }, + { + "Type": "NodeNet", + "interface": "Microsoft IP-HTTPS Platform Interface", + }, + { + "Type": "NodeNet", + "interface": "Hyper-V Virtual Switch Extension Adapter", + }, + { + "Type": "NodeNet", + "interface": "Microsoft Kernel Debug Network Adapter", + }, + { + "Type": "NodeNet", + "interface": "6to4 Adapter", + }, + { + "Type": "Node", + }, + } + + assert.Equal(t, len(cMetrics), 9) + for i := range expectedFields { + cExtractor.AssertContainsTaggedField(t, cMetrics[i], expectedFields[i], expectedTags[i]) + } + require.NoError(t, extractor.Shutdown()) + + // pod type metrics + podRawMetric := ConvertPodToRaw(result.Pods[0]) + podRawMetric2 := ConvertPodToRaw(result2.Pods[0]) + + containerType = ci.TypePod + extractor = NewNetMetricExtractor(nil) + + if extractor.HasValue(podRawMetric) { + cMetrics = extractor.GetValue(podRawMetric, nil, containerType) + } + if extractor.HasValue(podRawMetric2) { + cMetrics = extractor.GetValue(podRawMetric2, nil, containerType) + } + + expectedFields = []map[string]any{ + { + "pod_interface_network_rx_bytes": float64(1735.9333333333334), + "pod_interface_network_rx_dropped": float64(0), + "pod_interface_network_rx_errors": float64(0), + "pod_interface_network_total_bytes": float64(1903.75), + "pod_interface_network_tx_bytes": float64(167.81666666666666), + "pod_interface_network_tx_dropped": float64(0), + "pod_interface_network_tx_errors": float64(0), + }, + + { + "pod_network_rx_bytes": float64(1735.9333333333334), + "pod_network_rx_dropped": float64(0), + "pod_network_rx_errors": float64(0), + "pod_network_total_bytes": float64(1903.75), + "pod_network_tx_bytes": float64(167.81666666666666), + "pod_network_tx_dropped": float64(0), + "pod_network_tx_errors": float64(0), + }, + } + + expectedTags = []map[string]string{ + { + "Type": "PodNet", + "interface": "cid-ccecfbf3-5ecf-4d93-9229-3f2f0df1b6b5", + }, + { + "Type": "Pod", + }, + } + + assert.Equal(t, len(cMetrics), 2) + for i := range expectedFields { + cExtractor.AssertContainsTaggedField(t, cMetrics[i], expectedFields[i], expectedTags[i]) + } + require.NoError(t, extractor.Shutdown()) +} diff --git a/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/testdata/CurSingleKubeletSummary.json b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/testdata/CurSingleKubeletSummary.json new file mode 100644 index 000000000000..e177d22789c0 --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/testdata/CurSingleKubeletSummary.json @@ -0,0 +1,206 @@ +{ + "node": { + "nodeName": "ip-192-168-44-84.us-west-2.compute.internal", + "systemContainers": [ + { + "name": "pods", + "startTime": "2023-12-21T15:20:59Z", + "cpu": { + "time": "2023-12-21T15:20:59Z", + "usageCoreNanoSeconds": 313109375000 + }, + "memory": { + "time": "2023-12-21T15:20:59Z", + "workingSetBytes": 316674048 + } + } + ], + "startTime": "2023-12-11T23:48:04Z", + "cpu": { + "time": "2023-12-21T15:20:58Z", + "usageNanoCores": 70000000, + "usageCoreNanoSeconds": 38910780000000 + }, + "memory": { + "time": "2023-12-21T15:20:58Z", + "availableBytes": 7248187392, + "usageBytes": 3572293632, + "workingSetBytes": 1026678784, + "rssBytes": 0, + "pageFaults": 0, + "majorPageFaults": 0 + }, + "network": { + "time": "2023-12-21T15:20:58Z", + "name": "", + "interfaces": [ + { + "name": "Amazon Elastic Network Adapter", + "rxBytes": 5620532878, + "rxErrors": 0, + "txBytes": 781180227, + "txErrors": 0 + }, + { + "name": "Hyper-V Virtual Ethernet Adapter", + "rxBytes": 5574303646, + "rxErrors": 0, + "txBytes": 773064167, + "txErrors": 0 + }, + { + "name": "Teredo Tunneling Pseudo-Interface", + "rxBytes": 0, + "rxErrors": 0, + "txBytes": 0, + "txErrors": 0 + }, + { + "name": "AWS PV Network Device", + "rxBytes": 0, + "rxErrors": 0, + "txBytes": 0, + "txErrors": 0 + }, + { + "name": "Microsoft IP-HTTPS Platform Interface", + "rxBytes": 0, + "rxErrors": 0, + "txBytes": 0, + "txErrors": 0 + }, + { + "name": "Hyper-V Virtual Switch Extension Adapter", + "rxBytes": 0, + "rxErrors": 0, + "txBytes": 0, + "txErrors": 0 + }, + { + "name": "Microsoft Kernel Debug Network Adapter", + "rxBytes": 0, + "rxErrors": 0, + "txBytes": 0, + "txErrors": 0 + }, + { + "name": "6to4 Adapter", + "rxBytes": 0, + "rxErrors": 0, + "txBytes": 0, + "txErrors": 0 + } + ] + }, + "fs": { + "time": "2023-12-21T15:20:58Z", + "availableBytes": 51230019584, + "capacityBytes": 85897244672, + "usedBytes": 34667225088 + }, + "runtime": { + "imageFs": { + "time": "2023-12-21T15:20:56Z", + "availableBytes": 51230019584, + "capacityBytes": 85897244672, + "usedBytes": 16901889814, + "inodesUsed": 0 + } + } + }, + "pods": [ + { + "podRef": { + "name": "windows-server-iis-ltsc2019-58d94b5844-6v2pg", + "namespace": "amazon-cloudwatch", + "uid": "01bfbe59-2925-4ad5-a8d3-a1b23e3ddd74" + }, + "startTime": "2023-12-20T16:52:55Z", + "containers": [ + { + "name": "windows-server-iis-ltsc2019", + "startTime": "2023-12-20T16:52:58Z", + "cpu": { + "time": "2023-12-21T15:20:59Z", + "usageNanoCores": 0, + "usageCoreNanoSeconds": 289812500000 + }, + "memory": { + "time": "2023-12-21T15:20:59Z", + "workingSetBytes": 209088512 + }, + "rootfs": { + "time": "2023-12-21T15:20:56Z", + "availableBytes": 51230019584, + "capacityBytes": 85897244672, + "usedBytes": 339738624, + "inodesUsed": 0 + }, + "logs": { + "time": "2023-12-21T15:20:59Z", + "availableBytes": 51230019584, + "capacityBytes": 85897244672, + "usedBytes": 920146, + "inodesUsed": 0 + } + } + ], + "cpu": { + "time": "2023-12-21T15:20:59Z", + "usageNanoCores": 0, + "usageCoreNanoSeconds": 289812500000 + }, + "memory": { + "time": "2023-12-21T15:20:59Z", + "availableBytes": 0, + "usageBytes": 0, + "workingSetBytes": 209088512, + "rssBytes": 0, + "pageFaults": 0, + "majorPageFaults": 0 + }, + "network": { + "time": "2023-12-21T15:20:59Z", + "name": "cid-ccecfbf3-5ecf-4d93-9229-3f2f0df1b6b5", + "rxBytes": 8891342, + "txBytes": 3433553, + "interfaces": [ + { + "name": "cid-ccecfbf3-5ecf-4d93-9229-3f2f0df1b6b5", + "rxBytes": 8891342, + "txBytes": 3433553 + } + ] + }, + "volume": [ + { + "time": "2023-12-21T15:20:56Z", + "availableBytes": 51230019584, + "capacityBytes": 85897244672, + "usedBytes": 4431, + "inodesFree": 0, + "inodes": 0, + "inodesUsed": 0, + "name": "cwagentconfig" + }, + { + "time": "2023-12-21T15:20:56Z", + "availableBytes": 51230019584, + "capacityBytes": 85897244672, + "usedBytes": 7448, + "inodesFree": 0, + "inodes": 0, + "inodesUsed": 0, + "name": "kube-api-access-8gtnx" + } + ], + "ephemeral-storage": { + "time": "2023-12-21T15:20:59Z", + "availableBytes": 51230019584, + "capacityBytes": 85897244672, + "usedBytes": 340663443, + "inodesUsed": 0 + } + } + ] +} \ No newline at end of file diff --git a/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/testdata/PreSingleKubeletSummary.json b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/testdata/PreSingleKubeletSummary.json new file mode 100644 index 000000000000..6ef061851cf5 --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors/testdata/PreSingleKubeletSummary.json @@ -0,0 +1,213 @@ +{ + "node": { + "nodeName": "ip-192-168-44-84.us-west-2.compute.internal", + "systemContainers": [ + { + "name": "pods", + "startTime": "2023-12-21T15:19:59Z", + "cpu": { + "time": "2023-12-21T15:19:59Z", + "usageCoreNanoSeconds": 312859375000 + }, + "memory": { + "time": "2023-12-21T15:19:59Z", + "workingSetBytes": 316026880 + } + } + ], + "startTime": "2023-12-11T23:48:04Z", + "cpu": { + "time": "2023-12-21T15:19:58Z", + "usageNanoCores": 20000000, + "usageCoreNanoSeconds": 38907680000000 + }, + "memory": { + "time": "2023-12-21T15:19:58Z", + "availableBytes": 7234662400, + "usageBytes": 3583389696, + "workingSetBytes": 1040203776, + "rssBytes": 0, + "pageFaults": 0, + "majorPageFaults": 0 + }, + "network": { + "time": "2023-12-21T15:19:58Z", + "name": "", + "interfaces": [ + { + "name": "Hyper-V Virtual Switch Extension Adapter", + "rxBytes": 0, + "rxErrors": 0, + "txBytes": 0, + "txErrors": 0 + }, + { + "name": "Microsoft Kernel Debug Network Adapter", + "rxBytes": 0, + "rxErrors": 0, + "txBytes": 0, + "txErrors": 0 + }, + { + "name": "6to4 Adapter", + "rxBytes": 0, + "rxErrors": 0, + "txBytes": 0, + "txErrors": 0 + }, + { + "name": "Intel[R]82599VirtualFunction", + "rxBytes": 0, + "rxErrors": 0, + "txBytes": 0, + "txErrors": 0 + }, + { + "name": "Microsoft IP-HTTPS Platform Interface", + "rxBytes": 0, + "rxErrors": 0, + "txBytes": 0, + "txErrors": 0 + }, + { + "name": "Hyper-V Virtual Ethernet Adapter", + "rxBytes": 5574166022, + "rxErrors": 0, + "txBytes": 772914904, + "txErrors": 0 + }, + { + "name": "Teredo Tunneling Pseudo-Interface", + "rxBytes": 0, + "rxErrors": 0, + "txBytes": 0, + "txErrors": 0 + }, + { + "name": "AWS PV Network Device", + "rxBytes": 0, + "rxErrors": 0, + "txBytes": 0, + "txErrors": 0 + }, + { + "name": "Amazon Elastic Network Adapter", + "rxBytes": 5620324400, + "rxErrors": 0, + "txBytes": 781060052, + "txErrors": 0 + } + ] + }, + "fs": { + "time": "2023-12-21T15:19:58Z", + "availableBytes": 51230154752, + "capacityBytes": 85897244672, + "usedBytes": 34667089920 + }, + "runtime": { + "imageFs": { + "time": "2023-12-21T15:19:56Z", + "availableBytes": 51230154752, + "capacityBytes": 85897244672, + "usedBytes": 16901889814, + "inodesUsed": 0 + } + } + }, + "pods": [ + { + "podRef": { + "name": "windows-server-iis-ltsc2019-58d94b5844-6v2pg", + "namespace": "amazon-cloudwatch", + "uid": "01bfbe59-2925-4ad5-a8d3-a1b23e3ddd74" + }, + "startTime": "2023-12-20T16:52:55Z", + "containers": [ + { + "name": "windows-server-iis-ltsc2019", + "startTime": "2023-12-20T16:52:58Z", + "cpu": { + "time": "2023-12-21T15:19:59Z", + "usageNanoCores": 0, + "usageCoreNanoSeconds": 289625000000 + }, + "memory": { + "time": "2023-12-21T15:19:59Z", + "workingSetBytes": 208949248 + }, + "rootfs": { + "time": "2023-12-21T15:19:56Z", + "availableBytes": 51230154752, + "capacityBytes": 85897244672, + "usedBytes": 339738624, + "inodesUsed": 0 + }, + "logs": { + "time": "2023-12-21T15:19:59Z", + "availableBytes": 51230154752, + "capacityBytes": 85897244672, + "usedBytes": 919463, + "inodesUsed": 0 + } + } + ], + "cpu": { + "time": "2023-12-21T15:19:59Z", + "usageNanoCores": 0, + "usageCoreNanoSeconds": 289625000000 + }, + "memory": { + "time": "2023-12-21T15:19:59Z", + "availableBytes": 0, + "usageBytes": 0, + "workingSetBytes": 208949248, + "rssBytes": 0, + "pageFaults": 0, + "majorPageFaults": 0 + }, + "network": { + "time": "2023-12-21T15:19:59Z", + "name": "cid-ccecfbf3-5ecf-4d93-9229-3f2f0df1b6b5", + "rxBytes": 8787186, + "txBytes": 3423484, + "interfaces": [ + { + "name": "cid-ccecfbf3-5ecf-4d93-9229-3f2f0df1b6b5", + "rxBytes": 8787186, + "txBytes": 3423484 + } + ] + }, + "volume": [ + { + "time": "2023-12-21T15:19:04Z", + "availableBytes": 51230248960, + "capacityBytes": 85897244672, + "usedBytes": 4431, + "inodesFree": 0, + "inodes": 0, + "inodesUsed": 0, + "name": "cwagentconfig" + }, + { + "time": "2023-12-21T15:19:04Z", + "availableBytes": 51230248960, + "capacityBytes": 85897244672, + "usedBytes": 7448, + "inodesFree": 0, + "inodes": 0, + "inodesUsed": 0, + "name": "kube-api-access-8gtnx" + } + ], + "ephemeral-storage": { + "time": "2023-12-21T15:19:59Z", + "availableBytes": 51230154752, + "capacityBytes": 85897244672, + "usedBytes": 340662760, + "inodesUsed": 0 + } + } + ] +} \ No newline at end of file diff --git a/receiver/awscontainerinsightreceiver/internal/k8swindows/hcsshim/client.go b/receiver/awscontainerinsightreceiver/internal/k8swindows/hcsshim/client.go new file mode 100644 index 000000000000..ec7baf990562 --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/k8swindows/hcsshim/client.go @@ -0,0 +1,54 @@ +package hcsshim + +import ( + "fmt" + + "github.com/Microsoft/hcsshim" + "go.uber.org/zap" +) + +type HCSClient interface { + GetContainerStats(containerId string) (hcsshim.Statistics, error) + GetEndpointList() ([]hcsshim.HNSEndpoint, error) + GetEndpointStat(endpointId string) (hcsshim.HNSEndpointStats, error) +} +type hCSClient struct { + logger *zap.Logger +} + +func (hc *hCSClient) GetContainerStats(containerId string) (hcsshim.Statistics, error) { + container, err := hcsshim.OpenContainer(containerId) + if err != nil { + hc.logger.Error("failed to open container using HCS shim APIs, ", zap.Error(err)) + return hcsshim.Statistics{}, err + } + defer container.Close() + cps, err := container.Statistics() + if err != nil { + hc.logger.Error("failed to get container stats from HCS shim APIs, ", zap.Error(err)) + return hcsshim.Statistics{}, err + } + + return cps, nil +} + +func (hc *hCSClient) GetEndpointList() ([]hcsshim.HNSEndpoint, error) { + endpointList, err := hcsshim.HNSListEndpointRequest() + if err != nil { + hc.logger.Error("failed to list endpoints using HNS APIs, ", zap.Error(err)) + return []hcsshim.HNSEndpoint{}, err + } + return endpointList, nil +} + +func (hc *hCSClient) GetEndpointStat(endpointId string) (hcsshim.HNSEndpointStats, error) { + endpointStat, err := hcsshim.GetHNSEndpointStats(endpointId) + if err != nil { + hc.logger.Error("failed to get HNS endpoint stats, ", zap.Error(err)) + return hcsshim.HNSEndpointStats{}, err + } + if endpointStat != nil { + return *endpointStat, nil + } + return hcsshim.HNSEndpointStats{}, fmt.Errorf("no stats for endpoint") +} diff --git a/receiver/awscontainerinsightreceiver/internal/k8swindows/hcsshim/hcsshim.go b/receiver/awscontainerinsightreceiver/internal/k8swindows/hcsshim/hcsshim.go new file mode 100644 index 000000000000..eb47bd840d31 --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/k8swindows/hcsshim/hcsshim.go @@ -0,0 +1,217 @@ +//go:build windows +// +build windows + +package hcsshim // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/k8s_windox2" + +import ( + "fmt" + "strconv" + "strings" + + ci "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight" + cExtractor "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/k8swindows/kubelet" + + "go.uber.org/zap" + "golang.org/x/exp/slices" +) + +type PodKey struct { + PodId, PodName, PodNamespace string + Containers []ContainerInfo +} + +type ContainerInfo struct { + Name, Id string +} + +type EndpointInfo struct { + Id, Name string +} + +type HCSStatsProvider struct { + logger *zap.Logger + hostInfo cExtractor.CPUMemInfoProvider + hcsClient HCSClient + kubeletProvider kubelet.KubeletProvider + metricExtractors []extractors.MetricExtractor + containerToEndpoint map[string]EndpointInfo +} + +func createHCSClient(logger *zap.Logger) HCSClient { + return &hCSClient{logger: logger} +} + +// Options decorates SummaryProvider struct. +type Options func(provider *HCSStatsProvider) + +func NewHnSProvider(logger *zap.Logger, info cExtractor.CPUMemInfoProvider, mextractor []extractors.MetricExtractor, opts ...Options) (*HCSStatsProvider, error) { + hp := &HCSStatsProvider{ + logger: logger, + hostInfo: info, + kubeletProvider: kubelet.CreateDefaultKubeletProvider(logger), + hcsClient: createHCSClient(logger), + metricExtractors: mextractor, + } + + for _, opt := range opts { + opt(hp) + } + + return hp, nil +} + +func (hp *HCSStatsProvider) GetMetrics() ([]*cExtractor.CAdvisorMetric, error) { + var metrics []*cExtractor.CAdvisorMetric + if ci.IsWindowsHostProcessContainer() { + containerToEndpointMap, err := hp.getContainerToEndpointMap() + if err != nil { + hp.logger.Error("failed to create container to endpoint map using HCS shim APIs, ", zap.Error(err)) + return nil, err + } + hp.containerToEndpoint = containerToEndpointMap + return hp.getPodMetrics() + } + return metrics, nil +} + +func (hp *HCSStatsProvider) getContainerMetrics(containerId string) (extractors.HCSStat, error) { + hp.logger.Debug("Getting Container stats using Microsoft HCS shim APIs") + + cps, err := hp.hcsClient.GetContainerStats(containerId) + if err != nil { + hp.logger.Error("failed to get container stats from HCS shim client, ", zap.Error(err)) + return extractors.HCSStat{}, err + } + + if _, ok := hp.containerToEndpoint[containerId]; !ok { + hp.logger.Warn("HNS endpoint not found ", zap.String("container", containerId)) + return extractors.HCSStat{}, nil + } + + endpoint := hp.containerToEndpoint[containerId] + enpointStat, err := hp.hcsClient.GetEndpointStat(endpoint.Id) + if err != nil { + hp.logger.Error("failed to get HNS endpoint stats, ", zap.Error(err)) + return extractors.HCSStat{}, err + } + var hnsNetworks []extractors.HCSNetworkStat + + hnsNetworks = append(hnsNetworks, extractors.HCSNetworkStat{ + Name: endpoint.Name, + BytesReceived: enpointStat.BytesReceived, + BytesSent: enpointStat.BytesSent, + DroppedPacketsOutgoing: enpointStat.DroppedPacketsOutgoing, + DroppedPacketsIncoming: enpointStat.DroppedPacketsIncoming, + }) + + stat := extractors.HCSStat{Time: cps.Timestamp, CPU: &cps.Processor, Network: &hnsNetworks} + hp.logger.Debug("Returning Container stats using Microsoft HCS shim APIs") + return stat, nil +} + +func (hp *HCSStatsProvider) getPodMetrics() ([]*cExtractor.CAdvisorMetric, error) { + hp.logger.Debug("Getting pod stats using Microsoft HCS shim APIs") + podToContainerMap, err := hp.getPodToContainerMap() + if err != nil { + hp.logger.Error("failed to create pod to container map using kubelet APIs, ", zap.Error(err)) + return nil, err + } + + var metrics []*cExtractor.CAdvisorMetric + var endpointMetricsCollected []string + + for _, pod := range podToContainerMap { + var metricsPerPod []*cExtractor.CAdvisorMetric + tags := map[string]string{} + + tags[ci.PodIDKey] = pod.PodId + tags[ci.K8sPodNameKey] = pod.PodName + tags[ci.K8sNamespace] = pod.PodNamespace + + for _, container := range pod.Containers { + if _, ok := hp.containerToEndpoint[container.Id]; !ok { + hp.logger.Debug("Skipping as endpoint don't exist for container") + continue + } + endpoint := hp.containerToEndpoint[container.Id] + if slices.Contains(endpointMetricsCollected, endpoint.Id) { + hp.logger.Debug("Skipping as metric already collected for HNS Endpoint") + continue + } + + containerStats, err := hp.getContainerMetrics(container.Id) + if err != nil { + hp.logger.Warn("failed to get container metrics using HCS shim APIs, ", zap.Error(err)) + continue + } + + rawMetric := extractors.ConvertHCSContainerToRaw(containerStats) + tags[ci.Timestamp] = strconv.FormatInt(rawMetric.Time.UnixNano(), 10) + + for _, extractor := range hp.metricExtractors { + if extractor.HasValue(rawMetric) { + metricsPerPod = append(metricsPerPod, extractor.GetValue(rawMetric, hp.hostInfo, ci.TypePod)...) + } + } + endpointMetricsCollected = append(endpointMetricsCollected, hp.containerToEndpoint[container.Id].Id) + } + for _, metric := range metricsPerPod { + metric.AddTags(tags) + } + metrics = append(metrics, metricsPerPod...) + } + + return metrics, nil +} + +func (hp *HCSStatsProvider) getPodToContainerMap() (map[string]PodKey, error) { + containerNameToIdMapping := make(map[string]PodKey) + podList, err := hp.kubeletProvider.GetPods() + if err != nil { + hp.logger.Error("failed to get pod list from kubelet provider, ", zap.Error(err)) + return nil, err + } + for _, pod := range podList { + podId := fmt.Sprintf("%s", pod.UID) + podKey := PodKey{ + PodId: podId, + PodName: pod.Name, + PodNamespace: pod.Namespace, + } + if _, ok := containerNameToIdMapping[podId]; !ok { + containerNameToIdMapping[podId] = podKey + } + + for _, container := range pod.Status.ContainerStatuses { + if strings.Contains(container.ContainerID, "containerd") { + cinfo := ContainerInfo{ + Id: strings.Split(container.ContainerID, "containerd://")[1], + Name: container.Name, + } + podKey.Containers = append(podKey.Containers, cinfo) + } + } + containerNameToIdMapping[podId] = podKey + } + + return containerNameToIdMapping, nil +} + +func (hp *HCSStatsProvider) getContainerToEndpointMap() (map[string]EndpointInfo, error) { + var containerToEndpointMap = make(map[string]EndpointInfo) + endpointList, err := hp.hcsClient.GetEndpointList() + if err != nil { + hp.logger.Error("failed to get endpoints list from HCS shim client, ", zap.Error(err)) + return containerToEndpointMap, err + } + + for _, endpoint := range endpointList { + for _, container := range endpoint.SharedContainers { + containerToEndpointMap[container] = EndpointInfo{Id: endpoint.Id, Name: endpoint.Name} + } + } + + return containerToEndpointMap, nil +} diff --git a/receiver/awscontainerinsightreceiver/internal/k8swindows/hcsshim/hcsshim_test.go b/receiver/awscontainerinsightreceiver/internal/k8swindows/hcsshim/hcsshim_test.go new file mode 100644 index 000000000000..39e52a359785 --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/k8swindows/hcsshim/hcsshim_test.go @@ -0,0 +1,147 @@ +package hcsshim + +import ( + "testing" + "time" + + ci "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight" + cTestUtils "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/cadvisor/testutils" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/k8swindows/testutils" + + "github.com/Microsoft/hcsshim" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "go.uber.org/zap/zaptest" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1" +) + +// MockKubeletProvider Mock provider implements KubeletProvider interface. +type MockHCSClient struct { + logger *zap.Logger + t *testing.T +} + +// MockKubeletProvider Mock provider implements KubeletProvider interface. +type MockKubeletProvider struct { + logger *zap.Logger + t *testing.T +} + +func (m *MockHCSClient) GetContainerStats(containerId string) (hcsshim.Statistics, error) { + return hcsshim.Statistics{ + Timestamp: time.Now(), + }, nil +} + +func (m *MockHCSClient) GetEndpointList() ([]hcsshim.HNSEndpoint, error) { + return []hcsshim.HNSEndpoint{{ + Id: "endpointId123456c6asdfasdf4354545", + Name: "cid-adfklq3qr43lj523l4daf", + SharedContainers: []string{"1234123412341afasdfa12342343134", "kaljsflasdjf1234123412341afasdfa12342343134"}, + }}, nil +} + +func (m *MockHCSClient) GetEndpointStat(endpointId string) (hcsshim.HNSEndpointStats, error) { + return hcsshim.HNSEndpointStats{ + BytesReceived: 44340, + BytesSent: 3432, + DroppedPacketsIncoming: 43, + DroppedPacketsOutgoing: 1, + EndpointID: "endpointId123456c6asdfasdf4354545", + }, nil +} + +func (m *MockKubeletProvider) GetSummary() (*stats.Summary, error) { + return testutils.LoadKubeletSummary(m.t, "./../extractors/testdata/CurSingleKubeletSummary.json"), nil +} + +func (m *MockKubeletProvider) GetPods() ([]corev1.Pod, error) { + mockPods := []corev1.Pod{} + + mockPods = append(mockPods, corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + UID: "podidq3erqwezdfa3q34q34dfdf", + Name: "mockPod", + Namespace: "default", + }, + Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{ + Name: "mockContainername", + ContainerID: "containerd://1234123412341afasdfa12342343134", + }}}}) + return mockPods, nil +} + +func createKubeletDecoratorWithMockKubeletProvider(t *testing.T, logger *zap.Logger) Options { + return func(provider *HCSStatsProvider) { + provider.kubeletProvider = &MockKubeletProvider{t: t, logger: logger} + } +} + +func createHCSDecoratorWithMockHCSClient(t *testing.T, logger *zap.Logger) Options { + return func(provider *HCSStatsProvider) { + provider.hcsClient = &MockHCSClient{t: t, logger: logger} + } +} + +func mockInfoProvider() cTestUtils.MockHostInfo { + hostInfo := cTestUtils.MockHostInfo{ClusterName: "cluster"} + return hostInfo +} + +func mockMetricExtractors(t *testing.T) []extractors.MetricExtractor { + metricsExtractors := []extractors.MetricExtractor{} + metricsExtractors = append(metricsExtractors, extractors.NewNetMetricExtractor(zaptest.NewLogger(t))) + return metricsExtractors +} + +func TestGetContainerToEndpointMap(t *testing.T) { + hsp, err := NewHnSProvider(zaptest.NewLogger(t), mockInfoProvider(), mockMetricExtractors(t), createKubeletDecoratorWithMockKubeletProvider(t, zaptest.NewLogger(t)), createHCSDecoratorWithMockHCSClient(t, zaptest.NewLogger(t))) + assert.NoError(t, err) + + containerEndpointMap, err := hsp.getContainerToEndpointMap() + + assert.NoError(t, err) + + assert.Len(t, containerEndpointMap, 2) + assert.Contains(t, containerEndpointMap, "1234123412341afasdfa12342343134") + assert.Contains(t, containerEndpointMap, "kaljsflasdjf1234123412341afasdfa12342343134") +} + +func TestGetPodToContainerMap(t *testing.T) { + hsp, err := NewHnSProvider(zaptest.NewLogger(t), mockInfoProvider(), mockMetricExtractors(t), createKubeletDecoratorWithMockKubeletProvider(t, zaptest.NewLogger(t)), createHCSDecoratorWithMockHCSClient(t, zaptest.NewLogger(t))) + assert.NoError(t, err) + + podContainerMap, err := hsp.getPodToContainerMap() + + assert.NoError(t, err) + + assert.Len(t, podContainerMap, 1) + assert.Contains(t, podContainerMap, "podidq3erqwezdfa3q34q34dfdf") + assert.Len(t, podContainerMap["podidq3erqwezdfa3q34q34dfdf"].Containers, 1) + assert.Equal(t, podContainerMap["podidq3erqwezdfa3q34q34dfdf"].Containers[0].Id, "1234123412341afasdfa12342343134") +} + +func TestGetPodMetrics(t *testing.T) { + hsp, err := NewHnSProvider(zaptest.NewLogger(t), mockInfoProvider(), mockMetricExtractors(t), createKubeletDecoratorWithMockKubeletProvider(t, zaptest.NewLogger(t)), createHCSDecoratorWithMockHCSClient(t, zaptest.NewLogger(t))) + assert.NoError(t, err) + + containerToEndpointMap, err := hsp.getContainerToEndpointMap() + assert.NoError(t, err) + hsp.containerToEndpoint = containerToEndpointMap + + metrics, err := hsp.getPodMetrics() + assert.NoError(t, err) + + assert.Equal(t, len(metrics), 1) + podMetric := metrics[0] + assert.Equal(t, podMetric.GetMetricType(), ci.TypePodNet) + assert.NotNil(t, podMetric.GetTag(ci.PodIDKey)) + assert.NotNil(t, podMetric.GetTag(ci.K8sPodNameKey)) + assert.NotNil(t, podMetric.GetTag(ci.K8sNamespace)) + assert.NotNil(t, podMetric.GetTag(ci.Timestamp)) + assert.NotNil(t, podMetric.GetTag(ci.SourcesKey)) + assert.NotNil(t, podMetric.GetTag(ci.NetIfce)) +} diff --git a/receiver/awscontainerinsightreceiver/internal/k8swindows/k8swindows.go b/receiver/awscontainerinsightreceiver/internal/k8swindows/k8swindows.go new file mode 100644 index 000000000000..d4b4a15415ce --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/k8swindows/k8swindows.go @@ -0,0 +1,147 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//go:build windows +// +build windows + +package k8swindows // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/k8swindows" + +import ( + "context" + "errors" + "os" + + ci "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight" + cExtractor "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/host" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/k8swindows/hcsshim" + kubeletsummaryprovider "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/k8swindows/kubelet" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/stores" + + "go.opentelemetry.io/collector/pdata/pmetric" + "go.uber.org/zap" +) + +type K8sWindows struct { + cancel context.CancelFunc + logger *zap.Logger + nodeName string `toml:"node_name"` + k8sDecorator stores.K8sDecorator + kubeletSummaryProvider *kubeletsummaryprovider.SummaryProvider + hcsshimStatsProvider *hcsshim.HCSStatsProvider + hostInfo host.Info + version string +} + +var metricsExtractors = []extractors.MetricExtractor{} + +func New(logger *zap.Logger, decorator *stores.K8sDecorator, hostInfo host.Info) (*K8sWindows, error) { + nodeName := os.Getenv("HOST_NAME") + if nodeName == "" { + return nil, errors.New("missing environment variable HOST_NAME. Please check your deployment YAML config") + } + + metricsExtractors = []extractors.MetricExtractor{} + metricsExtractors = append(metricsExtractors, extractors.NewCPUMetricExtractor(logger)) + metricsExtractors = append(metricsExtractors, extractors.NewMemMetricExtractor(logger)) + metricsExtractors = append(metricsExtractors, extractors.NewFileSystemMetricExtractor(logger)) + metricsExtractors = append(metricsExtractors, extractors.NewNetMetricExtractor(logger)) + + ksp, err := kubeletsummaryprovider.New(logger, &hostInfo, metricsExtractors) + if err != nil { + logger.Error("failed to initialize kubelet SummaryProvider, ", zap.Error(err)) + return nil, err + } + + hnsMetricsExtractors := append([]extractors.MetricExtractor{}, extractors.NewNetMetricExtractor(logger)) + hsp, err := hcsshim.NewHnSProvider(logger, &hostInfo, hnsMetricsExtractors) + if err != nil { + logger.Error("failed to initialize HCSShim SummaryProvider, ", zap.Error(err)) + return nil, err + } + + return &K8sWindows{ + logger: logger, + nodeName: nodeName, + k8sDecorator: *decorator, + kubeletSummaryProvider: ksp, + hcsshimStatsProvider: hsp, + hostInfo: hostInfo, + version: "0", + }, nil +} + +func (k *K8sWindows) GetMetrics() []pmetric.Metrics { + k.logger.Debug("D! called K8sWindows GetMetrics") + var result []pmetric.Metrics + + metrics, err := k.kubeletSummaryProvider.GetMetrics() + if err != nil { + k.logger.Error("failed to get metrics from kubelet SummaryProvider, ", zap.Error(err)) + return result + } + + hcsmetrics, err := k.hcsshimStatsProvider.GetMetrics() + if err != nil { + k.logger.Error("failed to get metrics from HCSShim StatsProvider, ", zap.Error(err)) + return result + } + + metrics = append(metrics, hcsmetrics...) + metrics = cExtractor.MergeMetrics(metrics) + metrics = k.decorateMetrics(metrics) + for _, ciMetric := range metrics { + md := ci.ConvertToOTLPMetrics(ciMetric.GetFields(), ciMetric.GetTags(), k.logger) + result = append(result, md) + } + + return result +} + +func (k *K8sWindows) decorateMetrics(cadvisormetrics []*cExtractor.CAdvisorMetric) []*cExtractor.CAdvisorMetric { + //ebsVolumeIdsUsedAsPV := c.hostInfo.ExtractEbsIDsUsedByKubernetes() + var result []*cExtractor.CAdvisorMetric + for _, m := range cadvisormetrics { + tags := m.GetTags() + //c.addEbsVolumeInfo(tags, ebsVolumeIdsUsedAsPV) + + // add version + tags[ci.Version] = k.version + + // add nodeName for node, pod and container + metricType := tags[ci.MetricType] + if k.nodeName != "" && (ci.IsNode(metricType) || ci.IsInstance(metricType) || + ci.IsPod(metricType) || ci.IsContainer(metricType)) { + tags[ci.NodeNameKey] = k.nodeName + } + + // add instance id and type + if instanceID := k.hostInfo.GetInstanceID(); instanceID != "" { + tags[ci.InstanceID] = instanceID + } + if instanceType := k.hostInfo.GetInstanceType(); instanceType != "" { + tags[ci.InstanceType] = instanceType + } + + // add scaling group name + tags[ci.AutoScalingGroupNameKey] = k.hostInfo.GetAutoScalingGroupName() + + // add tags for EKS + tags[ci.ClusterNameKey] = k.hostInfo.GetClusterName() + + // add tags for OS + tags[ci.OperatingSystem] = ci.OperatingSystemWindows + + out := k.k8sDecorator.Decorate(m) + if out != nil { + result = append(result, out) + } + } + return result +} + +func (k *K8sWindows) Shutdown() error { + k.logger.Debug("D! called K8sWindows Shutdown") + return nil +} diff --git a/receiver/awscontainerinsightreceiver/internal/k8swindows/k8swindows_nowindows.go b/receiver/awscontainerinsightreceiver/internal/k8swindows/k8swindows_nowindows.go new file mode 100644 index 000000000000..dbc75d936351 --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/k8swindows/k8swindows_nowindows.go @@ -0,0 +1,32 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//go:build !windows +// +build !windows + +package k8swindows // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/k8swindows" + +import ( + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/host" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/stores" + + "go.opentelemetry.io/collector/pdata/pmetric" + "go.uber.org/zap" +) + +type K8sWindows struct { +} + +// New is a dummy function to construct a dummy K8sWindows struct for linux +func New(_ *zap.Logger, _ *stores.K8sDecorator, _ host.Info) (*K8sWindows, error) { + return &K8sWindows{}, nil +} + +// GetMetrics is a dummy function to always returns empty metrics for linux +func (k *K8sWindows) GetMetrics() []pmetric.Metrics { + return []pmetric.Metrics{} +} + +func (k *K8sWindows) Shutdown() error { + return nil +} diff --git a/receiver/awscontainerinsightreceiver/internal/k8swindows/kubelet/client.go b/receiver/awscontainerinsightreceiver/internal/k8swindows/kubelet/client.go new file mode 100644 index 000000000000..d470b2f7175b --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/k8swindows/kubelet/client.go @@ -0,0 +1,64 @@ +//go:build windows +// +build windows + +package kubelet + +import ( + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/stores/kubeletutil" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1" +) + +// KubeletProvider Represents interface to kubelet. +type KubeletProvider interface { + GetSummary() (*stats.Summary, error) + GetPods() ([]corev1.Pod, error) +} + +type kubeletProvider struct { + logger *zap.Logger + hostIP string + hostPort string + client *kubeletutil.KubeletClient +} + +// getClient Returns singleton kubelet client. +func (kp *kubeletProvider) getClient() (*kubeletutil.KubeletClient, error) { + if kp.client != nil { + return kp.client, nil + } + kclient, err := kubeletutil.NewKubeletClient(kp.hostIP, kp.hostPort, kp.logger) + if err != nil { + kp.logger.Error("failed to initialize new kubelet client, ", zap.Error(err)) + return nil, err + } + kp.client = kclient + return kclient, nil +} + +// GetSummary Get Summary from kubelet API. +func (kp *kubeletProvider) GetSummary() (*stats.Summary, error) { + kclient, err := kp.getClient() + if err != nil { + kp.logger.Error("failed to get kubelet client, ", zap.Error(err)) + return nil, err + } + + summary, err := kclient.Summary(kp.logger) + if err != nil { + kp.logger.Error("failure from kubelet on getting summary, ", zap.Error(err)) + return nil, err + } + return summary, nil +} + +func (kp *kubeletProvider) GetPods() ([]corev1.Pod, error) { + kclient, err := kp.getClient() + if err != nil { + kp.logger.Error("failed to get kubelet client, ", zap.Error(err)) + return nil, err + } + return kclient.ListPods() +} diff --git a/receiver/awscontainerinsightreceiver/internal/k8swindows/kubelet/kubelet.go b/receiver/awscontainerinsightreceiver/internal/k8swindows/kubelet/kubelet.go new file mode 100644 index 000000000000..1fedf7c02236 --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/k8swindows/kubelet/kubelet.go @@ -0,0 +1,161 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//go:build windows +// +build windows + +package kubelet // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/k8swindows" + +import ( + "fmt" + "os" + "strconv" + + ci "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight" + cExtractor "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/cadvisor/extractors" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors" + "go.uber.org/zap" + stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1" +) + +// SummaryProvider represents receiver to get container metric from Kubelet. +type SummaryProvider struct { + logger *zap.Logger + kubeletProvider KubeletProvider + hostInfo cExtractor.CPUMemInfoProvider + metricExtractors []extractors.MetricExtractor +} + +func CreateDefaultKubeletProvider(logger *zap.Logger) KubeletProvider { + return &kubeletProvider{logger: logger, hostIP: os.Getenv("HOST_IP"), hostPort: ci.KubeSecurePort} +} + +// Options decorates SummaryProvider struct. +type Options func(provider *SummaryProvider) + +func New(logger *zap.Logger, info cExtractor.CPUMemInfoProvider, mextractor []extractors.MetricExtractor, opts ...Options) (*SummaryProvider, error) { + sp := &SummaryProvider{ + logger: logger, + hostInfo: info, + kubeletProvider: CreateDefaultKubeletProvider(logger), + metricExtractors: mextractor, + } + + for _, opt := range opts { + opt(sp) + } + + return sp, nil +} + +func (sp *SummaryProvider) GetMetrics() ([]*cExtractor.CAdvisorMetric, error) { + var metrics []*cExtractor.CAdvisorMetric + + summary, err := sp.kubeletProvider.GetSummary() + if err != nil { + sp.logger.Error("failed to get summary from kubeletProvider, ", zap.Error(err)) + return nil, err + } + outMetrics, err := sp.getPodMetrics(summary) + if err != nil { + sp.logger.Error("failed to get pod metrics using kubelet summary, ", zap.Error(err)) + return metrics, err + } + metrics = append(metrics, outMetrics...) + + nodeMetrics, err := sp.getNodeMetrics(summary) + if err != nil { + sp.logger.Error("failed to get node metrics using kubelet summary, ", zap.Error(err)) + return nodeMetrics, err + } + metrics = append(metrics, nodeMetrics...) + + return metrics, nil +} + +// getContainerMetrics returns container level metrics from kubelet summary. +func (sp *SummaryProvider) getContainerMetrics(pod stats.PodStats) ([]*cExtractor.CAdvisorMetric, error) { + var metrics []*cExtractor.CAdvisorMetric + + for _, container := range pod.Containers { + tags := map[string]string{} + + tags[ci.PodIDKey] = pod.PodRef.UID + tags[ci.K8sPodNameKey] = pod.PodRef.Name + tags[ci.K8sNamespace] = pod.PodRef.Namespace + tags[ci.ContainerNamekey] = container.Name + containerID := fmt.Sprintf("%s-%s", pod.PodRef.UID, container.Name) + tags[ci.ContainerIDkey] = containerID + + rawMetric := extractors.ConvertContainerToRaw(container, pod) + tags[ci.Timestamp] = strconv.FormatInt(rawMetric.Time.UnixNano(), 10) + + for _, extractor := range sp.metricExtractors { + if extractor.HasValue(rawMetric) { + metrics = append(metrics, extractor.GetValue(rawMetric, sp.hostInfo, ci.TypeContainer)...) + } + } + for _, metric := range metrics { + metric.AddTags(tags) + } + } + + return metrics, nil +} + +// getPodMetrics returns pod and container level metrics from kubelet summary. +func (sp *SummaryProvider) getPodMetrics(summary *stats.Summary) ([]*cExtractor.CAdvisorMetric, error) { + var metrics []*cExtractor.CAdvisorMetric + + if summary == nil { + return metrics, nil + } + + for _, pod := range summary.Pods { + var metricsPerPod []*cExtractor.CAdvisorMetric + + tags := map[string]string{} + + tags[ci.PodIDKey] = pod.PodRef.UID + tags[ci.K8sPodNameKey] = pod.PodRef.Name + tags[ci.K8sNamespace] = pod.PodRef.Namespace + + rawMetric := extractors.ConvertPodToRaw(pod) + tags[ci.Timestamp] = strconv.FormatInt(rawMetric.Time.UnixNano(), 10) + + for _, extractor := range sp.metricExtractors { + if extractor.HasValue(rawMetric) { + metricsPerPod = append(metricsPerPod, extractor.GetValue(rawMetric, sp.hostInfo, ci.TypePod)...) + } + } + for _, metric := range metricsPerPod { + metric.AddTags(tags) + } + metrics = append(metrics, metricsPerPod...) + + containerMetrics, err := sp.getContainerMetrics(pod) + if err != nil { + sp.logger.Error("failed to get container metrics, ", zap.Error(err)) + return containerMetrics, err + } + metrics = append(metrics, containerMetrics...) + } + return metrics, nil +} + +// getNodeMetrics returns Node level metrics from kubelet summary. +func (sp *SummaryProvider) getNodeMetrics(summary *stats.Summary) ([]*cExtractor.CAdvisorMetric, error) { + var metrics []*cExtractor.CAdvisorMetric + + if summary == nil { + return metrics, nil + } + + rawMetric := extractors.ConvertNodeToRaw(summary.Node) + for _, extractor := range sp.metricExtractors { + if extractor.HasValue(rawMetric) { + metrics = append(metrics, extractor.GetValue(rawMetric, sp.hostInfo, ci.TypeNode)...) + } + } + return metrics, nil +} diff --git a/receiver/awscontainerinsightreceiver/internal/k8swindows/kubelet/kubelet_test.go b/receiver/awscontainerinsightreceiver/internal/k8swindows/kubelet/kubelet_test.go new file mode 100644 index 000000000000..3d3a08b2fa10 --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/k8swindows/kubelet/kubelet_test.go @@ -0,0 +1,118 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//go:build windows +// +build windows + +package kubelet + +import ( + "testing" + + ci "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight" + cTestUtils "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/cadvisor/testutils" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/k8swindows/extractors" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/k8swindows/testutils" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1" +) + +// MockKubeletProvider Mock provider implements KubeletProvider interface. +type MockKubeletProvider struct { + logger *zap.Logger + t *testing.T +} + +func (m *MockKubeletProvider) GetSummary() (*stats.Summary, error) { + return testutils.LoadKubeletSummary(m.t, "./../extractors/testdata/CurSingleKubeletSummary.json"), nil +} + +func (m *MockKubeletProvider) GetPods() ([]corev1.Pod, error) { + return []corev1.Pod{}, nil +} + +func createKubeletDecoratorWithMockKubeletProvider(t *testing.T, logger *zap.Logger) Options { + return func(provider *SummaryProvider) { + provider.kubeletProvider = &MockKubeletProvider{t: t, logger: logger} + } +} + +func mockInfoProvider() cTestUtils.MockHostInfo { + hostInfo := cTestUtils.MockHostInfo{ClusterName: "cluster"} + return hostInfo +} + +func mockMetricExtractors() []extractors.MetricExtractor { + metricsExtractors := []extractors.MetricExtractor{} + metricsExtractors = append(metricsExtractors, extractors.NewCPUMetricExtractor(&zap.Logger{})) + metricsExtractors = append(metricsExtractors, extractors.NewMemMetricExtractor(&zap.Logger{})) + return metricsExtractors +} + +// TestGetPodMetrics Verify tags on pod and container levels metrics. +func TestGetPodMetrics(t *testing.T) { + + k8sSummaryProvider, err := New(&zap.Logger{}, mockInfoProvider(), mockMetricExtractors(), createKubeletDecoratorWithMockKubeletProvider(t, &zap.Logger{})) + summary, err := k8sSummaryProvider.kubeletProvider.GetSummary() + metrics, err := k8sSummaryProvider.getPodMetrics(summary) + + assert.NoError(t, err) + assert.NotNil(t, metrics) + + podMetric := metrics[1] + assert.Equal(t, podMetric.GetMetricType(), ci.TypePod) + assert.NotNil(t, podMetric.GetTag(ci.PodIDKey)) + assert.NotNil(t, podMetric.GetTag(ci.K8sPodNameKey)) + assert.NotNil(t, podMetric.GetTag(ci.K8sNamespace)) + assert.NotNil(t, podMetric.GetTag(ci.Timestamp)) + assert.NotNil(t, podMetric.GetTag(ci.SourcesKey)) + + containerMetric := metrics[len(metrics)-1] + assert.Equal(t, containerMetric.GetMetricType(), ci.TypeContainer) + assert.NotNil(t, containerMetric.GetTag(ci.PodIDKey)) + assert.NotNil(t, containerMetric.GetTag(ci.K8sPodNameKey)) + assert.NotNil(t, containerMetric.GetTag(ci.K8sNamespace)) + assert.NotNil(t, containerMetric.GetTag(ci.Timestamp)) + assert.NotNil(t, containerMetric.GetTag(ci.ContainerNamekey)) + assert.NotNil(t, containerMetric.GetTag(ci.ContainerIDkey)) + assert.NotNil(t, containerMetric.GetTag(ci.SourcesKey)) +} + +// TestGetContainerMetrics verify tags on container level metrics returned. +func TestGetContainerMetrics(t *testing.T) { + + k8sSummaryProvider, err := New(&zap.Logger{}, mockInfoProvider(), mockMetricExtractors(), createKubeletDecoratorWithMockKubeletProvider(t, &zap.Logger{})) + summary, err := k8sSummaryProvider.kubeletProvider.GetSummary() + + metrics, err := k8sSummaryProvider.getContainerMetrics(summary.Pods[0]) + assert.NoError(t, err) + assert.NotNil(t, metrics) + + containerMetric := metrics[1] + assert.Equal(t, containerMetric.GetMetricType(), ci.TypeContainer) + assert.NotNil(t, containerMetric.GetTag(ci.PodIDKey)) + assert.NotNil(t, containerMetric.GetTag(ci.K8sPodNameKey)) + assert.NotNil(t, containerMetric.GetTag(ci.K8sNamespace)) + assert.NotNil(t, containerMetric.GetTag(ci.Timestamp)) + assert.NotNil(t, containerMetric.GetTag(ci.ContainerNamekey)) + assert.NotNil(t, containerMetric.GetTag(ci.ContainerIDkey)) + assert.NotNil(t, containerMetric.GetTag(ci.SourcesKey)) +} + +// TestGetNodeMetrics verify tags on node level metrics. +func TestGetNodeMetrics(t *testing.T) { + + k8sSummaryProvider, err := New(&zap.Logger{}, mockInfoProvider(), mockMetricExtractors(), createKubeletDecoratorWithMockKubeletProvider(t, &zap.Logger{})) + summary, err := k8sSummaryProvider.kubeletProvider.GetSummary() + + metrics, err := k8sSummaryProvider.getNodeMetrics(summary) + assert.NoError(t, err) + assert.NotNil(t, metrics) + + nodeMetric := metrics[1] + assert.Equal(t, nodeMetric.GetMetricType(), ci.TypeNode) + assert.NotNil(t, nodeMetric.GetTag(ci.SourcesKey)) +} diff --git a/receiver/awscontainerinsightreceiver/internal/k8swindows/testutils/helpers.go b/receiver/awscontainerinsightreceiver/internal/k8swindows/testutils/helpers.go new file mode 100644 index 000000000000..66f828c91ba5 --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/k8swindows/testutils/helpers.go @@ -0,0 +1,22 @@ +package testutils + +import ( + "encoding/json" + "os" + "testing" + + stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1" + + "github.com/stretchr/testify/assert" +) + +func LoadKubeletSummary(t *testing.T, file string) *stats.Summary { + info, err := os.ReadFile(file) + assert.Nil(t, err, "Fail to read sample kubelet summary response file content") + + var kSummary stats.Summary + err = json.Unmarshal(info, &kSummary) + assert.Nil(t, err, "Fail to parse json string") + + return &kSummary +} diff --git a/receiver/awscontainerinsightreceiver/internal/stores/kubeletutil/kubeletclient.go b/receiver/awscontainerinsightreceiver/internal/stores/kubeletutil/kubeletclient.go index 8a36a35bbf5f..8ec921a04322 100644 --- a/receiver/awscontainerinsightreceiver/internal/stores/kubeletutil/kubeletclient.go +++ b/receiver/awscontainerinsightreceiver/internal/stores/kubeletutil/kubeletclient.go @@ -9,6 +9,7 @@ import ( "go.uber.org/zap" corev1 "k8s.io/api/core/v1" + stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1" "k8s.io/utils/net" "github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig" @@ -69,3 +70,22 @@ func (k *KubeletClient) ListPods() ([]corev1.Pod, error) { return pods.Items, nil } + +// Summary hits kubelet summary API using service account authentication. +// Summary API returns metrics at container, pod and node level for CPU, memory, networking and file system resources. +func (k *KubeletClient) Summary(logger *zap.Logger) (*stats.Summary, error) { + logger.Debug("Calling kubelet /stats/summary API") + + b, err := k.restClient.Get("/stats/summary") + if err != nil { + return nil, fmt.Errorf("call to kubelet /stats/summary API failed %w", err) + } + var out stats.Summary + err = json.Unmarshal(b, &out) + if err != nil { + return nil, fmt.Errorf("kubelet summary unmarshalling failed %w", err) + } + + logger.Debug("/stats/summary API response unmarshalled successfully") + return &out, nil +} diff --git a/receiver/awscontainerinsightreceiver/internal/stores/utils.go b/receiver/awscontainerinsightreceiver/internal/stores/utils.go index c1db9f3f9884..2ba72120946d 100644 --- a/receiver/awscontainerinsightreceiver/internal/stores/utils.go +++ b/receiver/awscontainerinsightreceiver/internal/stores/utils.go @@ -96,6 +96,14 @@ func stringInRuneset(name, subset string) bool { } func TagMetricSource(metric CIMetric) { + if metric.GetTag(ci.OperatingSystem) == ci.OperatingSystemWindows { + tagMetricSourceWindows(metric) + return + } + tagMetricSourceLinux(metric) +} + +func tagMetricSourceLinux(metric CIMetric) { metricType := metric.GetTag(ci.MetricType) if metricType == "" { return @@ -132,6 +140,43 @@ func TagMetricSource(metric CIMetric) { } } +func tagMetricSourceWindows(metric CIMetric) { + metricType := metric.GetTag(ci.MetricType) + if metricType == "" { + return + } + + var sources []string + switch metricType { + case ci.TypeNode: + sources = append(sources, []string{"kubelet", "pod", "calculated"}...) + case ci.TypeNodeFS: + sources = append(sources, []string{"kubelet", "calculated"}...) + case ci.TypeNodeNet: + sources = append(sources, []string{"kubelet", "calculated"}...) + case ci.TypeNodeDiskIO: + sources = append(sources, []string{"kubelet"}...) + case ci.TypePod: + sources = append(sources, []string{"kubelet", "pod", "calculated"}...) + case ci.TypePodNet: + sources = append(sources, []string{"kubelet", "calculated"}...) + case ci.TypeContainer: + sources = append(sources, []string{"kubelet", "pod", "calculated"}...) + case ci.TypeContainerFS: + sources = append(sources, []string{"kubelet", "calculated"}...) + case ci.TypeContainerDiskIO: + sources = append(sources, []string{"kubelet"}...) + } + + if len(sources) > 0 { + sourcesInfo, err := json.Marshal(sources) + if err != nil { + return + } + metric.AddTag(ci.SourcesKey, string(sourcesInfo)) + } +} + func AddKubernetesInfo(metric CIMetric, kubernetesBlob map[string]any, retainContainerNameTag bool) { needMoveToKubernetes := map[string]string{ci.K8sPodNameKey: "pod_name", ci.PodIDKey: "pod_id"} needCopyToKubernetes := map[string]string{ci.K8sNamespace: "namespace_name", ci.TypeService: "service_name", ci.NodeNameKey: "host"} diff --git a/receiver/awscontainerinsightreceiver/internal/stores/utils_test.go b/receiver/awscontainerinsightreceiver/internal/stores/utils_test.go index 397d325cc773..0b68664146d6 100644 --- a/receiver/awscontainerinsightreceiver/internal/stores/utils_test.go +++ b/receiver/awscontainerinsightreceiver/internal/stores/utils_test.go @@ -201,3 +201,43 @@ func TestUtils_TagMetricSource(t *testing.T) { assert.Equal(t, expectedSources[i], metric.tags[ci.SourcesKey]) } } + +func TestUtils_TagMetricSourceWindows(t *testing.T) { + types := []string{ + "", + ci.TypeNode, + ci.TypeNodeFS, + ci.TypeNodeNet, + ci.TypeNodeDiskIO, + ci.TypePod, + ci.TypePodNet, + ci.TypeContainer, + ci.TypeContainerFS, + ci.TypeContainerDiskIO, + } + + expectedSources := []string{ + "", + "[\"kubelet\",\"pod\",\"calculated\"]", + "[\"kubelet\",\"calculated\"]", + "[\"kubelet\",\"calculated\"]", + "[\"kubelet\"]", + "[\"kubelet\",\"pod\",\"calculated\"]", + "[\"kubelet\",\"calculated\"]", + "[\"kubelet\",\"pod\",\"calculated\"]", + "[\"kubelet\",\"calculated\"]", + "[\"kubelet\"]", + } + for i, mtype := range types { + tags := map[string]string{ + ci.MetricType: mtype, + ci.OperatingSystem: ci.OperatingSystemWindows, + } + + metric := &mockCIMetric{ + tags: tags, + } + TagMetricSource(metric) + assert.Equal(t, expectedSources[i], metric.tags[ci.SourcesKey]) + } +} diff --git a/receiver/awscontainerinsightreceiver/receiver.go b/receiver/awscontainerinsightreceiver/receiver.go index 9972acd01718..a237494cbb16 100644 --- a/receiver/awscontainerinsightreceiver/receiver.go +++ b/receiver/awscontainerinsightreceiver/receiver.go @@ -6,6 +6,7 @@ package awscontainerinsightreceiver // import "github.com/open-telemetry/opentel import ( "context" "errors" + "runtime" "time" "go.opentelemetry.io/collector/component" @@ -21,6 +22,7 @@ import ( ecsinfo "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/ecsInfo" hostInfo "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/host" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/k8sapiserver" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/k8swindows" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/stores" ) @@ -33,13 +35,13 @@ type metricsProvider interface { // awsContainerInsightReceiver implements the receiver.Metrics type awsContainerInsightReceiver struct { - settings component.TelemetrySettings - nextConsumer consumer.Metrics - config *Config - cancel context.CancelFunc - cadvisor metricsProvider - k8sapiserver metricsProvider - prometheusScraper *k8sapiserver.PrometheusScraper + settings component.TelemetrySettings + nextConsumer consumer.Metrics + config *Config + cancel context.CancelFunc + containerMetricsProvider metricsProvider + k8sapiserver metricsProvider + prometheusScraper *k8sapiserver.PrometheusScraper } // newAWSContainerInsightReceiver creates the aws container insight receiver with the given parameters. @@ -74,26 +76,33 @@ func (acir *awsContainerInsightReceiver) Start(ctx context.Context, host compone return err } - decoratorOption := cadvisor.WithDecorator(k8sDecorator) - acir.cadvisor, err = cadvisor.New(acir.config.ContainerOrchestrator, hostinfo, acir.settings.Logger, decoratorOption) - if err != nil { - return err - } + if runtime.GOOS == ci.OperatingSystemWindows { + acir.containerMetricsProvider, err = k8swindows.New(acir.settings.Logger, k8sDecorator, *hostinfo) + if err != nil { + return err + } + } else { + decoratorOption := cadvisor.WithDecorator(k8sDecorator) + acir.containerMetricsProvider, err = cadvisor.New(acir.config.ContainerOrchestrator, hostinfo, acir.settings.Logger, decoratorOption) + if err != nil { + return err + } - leaderElection, err := k8sapiserver.NewLeaderElection(acir.settings.Logger, k8sapiserver.WithLeaderLockName(acir.config.LeaderLockName), - k8sapiserver.WithLeaderLockUsingConfigMapOnly(acir.config.LeaderLockUsingConfigMapOnly)) - if err != nil { - return err - } + leaderElection, err := k8sapiserver.NewLeaderElection(acir.settings.Logger, k8sapiserver.WithLeaderLockName(acir.config.LeaderLockName), + k8sapiserver.WithLeaderLockUsingConfigMapOnly(acir.config.LeaderLockUsingConfigMapOnly)) + if err != nil { + return err + } - acir.k8sapiserver, err = k8sapiserver.NewK8sAPIServer(hostinfo, acir.settings.Logger, leaderElection, acir.config.AddFullPodNameMetricLabel, acir.config.EnableControlPlaneMetrics) - if err != nil { - return err - } + acir.k8sapiserver, err = k8sapiserver.NewK8sAPIServer(hostinfo, acir.settings.Logger, leaderElection, acir.config.AddFullPodNameMetricLabel, acir.config.EnableControlPlaneMetrics) + if err != nil { + return err + } - err = acir.startPrometheusScraper(ctx, host, hostinfo, leaderElection) - if err != nil { - acir.settings.Logger.Debug("Unable to start kube apiserver prometheus scraper", zap.Error(err)) + err = acir.startPrometheusScraper(ctx, host, hostinfo, leaderElection) + if err != nil { + acir.settings.Logger.Debug("Unable to start kube apiserver prometheus scraper", zap.Error(err)) + } } } if acir.config.ContainerOrchestrator == ci.ECS { @@ -105,7 +114,7 @@ func (acir *awsContainerInsightReceiver) Start(ctx context.Context, host compone ecsOption := cadvisor.WithECSInfoCreator(ecsInfo) - acir.cadvisor, err = cadvisor.New(acir.config.ContainerOrchestrator, hostinfo, acir.settings.Logger, ecsOption) + acir.containerMetricsProvider, err = cadvisor.New(acir.config.ContainerOrchestrator, hostinfo, acir.settings.Logger, ecsOption) if err != nil { return err } @@ -186,8 +195,8 @@ func (acir *awsContainerInsightReceiver) Shutdown(context.Context) error { if acir.k8sapiserver != nil { errs = errors.Join(errs, acir.k8sapiserver.Shutdown()) } - if acir.cadvisor != nil { - errs = errors.Join(errs, acir.cadvisor.Shutdown()) + if acir.containerMetricsProvider != nil { + errs = errors.Join(errs, acir.containerMetricsProvider.Shutdown()) } return errs @@ -197,14 +206,15 @@ func (acir *awsContainerInsightReceiver) Shutdown(context.Context) error { // collectData collects container stats from Amazon ECS Task Metadata Endpoint func (acir *awsContainerInsightReceiver) collectData(ctx context.Context) error { var mds []pmetric.Metrics - if acir.cadvisor == nil && acir.k8sapiserver == nil { + + if acir.containerMetricsProvider == nil && acir.k8sapiserver == nil { err := errors.New("both cadvisor and k8sapiserver failed to start") acir.settings.Logger.Error("Failed to collect stats", zap.Error(err)) return err } - if acir.cadvisor != nil { - mds = append(mds, acir.cadvisor.GetMetrics()...) + if acir.containerMetricsProvider != nil { + mds = append(mds, acir.containerMetricsProvider.GetMetrics()...) } if acir.k8sapiserver != nil { diff --git a/receiver/awscontainerinsightreceiver/receiver_test.go b/receiver/awscontainerinsightreceiver/receiver_test.go index 27686d194f1b..f3b251023d02 100644 --- a/receiver/awscontainerinsightreceiver/receiver_test.go +++ b/receiver/awscontainerinsightreceiver/receiver_test.go @@ -90,12 +90,12 @@ func TestCollectData(t *testing.T) { _ = r.Start(context.Background(), nil) ctx := context.Background() r.k8sapiserver = &mockK8sAPIServer{} - r.cadvisor = &mockCadvisor{} + r.containerMetricsProvider = &mockCadvisor{} err = r.collectData(ctx) require.Nil(t, err) // test the case when cadvisor and k8sapiserver failed to initialize - r.cadvisor = nil + r.containerMetricsProvider = nil r.k8sapiserver = nil err = r.collectData(ctx) require.NotNil(t, err) @@ -114,7 +114,7 @@ func TestCollectDataWithErrConsumer(t *testing.T) { r := metricsReceiver.(*awsContainerInsightReceiver) _ = r.Start(context.Background(), nil) - r.cadvisor = &mockCadvisor{} + r.containerMetricsProvider = &mockCadvisor{} r.k8sapiserver = &mockK8sAPIServer{} ctx := context.Background() @@ -138,12 +138,12 @@ func TestCollectDataWithECS(t *testing.T) { _ = r.Start(context.Background(), nil) ctx := context.Background() - r.cadvisor = &mockCadvisor{} + r.containerMetricsProvider = &mockCadvisor{} err = r.collectData(ctx) require.Nil(t, err) // test the case when cadvisor and k8sapiserver failed to initialize - r.cadvisor = nil + r.containerMetricsProvider = nil err = r.collectData(ctx) require.NotNil(t, err) }