Enable awscontainerinsights receiver to run inside Host Process conta…

…iner (amazon-contributing#153)

* Enable awscontainerinsights receiver to run inside Host Process container

1. Added workaround to fix ServiceAccount token and cert path for kubelet account inside HPC.
2. Added workarond to fix above issue in k8s clientset.

* Addressed chad's comments

* Addrssed pooja's comments

* Fix go.mod

cmd/otelcontribcol/go.mod

@@ -520,7 +520,7 @@ require (
 	github.com/nginxinc/nginx-prometheus-exporter v0.8.1-0.20201110005315-f5a5f8086c19 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer v0.89.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil v0.89.0 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight v0.89.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight v0.92.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs v0.89.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/ecsutil v0.89.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s v0.89.0 // indirect

go.mod

@@ -495,7 +495,7 @@ require (
 	github.com/nginxinc/nginx-prometheus-exporter v0.8.1-0.20201110005315-f5a5f8086c19 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer v0.89.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil v0.89.0 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight v0.89.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight v0.92.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs v0.89.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/ecsutil v0.89.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s v0.89.0 // indirect

internal/aws/containerinsight/const.go

@@ -12,6 +12,11 @@ const (
 	// We assume 50 micro-seconds is the minimal gap between two collected data sample to be valid to calculate delta
 	MinTimeDiff = 50 * time.Microsecond
 
+	// Environment variables
+	RunInContainer            = "RUN_IN_CONTAINER"
+	RunAsHostProcessContainer = "RUN_AS_HOST_PROCESS_CONTAINER"
+	TrueValue                 = "True"
+
 	// Attribute names
 	InstanceID              = "InstanceId"
 	InstanceType            = "InstanceType"

internal/aws/containerinsight/utils.go

@@ -5,6 +5,8 @@ package containerinsight // import "github.com/open-telemetry/opentelemetry-coll
 import (
 	"fmt"
 	"log"
+	"os"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
@@ -83,6 +85,15 @@ func IsPod(mType string) bool {
 	return false
 }
 
+func IsWindowsHostProcessContainer() bool {
+	// todo: Remove this workaround func when Windows AMIs has containerd 1.7 which solves upstream bug
+	// https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/#containerd-v1-6
+	if runtime.GOOS == "windows" && os.Getenv(RunInContainer) == TrueValue && os.Getenv(RunAsHostProcessContainer) == TrueValue {
+		return true
+	}
+	return false
+}
+
 func getPrefixByMetricType(mType string) string {
 	prefix := ""
 	instancePrefix := "instance_"

internal/aws/containerinsight/utils_test.go

@@ -5,6 +5,7 @@ package containerinsight
 import (
 	"fmt"
 	"log"
+	"os"
 	"strconv"
 	"strings"
 	"testing"
@@ -866,3 +867,14 @@ func TestConvertToOTLPMetricsForPodContainerStatusMetrics(t *testing.T) {
 	md = ConvertToOTLPMetrics(fields, tags, zap.NewNop())
 	checkMetricsAreExpected(t, md, fields, tags, expectedUnits)
 }
+
+func TestHostProcessContainer(t *testing.T) {
+	os.Setenv(RunInContainer, "True")
+	assert.Equal(t, IsWindowsHostProcessContainer(), false)
+
+	os.Setenv(RunAsHostProcessContainer, "True")
+	assert.Equal(t, IsWindowsHostProcessContainer(), true)
+
+	os.Unsetenv(RunInContainer)
+	os.Unsetenv(RunAsHostProcessContainer)
+}

internal/aws/k8s/go.mod

@@ -9,6 +9,7 @@ require (
 	k8s.io/api v0.28.3
 	k8s.io/apimachinery v0.28.3
 	k8s.io/client-go v0.28.3
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight v0.89.0
 )
 
 require (
@@ -56,6 +57,8 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
+replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => ../../aws/containerinsight
+
 retract (
 	v0.76.2
 	v0.76.1

internal/aws/k8s/k8sclient/clientset.go

@@ -5,6 +5,8 @@ package k8sclient // import "github.com/open-telemetry/opentelemetry-collector-c
 
 import (
 	"context"
+	"fmt"
+	"net"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -20,6 +22,9 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	certutil "k8s.io/client-go/util/cert"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight"
 )
 
 const (
@@ -251,7 +256,7 @@ func (c *K8sClient) init(logger *zap.Logger, options ...Option) error {
 		opt.set(c)
 	}
 
-	config, err := rest.InClusterConfig()
+	config, err := c.inClusterConfig()
 	if err != nil {
 		c.logger.Warn("cannot find in cluster config", zap.Error(err))
 		config, err = clientcmd.BuildConfigFromFlags("", c.kubeConfigPath)
@@ -459,3 +464,42 @@ func (c *K8sClient) Shutdown() {
 		}
 	}
 }
+
+// inClusterConfig is copy of rest.InClusterConfig.
+// There is known bug in rest.InClusterConfig on Windows when running it as host process container.
+// https://github.com/kubernetes/kubernetes/issues/104562
+// This copy fixes that bug by appending `CONTAINER_SANDBOX_MOUNT_POINT` in k8s token and cert file paths.
+// todo: Remove this workaround func when Windows AMIs has containerd 1.7 which solves upstream bug.
+func (c *K8sClient) inClusterConfig() (*rest.Config, error) {
+	if !containerinsight.IsWindowsHostProcessContainer() {
+		return rest.InClusterConfig()
+	}
+	var (
+		tokenFile  = filepath.Join(os.Getenv("CONTAINER_SANDBOX_MOUNT_POINT"), "/var/run/secrets/kubernetes.io/serviceaccount/token")
+		rootCAFile = filepath.Join(os.Getenv("CONTAINER_SANDBOX_MOUNT_POINT"), "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt")
+	)
+	host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT")
+	if len(host) == 0 || len(port) == 0 {
+		return nil, rest.ErrNotInCluster
+	}
+
+	token, err := os.ReadFile(tokenFile)
+	if err != nil {
+		return nil, err
+	}
+
+	tlsClientConfig := rest.TLSClientConfig{}
+
+	if _, err := certutil.NewPool(rootCAFile); err != nil {
+		c.logger.Error(fmt.Sprintf("Expected to load root CA config from %s, but got err: %v", rootCAFile, err))
+	} else {
+		tlsClientConfig.CAFile = rootCAFile
+	}
+
+	return &rest.Config{
+		Host:            "https://" + net.JoinHostPort(host, port),
+		TLSClientConfig: tlsClientConfig,
+		BearerToken:     string(token),
+		BearerTokenFile: tokenFile,
+	}, nil
+}

internal/kubelet/client.go

@@ -11,23 +11,39 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"go.uber.org/zap"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/transport"
 
+	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/common/sanitize"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig"
 )
 
-const (
+var (
 	svcAcctCACertPath   = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
 	svcAcctTokenPath    = "/var/run/secrets/kubernetes.io/serviceaccount/token" // #nosec
 	defaultSecurePort   = "10250"
 	defaultReadOnlyPort = "10255"
 )
 
+func init() {
+	updateSVCPath()
+}
+
+func updateSVCPath() {
+	// This is known that k8s token and cert file as available with CONTAINER_SANDBOX_MOUNT_POINT in path.
+	// https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/#containerd-v1-6
+	// todo: Remove this workaround func when Windows AMIs has containerd 1.7 which solves upstream bug
+	if containerinsight.IsWindowsHostProcessContainer() {
+		svcAcctCACertPath = filepath.Join(os.Getenv("CONTAINER_SANDBOX_MOUNT_POINT"), svcAcctCACertPath)
+		svcAcctTokenPath = filepath.Join(os.Getenv("CONTAINER_SANDBOX_MOUNT_POINT"), svcAcctTokenPath)
+	}
+}
+
 type Client interface {
 	Get(path string) ([]byte, error)
 }

internal/kubelet/client_test.go

@@ -15,6 +15,7 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -26,6 +27,7 @@ import (
 	"go.uber.org/zap"
 	"k8s.io/client-go/tools/clientcmd"
 
+	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig"
 )
 
@@ -71,6 +73,32 @@ func TestNewTLSClientProvider(t *testing.T) {
 	require.NotNil(t, tcc.RootCAs)
 }
 
+func TestSAPathInHostProcessContainer(t *testing.T) {
+	// todo: Remove this workaround func when Windows AMIs has containerd 1.7 which solves upstream bug.
+
+	// Test default SA cert and token.
+	assert.Equal(t, svcAcctCACertPath, "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt")
+	assert.Equal(t, svcAcctTokenPath, "/var/run/secrets/kubernetes.io/serviceaccount/token")
+
+	// Test SA cert and token when run inside container.
+	os.Setenv(containerinsight.RunInContainer, "True")
+	updateSVCPath()
+	assert.Equal(t, svcAcctCACertPath, "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt")
+	assert.Equal(t, svcAcctTokenPath, "/var/run/secrets/kubernetes.io/serviceaccount/token")
+
+	// Test SA cert and token when run inside host process container.
+	os.Setenv(containerinsight.RunAsHostProcessContainer, "True")
+	os.Setenv("CONTAINER_SANDBOX_MOUNT_POINT", "test123456")
+	updateSVCPath()
+	assert.Equal(t, svcAcctCACertPath, "test123456/var/run/secrets/kubernetes.io/serviceaccount/ca.crt")
+	assert.Equal(t, svcAcctTokenPath, "test123456/var/run/secrets/kubernetes.io/serviceaccount/token")
+
+	os.Unsetenv("CONTAINER_SANDBOX_MOUNT_POINT")
+	os.Unsetenv(containerinsight.RunInContainer)
+	os.Unsetenv(containerinsight.RunAsHostProcessContainer)
+	updateSVCPath()
+}
+
 func TestNewSAClientProvider(t *testing.T) {
 	p, err := NewClientProvider("localhost:9876", &ClientConfig{
 		APIConfig: k8sconfig.APIConfig{

internal/kubelet/go.mod

@@ -3,6 +3,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/internal/kubele
 go 1.20
 
 require (
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight v0.92.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.89.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig v0.89.0
 	github.com/stretchr/testify v1.8.4
@@ -37,6 +38,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.opentelemetry.io/collector/config/configopaque v0.89.0 // indirect
+	go.opentelemetry.io/collector/pdata v1.0.0-rcv0018 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/net v0.18.0 // indirect
 	golang.org/x/oauth2 v0.14.0 // indirect
@@ -45,6 +47,8 @@ require (
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.4.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
+	google.golang.org/grpc v1.59.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
@@ -63,6 +67,8 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/commo
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig => ../../internal/k8sconfig
 
+replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => ../../internal/aws/containerinsight
+
 // openshift removed all tags from their repo, use the pseudoversion from the release-3.9 branch HEAD
 replace github.com/openshift/api v3.9.0+incompatible => github.com/openshift/api v0.0.0-20180801171038-322a19404e37