attacher: try different kernel sources if there is none in default locations

build/Dockerfile

@@ -32,4 +32,10 @@ ADD https://raw.githubusercontent.com/sustainable-computing-io/kepler-model-serv
 
 ADD https://raw.githubusercontent.com/sustainable-computing-io/kepler-model-server/main/tests/test_models/AbsComponentModelWeight/Full/KerasCompWeightFullPipeline/KerasCompWeightFullPipeline.json /var/lib/kepler/data/KerasCompWeightFullPipeline.json
 
+# pre install kernel sources
+RUN mkdir -p /usr/share/kepler/kernel_sources
+
+COPY --from=quay.io/sustainable_computing_io/kepler_kernel_source_images:ubi8 /usr/src/kernels /usr/share/kepler/kernel_sources
+COPY --from=quay.io/sustainable_computing_io/kepler_kernel_source_images:ubi9 /usr/src/kernels /usr/share/kepler/kernel_sources
+
 ENTRYPOINT ["/usr/bin/kepler"]

build/kernel-source-images/Dockerfile.ubi

@@ -0,0 +1,4 @@
+FROM ImageName
+
+ARG ARCH=amd64
+RUN yum install -y kernel-devel

build/kernel-source-images/build-kernel-source-images.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -x
+IMAGE_BASE="quay.io/sustainable_computing_io/kepler_kernel_source_images"
+for i in "8" "9"; do
+    Image="registry.access.redhat.com/ubi${i}/ubi" 
+    echo "Building $i"
+    # podman doesn't support --build-arg
+    # replace ImageName with the actual image name using sed
+    sed "s|ImageName|${Image}|g" Dockerfile.ubi > Dockerfile.ubi.${i}
+
+    docker build --build-arg ImageName=${Image} -t ${IMAGE_BASE}:ubi${i}  -f Dockerfile.ubi.${i} .
+    docker push ${IMAGE_BASE}:ubi${i}
+    rm Dockerfile.ubi.${i}
+done

cmd/exporter.go

@@ -60,6 +60,7 @@ var (
 	profileDuration              = flag.Int("profile-duration", 60, "duration in seconds")
 	enabledMSR                   = flag.Bool("enable-msr", false, "whether MSR is allowed to obtain energy data")
 	enabledBPFBatchDelete        = flag.Bool("enable-bpf-batch-del", true, "bpf map batch deletion can be enabled for backported kernels older than 5.6")
+	kernelSourceDirPath          = flag.String("kernel-source-dir", "", "path to the kernel source directory")
 )
 
 func healthProbe(w http.ResponseWriter, req *http.Request) {
@@ -151,6 +152,7 @@ func main() {
 	config.SetEnabledHardwareCounterMetrics(*exposeHardwareCounterMetrics)
 	config.SetEnabledGPU(*enableGPU)
 	config.EnabledMSR = *enabledMSR
+	config.SetKernelSourceDir(*kernelSourceDirPath)
 
 	// the ebpf batch deletion operation was introduced in linux kernel 5.6, which provides better performance to delete keys.
 	// but the user can enable it if the kernel has backported this functionality.

pkg/bpfassets/attacher/bcc_attacher.go

@@ -21,6 +21,7 @@ package attacher
 
 import (
 	"fmt"
+	"os"
 	"runtime"
 	"strconv"
 
@@ -138,9 +139,24 @@ func AttachBPFAssets() (*BpfModuleTables, error) {
 	}
 	// TODO: verify if ebpf can run in the VM without hardware counter support, if not, we can disable the HC part and only collect the cpu time
 	m, err := loadModule(objProg, options)
+	if err != nil {
+		klog.Infof("failed to attach perf module with options %v: %v, from default kernel source.\n", options, err)
+		dirs := config.GetKernelSourceDir()
+		for _, dir := range dirs {
+			klog.Infof("try to load eBPF module with kernel source dir %s\n", dir)
+			os.Setenv("BCC_KERNEL_SOURCE", dir)
+			m, err = loadModule(objProg, options)
+			if err != nil {
+				klog.Infof("failed to attach perf module with options %v: %v, from kernel source %q\n", options, err, dir)
+			} else {
+				klog.Infof("Successfully load eBPF module with option: %s from kernel source %q", options, dir)
+				break
+			}
+		}
+	}
 	if err != nil {
 		klog.Infof("failed to attach perf module with options %v: %v, not able to load eBPF modules\n", options, err)
-		return nil, err
+		return nil, fmt.Errorf("failed to attach perf module with options %v: %v, not able to load eBPF modules", options, err)
 	}
 
 	tableId := m.TableId(tableProcessName)

pkg/config/config.go

@@ -87,6 +87,9 @@ var (
 
 	configPath = "/etc/kepler/kepler.config"
 
+	// dir of kernel sources for bcc
+	kernelSourceDir = []string{}
+
 	////////////////////////////////////
 	ModelServerEnable   = getBoolConfig("MODEL_SERVER_ENABLE", false)
 	ModelServerEndpoint = SetModelServerReqEndpoint()
@@ -148,6 +151,30 @@ func getConfig(configKey, defaultValue string) (result string) {
 	return
 }
 
+// SetKernelSourceDir sets the directory for all kernel source. This is used for bcc. Only the top level directory is needed.
+func SetKernelSourceDir(dir string) {
+	// read all the kernel source directories
+	if dir != "" {
+		// list all the directories under `dir` and store in kernelSourceDir
+		klog.V(4).Infoln("kernel source dir is set to", dir)
+		files, err := os.ReadDir(dir)
+		if err != nil {
+			klog.Warning("failed to read kernel source dir", err)
+		}
+		kernelVers := fmt.Sprintf("%v", getKernelVersion(c))
+		for _, file := range files {
+			// only store directories and the directory name should contain the kernel version
+			if strings.Contains(file.Name(), kernelVers) && file.IsDir() {
+				kernelSourceDir = append(kernelSourceDir, filepath.Join(dir, file.Name()))
+			}
+		}
+	}
+}
+
+func GetKernelSourceDir() []string {
+	return kernelSourceDir
+}
+
 func SetModelServerReqEndpoint() (modelServerReqEndpoint string) {
 	modelServerURL := getConfig("MODEL_SERVER_URL", modelServerService)
 	if modelServerURL == modelServerService {