bpf: load precompiled module

Signed-off-by: Huamin Chen <[email protected]>

cmd/exporter.go

@@ -60,6 +60,7 @@ var (
 	profileDuration              = flag.Int("profile-duration", 60, "duration in seconds")
 	enabledMSR                   = flag.Bool("enable-msr", false, "whether MSR is allowed to obtain energy data")
 	enabledBPFBatchDelete        = flag.Bool("enable-bpf-batch-del", true, "bpf map batch deletion can be enabled for backported kernels older than 5.6")
+	preCompiledModuleDir         = flag.String("pre-compiled-module-dir", "", "path to the pre-compiled eBPF module directory")
 )
 
 func healthProbe(w http.ResponseWriter, req *http.Request) {
@@ -151,6 +152,7 @@ func main() {
 	config.SetEnabledHardwareCounterMetrics(*exposeHardwareCounterMetrics)
 	config.SetEnabledGPU(*enableGPU)
 	config.EnabledMSR = *enabledMSR
+	config.SetPreCompiledModuleDir(*preCompiledModuleDir)
 
 	// the ebpf batch deletion operation was introduced in linux kernel 5.6, which provides better performance to delete keys.
 	// but the user can enable it if the kernel has backported this functionality.

pkg/bpfassets/attacher/bcc_attacher.go

@@ -30,6 +30,7 @@ import (
 	"github.com/sustainable-computing-io/kepler/pkg/config"
 
 	bpf "github.com/iovisor/gobpf/bcc"
+	elf "github.com/iovisor/gobpf/elf"
 
 	"github.com/jaypipes/ghw"
 
@@ -50,9 +51,10 @@ type perfCounter struct {
 }
 
 const (
-	tableProcessName = "processes"
-	tableCPUFreqName = "cpu_freq_array"
-	mapSize          = 10240
+	tableProcessName  = "processes"
+	tableCPUFreqName  = "cpu_freq_array"
+	mapSize           = 10240
+	skipCompileModule = true
 )
 
 var (
@@ -116,45 +118,73 @@ func loadModule(objProg []byte, options []string) (m *bpf.Module, err error) {
 
 func AttachBPFAssets() (*BpfModuleTables, error) {
 	bpfModules := &BpfModuleTables{}
-	program := assets.Program
-	objProg, err := assets.Asset(program)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get program %q: %v", program, err)
-	}
-	// builtin runtime.NumCPU() returns the number of logical CPUs usable by the current process
-	cores := runtime.NumCPU()
-	if cpu, err := ghw.CPU(); err == nil {
-		// we need to get the number of all CPUs,
-		// so if /proc/cpuinfo is available, we can get the number of all CPUs
-		cores = int(cpu.TotalThreads)
-	}
 
-	options := []string{
-		"-D__BCC__",
-		"-DMAP_SIZE=" + strconv.Itoa(mapSize),
-		"-DNUM_CPUS=" + strconv.Itoa(cores),
-	}
-	if config.EnabledEBPFCgroupID {
-		options = append(options, "-DSET_GROUP_ID")
+	loadPreCompiled := false
+	if skipCompileModule {
+		loadPreCompiled = true
+	} else {
+		// first try to compile the eBPF module
+		program := assets.Program
+		objProg, err := assets.Asset(program)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get program %q: %v", program, err)
+		}
+		// builtin runtime.NumCPU() returns the number of logical CPUs usable by the current process
+		cores := runtime.NumCPU()
+		if cpu, err := ghw.CPU(); err == nil {
+			// we need to get the number of all CPUs,
+			// so if /proc/cpuinfo is available, we can get the number of all CPUs
+			cores = int(cpu.TotalThreads)
+		}
+
+		options := []string{
+			"-D__BCC__",
+			"-DMAP_SIZE=" + strconv.Itoa(mapSize),
+			"-DNUM_CPUS=" + strconv.Itoa(cores),
+		}
+		if config.EnabledEBPFCgroupID {
+			options = append(options, "-DSET_GROUP_ID")
+		}
+		// TODO: verify if ebpf can run in the VM without hardware counter support, if not, we can disable the HC part and only collect the cpu time
+		m, err := loadModule(objProg, options)
+		if err == nil {
+			bpfModules.Module = m
+			klog.Infof("Successfully load eBPF module with option: %s", options)
+		} else {
+			klog.Infof("failed to attach perf module with options %v: %v, not able to compile and load eBPF modules\n", options, err)
+			// if we failed to load the eBPF module, we will try to load the pre-compiled eBPF module
+			loadPreCompiled = true
+		}
 	}
-	// TODO: verify if ebpf can run in the VM without hardware counter support, if not, we can disable the HC part and only collect the cpu time
-	m, err := loadModule(objProg, options)
-	if err != nil {
-		klog.Infof("failed to attach perf module with options %v: %v, not able to load eBPF modules\n", options, err)
-		return nil, err
+	if loadPreCompiled {
+		// load pre-compiled eBPF module if exists
+		modules := config.GetPreCompiledModules()
+		loaded := false
+		for _, module := range modules {
+			if module != "" {
+				m := elf.NewModule(module)
+				if err := m.Load(nil); err != nil {
+					klog.Infof("failed to load pre-compiled eBPF module %s: %v\n", module, err)
+					continue
+				}
+				loaded = true
+				//bpfModules.Module = m
+				break
+			}
+		}
+		if !loaded {
+			return nil, fmt.Errorf("failed to load pre-compiled eBPF module")
+		}
 	}
-
+	m := bpfModules.Module
 	tableId := m.TableId(tableProcessName)
 	table := bpf.NewTable(tableId, m)
 	cpuFreqTable := bpf.NewTable(m.TableId(tableCPUFreqName), m)
 
-	bpfModules.Module = m
 	bpfModules.Table = table
 	bpfModules.TableName = tableProcessName
 	bpfModules.CPUFreqTable = cpuFreqTable
 
-	klog.Infof("Successfully load eBPF module with option: %s", options)
-
 	return bpfModules, nil
 }
 

pkg/config/config.go

@@ -87,6 +87,8 @@ var (
 
 	configPath = "/etc/kepler/kepler.config"
 
+	preCompiledModules = []string{}
+
 	////////////////////////////////////
 	ModelServerEnable   = getBoolConfig("MODEL_SERVER_ENABLE", false)
 	ModelServerEndpoint = SetModelServerReqEndpoint()
@@ -148,6 +150,23 @@ func getConfig(configKey, defaultValue string) (result string) {
 	return
 }
 
+// SetPreCompiledModuleDir sets the directory for pre-compiled eBPF module and return all the pre-complied module file path
+func SetPreCompiledModuleDir(dir string) {
+	// read all the module file path
+	if dir != "" {
+		filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+			if !info.IsDir() {
+				preCompiledModules = append(preCompiledModules, path)
+			}
+			return nil
+		})
+	}
+}
+
+func GetPreCompiledModules() []string {
+	return preCompiledModules
+}
+
 func SetModelServerReqEndpoint() (modelServerReqEndpoint string) {
 	modelServerURL := getConfig("MODEL_SERVER_URL", modelServerService)
 	if modelServerURL == modelServerService {

vendor/modules.txt

@@ -115,6 +115,10 @@ github.com/imdario/mergo
 # github.com/iovisor/gobpf v0.2.1-0.20221005153822-16120a1bf4d4
 ## explicit; go 1.15
 github.com/iovisor/gobpf/bcc
+github.com/iovisor/gobpf/elf
+github.com/iovisor/gobpf/elf/include
+github.com/iovisor/gobpf/elf/include/uapi/linux
+github.com/iovisor/gobpf/pkg/bpffs
 github.com/iovisor/gobpf/pkg/cpuonline
 github.com/iovisor/gobpf/pkg/cpupossible
 github.com/iovisor/gobpf/pkg/cpurange