From a48bc1578bb20074e6b1d3021dc56c12c9fd86cc Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Fri, 9 Jun 2023 15:20:14 -0400 Subject: [PATCH] bpf: load precompiled module Signed-off-by: Huamin Chen --- cmd/exporter.go | 2 + pkg/bpfassets/attacher/bcc_attacher.go | 92 +++++++++++++++++--------- pkg/config/config.go | 19 ++++++ vendor/modules.txt | 4 ++ 4 files changed, 86 insertions(+), 31 deletions(-) diff --git a/cmd/exporter.go b/cmd/exporter.go index 7d676ad748..c06eef2957 100644 --- a/cmd/exporter.go +++ b/cmd/exporter.go @@ -60,6 +60,7 @@ var ( profileDuration = flag.Int("profile-duration", 60, "duration in seconds") enabledMSR = flag.Bool("enable-msr", false, "whether MSR is allowed to obtain energy data") enabledBPFBatchDelete = flag.Bool("enable-bpf-batch-del", true, "bpf map batch deletion can be enabled for backported kernels older than 5.6") + preCompiledModuleDir = flag.String("pre-compiled-module-dir", "", "path to the pre-compiled eBPF module directory") ) func healthProbe(w http.ResponseWriter, req *http.Request) { @@ -151,6 +152,7 @@ func main() { config.SetEnabledHardwareCounterMetrics(*exposeHardwareCounterMetrics) config.SetEnabledGPU(*enableGPU) config.EnabledMSR = *enabledMSR + config.SetPreCompiledModuleDir(*preCompiledModuleDir) // the ebpf batch deletion operation was introduced in linux kernel 5.6, which provides better performance to delete keys. // but the user can enable it if the kernel has backported this functionality. diff --git a/pkg/bpfassets/attacher/bcc_attacher.go b/pkg/bpfassets/attacher/bcc_attacher.go index ca1c01487b..c3a572b91c 100644 --- a/pkg/bpfassets/attacher/bcc_attacher.go +++ b/pkg/bpfassets/attacher/bcc_attacher.go @@ -30,6 +30,7 @@ import ( "github.com/sustainable-computing-io/kepler/pkg/config" bpf "github.com/iovisor/gobpf/bcc" + elf "github.com/iovisor/gobpf/elf" "github.com/jaypipes/ghw" @@ -50,9 +51,10 @@ type perfCounter struct { } const ( - tableProcessName = "processes" - tableCPUFreqName = "cpu_freq_array" - mapSize = 10240 + tableProcessName = "processes" + tableCPUFreqName = "cpu_freq_array" + mapSize = 10240 + skipCompileModule = true ) var ( @@ -116,45 +118,73 @@ func loadModule(objProg []byte, options []string) (m *bpf.Module, err error) { func AttachBPFAssets() (*BpfModuleTables, error) { bpfModules := &BpfModuleTables{} - program := assets.Program - objProg, err := assets.Asset(program) - if err != nil { - return nil, fmt.Errorf("failed to get program %q: %v", program, err) - } - // builtin runtime.NumCPU() returns the number of logical CPUs usable by the current process - cores := runtime.NumCPU() - if cpu, err := ghw.CPU(); err == nil { - // we need to get the number of all CPUs, - // so if /proc/cpuinfo is available, we can get the number of all CPUs - cores = int(cpu.TotalThreads) - } - options := []string{ - "-D__BCC__", - "-DMAP_SIZE=" + strconv.Itoa(mapSize), - "-DNUM_CPUS=" + strconv.Itoa(cores), - } - if config.EnabledEBPFCgroupID { - options = append(options, "-DSET_GROUP_ID") + loadPreCompiled := false + if skipCompileModule { + loadPreCompiled = true + } else { + // first try to compile the eBPF module + program := assets.Program + objProg, err := assets.Asset(program) + if err != nil { + return nil, fmt.Errorf("failed to get program %q: %v", program, err) + } + // builtin runtime.NumCPU() returns the number of logical CPUs usable by the current process + cores := runtime.NumCPU() + if cpu, err := ghw.CPU(); err == nil { + // we need to get the number of all CPUs, + // so if /proc/cpuinfo is available, we can get the number of all CPUs + cores = int(cpu.TotalThreads) + } + + options := []string{ + "-D__BCC__", + "-DMAP_SIZE=" + strconv.Itoa(mapSize), + "-DNUM_CPUS=" + strconv.Itoa(cores), + } + if config.EnabledEBPFCgroupID { + options = append(options, "-DSET_GROUP_ID") + } + // TODO: verify if ebpf can run in the VM without hardware counter support, if not, we can disable the HC part and only collect the cpu time + m, err := loadModule(objProg, options) + if err == nil { + bpfModules.Module = m + klog.Infof("Successfully load eBPF module with option: %s", options) + } else { + klog.Infof("failed to attach perf module with options %v: %v, not able to compile and load eBPF modules\n", options, err) + // if we failed to load the eBPF module, we will try to load the pre-compiled eBPF module + loadPreCompiled = true + } } - // TODO: verify if ebpf can run in the VM without hardware counter support, if not, we can disable the HC part and only collect the cpu time - m, err := loadModule(objProg, options) - if err != nil { - klog.Infof("failed to attach perf module with options %v: %v, not able to load eBPF modules\n", options, err) - return nil, err + if loadPreCompiled { + // load pre-compiled eBPF module if exists + modules := config.GetPreCompiledModules() + loaded := false + for _, module := range modules { + if module != "" { + m := elf.NewModule(module) + if err := m.Load(nil); err != nil { + klog.Infof("failed to load pre-compiled eBPF module %s: %v\n", module, err) + continue + } + loaded = true + //bpfModules.Module = m + break + } + } + if !loaded { + return nil, fmt.Errorf("failed to load pre-compiled eBPF module") + } } - + m := bpfModules.Module tableId := m.TableId(tableProcessName) table := bpf.NewTable(tableId, m) cpuFreqTable := bpf.NewTable(m.TableId(tableCPUFreqName), m) - bpfModules.Module = m bpfModules.Table = table bpfModules.TableName = tableProcessName bpfModules.CPUFreqTable = cpuFreqTable - klog.Infof("Successfully load eBPF module with option: %s", options) - return bpfModules, nil } diff --git a/pkg/config/config.go b/pkg/config/config.go index cec3161892..d38a077013 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -87,6 +87,8 @@ var ( configPath = "/etc/kepler/kepler.config" + preCompiledModules = []string{} + //////////////////////////////////// ModelServerEnable = getBoolConfig("MODEL_SERVER_ENABLE", false) ModelServerEndpoint = SetModelServerReqEndpoint() @@ -148,6 +150,23 @@ func getConfig(configKey, defaultValue string) (result string) { return } +// SetPreCompiledModuleDir sets the directory for pre-compiled eBPF module and return all the pre-complied module file path +func SetPreCompiledModuleDir(dir string) { + // read all the module file path + if dir != "" { + filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if !info.IsDir() { + preCompiledModules = append(preCompiledModules, path) + } + return nil + }) + } +} + +func GetPreCompiledModules() []string { + return preCompiledModules +} + func SetModelServerReqEndpoint() (modelServerReqEndpoint string) { modelServerURL := getConfig("MODEL_SERVER_URL", modelServerService) if modelServerURL == modelServerService { diff --git a/vendor/modules.txt b/vendor/modules.txt index 752618144f..a0f78b6d1e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -115,6 +115,10 @@ github.com/imdario/mergo # github.com/iovisor/gobpf v0.2.1-0.20221005153822-16120a1bf4d4 ## explicit; go 1.15 github.com/iovisor/gobpf/bcc +github.com/iovisor/gobpf/elf +github.com/iovisor/gobpf/elf/include +github.com/iovisor/gobpf/elf/include/uapi/linux +github.com/iovisor/gobpf/pkg/bpffs github.com/iovisor/gobpf/pkg/cpuonline github.com/iovisor/gobpf/pkg/cpupossible github.com/iovisor/gobpf/pkg/cpurange