Skip to content

Commit

Permalink
Use cilium/ebpfs kallsyms resolver
Browse files Browse the repository at this point in the history
Instead of implementing custom logic for loading kallsyms, kvisor now
uses cilium/ebpfs logic, which has been introduced with cilium v0.17.

Additionally, the netflow related headers had to be broken out, as they
broke the debug tool, since a map was not correctly initialized.

Co-authored-by: Samuel Veloso <[email protected]>
  • Loading branch information
patrickpichler and samuelvl committed Feb 10, 2025
1 parent cc656ef commit 3423e5a
Show file tree
Hide file tree
Showing 21 changed files with 235 additions and 411 deletions.
1 change: 1 addition & 0 deletions charts/kvisor/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ agent:
- BPF
- PERFMON
- IPC_LOCK # Needed for mmap.
- SYSLOG # Required to resolve /proc/kallsyms.
drop:
- ALL
seccompProfile:
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ require (
github.com/cenkalti/backoff/v5 v5.0.1
github.com/cespare/xxhash/v2 v2.3.0
github.com/cilium/cilium v1.17.0-pre.2
github.com/cilium/ebpf v0.16.0
github.com/cilium/ebpf v0.17.2
github.com/containerd/containerd v1.7.24
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
github.com/elastic/go-freelru v0.16.0
Expand Down Expand Up @@ -43,7 +43,7 @@ require (
go.uber.org/goleak v1.3.0
golang.org/x/net v0.34.0
golang.org/x/sync v0.10.0
golang.org/x/sys v0.29.0
golang.org/x/sys v0.30.0
golang.org/x/time v0.8.0
golang.stackrox.io/kube-linter v0.7.1
google.golang.org/grpc v1.68.0
Expand Down
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,8 @@ github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ
github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
github.com/cilium/ebpf v0.16.0 h1:+BiEnHL6Z7lXnlGUsXQPPAE7+kenAd4ES8MQ5min0Ok=
github.com/cilium/ebpf v0.16.0/go.mod h1:L7u2Blt2jMM/vLAVgjxluxtBKlz3/GWjB0dMOEngfwE=
github.com/cilium/ebpf v0.17.2 h1:IQTaTVu0vKA8WTemFuBnxW9YbAwMkJVKHsNHW4lHv/g=
github.com/cilium/ebpf v0.17.2/go.mod h1:9X5VAsIOck/nCAp0+nCSVzub1Q7x+zKXXItTMYfNE+E=
github.com/cilium/fake v0.6.1 h1:cLkNx1nkF0b0pPW79JaQxaI5oG2/rBzRKpp0YUg1fTA=
github.com/cilium/fake v0.6.1/go.mod h1:V9lCbbcsnSf3vB6sdOP7Q0bsUUJ/jyHPZxnFAw5nPUc=
github.com/cilium/hive v0.0.0-20241021113747-bb8f3c0bede4 h1:dTnQNUDijFP+hf7soSZtoBYZ1OTV7qATqE+qbb//zUQ=
Expand Down Expand Up @@ -1704,6 +1706,8 @@ golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
Expand Down
2 changes: 0 additions & 2 deletions pkg/ebpftracer/c/headers/common/common.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@

#include <bpf/bpf_core_read.h>

#include <maps.h>

#define READ_KERNEL(P) ({ typeof(P) _val; \
bpf_probe_read_kernel(&_val, sizeof(_val), &P); \
_val; \
Expand Down
3 changes: 0 additions & 3 deletions pkg/ebpftracer/c/headers/common/consts.h
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,6 @@ typedef struct {
bool track_syscall_stats;
bool export_metrics;
bool cgroup_v1;
// TODO(patrick.pichler): replace this with a `__ksym` defined variable once
// this PR gets merged https://github.com/cilium/ebpf/pull/1587
u64 socket_file_ops_addr;
} global_config_t;

volatile const global_config_t global_config;
Expand Down
72 changes: 72 additions & 0 deletions pkg/ebpftracer/c/headers/common/netflow.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
#ifndef __COMMON_NETFLOW_H__
#define __COMMON_NETFLOW_H__

#include "network.h"
#include "maps.h"

statfunc void
update_traffic_summary(struct traffic_summary *val, u64 bytes, enum flow_direction direction)
{
if (unlikely(!val)) {
return;
}

val->last_packet_ts = bpf_ktime_get_ns();

switch (direction) {
case INGRESS:
__sync_fetch_and_add(&val->rx_bytes, bytes);
__sync_fetch_and_add(&val->rx_packets, 1);
break;
case EGRESS:
__sync_fetch_and_add(&val->tx_bytes, bytes);
__sync_fetch_and_add(&val->tx_packets, 1);
break;
}
}

statfunc void record_netflow(struct __sk_buff *ctx,
task_context_t *task_ctx,
nethdrs *nethdrs,
enum flow_direction direction)
{
process_identity_t identity = {
.pid = task_ctx->pid,
.pid_start_time = task_ctx->start_time,
.cgroup_id = task_ctx->cgroup_id,
};

__builtin_memcpy(identity.comm, task_ctx->comm, sizeof(identity.comm));

int zero = 0;
config_t *config = bpf_map_lookup_elem(&config_map, &zero);
if (!config)
return;

struct ip_key key = {0};

if (!load_ip_key(&key, ctx->sk, nethdrs, identity, direction)) {
return;
}

void *sum_map = bpf_map_lookup_elem(&network_traffic_buffer_map, &config->summary_map_index);
if (!sum_map)
return;

struct traffic_summary *summary = bpf_map_lookup_elem(sum_map, &key);
if (summary == NULL) {
struct traffic_summary empty = {0};

// We do not really care if the update fails, as it would mean, another thread added the
// entry.
bpf_map_update_elem(sum_map, &key, &empty, BPF_NOEXIST);

summary = bpf_map_lookup_elem(sum_map, &key);
if (summary == NULL) // Something went terribly wrong...
return;
}

update_traffic_summary(summary, ctx->len, direction);
}

#endif
65 changes: 0 additions & 65 deletions pkg/ebpftracer/c/headers/common/network.h
Original file line number Diff line number Diff line change
Expand Up @@ -398,69 +398,4 @@ statfunc bool load_ip_key(struct ip_key *key,
return true;
}

statfunc void
update_traffic_summary(struct traffic_summary *val, u64 bytes, enum flow_direction direction)
{
if (unlikely(!val)) {
return;
}

val->last_packet_ts = bpf_ktime_get_ns();

switch (direction) {
case INGRESS:
__sync_fetch_and_add(&val->rx_bytes, bytes);
__sync_fetch_and_add(&val->rx_packets, 1);
break;
case EGRESS:
__sync_fetch_and_add(&val->tx_bytes, bytes);
__sync_fetch_and_add(&val->tx_packets, 1);
break;
}
}

statfunc void record_netflow(struct __sk_buff *ctx,
task_context_t *task_ctx,
nethdrs *nethdrs,
enum flow_direction direction)
{
process_identity_t identity = {
.pid = task_ctx->pid,
.pid_start_time = task_ctx->start_time,
.cgroup_id = task_ctx->cgroup_id,
};

__builtin_memcpy(identity.comm, task_ctx->comm, sizeof(identity.comm));

int zero = 0;
config_t *config = bpf_map_lookup_elem(&config_map, &zero);
if (!config)
return;

struct ip_key key = {0};

if (!load_ip_key(&key, ctx->sk, nethdrs, identity, direction)) {
return;
}

void *sum_map = bpf_map_lookup_elem(&network_traffic_buffer_map, &config->summary_map_index);
if (!sum_map)
return;

struct traffic_summary *summary = bpf_map_lookup_elem(sum_map, &key);
if (summary == NULL) {
struct traffic_summary empty = {0};

// We do not really care if the update fails, as it would mean, another thread added the
// entry.
bpf_map_update_elem(sum_map, &key, &empty, BPF_NOEXIST);

summary = bpf_map_lookup_elem(sum_map, &key);
if (summary == NULL) // Something went terribly wrong...
return;
}

update_traffic_summary(summary, ctx->len, direction);
}

#endif
6 changes: 4 additions & 2 deletions pkg/ebpftracer/c/tracee.bpf.c
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
#include <common/logging.h>
#include <common/memory.h>
#include <common/network.h>
#include <common/netflow.h>
#include <common/stats.h>
#include <common/metrics.h>
#include <common/signatures.h>
Expand All @@ -39,6 +40,8 @@ char LICENSE[] SEC("license") = "GPL";

extern _Bool LINUX_HAS_SYSCALL_WRAPPER __kconfig;

extern void socket_file_ops __ksym;

// trace/events/syscalls.h: TP_PROTO(struct pt_regs *regs, long id)
// initial entry for sys_enter syscall logic
SEC("raw_tracepoint/sys_enter")
Expand Down Expand Up @@ -1748,7 +1751,6 @@ CGROUP_SKB_HANDLE_FUNCTION(proto_udp_dns);
return 0; \
}


SKB_NET_EVENT_CONTEXT(bucket1, SKB_PAYLOAD_BUCKET1);
SKB_NET_EVENT_CONTEXT(bucket2, SKB_PAYLOAD_BUCKET2);
SKB_NET_EVENT_CONTEXT(bucket3, SKB_PAYLOAD_BUCKET3);
Expand Down Expand Up @@ -1853,7 +1855,7 @@ int socket_task_file_iter(struct bpf_iter__task_file *ctx)
return 0;

// // We only care about sockets.
if ((u64) file->f_op != global_config.socket_file_ops_addr) {
if (file->f_op != &socket_file_ops) {
return 0;
}
net_task_context_t netctx = {0};
Expand Down
11 changes: 4 additions & 7 deletions pkg/ebpftracer/debug/c/debug.bpf.c
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,9 @@

char LICENSE[] SEC("license") = "GPL";

static volatile const pid_t target_pid SEC(".rodata.target_pid");
// For whatever reason the compiler doesn't but the bpf_map_fops variable
// in the .rodata section of the elf binary. This causes the rewrite in the
// go binary to fail. This can be fixed by forcing section via the SEC
// annotation.
static volatile const void *bpf_map_fops SEC(".rodata.bpf_map_fops");
volatile const pid_t target_pid;

extern void bpf_map_fops __ksym;

struct socket_info {
tuple_t tuple;
Expand Down Expand Up @@ -88,7 +85,7 @@ int iter_maps(struct bpf_iter__task_file *ctx)
}

// We are only intersted in maps for a specific PID.
if (file->f_op != bpf_map_fops || task->pid != target_pid) {
if (file->f_op != &bpf_map_fops || task->pid != target_pid) {
return 0;
}

Expand Down
13 changes: 3 additions & 10 deletions pkg/ebpftracer/debug/debug.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"strings"
"sync/atomic"

"github.com/castai/kvisor/pkg/ebpftracer/helpers"
"github.com/castai/kvisor/pkg/logging"
"github.com/castai/kvisor/pkg/proc"
"github.com/cilium/ebpf"
Expand Down Expand Up @@ -107,16 +108,8 @@ func (d *Debug) Load() error {

var objects debugObjects

ksymAddrs := map[string]uint64{"bpf_map_fops": 0}
if err := proc.LoadSymbolAddresses(ksymAddrs); err != nil {
return fmt.Errorf("error while resolving kallsym addresses: %w", err)
}

if err := spec.RewriteConstants(map[string]any{
"target_pid": d.cfg.TargetPID,
"bpf_map_fops": ksymAddrs["bpf_map_fops"],
}); err != nil {
return fmt.Errorf("error while rewriting constants: %w", err)
if err := helpers.SetVariable(spec, "target_pid", d.cfg.TargetPID); err != nil {
return err
}

if err := spec.LoadAndAssign(&objects, &ebpf.CollectionOptions{
Expand Down
Loading

0 comments on commit 3423e5a

Please sign in to comment.