Using bpf_perf_event_read_value causes verifier error #1411

dave-tucker · 2024-05-08T09:33:13Z

What happened?

With this patch applied the bpf probes are unable to be loaded due to a verifier error. There is a TODO in the code to track fixing this.

What did you expect to happen?

For the probes to load correctly.

How can we reproduce it (as minimally and precisely as possible)?

Checkout my branch with the patch applied
make build
sudo ./_output/amd64_linux/kepler and see the logs for the verifier error

Anything else we need to know?

-- BEGIN PROG LOAD LOG --
reg type unsupported for arg#0 function kepler_sched_switch_trace#57
0: R1=ctx(off=0,imm=0) R10=fp0
; if (SAMPLE_RATE > 0) {
0: (18) r2 = 0xffff956968fea310       ; R2_w=map_value(off=0,ks=4,vs=12,imm=0)
2: (61) r2 = *(u32 *)(r2 +0)          ; R2_w=0
3: (67) r2 <<= 32                     ; R2_w=0
4: (c7) r2 s>>= 32                    ; R2_w=0
5: (b7) r4 = 1                        ; R4_w=1
; if (SAMPLE_RATE > 0) {
6: (6d) if r4 s> r2 goto pc+13        ; R2_w=0 R4_w=1
; prev_pid = ctx->prev_pid;
20: (61) r6 = *(u32 *)(r1 +24)        ; R1=ctx(off=0,imm=0) R6_w=scalar(smin=0,smax=umax=4294967295,var_off=(0x0; 0xffffffff))
; prev_pid = ctx->prev_pid;
21: (63) *(u32 *)(r10 -20) = r6       ; R6_w=scalar(smin=0,smax=umax=4294967295,var_off=(0x0; 0xffffffff)) R10=fp0 fp-24=mmmm????
; pid_tgid = bpf_get_current_pid_tgid();
22: (85) call bpf_get_current_pid_tgid#14     ; R0=scalar()
23: (7b) *(u64 *)(r10 -152) = r0      ; R0=scalar() R10=fp0 fp-152_w=mmmmmmmm
; cur_pid = pid_tgid & 0xffffffff;
24: (63) *(u32 *)(r10 -28) = r0       ; R0=scalar() R10=fp0 fp-32=mmmm????
; cgroup_id = bpf_get_current_cgroup_id();
25: (85) call bpf_get_current_cgroup_id#80    ; R0_w=scalar()
26: (7b) *(u64 *)(r10 -160) = r0      ; R0_w=scalar() R10=fp0 fp-160_w=mmmmmmmm
; cpu_id = bpf_get_smp_processor_id();
27: (85) call bpf_get_smp_processor_id#8      ; R0_w=scalar(smin=smin32=0,smax=umax=smax32=umax32=31,var_off=(0x0; 0x1f))
28: (bf) r9 = r0                      ; R0_w=scalar(id=1,smin=smin32=0,smax=umax=smax32=umax32=31,var_off=(0x0; 0x1f)) R9_w=scalar(id=1,smin=smin32=0,smax=umax=smax32=umax32=31,var_off=(0x0; 0x1f))
; cpu_id = bpf_get_smp_processor_id();
29: (63) *(u32 *)(r10 -24) = r9       ; R9_w=scalar(id=1,smin=smin32=0,smax=umax=smax32=umax32=31,var_off=(0x0; 0x1f)) R10=fp0 fp-24=
; cur_ts = bpf_ktime_get_ns();
30: (85) call bpf_ktime_get_ns#5      ; R0=scalar()
31: (bf) r8 = r0                      ; R0=scalar(id=2) R8_w=scalar(id=2)
; &cpu_cycles_event_reader, *cpu_id, &c, sizeof(c));
32: (67) r9 <<= 32                    ; R9_w=scalar(smin=smin32=0,smax=umax=133143986176,smax32=umax32=0,var_off=(0x0; 0x1f00000000))
33: (77) r9 >>= 32                    ; R9_w=scalar(smin=smin32=0,smax=umax=smax32=umax32=31,var_off=(0x0; 0x1f))
34: (bf) r3 = r10                     ; R3_w=fp0 R10=fp0
; prev_pid = ctx->prev_pid;
35: (07) r3 += -144                   ; R3_w=fp-144
; error = bpf_perf_event_read_value(
36: (18) r1 = 0xffff956968fea600      ; R1_w=map_ptr(off=0,ks=4,vs=4,imm=0)
38: (bf) r2 = r9                      ; R2_w=scalar(id=3,smin=smin32=0,smax=umax=smax32=umax32=31,var_off=(0x0; 0x1f)) R9_w=scalar(id=3,smin=smin32=0,smax=umax=smax32=umax32=31,var_off=(0x0; 0x1f))
39: (b7) r4 = 24                      ; R4_w=24
40: (85) call bpf_perf_event_read_value#55    ; R0_w=scalar() fp-128=mmmmmmmm fp-136=mmmmmmmm fp-144=mmmmmmmm
41: (18) r1 = 0x80000000              ; R1_w=2147483648
; if (error < 0) {
43: (5f) r0 &= r1                     ; R0_w=scalar(smin=0,smax=umax=umax32=2147483648,smax32=0,var_off=(0x0; 0x80000000)) R1_w=2147483648
; if (error < 0) {
44: (55) if r0 != 0x0 goto pc+20 65: R0_w=scalar(smin=0,smax=umax=umax32=2147483648,smax32=0,var_off=(0x0; 0x80000000)) R1_w=2147483648 R6=scalar(smin=0,smax=umax=4294967295,var_off=(0x0; 0xffffffff)) R8_w=scalar(id=2) R9_w=scalar(id=3,smin=smin32=0,smax=umax=smax32=umax32=31,var_off=(0x0; 0x1f)) R10=fp0 fp-24= fp-32=mmmm???? fp-128=mmmmmmmm fp-136=mmmmmmmm fp-144=mmmmmmmm fp-152=mmmmmmmm fp-160=mmmmmmmm
; &cpu_instructions_event_reader, *cpu_id, &c, sizeof(c));
65: (bf) r3 = r10                     ; R3_w=fp0 R10=fp0
; 
66: (07) r3 += -144                   ; R3_w=fp-144
; error = bpf_perf_event_read_value(
67: (18) r1 = 0xffff956968feb200      ; R1_w=map_ptr(off=0,ks=4,vs=4,imm=0)
69: (bf) r2 = r9                      ; R2_w=scalar(id=3,smin=smin32=0,smax=umax=smax32=umax32=31,var_off=(0x0; 0x1f)) R9_w=scalar(id=3,smin=smin32=0,smax=umax=smax32=umax32=31,var_off=(0x0; 0x1f))
70: (b7) r4 = 24                      ; R4_w=24
71: (85) call bpf_perf_event_read_value#55    ; R0=scalar() fp-128=mmmmmmmm fp-136=mmmmmmmm fp-144=mmmmmmmm
; if (error) {
72: (67) r0 <<= 32                    ; R0_w=scalar(smax=9223372032559808512,umax=18446744069414584320,smin32=0,smax32=umax32=0,var_off=(0x0; 0xffffffff00000000))
73: (77) r0 >>= 32                    ; R0_w=scalar(smin=0,smax=umax=4294967295,var_off=(0x0; 0xffffffff))
; if (error) {
74: (55) if r0 != 0x0 goto pc+20      ; R0_w=0
; val = c.counter;
75: (79) r9 = *(u64 *)(r10 -144)      ; R9_w=scalar() R10=fp0 fp-144=mmmmmmmm
; val = c.counter;
76: (7b) *(u64 *)(r10 -16) = r9       ; R9_w=scalar() R10=fp0 fp-16_w=mmmmmmmm
77: (bf) r2 = r10                     ; R2_w=fp0 R10=fp0
; val = c.counter;
78: (07) r2 += -24                    ; R2_w=fp-24
; prev_val = bpf_map_lookup_elem(&cpu_instructions, cpu_id);
79: (18) r1 = 0xffff956968febc00      ; R1_w=map_ptr(off=0,ks=4,vs=8,imm=0)
81: (85) call bpf_map_lookup_elem#1   ; R0=map_value_or_null(id=11,off=0,ks=4,vs=8,imm=0)
; if (prev_val) {
82: (15) if r0 == 0x0 goto pc+3       ; R0=map_value(off=0,ks=4,vs=8,imm=0)
; if (*val > *prev_val)
83: (79) r1 = *(u64 *)(r0 +0)         ; R0=map_value(off=0,ks=4,vs=8,imm=0) R1_w=scalar()
; if (*val > *prev_val)
84: (3d) if r1 >= r9 goto pc+1        ; R1_w=scalar() R9=scalar()
; delta = *val - *prev_val;
85: (1f) r9 -= r1                     ; R1_w=scalar() R9_w=scalar()
86: (7b) *(u64 *)(r10 -168) = r9      ; R9_w=scalar() R10=fp0 fp-168_w=mmmmmmmm
87: (bf) r2 = r10                     ; R2_w=fp0 R10=fp0
; 
88: (07) r2 += -24                    ; R2_w=fp-24
89: (bf) r3 = r10                     ; R3_w=fp0 R10=fp0
90: (07) r3 += -16                    ; R3_w=fp-16
; bpf_map_update_elem(&cpu_instructions, cpu_id, &val, BPF_ANY);
91: (18) r1 = 0xffff956968febc00      ; R1_w=map_ptr(off=0,ks=4,vs=8,imm=0)
93: (b7) r4 = 0                       ; R4_w=0
94: (85) call bpf_map_update_elem#2   ; R0=scalar()
; &cache_miss_event_reader, *cpu_id, &c, sizeof(c));
95: (61) r2 = *(u32 *)(r10 -24)       ; R2_w=scalar(smin=0,smax=umax=4294967295,var_off=(0x0; 0xffffffff)) R10=fp0 fp-24=mmmmmmmm
96: (bf) r3 = r10                     ; R3_w=fp0 R10=fp0
; 
97: (07) r3 += -144                   ; R3_w=fp-144
; error = bpf_perf_event_read_value(
98: (18) r1 = 0xffff956968fe8000      ; R1_w=map_ptr(off=0,ks=4,vs=4,imm=0)
100: (b7) r4 = 24                     ; R4_w=24
101: (85) call bpf_perf_event_read_value#55   ; R0_w=scalar() fp-128=mmmmmmmm fp-136=mmmmmmmm fp-144=mmmmmmmm
; if (error) {
102: (67) r0 <<= 32                   ; R0_w=scalar(smax=9223372032559808512,umax=18446744069414584320,smin32=0,smax32=umax32=0,var_off=(0x0; 0xffffffff00000000))
103: (77) r0 >>= 32                   ; R0_w=scalar(smin=0,smax=umax=4294967295,var_off=(0x0; 0xffffffff))
; if (error) {
104: (55) if r0 != 0x0 goto pc+19     ; R0_w=0
; val = c.counter;
105: (79) r9 = *(u64 *)(r10 -144)     ; R9_w=scalar() R10=fp0 fp-144=mmmmmmmm
; val = c.counter;
106: (7b) *(u64 *)(r10 -16) = r9      ; R9_w=scalar() R10=fp0 fp-16_w=mmmmmmmm
107: (bf) r2 = r10                    ; R2_w=fp0 R10=fp0
; val = c.counter;
108: (07) r2 += -24                   ; R2_w=fp-24
; prev_val = bpf_map_lookup_elem(&cache_miss, cpu_id);
109: (18) r1 = 0xffff956968feb000     ; R1_w=map_ptr(off=0,ks=4,vs=8,imm=0)
111: (85) call bpf_map_lookup_elem#1          ; R0=map_value_or_null(id=12,off=0,ks=4,vs=8,imm=0)
; if (prev_val) {
112: (15) if r0 == 0x0 goto pc+3      ; R0=map_value(off=0,ks=4,vs=8,imm=0)
; if (*val > *prev_val)
113: (79) r1 = *(u64 *)(r0 +0)        ; R0=map_value(off=0,ks=4,vs=8,imm=0) R1_w=scalar()
; if (*val > *prev_val)
114: (3d) if r1 >= r9 goto pc+1       ; R1_w=scalar() R9=scalar()
; delta = *val - *prev_val;
115: (1f) r9 -= r1                    ; R1_w=scalar() R9_w=scalar()
116: (bf) r2 = r10                    ; R2_w=fp0 R10=fp0
; 
117: (07) r2 += -24                   ; R2_w=fp-24
118: (bf) r3 = r10                    ; R3_w=fp0 R10=fp0
119: (07) r3 += -16                   ; R3_w=fp-16
; bpf_map_update_elem(&cache_miss, cpu_id, &val, BPF_ANY);
120: (18) r1 = 0xffff956968feb000     ; R1_w=map_ptr(off=0,ks=4,vs=8,imm=0)
122: (b7) r4 = 0                      ; R4_w=0
123: (85) call bpf_map_update_elem#2          ; R0=scalar()
124: (7b) *(u64 *)(r10 -144) = r8     ; R8=scalar(id=2) R10=fp0 fp-144_w=mmmmmmmm
; prev_pid_key.pid = prev_pid;
125: (63) *(u32 *)(r10 -16) = r6      ; R6=scalar(smin=0,smax=umax=4294967295,var_off=(0x0; 0xffffffff)) R10=fp0 fp-16=
126: (bf) r2 = r10                    ; R2_w=fp0 R10=fp0
; 
127: (07) r2 += -16                   ; R2_w=fp-16
; prev_ts = bpf_map_lookup_elem(&pid_time, &prev_pid_key);
128: (18) r1 = 0xffff9562b0f8f800     ; R1_w=map_ptr(off=0,ks=4,vs=8,imm=0)
130: (85) call bpf_map_lookup_elem#1          ; R0_w=map_value_or_null(id=13,off=0,ks=4,vs=8,imm=0)
; if (prev_ts) {
131: (15) if r0 == 0x0 goto pc+9      ; R0_w=map_value(off=0,ks=4,vs=8,imm=0)
; if (cur_ts > *prev_ts) {
132: (79) r6 = *(u64 *)(r0 +0)        ; R0=map_value(off=0,ks=4,vs=8,imm=0) R6=scalar()
; if (cur_ts > *prev_ts) {
133: (3d) if r6 >= r8 goto pc+7       ; R6=scalar() R8=scalar(id=2)
134: (bf) r2 = r10                    ; R2_w=fp0 R10=fp0
; bpf_map_delete_elem(&pid_time, &prev_pid_key);
135: (07) r2 += -16                   ; R2_w=fp-16
136: (18) r1 = 0xffff9562b0f8f800     ; R1_w=map_ptr(off=0,ks=4,vs=8,imm=0)
138: (85) call bpf_map_delete_elem#3          ; R0_w=scalar()
; cpu_time = (cur_ts - *prev_ts) / 1000000; // convert to ms
139: (1f) r8 -= r6                    ; R6=scalar() R8_w=scalar()
; cpu_time = (cur_ts - *prev_ts) / 1000000; // convert to ms
140: (37) r8 /= 1000000               ; R8_w=scalar()
141: (79) r6 = *(u64 *)(r10 -152)     ; R6_w=scalar() R10=fp0 fp-152=mmmmmmmm
; new_pid_key.pid = cur_pid;
142: (63) *(u32 *)(r10 -8) = r6       ; R6_w=scalar() R10=fp0 fp-8=????mmmm
143: (bf) r2 = r10                    ; R2_w=fp0 R10=fp0
; 
144: (07) r2 += -8                    ; R2_w=fp-8
145: (bf) r3 = r10                    ; R3_w=fp0 R10=fp0
146: (07) r3 += -144                  ; R3_w=fp-144
; bpf_map_update_elem(&pid_time, &new_pid_key, &cur_ts, BPF_NOEXIST);
147: (18) r1 = 0xffff9562b0f8f800     ; R1_w=map_ptr(off=0,ks=4,vs=8,imm=0)
149: (b7) r4 = 1                      ; R4_w=1
150: (85) call bpf_map_update_elem#2          ; R0=scalar()
151: (bf) r2 = r10                    ; R2_w=fp0 R10=fp0
; 
152: (07) r2 += -20                   ; R2_w=fp-20
; prev_pid_metrics = bpf_map_lookup_elem(&processes, &prev_pid);
153: (18) r1 = 0xffff9562b0f8f400     ; R1_w=map_ptr(off=0,ks=4,vs=112,imm=0)
155: (85) call bpf_map_lookup_elem#1          ; R0_w=map_value_or_null(id=14,off=0,ks=4,vs=112,imm=0)
; if (prev_pid_metrics) {
156: (15) if r0 == 0x0 goto pc+17     ; R0_w=map_value(off=0,ks=4,vs=112,imm=0)
; prev_pid_metrics->cpu_cycles += buf.cpu_cycles;
157: (79) r1 = *(u64 *)(r0 +40)       ; R0_w=map_value(off=0,ks=4,vs=112,imm=0) R1_w=scalar()
158: (0f) r1 += r7
R7 !read_ok
processed 343 insns (limit 1000000) max_states_per_insn 1 total_states 24 peak_states 24 mark_read 10
-- END PROG LOAD LOG --

Kepler image tag

main

Kubernetes version

N/A

Cloud provider or bare metal

BM

OS version

# On Linux:
$ cat /etc/os-release
# paste output here
$ uname -a
# paste output here

# On Windows:
C:\> wmic os get Caption, Version, BuildNumber, OSArchitecture
# paste output here

Install tools

Kepler deployment config

For on kubernetes:

$ KEPLER_NAMESPACE=kepler

# provide kepler configmap
$ kubectl get configmap kepler-cfm -n ${KEPLER_NAMESPACE} 
# paste output here

# provide kepler deployment description
$ kubectl describe deployment kepler-exporter -n ${KEPLER_NAMESPACE}

For standalone:

put your Kepler command argument here

Container runtime (CRI) and version (if applicable)

Related plugins (CNI, CSI, ...) and versions (if applicable)

The text was updated successfully, but these errors were encountered:

This fixes a few issues identified with the bpf code format and refactor. 1. Zero initialize all variables 2. Use the bpf_perf_event_read_value helper exlusively In addition, add some logging around eBPF array resizing. Fixes: sustainable-computing-io#1402 sustainable-computing-io#1411 Signed-off-by: Dave Tucker <dave@dtucker.co.uk>

* fix(bpf): Fix issue introduced with bpf refactor This fixes a few issues identified with the bpf code format and refactor. 1. Zero initialize all variables 2. Use the bpf_perf_event_read_value helper exlusively In addition, add some logging around eBPF array resizing. Fixes: #1402 #1411 Signed-off-by: Dave Tucker <dave@dtucker.co.uk> * fix(bpf): Incorrect map size for processes and pid_time Signed-off-by: Dave Tucker <dave@dtucker.co.uk> --------- Signed-off-by: Dave Tucker <dave@dtucker.co.uk>

dave-tucker added the kind/bug report bug issue label May 8, 2024

dave-tucker mentioned this issue May 8, 2024

fix(bpf): Fix issue(s) introduced with bpf refactor #1407

Merged

dave-tucker closed this as completed Jun 7, 2024

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Using bpf_perf_event_read_value causes verifier error #1411

Using bpf_perf_event_read_value causes verifier error #1411

dave-tucker commented May 8, 2024

put your Kepler command argument here

Using bpf_perf_event_read_value causes verifier error #1411

Using bpf_perf_event_read_value causes verifier error #1411

Comments

dave-tucker commented May 8, 2024

What happened?

What did you expect to happen?

How can we reproduce it (as minimally and precisely as possible)?

Anything else we need to know?

Kepler image tag

Kubernetes version

Cloud provider or bare metal

OS version

Install tools

Kepler deployment config

put your Kepler command argument here

Container runtime (CRI) and version (if applicable)

Related plugins (CNI, CSI, ...) and versions (if applicable)