Initial try for ebpf stack unwinding

Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com>

Split simple cpu profiling and cpu profiling with unwinding

Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com>

Fix communication issues with kernel space

Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com>

Send eh_frame of all mappings

Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com>

Add debug prints back

Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com>

Remove build id flag

Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com>

Use fork of delve, rather than vendoring a single package

Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com>

.dockerignore

@@ -6,3 +6,6 @@ Dockerfile
 /images
 /.github
 /tmp
+/local_workspace
+/.git
+/pkg/agent/*.bpf.o

.gitignore

@@ -4,5 +4,12 @@
 /tmp
 /out
 /bin
-pkg/agent/parca-agent.bpf.o
+pkg/agent/*.bpf.o
 TODO.md
+local_workspace
+NOTES.md
+minikube.iso
+minikube-*.iso
+/*.txt
+.kernel_config
+/*.service

.golangci.yml

@@ -3,6 +3,7 @@ run:
   skip-dirs:
     - internal/pprof
     - internal/go
+    - pkg/stack/frame
 linters-settings:
   errcheck:
     exclude: ./.errcheck_excludes.txt

Dockerfile

@@ -17,7 +17,8 @@ WORKDIR /parca-agent
 COPY go.mod go.sum /parca-agent/
 RUN go mod download -modcacherw
 
-COPY parca-agent.bpf.c vmlinux.h Makefile /parca-agent/
+COPY Makefile /parca-agent/
+COPY ./bpf /parca-agent/bpf
 COPY ./3rdparty /parca-agent/3rdparty
 RUN make bpf
 

Dockerfile.dev

@@ -21,15 +21,19 @@ WORKDIR /parca-agent
 COPY go.mod go.sum /parca-agent/
 RUN go mod download -modcacherw
 
-COPY parca-agent.bpf.c vmlinux.h Makefile /parca-agent/
+COPY Makefile /parca-agent/
+COPY ./bpf /parca-agent/bpf
 COPY ./3rdparty /parca-agent/3rdparty
-RUN make bpf
+RUN make clean bpf
 
 COPY . /parca-agent
 RUN make build
 
-# TODO(kakkoyun): Add Delve
+RUN go install github.com/go-delve/delve/cmd/dlv@v1.7.2
 
+RUN cp /go/bin/dlv /bin/dlv
 RUN cp /parca-agent/dist/parca-agent /bin/parca-agent
 
-CMD ["/bin/parca-agent"]
+EXPOSE 7071
+
+ENTRYPOINT ["/bin/dlv", "--listen=:40000", "--headless=true", "--api-version=2", "--accept-multiclient", "exec", "--continue", "--"]

Makefile

@@ -1,5 +1,5 @@
 .PHONY: all
-all: bpf build
+all: clean bpf build
 
 # tools:
 CMD_LLC ?= llc
@@ -29,9 +29,10 @@ OUT_DIR ?= dist
 GO_SRC := $(shell find . -type f -name '*.go')
 OUT_BIN := $(OUT_DIR)/parca-agent
 OUT_BIN_DEBUG_INFO := $(OUT_DIR)/debug-info
-BPF_SRC := parca-agent.bpf.c
-VMLINUX := vmlinux.h
-OUT_BPF := pkg/agent/parca-agent.bpf.o
+BPF_ROOT := bpf
+BPF_SRC := $(BPF_ROOT)/*.bpf.c
+VMLINUX := $(BPF_ROOT)/vmlinux.h
+BPF_OBJECTS := cpu_profiler cpu_profiler_with_unwinding
 BPF_HEADERS := 3rdparty/include
 BPF_BUNDLE := $(OUT_DIR)/parca-agent.bpf.tar.gz
 LIBBPF_SRC := 3rdparty/libbpf/src
@@ -98,11 +99,11 @@ $(BPF_BUNDLE): $(BPF_SRC) $(LIBBPF_HEADERS)/bpf $(BPF_HEADERS)
 	cp $$(find $^ -type f) $(bpf_bundle_dir)
 
 .PHONY: bpf
-bpf: $(OUT_BPF)
+bpf: $(BPF_OBJECTS)
 
 linux_arch := $(ARCH:x86_64=x86)
 ifndef DOCKER
-$(OUT_BPF): $(BPF_SRC) $(LIBBPF_HEADERS) $(LIBBPF_OBJ) | $(OUT_DIR) $(bpf_compile_tools)
+$(BPF_OBJECTS): %: bpf/%.bpf.c $(LIBBPF_HEADERS) $(LIBBPF_OBJ) | $(OUT_DIR) $(bpf_compile_tools)
 	mkdir -p pkg/agent
 	@v=$$($(CMD_CLANG) --version); test $$(echo $${v#*version} | head -n1 | cut -d '.' -f1) -ge '9' || (echo 'required minimum clang version: 9' ; false)
 	$(CMD_CLANG) -S \
@@ -130,7 +131,7 @@ $(OUT_BPF): $(BPF_SRC) $(LIBBPF_HEADERS) $(LIBBPF_OBJ) | $(OUT_DIR) $(bpf_compil
 		-nostdinc \
 		-target bpf \
 		-O2 -emit-llvm -c -g $< -o $(@:.o=.ll)
-	$(CMD_LLC) -march=bpf -filetype=obj -o $@ $(@:.o=.ll)
+	$(CMD_LLC) -march=bpf -filetype=obj -o pkg/agent/$@.bpf.o $(@:.o=.ll)
 	rm $(@:.o=.ll)
 else
 $(OUT_BPF): $(DOCKER_BUILDER) | $(OUT_DIR)
@@ -180,7 +181,7 @@ mostlyclean:
 
 .PHONY: clean
 clean:
-	rm pkg/agent/parca-agent.bpf.o
+	-rm pkg/agent/*.bpf.o
 	-FILE="$(docker_builder_file)" ; \
 	if [ -r "$$FILE" ] ; then \
 		$(CMD_DOCKER) rmi "$$(< $$FILE)" ; \
@@ -227,15 +228,15 @@ README.md: $(CMD_EMBEDMD) $(OUT_DIR)/help.txt deploy/manifests
 	$(CMD_EMBEDMD) -w README.md
 
 .PHONY: format
-format: go-fmt check-license
+format: go-fmt c-fmt check-license
 
 .PHONY: c-fmt
 c-fmt:
 	clang-format -i --style=GNU $(BPF_SRC)
 
 .PHONY: go-fmt
 go-fmt:
-	go fmt $(shell go list ./... | grep -E -v "pkg/internal/pprof|pkg/internal/go")
+	go fmt $(shell go list ./... | grep -E -v "pkg/internal/pprof|pkg/internal/go|pkg/stack/frame")
 
 .PHONY: check-license
 check-license:

Tiltfile

@@ -1,11 +1,14 @@
-docker_prune_settings(num_builds=5)
+docker_prune_settings(num_builds=2)
+
+# allow_k8s_contexts('admin@k8s-festive-perlman')
+# default_registry('ttl.sh/tilt-parca-dev')
 
 ## Parca Agent
 
 docker_build(
     'parca.io/parca/parca-agent:dev', '',
      dockerfile='Dockerfile.dev',
-     only=['./cmd', './pkg', './3rdparty', './go.mod', './go.sum', './parca-agent.bpf.c', './vmlinux.h', './Makefile'],
+     only=['./cmd', './pkg', './3rdparty', './go.mod', './go.sum', './bpf', './Makefile'],
 )
 k8s_yaml('deploy/tilt/parca-agent-daemonSet.yaml')
-k8s_resource('parca-agent', port_forwards=[7071])
+k8s_resource('parca-agent', port_forwards=[7071, 40000])

parca-agent.bpf.c → bpf/cpu_profiler.bpf.c

@@ -17,7 +17,6 @@
 #define KBUILD_MODNAME "parca-agent"
 
 #undef container_of
-//#include "bpf_core_read.h"
 #include <bpf_core_read.h>
 #include <bpf_endian.h>
 #include <bpf_helpers.h>
@@ -33,6 +32,7 @@
 #define MAX_STACK_ADDRESSES 1024
 // Max depth of each stack trace to track
 #define MAX_STACK_DEPTH 127
+#define MAX_ENTRIES 10240
 
 #define BPF_MAP(_name, _type, _key_type, _value_type, _max_entries)           \
   struct bpf_map_def SEC ("maps") _name = {                                   \
@@ -53,8 +53,26 @@
     .max_entries = _max_entries,                                              \
   };
 
-#define BPF_HASH(_name, _key_type, _value_type)                               \
-  BPF_MAP (_name, BPF_MAP_TYPE_HASH, _key_type, _value_type, 10240);
+#define BPF_HASH(_name, _key_type, _value_type, _max_entries)                 \
+  BPF_MAP (_name, BPF_MAP_TYPE_HASH, _key_type, _value_type, _max_entries);
+
+#define LOG(_pid, _msg)                                                       \
+  {                                                                           \
+    log_event_t e = { .pid = _pid, .message = _msg };                         \
+    bpf_ringbuf_output (&events, &e, sizeof (e), 0);                          \
+  }
+
+typedef struct log_event
+{
+  u32 pid;
+  char message[32];
+  // u64 addr;
+} log_event_t;
+
+struct bpf_map_def SEC ("maps") events = {
+  .type = BPF_MAP_TYPE_RINGBUF,
+  .max_entries = 1 << 24,
+};
 
 /*============================= INTERNAL STRUCTS ============================*/
 
@@ -67,7 +85,7 @@ typedef struct stack_count_key
 
 /*================================ MAPS =====================================*/
 
-BPF_HASH (counts, stack_count_key_t, u64);
+BPF_HASH (counts, stack_count_key_t, u64, MAX_ENTRIES);
 BPF_STACK_TRACE (stack_traces, MAX_STACK_ADDRESSES);
 
 /*=========================== HELPER FUNCTIONS ==============================*/
@@ -90,7 +108,8 @@ bpf_map_lookup_or_try_init (void *map, const void *key, const void *init)
   return bpf_map_lookup_elem (map, key);
 }
 
-// This code gets a bit complex. Probably not suitable for casual hacking.
+/*=========================== BPF FUNCTIONS ==============================*/
+
 SEC ("perf_event")
 int
 do_sample (struct bpf_perf_event_data *ctx)
@@ -105,18 +124,22 @@ do_sample (struct bpf_perf_event_data *ctx)
   // create map key
   stack_count_key_t key = { .pid = tgid };
 
-  // get stacks
-  key.user_stack_id = bpf_get_stackid (ctx, &stack_traces, BPF_F_USER_STACK);
+  // get user stack
+  u32 zero = 0; // First element is the PID to lookup.
+  key.user_stack_id = 0;
+  int stack_id = bpf_get_stackid (ctx, &stack_traces, BPF_F_USER_STACK);
+  if (stack_id >= 0)
+    key.user_stack_id = stack_id;
+
+  // get kernel stack
   key.kernel_stack_id = bpf_get_stackid (ctx, &stack_traces, 0);
 
-  u64 zero = 0;
   u64 *count;
   count = bpf_map_lookup_or_try_init (&counts, &key, &zero);
   if (!count)
     return 0;
 
   __sync_fetch_and_add (count, 1);
-
   return 0;
 }