Use forked libbpf for debugging

Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com>

Makefile

@@ -101,6 +101,8 @@ bpf_compile_tools = $(CMD_LLC) $(CMD_CLANG)
 .PHONY: $(bpf_compile_tools)
 $(bpf_compile_tools): % : check_%
 
+# TODO(kakkoyun): To prevent out of sync libbpf dependency, we nmight want to try directly linking/updating the submodule in the libbpf-go.
+# - Determining the location of the go module cache dir and initializing the submodule in there and linking in here, should be doable.
 $(LIBBPF_SRC):
 	test -d $(LIBBPF_SRC) || (echo "missing libbpf source - maybe do 'git submodule init && git submodule update'" ; false)
 

go.mod

@@ -149,3 +149,5 @@ require (
 
 // Prometheus v2.32.1
 replace github.com/prometheus/prometheus => github.com/prometheus/prometheus v1.8.2-0.20211217191541-41f1a8125e66
+
+replace github.com/aquasecurity/libbpfgo => github.com/kakkoyun/libbpfgo v0.2.5-libbpf-0.7.0.0.20220407163225-d697d4f75434

go.sum

@@ -160,8 +160,6 @@ github.com/apache/arrow/go/arrow v0.0.0-20200601151325-b2287a20f230/go.mod h1:QN
 github.com/apache/arrow/go/arrow v0.0.0-20200923215132-ac86123a3f01/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
-github.com/aquasecurity/libbpfgo v0.2.5-libbpf-0.7.0.0.20220323152305-5ced2a6e6c13 h1:YqnctcSNOGQ3vnzsNqTfehRixFal1MWYfrFJGPGJyY8=
-github.com/aquasecurity/libbpfgo v0.2.5-libbpf-0.7.0.0.20220323152305-5ced2a6e6c13/go.mod h1:/+clceXE103FaXvVTIY2HAkQjxNtkra4DRWvZYr2SKw=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
@@ -941,6 +939,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0=
+github.com/kakkoyun/libbpfgo v0.2.5-libbpf-0.7.0.0.20220407163225-d697d4f75434 h1:CBqiBxo2Ck/bdjuh5+nowMnQl2eu83D7qaVwGi4e4MI=
+github.com/kakkoyun/libbpfgo v0.2.5-libbpf-0.7.0.0.20220407163225-d697d4f75434/go.mod h1:/+clceXE103FaXvVTIY2HAkQjxNtkra4DRWvZYr2SKw=
 github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
 github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=

pkg/agent/write_client.go

@@ -105,7 +105,7 @@ func (b *Batcher) batchLoop(ctx context.Context) error {
 	}
 
 	if len(batch) > 0 {
-		level.Debug(b.logger).Log("msg", "batch write client sent profiles", "count", len(batch))
+		level.Debug(b.logger).Log("msg", "batch write client has sent profiles", "count", len(batch))
 	}
 	return nil
 }

pkg/maps/maps.go

@@ -95,7 +95,6 @@ func (c *PIDMappingFileCache) mappingForPID(pid uint32) ([]*profile.Mapping, err
 	for _, m := range mapping {
 		// Try our best to have the BuildID.
 		if m.BuildID == "" {
-			// TODO(brancz): These need special cases. See pseudo-paths in proc's man page
 			//  m.File == "[vdso]" || m.File == "[vsyscall]" || m.File == "[stack]" || m.File == "[heap]"
 			if m.Unsymbolizable() || m.File == "" {
 				continue

pkg/profiler/profiler.go

@@ -68,9 +68,8 @@ type stackCountKey struct {
 }
 
 type bpfMaps struct {
-	counts      *bpf.BPFMap
-	stackTraces *bpf.BPFMap
-	countKeys   []stackCountKey
+	counts *bpf.BPFMap
+	traces *bpf.BPFMap
 }
 
 type CgroupProfiler struct {
@@ -85,7 +84,8 @@ type CgroupProfiler struct {
 	ksymCache           *ksym.Cache
 	objCache            objectfile.Cache
 
-	bpfMaps *bpfMaps
+	bpfMaps   *bpfMaps
+	countKeys []stackCountKey
 
 	missingStacks      *prometheus.CounterVec
 	lastError          error
@@ -248,11 +248,12 @@ func (p *CgroupProfiler) Run(ctx context.Context) error {
 	// smaller batch sizes.
 	countKeys := make([]stackCountKey, counts.GetMaxEntries())
 
-	stackTraces, err := m.GetMap("stack_traces")
+	traces, err := m.GetMap("stack_traces")
 	if err != nil {
 		return fmt.Errorf("get stack traces map: %w", err)
 	}
-	p.bpfMaps = &bpfMaps{counts: counts, countKeys: countKeys, stackTraces: stackTraces}
+	p.bpfMaps = &bpfMaps{counts: counts, traces: traces}
+	p.countKeys = countKeys
 
 	ticker := time.NewTicker(p.profilingDuration)
 	defer ticker.Stop()
@@ -307,23 +308,34 @@ func (p *CgroupProfiler) profileLoop(ctx context.Context, captureTime time.Time)
 		byteOrder       = byteorder.GetHostByteOrder()
 
 		// Variables needed for eBPF map batch iteration.
-		keysPtr = unsafe.Pointer(&p.bpfMaps.countKeys[0])
+		keysPtr = unsafe.Pointer(&p.countKeys[0])
 		nextKey = uintptr(1)
 	)
 
-	memsetCountKeys(p.bpfMaps.countKeys, stackCountKey{})
+	memsetCountKeys(p.countKeys, stackCountKey{})
 
+	var (
+		values [][]byte
+		err    error
+	)
 	counts := p.bpfMaps.counts
-	vals, err := counts.GetValueAndDeleteBatch(keysPtr, nil, unsafe.Pointer(&nextKey), counts.GetMaxEntries())
+	values, err = counts.GetValueAndDeleteBatch(keysPtr, nil, unsafe.Pointer(&nextKey), counts.GetMaxEntries())
 	if err != nil {
 		// TODO(kakkoyun): Check if errors are correct!
-		if !errors.Is(err, syscall.ENOENT) { // Map is empty or we got all keys in the last batch.
-			return err
+		if !errors.Is(err, syscall.ENOENT) { // Map is empty, or we got all keys in the last batch.
+			level.Error(p.logger).Log("msg", "get value and delete batch", "err", err)
+		}
+	}
+	// TODO(kakkoyun): Remove when error handling with eBPF map batch operations is fixed.
+	if len(values) == 0 {
+		const msg = "get value and delete batch, nothing received"
+		if err != nil {
+			return fmt.Errorf(msg+": %w", err)
 		}
+		return errors.New(msg)
 	}
 
-	// keys := p.bpfMaps.counts
-	for i, key := range p.bpfMaps.countKeys {
+	for i, key := range p.countKeys {
 		var (
 			pid           = key.pid
 			userStackID   = key.userStackID
@@ -334,9 +346,9 @@ func (p *CgroupProfiler) profileLoop(ctx context.Context, captureTime time.Time)
 			break
 		}
 
-		value := byteOrder.Uint64(vals[i])
+		value := byteOrder.Uint64(values[i])
 
-		stackTraces := p.bpfMaps.stackTraces
+		stackTraces := p.bpfMaps.traces
 		stackBytes, err := stackTraces.GetValue(unsafe.Pointer(&userStackID))
 		if err != nil {
 			p.missingStacks.WithLabelValues("user").Inc()