diff --git a/attachtype_string.go b/attachtype_string.go index de355ed90..add2a3b5c 100644 --- a/attachtype_string.go +++ b/attachtype_string.go @@ -51,11 +51,12 @@ func _() { _ = x[AttachSkReuseportSelect-39] _ = x[AttachSkReuseportSelectOrMigrate-40] _ = x[AttachPerfEvent-41] + _ = x[AttachTraceKprobeMulti-42] } -const _AttachType_name = "NoneCGroupInetEgressCGroupInetSockCreateCGroupSockOpsSkSKBStreamParserSkSKBStreamVerdictCGroupDeviceSkMsgVerdictCGroupInet4BindCGroupInet6BindCGroupInet4ConnectCGroupInet6ConnectCGroupInet4PostBindCGroupInet6PostBindCGroupUDP4SendmsgCGroupUDP6SendmsgLircMode2FlowDissectorCGroupSysctlCGroupUDP4RecvmsgCGroupUDP6RecvmsgCGroupGetsockoptCGroupSetsockoptTraceRawTpTraceFEntryTraceFExitModifyReturnLSMMacTraceIterCgroupInet4GetPeernameCgroupInet6GetPeernameCgroupInet4GetSocknameCgroupInet6GetSocknameXDPDevMapCgroupInetSockReleaseXDPCPUMapSkLookupXDPSkSKBVerdictSkReuseportSelectSkReuseportSelectOrMigratePerfEvent" +const _AttachType_name = "NoneCGroupInetEgressCGroupInetSockCreateCGroupSockOpsSkSKBStreamParserSkSKBStreamVerdictCGroupDeviceSkMsgVerdictCGroupInet4BindCGroupInet6BindCGroupInet4ConnectCGroupInet6ConnectCGroupInet4PostBindCGroupInet6PostBindCGroupUDP4SendmsgCGroupUDP6SendmsgLircMode2FlowDissectorCGroupSysctlCGroupUDP4RecvmsgCGroupUDP6RecvmsgCGroupGetsockoptCGroupSetsockoptTraceRawTpTraceFEntryTraceFExitModifyReturnLSMMacTraceIterCgroupInet4GetPeernameCgroupInet6GetPeernameCgroupInet4GetSocknameCgroupInet6GetSocknameXDPDevMapCgroupInetSockReleaseXDPCPUMapSkLookupXDPSkSKBVerdictSkReuseportSelectSkReuseportSelectOrMigratePerfEventTraceKprobeMulti" -var _AttachType_index = [...]uint16{0, 4, 20, 40, 53, 70, 88, 100, 112, 127, 142, 160, 178, 197, 216, 233, 250, 259, 272, 284, 301, 318, 334, 350, 360, 371, 381, 393, 399, 408, 430, 452, 474, 496, 505, 526, 535, 543, 546, 558, 575, 601, 610} +var _AttachType_index = [...]uint16{0, 4, 20, 40, 53, 70, 88, 100, 112, 127, 142, 160, 178, 197, 216, 233, 250, 259, 272, 284, 301, 318, 334, 350, 360, 371, 381, 393, 399, 408, 430, 452, 474, 496, 505, 526, 535, 543, 546, 558, 575, 601, 610, 626} func (i AttachType) String() string { if i >= AttachType(len(_AttachType_index)-1) { diff --git a/elf_reader.go b/elf_reader.go index e0e8fd612..a5b63538f 100644 --- a/elf_reader.go +++ b/elf_reader.go @@ -1153,8 +1153,9 @@ func getProgType(sectionName string) (ProgramType, AttachType, uint32, string) { {"cgroup/setsockopt", CGroupSockopt, AttachCGroupSetsockopt, 0}, {"struct_ops+", StructOps, AttachNone, 0}, {"sk_lookup/", SkLookup, AttachSkLookup, 0}, - {"seccomp", SocketFilter, AttachNone, 0}, + {"kprobe.multi", Kprobe, AttachTraceKprobeMulti, 0}, + {"kretprobe.multi", Kprobe, AttachTraceKprobeMulti, 0}, } for _, t := range types { diff --git a/internal/sys/ptr.go b/internal/sys/ptr.go index a22100688..bc7ebb444 100644 --- a/internal/sys/ptr.go +++ b/internal/sys/ptr.go @@ -36,3 +36,17 @@ func NewStringPointer(str string) Pointer { return Pointer{ptr: unsafe.Pointer(p)} } + +// NewStringSlicePointer allocates an array of Pointers to each string in the +// given slice of strings and returns a 64-bit pointer to the start of the +// resulting array. +// +// Use this function to pass arrays of strings as syscall arguments. +func NewStringSlicePointer(strings []string) Pointer { + sp := make([]Pointer, 0, len(strings)) + for _, s := range strings { + sp = append(sp, NewStringPointer(s)) + } + + return Pointer{ptr: unsafe.Pointer(&sp[0])} +} diff --git a/link/kprobe_multi.go b/link/kprobe_multi.go new file mode 100644 index 000000000..06db5185c --- /dev/null +++ b/link/kprobe_multi.go @@ -0,0 +1,176 @@ +package link + +import ( + "errors" + "fmt" + "os" + "unsafe" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/asm" + "github.com/cilium/ebpf/internal" + "github.com/cilium/ebpf/internal/sys" + "github.com/cilium/ebpf/internal/unix" +) + +// KprobeMultiOptions defines additional parameters that will be used +// when opening a KprobeMulti Link. +type KprobeMultiOptions struct { + // Symbols takes a list of kernel symbol names to attach an ebpf program to. + // + // Mutually exclusive with Addresses. + Symbols []string + + // Addresses takes a list of kernel symbol addresses in case they can not + // be referred to by name. + // + // Note that only start addresses can be specified, since the fprobe API + // limits the attach point to the function entry or return. + // + // Mutually exclusive with Symbols. + Addresses []uint64 + + // Cookies specifies arbitrary values that can be fetched from an eBPF + // program via `bpf_get_attach_cookie()`. + // + // If set, its length should be equal to the length of Symbols or Addresses. + // Each Cookie is assigned to the Symbol or Address specified at the + // corresponding slice index. + Cookies []uint64 +} + +// KprobeMulti attaches the given eBPF program to the entry point of a given set +// of kernel symbols. +// +// The difference with Kprobe() is that multi-kprobe accomplishes this in a +// single system call, making it significantly faster than attaching many +// probes one at a time. +// +// Requires at least Linux 5.18. +func KprobeMulti(prog *ebpf.Program, opts KprobeMultiOptions) (Link, error) { + return kprobeMulti(prog, opts, 0) +} + +// KretprobeMulti attaches the given eBPF program to the return point of a given +// set of kernel symbols. +// +// The difference with Kretprobe() is that multi-kprobe accomplishes this in a +// single system call, making it significantly faster than attaching many +// probes one at a time. +// +// Requires at least Linux 5.18. +func KretprobeMulti(prog *ebpf.Program, opts KprobeMultiOptions) (Link, error) { + return kprobeMulti(prog, opts, unix.BPF_F_KPROBE_MULTI_RETURN) +} + +func kprobeMulti(prog *ebpf.Program, opts KprobeMultiOptions, flags uint32) (Link, error) { + if prog == nil { + return nil, errors.New("cannot attach a nil program") + } + + syms := uint32(len(opts.Symbols)) + addrs := uint32(len(opts.Addresses)) + cookies := uint32(len(opts.Cookies)) + + if syms == 0 && addrs == 0 { + return nil, fmt.Errorf("one of Symbols or Addresses is required: %w", errInvalidInput) + } + if syms != 0 && addrs != 0 { + return nil, fmt.Errorf("Symbols and Addresses are mutually exclusive: %w", errInvalidInput) + } + if cookies > 0 && cookies != syms && cookies != addrs { + return nil, fmt.Errorf("Cookies must be exactly Symbols or Addresses in length: %w", errInvalidInput) + } + + if err := haveBPFLinkKprobeMulti(); err != nil { + return nil, err + } + + attr := &sys.LinkCreateKprobeMultiAttr{ + ProgFd: uint32(prog.FD()), + AttachType: sys.BPF_TRACE_KPROBE_MULTI, + KprobeMultiFlags: flags, + } + + switch { + case syms != 0: + attr.Count = syms + attr.Syms = sys.NewStringSlicePointer(opts.Symbols) + + case addrs != 0: + attr.Count = addrs + attr.Addrs = sys.NewPointer(unsafe.Pointer(&opts.Addresses[0])) + } + + if cookies != 0 { + attr.Cookies = sys.NewPointer(unsafe.Pointer(&opts.Cookies[0])) + } + + fd, err := sys.LinkCreateKprobeMulti(attr) + if errors.Is(err, unix.ESRCH) { + return nil, fmt.Errorf("couldn't find one or more symbols: %w", os.ErrNotExist) + } + if errors.Is(err, unix.EINVAL) { + return nil, fmt.Errorf("%w (missing kernel symbol or prog's AttachType not AttachTraceKprobeMulti?)", err) + } + if err != nil { + return nil, err + } + + return &kprobeMultiLink{RawLink{fd, ""}}, nil +} + +type kprobeMultiLink struct { + RawLink +} + +var _ Link = (*kprobeMultiLink)(nil) + +func (kml *kprobeMultiLink) Update(prog *ebpf.Program) error { + return fmt.Errorf("update kprobe_multi: %w", ErrNotSupported) +} + +func (kml *kprobeMultiLink) Pin(string) error { + return fmt.Errorf("pin kprobe_multi: %w", ErrNotSupported) +} + +func (kml *kprobeMultiLink) Unpin() error { + return fmt.Errorf("unpin kprobe_multi: %w", ErrNotSupported) +} + +var haveBPFLinkKprobeMulti = internal.FeatureTest("bpf_link_kprobe_multi", "5.18", func() error { + prog, err := ebpf.NewProgram(&ebpf.ProgramSpec{ + Name: "probe_kpm_link", + Type: ebpf.Kprobe, + Instructions: asm.Instructions{ + asm.Mov.Imm(asm.R0, 0), + asm.Return(), + }, + AttachType: ebpf.AttachTraceKprobeMulti, + License: "MIT", + }) + if errors.Is(err, unix.E2BIG) { + // Kernel doesn't support AttachType field. + return internal.ErrNotSupported + } + if err != nil { + return err + } + defer prog.Close() + + fd, err := sys.LinkCreateKprobeMulti(&sys.LinkCreateKprobeMultiAttr{ + ProgFd: uint32(prog.FD()), + AttachType: sys.BPF_TRACE_KPROBE_MULTI, + Count: 1, + Syms: sys.NewStringSlicePointer([]string{"vprintk"}), + }) + if errors.Is(err, unix.EINVAL) { + return internal.ErrNotSupported + } + if err != nil { + return err + } + fd.Close() + + return nil +}) diff --git a/link/kprobe_multi_test.go b/link/kprobe_multi_test.go new file mode 100644 index 000000000..57c91a26c --- /dev/null +++ b/link/kprobe_multi_test.go @@ -0,0 +1,131 @@ +package link + +import ( + "errors" + "math" + "os" + "testing" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/internal/testutils" + "github.com/cilium/ebpf/internal/unix" +) + +var kprobeMultiSyms = []string{"vprintk", "inet6_release"} + +func TestKprobeMulti(t *testing.T) { + testutils.SkipIfNotSupported(t, haveBPFLinkKprobeMulti()) + + prog := mustLoadProgram(t, ebpf.Kprobe, ebpf.AttachTraceKprobeMulti, "") + + km, err := KprobeMulti(prog, KprobeMultiOptions{Symbols: kprobeMultiSyms}) + if err != nil { + t.Fatal(err) + } + defer km.Close() + + testLink(t, km, prog) +} + +func TestKprobeMultiInput(t *testing.T) { + // Program type that loads on all kernels. Not expected to link successfully. + prog := mustLoadProgram(t, ebpf.SocketFilter, 0, "") + + // One of Symbols or Addresses must be given. + _, err := KprobeMulti(prog, KprobeMultiOptions{}) + if !errors.Is(err, errInvalidInput) { + t.Fatalf("expected errInvalidInput, got: %v", err) + } + + // Symbols and Addresses are mutually exclusive. + _, err = KprobeMulti(prog, KprobeMultiOptions{ + Symbols: []string{"foo"}, + Addresses: []uint64{1}, + }) + if !errors.Is(err, errInvalidInput) { + t.Fatalf("expected errInvalidInput, got: %v", err) + } + + // One Symbol, two cookies.. + _, err = KprobeMulti(prog, KprobeMultiOptions{ + Symbols: []string{"one"}, + Cookies: []uint64{2, 3}, + }) + if !errors.Is(err, errInvalidInput) { + t.Fatalf("expected errInvalidInput, got: %v", err) + } +} + +func TestKprobeMultiErrors(t *testing.T) { + testutils.SkipIfNotSupported(t, haveBPFLinkKprobeMulti()) + + prog := mustLoadProgram(t, ebpf.Kprobe, ebpf.AttachTraceKprobeMulti, "") + + // Nonexistent kernel symbol. + _, err := KprobeMulti(prog, KprobeMultiOptions{Symbols: []string{"bogus"}}) + if !errors.Is(err, os.ErrNotExist) && !errors.Is(err, unix.EINVAL) { + t.Fatalf("expected ErrNotExist or EINVAL, got: %s", err) + } + + // Only have a negative test for addresses as it would be hard to maintain a + // proper one. + if _, err := KprobeMulti(prog, KprobeMultiOptions{ + Addresses: []uint64{math.MaxUint64}, + }); !errors.Is(err, unix.EINVAL) { + t.Fatalf("expected EINVAL, got: %s", err) + } +} + +func TestKprobeMultiCookie(t *testing.T) { + testutils.SkipIfNotSupported(t, haveBPFLinkKprobeMulti()) + + prog := mustLoadProgram(t, ebpf.Kprobe, ebpf.AttachTraceKprobeMulti, "") + + if _, err := KprobeMulti(prog, KprobeMultiOptions{ + Symbols: kprobeMultiSyms, + Cookies: []uint64{0, 1}, + }); err != nil { + t.Fatal(err) + } +} + +func TestKprobeMultiProgramCall(t *testing.T) { + testutils.SkipIfNotSupported(t, haveBPFLinkKprobeMulti()) + + m, p := newUpdaterMapProg(t, ebpf.Kprobe, ebpf.AttachTraceKprobeMulti) + + // For simplicity, just assert the increment happens with any symbol in the array. + opts := KprobeMultiOptions{ + Symbols: []string{"__do_sys_getpid"}, + } + km, err := KprobeMulti(p, opts) + if err != nil { + t.Fatal(err) + } + + // Trigger ebpf program call. + unix.Getpid() + + // Assert that the value at index 0 has been updated to 1. + assertMapValue(t, m, 0, 1) + + // Close the link. + if err := km.Close(); err != nil { + t.Fatal(err) + } + + // Reset map value to 0 at index 0. + if err := m.Update(uint32(0), uint32(0), ebpf.UpdateExist); err != nil { + t.Fatal(err) + } + + // Retrigger the ebpf program call. + unix.Getpid() + + // Assert that this time the value has not been updated. + assertMapValue(t, m, 0, 0) +} + +func TestHaveBPFLinkKprobeMulti(t *testing.T) { + testutils.CheckFeatureTest(t, haveBPFLinkKprobeMulti) +} diff --git a/link/kprobe_test.go b/link/kprobe_test.go index a12378ee5..e9ba12af5 100644 --- a/link/kprobe_test.go +++ b/link/kprobe_test.go @@ -308,7 +308,7 @@ func TestKprobeTraceFSGroup(t *testing.T) { } func TestKprobeProgramCall(t *testing.T) { - m, p := newUpdaterMapProg(t, ebpf.Kprobe) + m, p := newUpdaterMapProg(t, ebpf.Kprobe, 0) // Open Kprobe on `sys_getpid` and attach it // to the ebpf program created above. @@ -340,7 +340,7 @@ func TestKprobeProgramCall(t *testing.T) { assertMapValue(t, m, 0, 0) } -func newUpdaterMapProg(t *testing.T, typ ebpf.ProgramType) (*ebpf.Map, *ebpf.Program) { +func newUpdaterMapProg(t *testing.T, typ ebpf.ProgramType, attach ebpf.AttachType) (*ebpf.Map, *ebpf.Program) { // Create ebpf map. Will contain only one key with initial value 0. m, err := ebpf.NewMap(&ebpf.MapSpec{ Type: ebpf.Array, @@ -378,7 +378,8 @@ func newUpdaterMapProg(t *testing.T, typ ebpf.ProgramType) (*ebpf.Map, *ebpf.Pro asm.Mov.Imm(asm.R0, 0), asm.Return(), }, - License: "Dual MIT/GPL", + AttachType: attach, + License: "Dual MIT/GPL", }) if err != nil { t.Fatal(err) diff --git a/link/link.go b/link/link.go index b6f61a7ad..112785004 100644 --- a/link/link.go +++ b/link/link.go @@ -286,7 +286,8 @@ func (l *RawLink) Info() (*Info, error) { extra = &TracingInfo{} case XDPType: extra = &XDPInfo{} - case RawTracepointType, IterType, PerfEventType: + case RawTracepointType, IterType, + PerfEventType, KprobeMultiType: // Extra metadata not supported. default: return nil, fmt.Errorf("unknown link info type: %d", info.Type) @@ -296,7 +297,7 @@ func (l *RawLink) Info() (*Info, error) { buf := bytes.NewReader(info.Extra[:]) err := binary.Read(buf, internal.NativeEndian, extra) if err != nil { - return nil, fmt.Errorf("can not read extra link info: %w", err) + return nil, fmt.Errorf("cannot read extra link info: %w", err) } } diff --git a/link/syscalls.go b/link/syscalls.go index a661395b3..035d8759f 100644 --- a/link/syscalls.go +++ b/link/syscalls.go @@ -23,6 +23,7 @@ const ( NetNsType = sys.BPF_LINK_TYPE_NETNS XDPType = sys.BPF_LINK_TYPE_XDP PerfEventType = sys.BPF_LINK_TYPE_PERF_EVENT + KprobeMultiType = sys.BPF_LINK_TYPE_KPROBE_MULTI ) var haveProgAttach = internal.FeatureTest("BPF_PROG_ATTACH", "4.10", func() error { diff --git a/link/tracepoint_test.go b/link/tracepoint_test.go index f7c054af1..18d36121e 100644 --- a/link/tracepoint_test.go +++ b/link/tracepoint_test.go @@ -75,7 +75,7 @@ func TestTracepointProgramCall(t *testing.T) { // Kernels before 4.14 don't support attaching to syscall tracepoints. testutils.SkipOnOldKernel(t, "4.14", "syscalls tracepoint support") - m, p := newUpdaterMapProg(t, ebpf.TracePoint) + m, p := newUpdaterMapProg(t, ebpf.TracePoint, 0) // Open Tracepoint at /sys/kernel/debug/tracing/events/syscalls/sys_enter_getpid // and attach it to the ebpf program created above. diff --git a/link/uprobe_test.go b/link/uprobe_test.go index 33f57a86c..f0ef04099 100644 --- a/link/uprobe_test.go +++ b/link/uprobe_test.go @@ -379,7 +379,7 @@ func TestUprobeProgramCall(t *testing.T) { testutils.SkipOnOldKernel(t, "4.14", "uprobes on Go binaries silently fail on kernel < 4.14") } - m, p := newUpdaterMapProg(t, ebpf.Kprobe) + m, p := newUpdaterMapProg(t, ebpf.Kprobe, 0) // Load the executable. ex, err := OpenExecutable(tt.elf) @@ -431,7 +431,7 @@ func TestUprobeProgramCall(t *testing.T) { } func TestUprobeProgramWrongPID(t *testing.T) { - m, p := newUpdaterMapProg(t, ebpf.Kprobe) + m, p := newUpdaterMapProg(t, ebpf.Kprobe, 0) // Load the '/bin/bash' executable. ex, err := OpenExecutable("/bin/bash") diff --git a/types.go b/types.go index 4659f1b4d..03b7f11f3 100644 --- a/types.go +++ b/types.go @@ -230,6 +230,7 @@ const ( AttachSkReuseportSelect AttachSkReuseportSelectOrMigrate AttachPerfEvent + AttachTraceKprobeMulti ) // AttachFlags of the eBPF program used in BPF_PROG_ATTACH command