From cf62eac8f9e3ba7ca2f81eca7e4f339a04a111fc Mon Sep 17 00:00:00 2001 From: Steve Zhang Date: Mon, 21 Nov 2022 21:28:36 -0700 Subject: [PATCH 1/2] The current codes miss below statistic data under solaris/illumos: 1. the disk io statistic data as: nread, nwritten, reads, writes, rtime, wtime; 2. the free memory under global zone; 3. the net io statistic data as: rbytes64, ipackets64, idrops64, ierrors, obytes64, opackets64, odrops64, oerrors. The new feature branch adds the above missing statistic data based on the psutil project (https://psutil.readthedocs.io/), it has been tested under solaris ( Oracle Solaris 11.4 X86) and illumos (OmniOS v11 r151044). --- disk/disk_solaris.go | 119 ++++++++++++++++++++++++++++++++++- mem/mem_solaris.go | 28 +++++++++ mem/mem_test.go | 4 +- net/net_fallback.go | 4 +- net/net_solaris.go | 143 +++++++++++++++++++++++++++++++++++++++++++ net/net_test.go | 11 +++- 6 files changed, 299 insertions(+), 10 deletions(-) create mode 100644 net/net_solaris.go diff --git a/disk/disk_solaris.go b/disk/disk_solaris.go index 9c4a798d0..934d651ff 100644 --- a/disk/disk_solaris.go +++ b/disk/disk_solaris.go @@ -10,6 +10,10 @@ import ( "fmt" "math" "os" + "path/filepath" + "regexp" + "runtime" + "strconv" "strings" "github.com/shirou/gopsutil/v3/internal/common" @@ -73,20 +77,129 @@ func PartitionsWithContext(ctx context.Context, all bool) ([]PartitionStat, erro }) } if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("unable to scan %q: %v", _MNTTAB, err) + return nil, fmt.Errorf("unable to scan %q: %w", _MNTTAB, err) } return ret, err } func IOCountersWithContext(ctx context.Context, names ...string) (map[string]IOCountersStat, error) { - return nil, common.ErrNotImplementedError + var issolaris bool + if runtime.GOOS == "illumos" { + issolaris = false + } else { + issolaris = true + } + // check disks instead of zfs pools + filterstr := "/[^zfs]/:::/^nread$|^nwritten$|^reads$|^writes$|^rtime$|^wtime$/" + kstatSysOut, err := invoke.CommandWithContext(ctx, "kstat", "-c", "disk", "-p", filterstr) + if err != nil { + return nil, fmt.Errorf("cannot execute kstat: %w", err) + } + lines := strings.Split(strings.TrimSpace(string(kstatSysOut)), "\n") + if len(lines) == 0 { + return nil, fmt.Errorf("no disk class found") + } + dnamearr := make(map[string]string) + nreadarr := make(map[string]uint64) + nwrittenarr := make(map[string]uint64) + readsarr := make(map[string]uint64) + writesarr := make(map[string]uint64) + rtimearr := make(map[string]uint64) + wtimearr := make(map[string]uint64) + re := regexp.MustCompile(`[:\s]+`) + + // in case the name is "/dev/sda1", then convert to "sda1" + for i, name := range names { + names[i] = filepath.Base(name) + } + + for _, line := range lines { + fields := re.Split(line, -1) + if len(fields) == 0 { + continue + } + moduleName := fields[0] + instance := fields[1] + dname := fields[2] + + if len(names) > 0 && !common.StringsHas(names, dname) { + continue + } + dnamearr[moduleName+instance] = dname + // fields[3] is the statistic label, fields[4] is the value + switch fields[3] { + case "nread": + nreadarr[moduleName+instance], err = strconv.ParseUint((fields[4]), 10, 64) + if err != nil { + return nil, err + } + case "nwritten": + nwrittenarr[moduleName+instance], err = strconv.ParseUint((fields[4]), 10, 64) + if err != nil { + return nil, err + } + case "reads": + readsarr[moduleName+instance], err = strconv.ParseUint((fields[4]), 10, 64) + if err != nil { + return nil, err + } + case "writes": + writesarr[moduleName+instance], err = strconv.ParseUint((fields[4]), 10, 64) + if err != nil { + return nil, err + } + case "rtime": + if issolaris { + // from sec to milli secs + var frtime float64 + frtime, err = strconv.ParseFloat((fields[4]), 64) + rtimearr[moduleName+instance] = uint64(frtime * 1000) + } else { + // from nano to milli secs + rtimearr[moduleName+instance], err = strconv.ParseUint((fields[4]), 10, 64) + rtimearr[moduleName+instance] = rtimearr[moduleName+instance] / 1000 / 1000 + } + if err != nil { + return nil, err + } + case "wtime": + if issolaris { + // from sec to milli secs + var fwtime float64 + fwtime, err = strconv.ParseFloat((fields[4]), 64) + wtimearr[moduleName+instance] = uint64(fwtime * 1000) + } else { + // from nano to milli secs + wtimearr[moduleName+instance], err = strconv.ParseUint((fields[4]), 10, 64) + wtimearr[moduleName+instance] = wtimearr[moduleName+instance] / 1000 / 1000 + } + if err != nil { + return nil, err + } + } + } + + ret := make(map[string]IOCountersStat, 0) + for k := range dnamearr { + d := IOCountersStat{ + Name: dnamearr[k], + ReadBytes: nreadarr[k], + WriteBytes: nwrittenarr[k], + ReadCount: readsarr[k], + WriteCount: writesarr[k], + ReadTime: rtimearr[k], + WriteTime: wtimearr[k], + } + ret[d.Name] = d + } + return ret, nil } func UsageWithContext(ctx context.Context, path string) (*UsageStat, error) { statvfs := unix.Statvfs_t{} if err := unix.Statvfs(path, &statvfs); err != nil { - return nil, fmt.Errorf("unable to call statvfs(2) on %q: %v", path, err) + return nil, fmt.Errorf("unable to call statvfs(2) on %q: %w", path, err) } usageStat := &UsageStat{ diff --git a/mem/mem_solaris.go b/mem/mem_solaris.go index 88f05f65d..d22cf0b43 100644 --- a/mem/mem_solaris.go +++ b/mem/mem_solaris.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/shirou/gopsutil/v3/internal/common" + "github.com/tklauser/go-sysconf" ) // VirtualMemory for Solaris is a minimal implementation which only returns @@ -34,6 +35,13 @@ func VirtualMemoryWithContext(ctx context.Context) (*VirtualMemoryStat, error) { return nil, err } result.Total = cap + freemem, err := globalZoneFreeMemory() + if err != nil { + return nil, err + } + result.Available = freemem + result.Free = freemem + result.Used = result.Total - result.Free } else { cap, err := nonGlobalZoneMemoryCapacity() if err != nil { @@ -85,6 +93,26 @@ func globalZoneMemoryCapacity() (uint64, error) { return totalMB * 1024 * 1024, nil } +func globalZoneFreeMemory() (uint64, error) { + ctx := context.Background() + output, err := invoke.CommandWithContext(ctx, "pagesize") + if err != nil { + return 0, err + } + + pagesize, err := strconv.ParseUint(strings.TrimSpace(string(output)), 10, 64) + if err != nil { + return 0, err + } + + free, err := sysconf.Sysconf(sysconf.SC_AVPHYS_PAGES) + if err != nil { + return 0, err + } + + return uint64(free) * pagesize, nil +} + var kstatMatch = regexp.MustCompile(`(\S+)\s+(\S*)`) func nonGlobalZoneMemoryCapacity() (uint64, error) { diff --git a/mem/mem_test.go b/mem/mem_test.go index d469674a6..57734d778 100644 --- a/mem/mem_test.go +++ b/mem/mem_test.go @@ -17,8 +17,8 @@ func skipIfNotImplementedErr(t *testing.T, err error) { } func TestVirtual_memory(t *testing.T) { - if runtime.GOOS == "solaris" { - t.Skip("Only .Total is supported on Solaris") + if runtime.GOOS == "solaris" || runtime.GOOS == "illumos" { + t.Skip("Only .Total .Available are supported on Solaris/illumos") } v, err := VirtualMemory() diff --git a/net/net_fallback.go b/net/net_fallback.go index 58325f655..e136be1ba 100644 --- a/net/net_fallback.go +++ b/net/net_fallback.go @@ -1,5 +1,5 @@ -//go:build !aix && !darwin && !linux && !freebsd && !openbsd && !windows -// +build !aix,!darwin,!linux,!freebsd,!openbsd,!windows +//go:build !aix && !darwin && !linux && !freebsd && !openbsd && !windows && !solaris +// +build !aix,!darwin,!linux,!freebsd,!openbsd,!windows,!solaris package net diff --git a/net/net_solaris.go b/net/net_solaris.go new file mode 100644 index 000000000..7f1f5c86f --- /dev/null +++ b/net/net_solaris.go @@ -0,0 +1,143 @@ +//go:build solaris +// +build solaris + +package net + +import ( + "context" + "fmt" + "regexp" + "runtime" + "strconv" + "strings" + + "github.com/shirou/gopsutil/v3/internal/common" +) + +// NetIOCounters returnes network I/O statistics for every network +// interface installed on the system. If pernic argument is false, +// return only sum of all information (which name is 'all'). If true, +// every network interface installed on the system is returned +// separately. +func IOCounters(pernic bool) ([]IOCountersStat, error) { + return IOCountersWithContext(context.Background(), pernic) +} + +func IOCountersWithContext(ctx context.Context, pernic bool) ([]IOCountersStat, error) { + // collect all the net class's links with below statistics + filterstr := "/^(?!vnic)/::phys:/^rbytes64$|^ipackets64$|^idrops64$|^ierrors$|^obytes64$|^opackets64$|^odrops64$|^oerrors$/" + if runtime.GOOS == "illumos" { + filterstr = "/[^vnic]/::mac:/^rbytes64$|^ipackets64$|^idrops64$|^ierrors$|^obytes64$|^opackets64$|^odrops64$|^oerrors$/" + } + kstatSysOut, err := invoke.CommandWithContext(ctx, "kstat", "-c", "net", "-p", filterstr) + if err != nil { + return nil, fmt.Errorf("cannot execute kstat: %w", err) + } + + lines := strings.Split(strings.TrimSpace(string(kstatSysOut)), "\n") + if len(lines) == 0 { + return nil, fmt.Errorf("no interface found") + } + rbytes64arr := make(map[string]uint64) + ipackets64arr := make(map[string]uint64) + idrops64arr := make(map[string]uint64) + ierrorsarr := make(map[string]uint64) + obytes64arr := make(map[string]uint64) + opackets64arr := make(map[string]uint64) + odrops64arr := make(map[string]uint64) + oerrorsarr := make(map[string]uint64) + + re := regexp.MustCompile(`[:\s]+`) + for _, line := range lines { + fields := re.Split(line, -1) + interfaceName := fields[0] + instance := fields[1] + switch fields[3] { + case "rbytes64": + rbytes64arr[interfaceName+instance], err = strconv.ParseUint(fields[4], 10, 64) + if err != nil { + return nil, fmt.Errorf("cannot parse rbytes64: %w", err) + } + case "ipackets64": + ipackets64arr[interfaceName+instance], err = strconv.ParseUint(fields[4], 10, 64) + if err != nil { + return nil, fmt.Errorf("cannot parse ipackets64: %w", err) + } + case "idrops64": + idrops64arr[interfaceName+instance], err = strconv.ParseUint(fields[4], 10, 64) + if err != nil { + return nil, fmt.Errorf("cannot parse idrops64: %w", err) + } + case "ierrors": + ierrorsarr[interfaceName+instance], err = strconv.ParseUint(fields[4], 10, 64) + if err != nil { + return nil, fmt.Errorf("cannot parse ierrors: %w", err) + } + case "obytes64": + obytes64arr[interfaceName+instance], err = strconv.ParseUint(fields[4], 10, 64) + if err != nil { + return nil, fmt.Errorf("cannot parse obytes64: %w", err) + } + case "opackets64": + opackets64arr[interfaceName+instance], err = strconv.ParseUint(fields[4], 10, 64) + if err != nil { + return nil, fmt.Errorf("cannot parse opackets64: %w", err) + } + case "odrops64": + odrops64arr[interfaceName+instance], err = strconv.ParseUint(fields[4], 10, 64) + if err != nil { + return nil, fmt.Errorf("cannot parse odrops64: %w", err) + } + case "oerrors": + oerrorsarr[interfaceName+instance], err = strconv.ParseUint(fields[4], 10, 64) + if err != nil { + return nil, fmt.Errorf("cannot parse oerrors: %w", err) + } + } + } + ret := make([]IOCountersStat, 0) + for k := range rbytes64arr { + nic := IOCountersStat{ + Name: k, + BytesRecv: rbytes64arr[k], + PacketsRecv: ipackets64arr[k], + Errin: ierrorsarr[k], + Dropin: idrops64arr[k], + BytesSent: obytes64arr[k], + PacketsSent: opackets64arr[k], + Errout: oerrorsarr[k], + Dropout: odrops64arr[k], + } + ret = append(ret, nic) + } + + if !pernic { + return getIOCountersAll(ret) + } + + return ret, nil +} + +func Connections(kind string) ([]ConnectionStat, error) { + return ConnectionsWithContext(context.Background(), kind) +} + +func ConnectionsWithContext(ctx context.Context, kind string) ([]ConnectionStat, error) { + return []ConnectionStat{}, common.ErrNotImplementedError +} + +func FilterCounters() ([]FilterStat, error) { + return FilterCountersWithContext(context.Background()) +} + +func FilterCountersWithContext(ctx context.Context) ([]FilterStat, error) { + return []FilterStat{}, common.ErrNotImplementedError +} + +func ProtoCounters(protocols []string) ([]ProtoCountersStat, error) { + return ProtoCountersWithContext(context.Background(), protocols) +} + +func ProtoCountersWithContext(ctx context.Context, protocols []string) ([]ProtoCountersStat, error) { + return []ProtoCountersStat{}, common.ErrNotImplementedError +} diff --git a/net/net_test.go b/net/net_test.go index bacca04f8..72f4db9c5 100644 --- a/net/net_test.go +++ b/net/net_test.go @@ -3,7 +3,6 @@ package net import ( "errors" "fmt" - "math" "os" "runtime" "testing" @@ -86,8 +85,14 @@ func TestNetIOCountersAll(t *testing.T) { for _, p := range per { pr += p.PacketsRecv } - // small diff is ok - if math.Abs(float64(v[0].PacketsRecv-pr)) > 5 { + // small diff is ok, compare instead of math.Abs(subtraction) with uint64 + var diff uint64 + if v[0].PacketsRecv > pr { + diff = v[0].PacketsRecv - pr + } else { + diff = pr - v[0].PacketsRecv + } + if diff > 5 { if ci := os.Getenv("CI"); ci != "" { // This test often fails in CI. so just print even if failed. fmt.Printf("invalid sum value: %v, %v", v[0].PacketsRecv, pr) From ccb11cf45e4e253e0a2d01cb0699f70151a53f3d Mon Sep 17 00:00:00 2001 From: Steve Zhang Date: Mon, 28 Nov 2022 22:30:09 -0700 Subject: [PATCH 2/2] reuse the context.Context instead of creating a new Context --- mem/mem_solaris.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mem/mem_solaris.go b/mem/mem_solaris.go index d22cf0b43..c911267e1 100644 --- a/mem/mem_solaris.go +++ b/mem/mem_solaris.go @@ -35,7 +35,7 @@ func VirtualMemoryWithContext(ctx context.Context) (*VirtualMemoryStat, error) { return nil, err } result.Total = cap - freemem, err := globalZoneFreeMemory() + freemem, err := globalZoneFreeMemory(ctx) if err != nil { return nil, err } @@ -93,8 +93,7 @@ func globalZoneMemoryCapacity() (uint64, error) { return totalMB * 1024 * 1024, nil } -func globalZoneFreeMemory() (uint64, error) { - ctx := context.Background() +func globalZoneFreeMemory(ctx context.Context) (uint64, error) { output, err := invoke.CommandWithContext(ctx, "pagesize") if err != nil { return 0, err