From d5679cac17e0d28fb3d762468e1acb64adbf8a0e Mon Sep 17 00:00:00 2001 From: Johannes Ziemke Date: Tue, 19 Jul 2022 14:35:03 +0200 Subject: [PATCH 1/4] Fix openbsd syscalls Apparently these are not int8 slices anymore Signed-off-by: Johannes Ziemke --- collector/filesystem_openbsd_amd64.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/collector/filesystem_openbsd_amd64.go b/collector/filesystem_openbsd_amd64.go index 448b3ef36e..75fc844053 100644 --- a/collector/filesystem_openbsd_amd64.go +++ b/collector/filesystem_openbsd_amd64.go @@ -41,14 +41,14 @@ func (c *filesystemCollector) GetStats() (stats []filesystemStats, err error) { stats = []filesystemStats{} for _, v := range mnt { - mountpoint := int8ToString(v.F_mntonname[:]) + mountpoint := string(v.F_mntonname[:]) if c.excludedMountPointsPattern.MatchString(mountpoint) { level.Debug(c.logger).Log("msg", "Ignoring mount point", "mountpoint", mountpoint) continue } - device := int8ToString(v.F_mntfromname[:]) - fstype := int8ToString(v.F_fstypename[:]) + device := string(v.F_mntfromname[:]) + fstype := string(v.F_fstypename[:]) if c.excludedFSTypesPattern.MatchString(fstype) { level.Debug(c.logger).Log("msg", "Ignoring fs type", "type", fstype) continue From 7623daf03807276b1baedc82d43e17453e45dd18 Mon Sep 17 00:00:00 2001 From: Johannes Ziemke Date: Mon, 25 Jul 2022 16:32:43 +0200 Subject: [PATCH 2/4] Sort collectors in end-to-end-test.sh Signed-off-by: Johannes Ziemke --- end-to-end-test.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/end-to-end-test.sh b/end-to-end-test.sh index 4eb4745e0b..bcbac485c1 100755 --- a/end-to-end-test.sh +++ b/end-to-end-test.sh @@ -5,10 +5,11 @@ set -euf -o pipefail enabled_collectors=$(cat << COLLECTORS arp bcache + bonding btrfs buddyinfo - conntrack cgroups + conntrack cpu cpufreq diskstats @@ -34,6 +35,7 @@ enabled_collectors=$(cat << COLLECTORS nfs nfsd pressure + processes qdisc rapl schedstat @@ -41,15 +43,13 @@ enabled_collectors=$(cat << COLLECTORS slabinfo sockstat stat - thermal_zone textfile - bonding + thermal_zone udp_queues vmstat wifi xfs zfs - processes zoneinfo COLLECTORS ) From 57db98e899d8666b47dbcb554e550d124e4c7d5f Mon Sep 17 00:00:00 2001 From: Johannes 'fish' Ziemke Date: Thu, 14 Jul 2022 18:51:55 +0200 Subject: [PATCH 3/4] Add sysctl collector Signed-off-by: Johannes Ziemke --- CHANGELOG.md | 1 + README.md | 1 + collector/fixtures/e2e-64k-page-output.txt | 28 +++ collector/fixtures/e2e-output.txt | 28 +++ .../proc/sys/kernel/seccomp/actions_avail | 1 + collector/sysctl_linux.go | 218 ++++++++++++++++++ end-to-end-test.sh | 5 + go.mod | 8 +- go.sum | 14 +- 9 files changed, 294 insertions(+), 10 deletions(-) create mode 100644 collector/fixtures/proc/sys/kernel/seccomp/actions_avail create mode 100644 collector/sysctl_linux.go diff --git a/CHANGELOG.md b/CHANGELOG.md index b4d6411cd3..f7fe71bb51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ * [ENHANCEMENT] * [BUGFIX] +* [FEATURE] Add sysctl collector #2425 * [ENHANCEMENT] Add node_softirqs_total metric #2221 * [ENHANCEMENT] Add device filter flags to arp collector #2254 * [ENHANCEMENT] Add rapl zone name label option #2401 diff --git a/README.md b/README.md index 0c2a29cd87..434d5ef95e 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,7 @@ selinux | Exposes SELinux statistics. | Linux sockstat | Exposes various statistics from `/proc/net/sockstat`. | Linux softnet | Exposes statistics from `/proc/net/softnet_stat`. | Linux stat | Exposes various statistics from `/proc/stat`. This includes boot time, forks and interrupts. | Linux +sysctl | Expose sysctl values from `/proc/sys`. Use `--collector.sysctl.include(-info)` to configure. | Linux tapestats | Exposes statistics from `/sys/class/scsi_tape`. | Linux textfile | Exposes statistics read from local disk. The `--collector.textfile.directory` flag must be set. | _any_ thermal | Exposes thermal statistics like `pmset -g therm`. | Darwin diff --git a/collector/fixtures/e2e-64k-page-output.txt b/collector/fixtures/e2e-64k-page-output.txt index 05f1572204..7e0ef4c64a 100644 --- a/collector/fixtures/e2e-64k-page-output.txt +++ b/collector/fixtures/e2e-64k-page-output.txt @@ -3052,6 +3052,7 @@ node_scrape_collector_success{collector="slabinfo"} 1 node_scrape_collector_success{collector="sockstat"} 1 node_scrape_collector_success{collector="softnet"} 1 node_scrape_collector_success{collector="stat"} 1 +node_scrape_collector_success{collector="sysctl"} 1 node_scrape_collector_success{collector="tapestats"} 1 node_scrape_collector_success{collector="textfile"} 1 node_scrape_collector_success{collector="thermal_zone"} 1 @@ -3185,6 +3186,33 @@ node_softnet_times_squeezed_total{cpu="0"} 1 node_softnet_times_squeezed_total{cpu="1"} 10 node_softnet_times_squeezed_total{cpu="2"} 85 node_softnet_times_squeezed_total{cpu="3"} 50 +# HELP node_sysctl_fs_file_nr sysctl fs.file-nr +# TYPE node_sysctl_fs_file_nr untyped +node_sysctl_fs_file_nr{index="0"} 1024 +node_sysctl_fs_file_nr{index="1"} 0 +node_sysctl_fs_file_nr{index="2"} 1.631329e+06 +# HELP node_sysctl_fs_file_nr_current sysctl fs.file-nr, field 1 +# TYPE node_sysctl_fs_file_nr_current untyped +node_sysctl_fs_file_nr_current 0 +# HELP node_sysctl_fs_file_nr_max sysctl fs.file-nr, field 2 +# TYPE node_sysctl_fs_file_nr_max untyped +node_sysctl_fs_file_nr_max 1.631329e+06 +# HELP node_sysctl_fs_file_nr_total sysctl fs.file-nr, field 0 +# TYPE node_sysctl_fs_file_nr_total untyped +node_sysctl_fs_file_nr_total 1024 +# HELP node_sysctl_info sysctl info +# TYPE node_sysctl_info gauge +node_sysctl_info{index="0",name="kernel.seccomp.actions_avail",value="kill_process"} 1 +node_sysctl_info{index="1",name="kernel.seccomp.actions_avail",value="kill_thread"} 1 +node_sysctl_info{index="2",name="kernel.seccomp.actions_avail",value="trap"} 1 +node_sysctl_info{index="3",name="kernel.seccomp.actions_avail",value="errno"} 1 +node_sysctl_info{index="4",name="kernel.seccomp.actions_avail",value="user_notif"} 1 +node_sysctl_info{index="5",name="kernel.seccomp.actions_avail",value="trace"} 1 +node_sysctl_info{index="6",name="kernel.seccomp.actions_avail",value="log"} 1 +node_sysctl_info{index="7",name="kernel.seccomp.actions_avail",value="allow"} 1 +# HELP node_sysctl_kernel_threads_max sysctl kernel.threads-max +# TYPE node_sysctl_kernel_threads_max untyped +node_sysctl_kernel_threads_max 7801 # HELP node_tape_io_now The number of I/Os currently outstanding to this device. # TYPE node_tape_io_now gauge node_tape_io_now{device="st0"} 1 diff --git a/collector/fixtures/e2e-output.txt b/collector/fixtures/e2e-output.txt index 124e923edd..97c899acb2 100644 --- a/collector/fixtures/e2e-output.txt +++ b/collector/fixtures/e2e-output.txt @@ -3074,6 +3074,7 @@ node_scrape_collector_success{collector="slabinfo"} 1 node_scrape_collector_success{collector="sockstat"} 1 node_scrape_collector_success{collector="softnet"} 1 node_scrape_collector_success{collector="stat"} 1 +node_scrape_collector_success{collector="sysctl"} 1 node_scrape_collector_success{collector="tapestats"} 1 node_scrape_collector_success{collector="textfile"} 1 node_scrape_collector_success{collector="thermal_zone"} 1 @@ -3207,6 +3208,33 @@ node_softnet_times_squeezed_total{cpu="0"} 1 node_softnet_times_squeezed_total{cpu="1"} 10 node_softnet_times_squeezed_total{cpu="2"} 85 node_softnet_times_squeezed_total{cpu="3"} 50 +# HELP node_sysctl_fs_file_nr sysctl fs.file-nr +# TYPE node_sysctl_fs_file_nr untyped +node_sysctl_fs_file_nr{index="0"} 1024 +node_sysctl_fs_file_nr{index="1"} 0 +node_sysctl_fs_file_nr{index="2"} 1.631329e+06 +# HELP node_sysctl_fs_file_nr_current sysctl fs.file-nr, field 1 +# TYPE node_sysctl_fs_file_nr_current untyped +node_sysctl_fs_file_nr_current 0 +# HELP node_sysctl_fs_file_nr_max sysctl fs.file-nr, field 2 +# TYPE node_sysctl_fs_file_nr_max untyped +node_sysctl_fs_file_nr_max 1.631329e+06 +# HELP node_sysctl_fs_file_nr_total sysctl fs.file-nr, field 0 +# TYPE node_sysctl_fs_file_nr_total untyped +node_sysctl_fs_file_nr_total 1024 +# HELP node_sysctl_info sysctl info +# TYPE node_sysctl_info gauge +node_sysctl_info{index="0",name="kernel.seccomp.actions_avail",value="kill_process"} 1 +node_sysctl_info{index="1",name="kernel.seccomp.actions_avail",value="kill_thread"} 1 +node_sysctl_info{index="2",name="kernel.seccomp.actions_avail",value="trap"} 1 +node_sysctl_info{index="3",name="kernel.seccomp.actions_avail",value="errno"} 1 +node_sysctl_info{index="4",name="kernel.seccomp.actions_avail",value="user_notif"} 1 +node_sysctl_info{index="5",name="kernel.seccomp.actions_avail",value="trace"} 1 +node_sysctl_info{index="6",name="kernel.seccomp.actions_avail",value="log"} 1 +node_sysctl_info{index="7",name="kernel.seccomp.actions_avail",value="allow"} 1 +# HELP node_sysctl_kernel_threads_max sysctl kernel.threads-max +# TYPE node_sysctl_kernel_threads_max untyped +node_sysctl_kernel_threads_max 7801 # HELP node_tape_io_now The number of I/Os currently outstanding to this device. # TYPE node_tape_io_now gauge node_tape_io_now{device="st0"} 1 diff --git a/collector/fixtures/proc/sys/kernel/seccomp/actions_avail b/collector/fixtures/proc/sys/kernel/seccomp/actions_avail new file mode 100644 index 0000000000..a608f9fd07 --- /dev/null +++ b/collector/fixtures/proc/sys/kernel/seccomp/actions_avail @@ -0,0 +1 @@ +kill_process kill_thread trap errno user_notif trace log allow diff --git a/collector/sysctl_linux.go b/collector/sysctl_linux.go new file mode 100644 index 0000000000..4b0d105333 --- /dev/null +++ b/collector/sysctl_linux.go @@ -0,0 +1,218 @@ +// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "fmt" + "strconv" + "strings" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/procfs" + "gopkg.in/alecthomas/kingpin.v2" +) + +var ( + sysctlInclude = kingpin.Flag("collector.sysctl.include", "Select sysctl metrics to include").Strings() + sysctlIncludeInfo = kingpin.Flag("collector.sysctl.include-info", "Select sysctl metrics to include as info metrics").Strings() + + sysctlInfoDesc = prometheus.NewDesc(prometheus.BuildFQName(namespace, "sysctl", "info"), "sysctl info", []string{"name", "value", "index"}, nil) +) + +type sysctlCollector struct { + fs procfs.FS + logger log.Logger + sysctls []*sysctl +} + +func init() { + registerCollector("sysctl", defaultDisabled, NewSysctlCollector) +} + +func NewSysctlCollector(logger log.Logger) (Collector, error) { + fs, err := procfs.NewFS(*procPath) + if err != nil { + return nil, fmt.Errorf("failed to open sysfs: %w", err) + } + c := &sysctlCollector{ + logger: logger, + fs: fs, + sysctls: []*sysctl{}, + } + + for _, include := range *sysctlInclude { + sysctl, err := newSysctl(include, true) + if err != nil { + return nil, err + } + c.sysctls = append(c.sysctls, sysctl) + } + + for _, include := range *sysctlIncludeInfo { + sysctl, err := newSysctl(include, false) + if err != nil { + return nil, err + } + c.sysctls = append(c.sysctls, sysctl) + } + return c, nil +} + +func (c *sysctlCollector) Update(ch chan<- prometheus.Metric) error { + for _, sysctl := range c.sysctls { + metrics, err := c.newMetrics(sysctl) + if err != nil { + return err + } + + for _, metric := range metrics { + ch <- metric + } + } + return nil +} + +func (c *sysctlCollector) newMetrics(s *sysctl) ([]prometheus.Metric, error) { + var ( + values interface{} + length int + err error + ) + + if s.numeric { + values, err = c.fs.SysctlInts(s.name) + if err != nil { + return nil, fmt.Errorf("error obtaining sysctl info: %w", err) + } + length = len(values.([]int)) + } else { + values, err = c.fs.SysctlStrings(s.name) + if err != nil { + return nil, fmt.Errorf("error obtaining sysctl info: %w", err) + } + length = len(values.([]string)) + } + + switch length { + case 0: + return nil, fmt.Errorf("sysctl %s has no values", s.name) + case 1: + if len(s.keys) > 0 { + return nil, fmt.Errorf("sysctl %s has only one value, but expected %v", s.name, s.keys) + } + return []prometheus.Metric{s.newConstMetric(values)}, nil + + default: + + if len(s.keys) == 0 { + return s.newIndexedMetrics(values), nil + } + + if length != len(s.keys) { + return nil, fmt.Errorf("sysctl %s has %d keys but only %d defined in f lag", s.name, length, len(s.keys)) + } + + return s.newMappedMetrics(values) + } +} + +type sysctl struct { + numeric bool + name string + keys []string +} + +func newSysctl(include string, numeric bool) (*sysctl, error) { + parts := strings.SplitN(include, ":", 2) + s := &sysctl{ + numeric: numeric, + name: parts[0], + } + if len(parts) == 2 { + s.keys = strings.Split(parts[1], ",") + s.name = parts[0] + } + return s, nil +} + +func (s *sysctl) metricName() string { + return SanitizeMetricName(s.name) +} + +func (s *sysctl) newConstMetric(v interface{}) prometheus.Metric { + if s.numeric { + return prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(namespace, "sysctl", s.metricName()), + fmt.Sprintf("sysctl %s", s.name), + nil, nil), + prometheus.UntypedValue, + float64(v.([]int)[0])) + } + return prometheus.MustNewConstMetric( + sysctlInfoDesc, + prometheus.GaugeValue, + 1.0, + s.name, + v.([]string)[0], + "0", + ) +} + +func (s *sysctl) newIndexedMetrics(v interface{}) []prometheus.Metric { + desc := prometheus.NewDesc( + prometheus.BuildFQName(namespace, "sysctl", s.metricName()), + fmt.Sprintf("sysctl %s", s.name), + []string{"index"}, nil, + ) + switch values := v.(type) { + case []int: + metrics := make([]prometheus.Metric, len(values)) + for i, n := range values { + metrics[i] = prometheus.MustNewConstMetric(desc, prometheus.UntypedValue, float64(n), strconv.Itoa(i)) + } + return metrics + case []string: + metrics := make([]prometheus.Metric, len(values)) + for i, str := range values { + metrics[i] = prometheus.MustNewConstMetric(sysctlInfoDesc, prometheus.GaugeValue, 1.0, s.name, str, strconv.Itoa(i)) + } + return metrics + default: + panic(fmt.Sprintf("unexpected type %T", values)) + } +} + +func (s *sysctl) newMappedMetrics(v interface{}) ([]prometheus.Metric, error) { + switch values := v.(type) { + case []int: + metrics := make([]prometheus.Metric, len(values)) + for i, n := range values { + key := s.keys[i] + desc := prometheus.NewDesc( + prometheus.BuildFQName(namespace, "sysctl", s.metricName()+"_"+key), + fmt.Sprintf("sysctl %s, field %d", s.name, i), + nil, + nil, + ) + metrics[i] = prometheus.MustNewConstMetric(desc, prometheus.UntypedValue, float64(n)) + } + return metrics, nil + case []string: + return nil, fmt.Errorf("mapped sysctl string values not supported") + default: + return nil, fmt.Errorf("unexpected type %T", values) + } +} diff --git a/end-to-end-test.sh b/end-to-end-test.sh index bcbac485c1..555cd1e12a 100755 --- a/end-to-end-test.sh +++ b/end-to-end-test.sh @@ -43,6 +43,7 @@ enabled_collectors=$(cat << COLLECTORS slabinfo sockstat stat + sysctl textfile thermal_zone udp_queues @@ -135,6 +136,10 @@ fi --collector.cpu.info.bugs-include="${cpu_info_bugs}" \ --collector.cpu.info.flags-include="${cpu_info_flags}" \ --collector.stat.softirq \ + --collector.sysctl.include="kernel.threads-max" \ + --collector.sysctl.include="fs.file-nr" \ + --collector.sysctl.include="fs.file-nr:total,current,max" \ + --collector.sysctl.include-info="kernel.seccomp.actions_avail" \ --web.listen-address "127.0.0.1:${port}" \ --log.level="debug" > "${tmpdir}/node_exporter.log" 2>&1 & diff --git a/go.mod b/go.mod index 8ccb37b256..bef66e8dce 100644 --- a/go.mod +++ b/go.mod @@ -21,10 +21,10 @@ require ( github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.35.0 github.com/prometheus/exporter-toolkit v0.7.1 - github.com/prometheus/procfs v0.7.4-0.20211209105546-0f8a320e1e1f + github.com/prometheus/procfs v0.7.4-0.20220719120938-9f9c419a6490 github.com/safchain/ethtool v0.2.0 github.com/soundcloud/go-runit v0.0.0-20150630195641-06ad41a06c4a - golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f + golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a gopkg.in/alecthomas/kingpin.v2 v2.2.6 ) @@ -35,7 +35,7 @@ require ( github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/google/go-cmp v0.5.7 // indirect + github.com/google/go-cmp v0.5.8 // indirect github.com/josharian/native v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect @@ -49,7 +49,7 @@ require ( golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect - golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect + golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect golang.org/x/text v0.3.7 // indirect google.golang.org/appengine v1.6.6 // indirect google.golang.org/protobuf v1.26.0 // indirect diff --git a/go.sum b/go.sum index 9905031ef9..6e45a64325 100644 --- a/go.sum +++ b/go.sum @@ -129,8 +129,9 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -239,8 +240,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.4-0.20211209105546-0f8a320e1e1f h1:tg+5gOV1AkhTfWNi94UA5rvn+Mat/thyB2WASYAzPZo= -github.com/prometheus/procfs v0.7.4-0.20211209105546-0f8a320e1e1f/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.4-0.20220719120938-9f9c419a6490 h1:r8tvHkY1WUgbABOocq3OtGE3rFK8pdSA30NZNSxpBIU= +github.com/prometheus/procfs v0.7.4-0.20220719120938-9f9c419a6490/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -363,8 +364,9 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -414,8 +416,9 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220307203707-22a9840ba4d7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f h1:8w7RhxzTVgUzw/AH/9mUV5q0vMgy40SQRursCcfmkCw= golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -472,7 +475,6 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= From 6421c5855623b958f51a0f5e58ec17747e193354 Mon Sep 17 00:00:00 2001 From: Johannes Ziemke Date: Mon, 25 Jul 2022 16:26:50 +0200 Subject: [PATCH 4/4] Add sysctl docs to README.md Signed-off-by: Johannes Ziemke --- README.md | 103 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 77 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 434d5ef95e..d83a003a86 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,6 @@ selinux | Exposes SELinux statistics. | Linux sockstat | Exposes various statistics from `/proc/net/sockstat`. | Linux softnet | Exposes statistics from `/proc/net/softnet_stat`. | Linux stat | Exposes various statistics from `/proc/stat`. This includes boot time, forks and interrupts. | Linux -sysctl | Expose sysctl values from `/proc/sys`. Use `--collector.sysctl.include(-info)` to configure. | Linux tapestats | Exposes statistics from `/sys/class/scsi_tape`. | Linux textfile | Exposes statistics read from local disk. The `--collector.textfile.directory` flag must be set. | _any_ thermal | Exposes thermal statistics like `pmset -g therm`. | Darwin @@ -156,6 +155,35 @@ and does not time out. In addition, monitor the `scrape_samples_post_metric_relabeling` metric to see the changes in cardinality. +Name | Description | OS +---------|-------------|---- +buddyinfo | Exposes statistics of memory fragments as reported by /proc/buddyinfo. | Linux +cgroups | A summary of the number of active and enabled cgroups | Linux +devstat | Exposes device statistics | Dragonfly, FreeBSD +drbd | Exposes Distributed Replicated Block Device statistics (to version 8.4) | Linux +ethtool | Exposes network interface information and network driver statistics equivalent to `ethtool`, `ethtool -S`, and `ethtool -i`. | Linux +interrupts | Exposes detailed interrupts statistics. | Linux, OpenBSD +ksmd | Exposes kernel and system statistics from `/sys/kernel/mm/ksm`. | Linux +lnstat | Exposes stats from `/proc/net/stat/`. | Linux +logind | Exposes session counts from [logind](http://www.freedesktop.org/wiki/Software/systemd/logind/). | Linux +meminfo\_numa | Exposes memory statistics from `/proc/meminfo_numa`. | Linux +mountstats | Exposes filesystem statistics from `/proc/self/mountstats`. Exposes detailed NFS client statistics. | Linux +network_route | Exposes the routing table as metrics | Linux +ntp | Exposes local NTP daemon health to check [time](./docs/TIME.md) | _any_ +perf | Exposes perf based metrics (Warning: Metrics are dependent on kernel configuration and settings). | Linux +processes | Exposes aggregate process statistics from `/proc`. | Linux +qdisc | Exposes [queuing discipline](https://en.wikipedia.org/wiki/Network_scheduler#Linux_kernel) statistics | Linux +runit | Exposes service status from [runit](http://smarden.org/runit/). | _any_ +slabinfo | Exposes slab statistics from `/proc/slabinfo`. Note that permission of `/proc/slabinfo` is usually 0400, so set it appropriately. | Linux +supervisord | Exposes service status from [supervisord](http://supervisord.org/). | _any_ +sysctl | Expose sysctl values from `/proc/sys`. Use `--collector.sysctl.include(-info)` to configure. | Linux +systemd | Exposes service and system status from [systemd](http://www.freedesktop.org/wiki/Software/systemd/). | Linux +tcpstat | Exposes TCP connection status information from `/proc/net/tcp` and `/proc/net/tcp6`. (Warning: the current version has potential performance issues in high load situations.) | Linux +wifi | Exposes WiFi device and station statistics. | Linux +zoneinfo | Exposes NUMA memory zone metrics. | Linux + +### Perf Collector + The `perf` collector may not work out of the box on some Linux systems due to kernel configuration and security settings. To allow access, set the following `sysctl` parameter: @@ -191,33 +219,56 @@ found using [`perf list`](http://man7.org/linux/man-pages/man1/perf.1.html) or from debugfs. And example usage of this would be `--collector.perf.tracepoint="sched:sched_process_exec"`. +### Sysctl Collector -Name | Description | OS ----------|-------------|---- -buddyinfo | Exposes statistics of memory fragments as reported by /proc/buddyinfo. | Linux -cgroups | A summary of the number of active and enabled cgroups | Linux -devstat | Exposes device statistics | Dragonfly, FreeBSD -drbd | Exposes Distributed Replicated Block Device statistics (to version 8.4) | Linux -ethtool | Exposes network interface information and network driver statistics equivalent to `ethtool`, `ethtool -S`, and `ethtool -i`. | Linux -interrupts | Exposes detailed interrupts statistics. | Linux, OpenBSD -ksmd | Exposes kernel and system statistics from `/sys/kernel/mm/ksm`. | Linux -lnstat | Exposes stats from `/proc/net/stat/`. | Linux -logind | Exposes session counts from [logind](http://www.freedesktop.org/wiki/Software/systemd/logind/). | Linux -meminfo\_numa | Exposes memory statistics from `/proc/meminfo_numa`. | Linux -mountstats | Exposes filesystem statistics from `/proc/self/mountstats`. Exposes detailed NFS client statistics. | Linux -network_route | Exposes the routing table as metrics | Linux -ntp | Exposes local NTP daemon health to check [time](./docs/TIME.md) | _any_ -perf | Exposes perf based metrics (Warning: Metrics are dependent on kernel configuration and settings). | Linux -processes | Exposes aggregate process statistics from `/proc`. | Linux -qdisc | Exposes [queuing discipline](https://en.wikipedia.org/wiki/Network_scheduler#Linux_kernel) statistics | Linux -runit | Exposes service status from [runit](http://smarden.org/runit/). | _any_ -slabinfo | Exposes slab statistics from `/proc/slabinfo`. Note that permission of `/proc/slabinfo` is usually 0400, so set it appropriately. | Linux -supervisord | Exposes service status from [supervisord](http://supervisord.org/). | _any_ -systemd | Exposes service and system status from [systemd](http://www.freedesktop.org/wiki/Software/systemd/). | Linux -tcpstat | Exposes TCP connection status information from `/proc/net/tcp` and `/proc/net/tcp6`. (Warning: the current version has potential performance issues in high load situations.) | Linux -wifi | Exposes WiFi device and station statistics. | Linux -zoneinfo | Exposes NUMA memory zone metrics. | Linux +The `sysctl` collector can be enabled with `--collector.sysctl`. It supports exposing numeric sysctl values +as metrics using the `--collector.sysctl.include` flag and string values as info metrics by using the +`--collector.sysctl.include-info` flag. The flags can be repeated. For sysctl with multiple numeric values, +an optional mapping can be given to expose each value as its own metric. Otherwise an `index` label is used +to identify the different fields. +#### Examples +##### Numeric values +###### Single values +Using `--collector.sysctl.include=vm.user_reserve_kbytes`: +`vm.user_reserve_kbytes = 131072` -> `node_sysctl_vm_user_reserve_kbytes 131072` + +###### Multiple values +A sysctl can contain multiple values, for example: +``` +net.ipv4.tcp_rmem = 4096 131072 6291456 +``` +Using `--collector.sysctl.include=net.ipv4.tcp_rmem` the collector will expose: +``` +node_sysctl_net_ipv4_tcp_rmem{index="0"} 4096 +node_sysctl_net_ipv4_tcp_rmem{index="1"} 131072 +node_sysctl_net_ipv4_tcp_rmem{index="2"} 6291456 +``` +If the indexes have defined meaning like in this case, the values can be mapped to multiple metrics by appending the mapping to the --collector.sysctl.include flag: +Using `--collector.sysctl.include=net.ipv4.tcp_rmem:min,default,max` the collector will expose: +``` +node_sysctl_net_ipv4_tcp_rmem_min 4096 +node_sysctl_net_ipv4_tcp_rmem_default 131072 +node_sysctl_net_ipv4_tcp_rmem_max 6291456 +``` + +##### String values +String values need to be exposed as info metric. The user selects them by using the `--collector.sysctl.include-info` flag. + +###### Single values +`kernel.core_pattern = core` -> `node_sysctl_info{key="kernel.core_pattern_info", value="core"} 1` + +###### Multiple values +Given the following sysctl: +``` +kernel.seccomp.actions_avail = kill_process kill_thread trap errno trace log allow +``` +Setting `--collector.sysctl.include-info=kernel.seccomp.actions_avail` will yield: +``` +node_sysctl_info{key="kernel.seccomp.actions_avail", index="0", value="kill_process"} 1 +node_sysctl_info{key="kernel.seccomp.actions_avail", index="1", value="kill_thread"} 1 +... +``` ### Textfile Collector