Skip to content

Commit

Permalink
Add sysctl collector
Browse files Browse the repository at this point in the history
Signed-off-by: Johannes Ziemke <github@5pi.de>
  • Loading branch information
discordianfish committed Jul 19, 2022
1 parent 690efa6 commit b2dbd2d
Show file tree
Hide file tree
Showing 9 changed files with 293 additions and 10 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
* [ENHANCEMENT]
* [BUGFIX]

* [FEATURE] Add sysctl collector #2425
* [ENHANCEMENT] Add node_softirqs_total metric #2221
* [ENHANCEMENT] Add device filter flags to arp collector #2254
* [ENHANCEMENT] Add rapl zone name label option #2401
Expand Down
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ selinux | Exposes SELinux statistics. | Linux
sockstat | Exposes various statistics from `/proc/net/sockstat`. | Linux
softnet | Exposes statistics from `/proc/net/softnet_stat`. | Linux
stat | Exposes various statistics from `/proc/stat`. This includes boot time, forks and interrupts. | Linux
sysctl | Expose sysctl values from `/proc/sys`. Use `--collector.sysctl.include(-info)` to configure. | Linux
tapestats | Exposes statistics from `/sys/class/scsi_tape`. | Linux
textfile | Exposes statistics read from local disk. The `--collector.textfile.directory` flag must be set. | _any_
thermal | Exposes thermal statistics like `pmset -g therm`. | Darwin
Expand Down
28 changes: 28 additions & 0 deletions collector/fixtures/e2e-64k-page-output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3052,6 +3052,7 @@ node_scrape_collector_success{collector="slabinfo"} 1
node_scrape_collector_success{collector="sockstat"} 1
node_scrape_collector_success{collector="softnet"} 1
node_scrape_collector_success{collector="stat"} 1
node_scrape_collector_success{collector="sysctl"} 1
node_scrape_collector_success{collector="tapestats"} 1
node_scrape_collector_success{collector="textfile"} 1
node_scrape_collector_success{collector="thermal_zone"} 1
Expand Down Expand Up @@ -3185,6 +3186,33 @@ node_softnet_times_squeezed_total{cpu="0"} 1
node_softnet_times_squeezed_total{cpu="1"} 10
node_softnet_times_squeezed_total{cpu="2"} 85
node_softnet_times_squeezed_total{cpu="3"} 50
# HELP node_sysctl_fs_file_nr sysctl fs.file-nr
# TYPE node_sysctl_fs_file_nr untyped
node_sysctl_fs_file_nr{index="0"} 1024
node_sysctl_fs_file_nr{index="1"} 0
node_sysctl_fs_file_nr{index="2"} 1.631329e+06
# HELP node_sysctl_fs_file_nr_current sysctl fs.file-nr, field 1
# TYPE node_sysctl_fs_file_nr_current untyped
node_sysctl_fs_file_nr_current 0
# HELP node_sysctl_fs_file_nr_max sysctl fs.file-nr, field 2
# TYPE node_sysctl_fs_file_nr_max untyped
node_sysctl_fs_file_nr_max 1.631329e+06
# HELP node_sysctl_fs_file_nr_total sysctl fs.file-nr, field 0
# TYPE node_sysctl_fs_file_nr_total untyped
node_sysctl_fs_file_nr_total 1024
# HELP node_sysctl_info sysctl info
# TYPE node_sysctl_info untyped
node_sysctl_info{index="0",name="kernel.seccomp.actions_avail",value="kill_process"} 1
node_sysctl_info{index="1",name="kernel.seccomp.actions_avail",value="kill_thread"} 1
node_sysctl_info{index="2",name="kernel.seccomp.actions_avail",value="trap"} 1
node_sysctl_info{index="3",name="kernel.seccomp.actions_avail",value="errno"} 1
node_sysctl_info{index="4",name="kernel.seccomp.actions_avail",value="user_notif"} 1
node_sysctl_info{index="5",name="kernel.seccomp.actions_avail",value="trace"} 1
node_sysctl_info{index="6",name="kernel.seccomp.actions_avail",value="log"} 1
node_sysctl_info{index="7",name="kernel.seccomp.actions_avail",value="allow"} 1
# HELP node_sysctl_kernel_threads_max sysctl kernel.threads-max
# TYPE node_sysctl_kernel_threads_max untyped
node_sysctl_kernel_threads_max 7801
# HELP node_tape_io_now The number of I/Os currently outstanding to this device.
# TYPE node_tape_io_now gauge
node_tape_io_now{device="st0"} 1
Expand Down
28 changes: 28 additions & 0 deletions collector/fixtures/e2e-output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3074,6 +3074,7 @@ node_scrape_collector_success{collector="slabinfo"} 1
node_scrape_collector_success{collector="sockstat"} 1
node_scrape_collector_success{collector="softnet"} 1
node_scrape_collector_success{collector="stat"} 1
node_scrape_collector_success{collector="sysctl"} 1
node_scrape_collector_success{collector="tapestats"} 1
node_scrape_collector_success{collector="textfile"} 1
node_scrape_collector_success{collector="thermal_zone"} 1
Expand Down Expand Up @@ -3207,6 +3208,33 @@ node_softnet_times_squeezed_total{cpu="0"} 1
node_softnet_times_squeezed_total{cpu="1"} 10
node_softnet_times_squeezed_total{cpu="2"} 85
node_softnet_times_squeezed_total{cpu="3"} 50
# HELP node_sysctl_fs_file_nr sysctl fs.file-nr
# TYPE node_sysctl_fs_file_nr untyped
node_sysctl_fs_file_nr{index="0"} 1024
node_sysctl_fs_file_nr{index="1"} 0
node_sysctl_fs_file_nr{index="2"} 1.631329e+06
# HELP node_sysctl_fs_file_nr_current sysctl fs.file-nr, field 1
# TYPE node_sysctl_fs_file_nr_current untyped
node_sysctl_fs_file_nr_current 0
# HELP node_sysctl_fs_file_nr_max sysctl fs.file-nr, field 2
# TYPE node_sysctl_fs_file_nr_max untyped
node_sysctl_fs_file_nr_max 1.631329e+06
# HELP node_sysctl_fs_file_nr_total sysctl fs.file-nr, field 0
# TYPE node_sysctl_fs_file_nr_total untyped
node_sysctl_fs_file_nr_total 1024
# HELP node_sysctl_info sysctl info
# TYPE node_sysctl_info untyped
node_sysctl_info{index="0",name="kernel.seccomp.actions_avail",value="kill_process"} 1
node_sysctl_info{index="1",name="kernel.seccomp.actions_avail",value="kill_thread"} 1
node_sysctl_info{index="2",name="kernel.seccomp.actions_avail",value="trap"} 1
node_sysctl_info{index="3",name="kernel.seccomp.actions_avail",value="errno"} 1
node_sysctl_info{index="4",name="kernel.seccomp.actions_avail",value="user_notif"} 1
node_sysctl_info{index="5",name="kernel.seccomp.actions_avail",value="trace"} 1
node_sysctl_info{index="6",name="kernel.seccomp.actions_avail",value="log"} 1
node_sysctl_info{index="7",name="kernel.seccomp.actions_avail",value="allow"} 1
# HELP node_sysctl_kernel_threads_max sysctl kernel.threads-max
# TYPE node_sysctl_kernel_threads_max untyped
node_sysctl_kernel_threads_max 7801
# HELP node_tape_io_now The number of I/Os currently outstanding to this device.
# TYPE node_tape_io_now gauge
node_tape_io_now{device="st0"} 1
Expand Down
1 change: 1 addition & 0 deletions collector/fixtures/proc/sys/kernel/seccomp/actions_avail
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
kill_process kill_thread trap errno user_notif trace log allow
218 changes: 218 additions & 0 deletions collector/sysctl_linux.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,218 @@
// Copyright 2022 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package collector

import (
"fmt"
"strconv"
"strings"

"github.com/go-kit/log"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/procfs"
"gopkg.in/alecthomas/kingpin.v2"
)

var (
sysctlInclude = kingpin.Flag("collector.sysctl.include", "Select sysctl metrics to include").Strings()
sysctlIncludeInfo = kingpin.Flag("collector.sysctl.include-info", "Select sysctl metrics to include as info metrics").Strings()

sysctlInfoDesc = prometheus.NewDesc(prometheus.BuildFQName(namespace, "sysctl", "info"), "sysctl info", []string{"name", "value", "index"}, nil)
)

type sysctlCollector struct {
fs procfs.FS
logger log.Logger
sysctls []*Sysctl
}

func init() {
registerCollector("sysctl", defaultEnabled, NewSysctlCollector)
}

func NewSysctlCollector(logger log.Logger) (Collector, error) {
fs, err := procfs.NewFS(*procPath)
if err != nil {
return nil, fmt.Errorf("failed to open sysfs: %w", err)
}
c := &sysctlCollector{
logger: logger,
fs: fs,
sysctls: []*Sysctl{},
}

for _, include := range *sysctlInclude {
sysctl, err := NewSysctl(include, true)
if err != nil {
return nil, err
}
c.sysctls = append(c.sysctls, sysctl)
}

for _, include := range *sysctlIncludeInfo {
sysctl, err := NewSysctl(include, false)
if err != nil {
return nil, err
}
c.sysctls = append(c.sysctls, sysctl)
}
return c, nil
}

func (c *sysctlCollector) Update(ch chan<- prometheus.Metric) error {
for _, sysctl := range c.sysctls {
metrics, err := c.NewMetrics(sysctl)
if err != nil {
return err
}

for _, metric := range metrics {
ch <- metric
}
}
return nil
}

func (c *sysctlCollector) NewMetrics(sysctl *Sysctl) ([]prometheus.Metric, error) {
var (
values interface{}
length int
err error
)

if sysctl.numeric {
values, err = c.fs.SysctlInts(sysctl.name)
if err != nil {
return nil, fmt.Errorf("error obtaining sysctl info: %w", err)
}
length = len(values.([]int))
} else {
values, err = c.fs.SysctlStrings(sysctl.name)
if err != nil {
return nil, fmt.Errorf("error obtaining sysctl info: %w", err)
}
length = len(values.([]string))
}

switch length {
case 0:
return nil, fmt.Errorf("sysctl %s has no values", sysctl.name)
case 1:
if len(sysctl.keys) > 0 {
return nil, fmt.Errorf("sysctl %s has only one value, but expected %v", sysctl.name, sysctl.keys)
}
return []prometheus.Metric{sysctl.NewConstMetric(values)}, nil

default:

if len(sysctl.keys) == 0 {
return sysctl.NewIndexedMetrics(values), nil
}

if length != len(sysctl.keys) {
return nil, fmt.Errorf("sysctl %s has %d keys but only %d defined in f lag", sysctl.name, length, len(sysctl.keys))
}

return sysctl.NewMappedMetrics(values)
}
}

type Sysctl struct {
numeric bool
name string
keys []string
}

func NewSysctl(include string, numeric bool) (*Sysctl, error) {
parts := strings.SplitN(include, ":", 2)
s := &Sysctl{
numeric: numeric,
name: parts[0],
}
if len(parts) == 2 {
s.keys = strings.Split(parts[1], ",")
s.name = parts[0]
}
return s, nil
}

func (sysctl *Sysctl) MetricName() string {
return SanitizeMetricName(sysctl.name)
}

func (sysctl *Sysctl) NewConstMetric(v interface{}) prometheus.Metric {
if sysctl.numeric {
return prometheus.MustNewConstMetric(
prometheus.NewDesc(
prometheus.BuildFQName(namespace, "sysctl", sysctl.MetricName()),
fmt.Sprintf("sysctl %s", sysctl.name),
nil, nil),
prometheus.UntypedValue,
float64(v.([]int)[0]))
}
return prometheus.MustNewConstMetric(
sysctlInfoDesc,
prometheus.UntypedValue,
1.0,
sysctl.name,
v.([]string)[0],
"0",
)
}

func (sysctl *Sysctl) NewIndexedMetrics(v interface{}) []prometheus.Metric {
desc := prometheus.NewDesc(
prometheus.BuildFQName(namespace, "sysctl", sysctl.MetricName()),
fmt.Sprintf("sysctl %s", sysctl.name),
[]string{"index"}, nil,
)
switch values := v.(type) {
case []int:
metrics := make([]prometheus.Metric, len(values))
for i, n := range values {
metrics[i] = prometheus.MustNewConstMetric(desc, prometheus.UntypedValue, float64(n), strconv.Itoa(i))
}
return metrics
case []string:
metrics := make([]prometheus.Metric, len(values))
for i, str := range values {
metrics[i] = prometheus.MustNewConstMetric(sysctlInfoDesc, prometheus.UntypedValue, 1.0, sysctl.name, str, strconv.Itoa(i))
}
return metrics
default:
panic(fmt.Sprintf("unexpected type %T", values))
}
}

func (sysctl *Sysctl) NewMappedMetrics(v interface{}) ([]prometheus.Metric, error) {
switch values := v.(type) {
case []int:
metrics := make([]prometheus.Metric, len(values))
for i, n := range values {
key := sysctl.keys[i]
desc := prometheus.NewDesc(
prometheus.BuildFQName(namespace, "sysctl", sysctl.MetricName()+"_"+key),
fmt.Sprintf("sysctl %s, field %d", sysctl.name, i),
nil,
nil,
)
metrics[i] = prometheus.MustNewConstMetric(desc, prometheus.UntypedValue, float64(n))
}
return metrics, nil
case []string:
return nil, fmt.Errorf("mapped sysctl string values not supported")
default:
return nil, fmt.Errorf("unexpected type %T", values)
}
}
4 changes: 4 additions & 0 deletions end-to-end-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,10 @@ fi
--collector.cpu.info.bugs-include="${cpu_info_bugs}" \
--collector.cpu.info.flags-include="${cpu_info_flags}" \
--collector.stat.softirq \
--collector.sysctl.include="kernel.threads-max" \
--collector.sysctl.include="fs.file-nr" \
--collector.sysctl.include="fs.file-nr:total,current,max" \
--collector.sysctl.include-info="kernel.seccomp.actions_avail" \
--web.listen-address "127.0.0.1:${port}" \
--log.level="debug" > "${tmpdir}/node_exporter.log" 2>&1 &

Expand Down
8 changes: 4 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,10 @@ require (
github.com/prometheus/client_model v0.2.0
github.com/prometheus/common v0.35.0
github.com/prometheus/exporter-toolkit v0.7.1
github.com/prometheus/procfs v0.7.4-0.20211209105546-0f8a320e1e1f
github.com/prometheus/procfs v0.7.4-0.20220719120938-9f9c419a6490
github.com/safchain/ethtool v0.2.0
github.com/soundcloud/go-runit v0.0.0-20150630195641-06ad41a06c4a
golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a
gopkg.in/alecthomas/kingpin.v2 v2.2.6
)

Expand All @@ -35,7 +35,7 @@ require (
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/go-logfmt/logfmt v0.5.1 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-cmp v0.5.7 // indirect
github.com/google/go-cmp v0.5.8 // indirect
github.com/josharian/native v1.0.0 // indirect
github.com/jpillora/backoff v1.0.0 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
Expand All @@ -49,7 +49,7 @@ require (
golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect
golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
golang.org/x/text v0.3.7 // indirect
google.golang.org/appengine v1.6.6 // indirect
google.golang.org/protobuf v1.26.0 // indirect
Expand Down

0 comments on commit b2dbd2d

Please sign in to comment.