Add qdisc collector for Linux (prometheus#580)

* Add qdisc collector for Linux

This collector gathers basic queueing discipline metrics via netlink,
similarly to what `tc -s qdisc show` does.

* qdisc collector: nl-specific code moved, names fixed

- netlink-specific parts moved to github.com/ema/qdisc
- avoid using shortened names
- counters renamed into XXX_total

* Get rid of parseMessage error checking leftover

* Add github.com/ema/qdisc to vendored packages

* Update help texts and comments

* Add qdisc collector to README file

* qdisc collector end-to-end testing

* Update qdisc dependency to latest version

Update github.com/ema/qdisc dependency to revision 2c7e72d, which
includes unit testing.

* qdisc collector: rename "iface" label into "device"

Author: ema

README.md

@@ -60,6 +60,7 @@ logind | Exposes session counts from [logind](http://www.freedesktop.org/wiki/So
 meminfo\_numa | Exposes memory statistics from `/proc/meminfo_numa`. | Linux
 mountstats | Exposes filesystem statistics from `/proc/self/mountstats`. Exposes detailed NFS client statistics. | Linux
 nfs | Exposes NFS client statistics from `/proc/net/rpc/nfs`. This is the same information as `nfsstat -c`. | Linux
+qdisc | Exposes [queuing discipline](https://en.wikipedia.org/wiki/Network_scheduler#Linux_kernel) statistics | Linux
 runit | Exposes service status from [runit](http://smarden.org/runit/). | _any_
 supervisord | Exposes service status from [supervisord](http://supervisord.org/). | _any_
 systemd | Exposes service and system status from [systemd](http://www.freedesktop.org/wiki/Software/systemd/). | Linux

collector/fixtures/e2e-output.txt

@@ -2137,6 +2137,26 @@ node_procs_blocked 0
 # HELP node_procs_running Number of processes in runnable state.
 # TYPE node_procs_running gauge
 node_procs_running 2
+# HELP node_qdisc_bytes_total Number of bytes sent.
+# TYPE node_qdisc_bytes_total counter
+node_qdisc_bytes_total{device="eth0",kind="pfifo_fast"} 83
+node_qdisc_bytes_total{device="wlan0",kind="fq"} 42
+# HELP node_qdisc_drops_total Number of packets dropped.
+# TYPE node_qdisc_drops_total counter
+node_qdisc_drops_total{device="eth0",kind="pfifo_fast"} 0
+node_qdisc_drops_total{device="wlan0",kind="fq"} 1
+# HELP node_qdisc_overlimits_total Number of overlimit packets.
+# TYPE node_qdisc_overlimits_total counter
+node_qdisc_overlimits_total{device="eth0",kind="pfifo_fast"} 0
+node_qdisc_overlimits_total{device="wlan0",kind="fq"} 0
+# HELP node_qdisc_packets_total Number of packets sent.
+# TYPE node_qdisc_packets_total counter
+node_qdisc_packets_total{device="eth0",kind="pfifo_fast"} 83
+node_qdisc_packets_total{device="wlan0",kind="fq"} 42
+# HELP node_qdisc_requeues_total Number of packets dequeued, not transmitted, and requeued.
+# TYPE node_qdisc_requeues_total counter
+node_qdisc_requeues_total{device="eth0",kind="pfifo_fast"} 2
+node_qdisc_requeues_total{device="wlan0",kind="fq"} 1
 # HELP node_scrape_collector_duration_seconds node_exporter: Duration of a collector scrape.
 # TYPE node_scrape_collector_duration_seconds gauge
 # HELP node_scrape_collector_success node_exporter: Whether a collector succeeded.
@@ -2162,6 +2182,7 @@ node_scrape_collector_success{collector="mountstats"} 1
 node_scrape_collector_success{collector="netdev"} 1
 node_scrape_collector_success{collector="netstat"} 1
 node_scrape_collector_success{collector="nfs"} 1
+node_scrape_collector_success{collector="qdisc"} 1
 node_scrape_collector_success{collector="sockstat"} 1
 node_scrape_collector_success{collector="stat"} 1
 node_scrape_collector_success{collector="textfile"} 1

collector/fixtures/qdisc/results.json

@@ -0,0 +1,17 @@
+[
+    {
+        "IfaceName": "wlan0",
+        "Bytes": 42,
+        "Packets": 42,
+        "Requeues": 1,
+        "Kind": "fq",
+        "Drops": 1
+    },
+    {
+        "IfaceName": "eth0",
+        "Bytes": 83,
+        "Packets": 83,
+        "Requeues": 2,
+        "Kind": "pfifo_fast"
+    }
+]

collector/qdisc_linux.go

@@ -0,0 +1,116 @@
+// Copyright 2017 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !noqdisc
+
+package collector
+
+import (
+	"encoding/json"
+	"flag"
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/ema/qdisc"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type qdiscStatCollector struct {
+	bytes      typedDesc
+	packets    typedDesc
+	drops      typedDesc
+	requeues   typedDesc
+	overlimits typedDesc
+}
+
+var (
+	collectorQdisc = flag.String("collector.qdisc", "", "test fixtures to use for qdisc collector end-to-end testing")
+)
+
+func init() {
+	Factories["qdisc"] = NewQdiscStatCollector
+}
+
+func NewQdiscStatCollector() (Collector, error) {
+	return &qdiscStatCollector{
+		bytes: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, "qdisc", "bytes_total"),
+			"Number of bytes sent.",
+			[]string{"device", "kind"}, nil,
+		), prometheus.CounterValue},
+		packets: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, "qdisc", "packets_total"),
+			"Number of packets sent.",
+			[]string{"device", "kind"}, nil,
+		), prometheus.CounterValue},
+		drops: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, "qdisc", "drops_total"),
+			"Number of packets dropped.",
+			[]string{"device", "kind"}, nil,
+		), prometheus.CounterValue},
+		requeues: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, "qdisc", "requeues_total"),
+			"Number of packets dequeued, not transmitted, and requeued.",
+			[]string{"device", "kind"}, nil,
+		), prometheus.CounterValue},
+		overlimits: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, "qdisc", "overlimits_total"),
+			"Number of overlimit packets.",
+			[]string{"device", "kind"}, nil,
+		), prometheus.CounterValue},
+	}, nil
+}
+
+func testQdiscGet(fixtures string) ([]qdisc.QdiscInfo, error) {
+	var res []qdisc.QdiscInfo
+
+	b, err := ioutil.ReadFile(filepath.Join(fixtures, "results.json"))
+	if err != nil {
+		return res, err
+	}
+
+	err = json.Unmarshal(b, &res)
+	return res, err
+}
+
+func (c *qdiscStatCollector) Update(ch chan<- prometheus.Metric) error {
+	var msgs []qdisc.QdiscInfo
+	var err error
+
+	fixtures := *collectorQdisc
+
+	if fixtures == "" {
+		msgs, err = qdisc.Get()
+	} else {
+		msgs, err = testQdiscGet(fixtures)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	for _, msg := range msgs {
+		// Only report root qdisc information.
+		if msg.Parent != 0 {
+			continue
+		}
+
+		ch <- c.bytes.mustNewConstMetric(float64(msg.Bytes), msg.IfaceName, msg.Kind)
+		ch <- c.packets.mustNewConstMetric(float64(msg.Packets), msg.IfaceName, msg.Kind)
+		ch <- c.drops.mustNewConstMetric(float64(msg.Drops), msg.IfaceName, msg.Kind)
+		ch <- c.requeues.mustNewConstMetric(float64(msg.Requeues), msg.IfaceName, msg.Kind)
+		ch <- c.overlimits.mustNewConstMetric(float64(msg.Overlimits), msg.IfaceName, msg.Kind)
+	}
+
+	return nil
+}

end-to-end-test.sh

@@ -22,6 +22,7 @@ collectors=$(cat << COLLECTORS
   netdev
   netstat
   nfs
+  qdisc
   sockstat
   stat
   textfile
@@ -76,6 +77,7 @@ fi
   -collector.textfile.directory="collector/fixtures/textfile/two_metric_files/" \
   -collector.megacli.command="collector/fixtures/megacli" \
   -collector.wifi="collector/fixtures/wifi" \
+  -collector.qdisc="collector/fixtures/qdisc/" \
   -web.listen-address "127.0.0.1:${port}" \
   -log.level="debug" > "${tmpdir}/node_exporter.log" 2>&1 &