Expose zfs zpool state

Create a metric node_zfs_zpool_state.

Signed-off-by: Artur Molchanov <artur.molchanov@easybrain.com>

Author: Artur Molchanov

collector/fixtures/e2e-output.txt

@@ -3827,6 +3827,20 @@ node_zfs_zpool_rtime{zpool="poolz1"} 9.82909164e+09
 # TYPE node_zfs_zpool_rupdate untyped
 node_zfs_zpool_rupdate{zpool="pool1"} 7.921048984922e+13
 node_zfs_zpool_rupdate{zpool="poolz1"} 1.10734831944501e+14
+# HELP node_zfs_zpool_state kstat.zfs.misc.state
+# TYPE node_zfs_zpool_state gauge
+node_zfs_zpool_state{state="degraded",zpool="pool1"} 0
+node_zfs_zpool_state{state="degraded",zpool="poolz1"} 1
+node_zfs_zpool_state{state="faulted",zpool="pool1"} 0
+node_zfs_zpool_state{state="faulted",zpool="poolz1"} 0
+node_zfs_zpool_state{state="offline",zpool="pool1"} 0
+node_zfs_zpool_state{state="offline",zpool="poolz1"} 0
+node_zfs_zpool_state{state="online",zpool="pool1"} 1
+node_zfs_zpool_state{state="online",zpool="poolz1"} 0
+node_zfs_zpool_state{state="removed",zpool="pool1"} 0
+node_zfs_zpool_state{state="removed",zpool="poolz1"} 0
+node_zfs_zpool_state{state="unavail",zpool="pool1"} 0
+node_zfs_zpool_state{state="unavail",zpool="poolz1"} 0
 # HELP node_zfs_zpool_wcnt kstat.zfs.misc.io.wcnt
 # TYPE node_zfs_zpool_wcnt untyped
 node_zfs_zpool_wcnt{zpool="pool1"} 0

collector/fixtures/proc/spl/kstat/zfs/pool1/state

@@ -0,0 +1 @@
+ONLINE

collector/fixtures/proc/spl/kstat/zfs/poolz1/state

@@ -0,0 +1 @@
+DEGRADED

collector/zfs.go

@@ -37,6 +37,7 @@ type zfsCollector struct {
 	linuxProcpathBase    string
 	linuxZpoolIoPath     string
 	linuxZpoolObjsetPath string
+	linuxZpoolStatePath  string
 	linuxPathMap         map[string]string
 	logger               log.Logger
 }
@@ -47,6 +48,7 @@ func NewZFSCollector(logger log.Logger) (Collector, error) {
 		linuxProcpathBase:    "spl/kstat/zfs",
 		linuxZpoolIoPath:     "/*/io",
 		linuxZpoolObjsetPath: "/*/objset-*",
+		linuxZpoolStatePath:  "/*/state",
 		linuxPathMap: map[string]string{
 			"zfs_abd":         "abdstats",
 			"zfs_arc":         "arcstats",
@@ -132,3 +134,18 @@ func (c *zfsCollector) constPoolObjsetMetric(poolName string, datasetName string
 		datasetName,
 	)
 }
+
+func (c *zfsCollector) constPoolStateMetric(poolName string, stateName string, isActive uint64) prometheus.Metric {
+	return prometheus.MustNewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, "zfs_zpool", "state"),
+			"kstat.zfs.misc.state",
+			[]string{"zpool", "state"},
+			nil,
+		),
+		prometheus.GaugeValue,
+		float64(isActive),
+		poolName,
+		stateName,
+	)
+}

collector/zfs_linux.go

@@ -41,6 +41,8 @@ const (
 	// kstatDataString = "7"
 )
 
+var zfsPoolStatesName = []string{"online", "degraded", "faulted", "offline", "removed", "unavail"}
+
 func (c *zfsCollector) openProcFile(path string) (*os.File, error) {
 	file, err := os.Open(procFilePath(path))
 	if err != nil {
@@ -97,10 +99,6 @@ func (c *zfsCollector) updatePoolStats(ch chan<- prometheus.Metric) error {
 		return err
 	}
 
-	if zpoolObjsetPaths == nil {
-		return nil
-	}
-
 	for _, zpoolPath := range zpoolObjsetPaths {
 		file, err := os.Open(zpoolPath)
 		if err != nil {
@@ -117,6 +115,34 @@ func (c *zfsCollector) updatePoolStats(ch chan<- prometheus.Metric) error {
 			return err
 		}
 	}
+
+	zpoolStatePaths, err := filepath.Glob(procFilePath(filepath.Join(c.linuxProcpathBase, c.linuxZpoolStatePath)))
+	if err != nil {
+		return err
+	}
+
+	if zpoolStatePaths == nil {
+		level.Warn(c.logger).Log("msg", "Not found pool state files")
+	}
+
+	for _, zpoolPath := range zpoolStatePaths {
+		file, err := os.Open(zpoolPath)
+		if err != nil {
+			// this file should exist, but there is a race where an exporting pool can remove the files -- ok to ignore
+			level.Debug(c.logger).Log("msg", "Cannot open file for reading", "path", zpoolPath)
+			return errZFSNotAvailable
+		}
+
+		err = c.parsePoolStateFile(file, zpoolPath, func(poolName string, stateName string, isActive uint64) {
+			ch <- c.constPoolStateMetric(poolName, stateName, isActive)
+		})
+
+		file.Close()
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -235,3 +261,35 @@ func (c *zfsCollector) parsePoolObjsetFile(reader io.Reader, zpoolPath string, h
 
 	return scanner.Err()
 }
+
+func (c *zfsCollector) parsePoolStateFile(reader io.Reader, zpoolPath string, handler func(string, string, uint64)) error {
+	scanner := bufio.NewScanner(reader)
+	scanner.Scan()
+
+	actualStateName, err := scanner.Text(), scanner.Err()
+	if err != nil {
+		return err
+	}
+
+	actualStateName = strings.ToLower(actualStateName)
+
+	zpoolPathElements := strings.Split(zpoolPath, "/")
+	pathLen := len(zpoolPathElements)
+	if pathLen < 2 {
+		return fmt.Errorf("zpool path did not return at least two elements")
+	}
+
+	zpoolName := zpoolPathElements[pathLen-2]
+
+	for _, stateName := range zfsPoolStatesName {
+		isActive := uint64(0)
+
+		if actualStateName == stateName {
+			isActive = 1
+		}
+
+		handler(zpoolName, stateName, isActive)
+	}
+
+	return nil
+}

collector/zfs_linux_test.go

@@ -494,3 +494,52 @@ func TestVdevMirrorstatsParsing(t *testing.T) {
 		t.Fatal("VdevMirrorStats parsing handler was not called for some expected sysctls")
 	}
 }
+
+func TestPoolStateParsing(t *testing.T) {
+	zpoolPaths, err := filepath.Glob("fixtures/proc/spl/kstat/zfs/*/state")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	c := zfsCollector{}
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	handlerCalled := false
+	for _, zpoolPath := range zpoolPaths {
+		file, err := os.Open(zpoolPath)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = c.parsePoolStateFile(file, zpoolPath, func(poolName string, stateName string, isActive uint64) {
+			handlerCalled = true
+
+			if poolName == "pool1" {
+				if isActive != uint64(1) && stateName == "online" {
+					t.Fatalf("Incorrect parsed value for online state")
+				}
+				if isActive != uint64(0) && stateName != "online" {
+					t.Fatalf("Incorrect parsed value for online state")
+				}
+			}
+			if poolName == "poolz1" {
+				if isActive != uint64(1) && stateName == "degraded" {
+					t.Fatalf("Incorrect parsed value for degraded state")
+				}
+				if isActive != uint64(0) && stateName != "degraded" {
+					t.Fatalf("Incorrect parsed value for degraded state")
+				}
+			}
+		})
+		file.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	if !handlerCalled {
+		t.Fatal("Zpool parsing handler was not called for some expected sysctls")
+	}
+
+}