[*] improve error messages

Author: pashagolub

checker/consul_leader_checker.go

@@ -23,7 +23,11 @@ func NewConsulLeaderChecker(con *vipconfig.Config) (lc *ConsulLeaderChecker, err
 
 	url, err := url.Parse(con.Endpoints[0])
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to parse consul endpoint URL %s: %w", con.Endpoints[0], err)
+	}
+
+	if url.Hostname() == "" {
+		return nil, fmt.Errorf("invalid consul endpoint URL: hostname is empty in %s", con.Endpoints[0])
 	}
 
 	config := &api.Config{
@@ -34,7 +38,7 @@ func NewConsulLeaderChecker(con *vipconfig.Config) (lc *ConsulLeaderChecker, err
 	}
 
 	if lc.Client, err = api.NewClient(config); err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to create consul client for endpoint %s: %w", con.Endpoints[0], err)
 	}
 
 	return lc, nil

checker/etcd_leader_checker.go

@@ -23,7 +23,7 @@ type EtcdLeaderChecker struct {
 func NewEtcdLeaderChecker(conf *vipconfig.Config) (*EtcdLeaderChecker, error) {
 	tlsConfig, err := getTransport(conf)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to create TLS transport for etcd: %w", err)
 	}
 	cfg := clientv3.Config{
 		Endpoints:            conf.Endpoints,
@@ -35,7 +35,10 @@ func NewEtcdLeaderChecker(conf *vipconfig.Config) (*EtcdLeaderChecker, error) {
 		Logger:               conf.Logger,
 	}
 	c, err := clientv3.New(cfg)
-	return &EtcdLeaderChecker{conf, c}, err
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to etcd at endpoints %v: %w", conf.Endpoints, err)
+	}
+	return &EtcdLeaderChecker{conf, c}, nil
 }
 
 func getTransport(conf *vipconfig.Config) (*tls.Config, error) {
@@ -74,7 +77,17 @@ func getTransport(conf *vipconfig.Config) (*tls.Config, error) {
 func (elc *EtcdLeaderChecker) get(ctx context.Context, out chan<- bool) {
 	resp, err := elc.Get(ctx, elc.TriggerKey)
 	if err != nil {
-		elc.Logger.Error("Failed to get etcd value:", zap.Error(err))
+		elc.Logger.Error("Failed to get etcd value", zap.String("key", elc.TriggerKey), zap.Error(err))
+		out <- false
+		return
+	}
+	if resp == nil {
+		elc.Logger.Error("Received nil response from etcd", zap.String("key", elc.TriggerKey))
+		out <- false
+		return
+	}
+	if len(resp.Kvs) == 0 {
+		elc.Logger.Sugar().Info("No value found for key ", elc.TriggerKey, " - DCS may not have a leader yet")
 		out <- false
 		return
 	}
@@ -99,6 +112,7 @@ func (elc *EtcdLeaderChecker) watch(ctx context.Context, out chan<- bool) error
 				continue
 			}
 			if err := watchResp.Err(); err != nil {
+				elc.Logger.Error("Watch error for key "+elc.TriggerKey+":", zap.Error(err))
 				elc.get(ctx, out) // RPC failed, try to get the key directly to be on the safe side
 				continue
 			}

checker/patroni_leader_checker.go

@@ -47,12 +47,17 @@ func (c *PatroniLeaderChecker) GetChangeNotificationStream(ctx context.Context,
 		case <-ctx.Done():
 			return nil
 		case <-time.After(time.Duration(c.Interval) * time.Millisecond):
-			r, err := c.Get(c.Endpoints[0] + c.TriggerKey)
+			url := c.Endpoints[0] + c.TriggerKey
+			r, err := c.Get(url)
 			if err != nil {
-				c.Logger.Sugar().Error("patroni REST API error:", err)
+				c.Logger.Sugar().Errorf("patroni REST API error connecting to %s: %v", url, err)
+				out <- false
 				continue
 			}
 			r.Body.Close() //throw away the body
+			if r.StatusCode < 200 || r.StatusCode >= 300 {
+				c.Logger.Sugar().Warnf("patroni REST API returned non-success status code %d for %s (expected %s)", r.StatusCode, url, c.TriggerValue)
+			}
 			out <- strconv.Itoa(r.StatusCode) == c.TriggerValue
 		}
 	}

ipmanager/hetznerConfigurer.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"net"
 	"os"
 	"os/exec"
@@ -40,17 +41,16 @@ func newHetznerConfigurer(config *IPConfiguration, verbose bool) (*HetznerConfig
  * In order to tell the Hetzner API to route the failover-ip to
  * this machine, we must attach our own IP address to the API request.
  */
-func getOutboundIP() net.IP {
+func getOutboundIP() (net.IP, error) {
 	conn, err := net.Dial("udp", "8.8.8.8:80")
 	if err != nil || conn == nil {
-		log.Error("error dialing 8.8.8.8 to retrieve preferred outbound IP", err)
-		return nil
+		return nil, fmt.Errorf("error dialing 8.8.8.8 to retrieve preferred outbound IP: %w", err)
 	}
 	defer conn.Close()
 
 	localAddr := conn.LocalAddr().(*net.UDPAddr)
 
-	return localAddr.IP
+	return localAddr.IP, nil
 }
 
 func (c *HetznerConfigurer) curlQueryFailover(post bool) (string, error) {
@@ -98,10 +98,10 @@ func (c *HetznerConfigurer) curlQueryFailover(post bool) (string, error) {
 	 */
 	var cmd *exec.Cmd
 	if post {
-		myOwnIP := getOutboundIP()
-		if myOwnIP == nil {
-			log.Error("Error determining this machine's IP address.")
-			return "", errors.New("error determining this machine's IP address")
+		myOwnIP, err := getOutboundIP()
+		if err != nil {
+			log.Error("Error determining this machine's IP address.", err)
+			return "", fmt.Errorf("error determining this machine's IP address: %w", err)
 		}
 		log.Infof("my_own_ip: %s\n", myOwnIP.String())
 
@@ -224,7 +224,14 @@ func (c *HetznerConfigurer) queryAddress() bool {
 		c.cachedState = unknown
 	}
 
-	if currentFailoverDestinationIP.Equal(getOutboundIP()) {
+	myOwnIP, err := getOutboundIP()
+	if err != nil {
+		log.Error("Error determining this machine's IP address.", err)
+		c.cachedState = unknown
+		return false
+	}
+
+	if currentFailoverDestinationIP.Equal(myOwnIP) {
 		//We "are" the current failover destination.
 		c.cachedState = configured
 		return true
@@ -262,7 +269,14 @@ func (c *HetznerConfigurer) runAddressConfiguration() bool {
 
 	c.lastAPICheck = time.Now()
 
-	if currentFailoverDestinationIP.Equal(getOutboundIP()) {
+	myOwnIP, err := getOutboundIP()
+	if err != nil {
+		log.Error("Error determining this machine's IP address.", err)
+		c.cachedState = unknown
+		return false
+	}
+
+	if currentFailoverDestinationIP.Equal(myOwnIP) {
 		//We "are" the current failover destination.
 		log.Info("Failover was successfully executed!")
 		c.cachedState = configured
@@ -271,7 +285,7 @@ func (c *HetznerConfigurer) runAddressConfiguration() bool {
 
 	log.Infof("The failover command was issued, but the current Failover destination (%s) is different from what it should be (%s).",
 		currentFailoverDestinationIP.String(),
-		getOutboundIP().String())
+		myOwnIP.String())
 	//Something must have gone wrong while trying to switch IP's...
 	c.cachedState = unknown
 	return false

ipmanager/ip_manager.go

@@ -2,6 +2,7 @@ package ipmanager
 
 import (
 	"context"
+	"fmt"
 	"net"
 	"net/netip"
 	"sync/atomic"
@@ -18,7 +19,7 @@ type ipConfigurer interface {
 	getCIDR() string
 }
 
-var log *zap.SugaredLogger = zap.L().Sugar()
+var log *zap.SugaredLogger
 
 // IPManager implements the main functionality of the VIP manager
 type IPManager struct {
@@ -40,22 +41,28 @@ func getMask(vip netip.Addr, mask int) net.IPMask {
 	return net.CIDRMask(mask, 128) //IPv6
 }
 
-func getNetIface(iface string) *net.Interface {
+func getNetIface(iface string) (*net.Interface, error) {
 	netIface, err := net.InterfaceByName(iface)
 	if err != nil {
-		log.Fatalf("Obtaining the interface raised an error: %s", err)
+		return nil, fmt.Errorf("failed to get interface %s: %w", iface, err)
 	}
-	return netIface
+	if netIface.Flags&net.FlagUp == 0 {
+		return nil, fmt.Errorf("interface %s is not up", iface)
+	}
+	return netIface, nil
 }
 
 // NewIPManager returns a new instance of IPManager
 func NewIPManager(conf *vipconfig.Config, states <-chan bool) (m *IPManager, err error) {
 	vip, err := netip.ParseAddr(conf.IP)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to parse VIP address: %w", err)
 	}
 	vipMask := getMask(vip, conf.Mask)
-	netIface := getNetIface(conf.Iface)
+	netIface, err := getNetIface(conf.Iface)
+	if err != nil {
+		return nil, err
+	}
 	ipConf := &IPConfiguration{
 		VIP:        vip,
 		Netmask:    vipMask,