Add metrics for real servers to ipvs (#4929)

plugins/inputs/ipvs/README.md

@@ -5,14 +5,14 @@ metrics about ipvs virtual and real servers.
 
 **Supported Platforms:** Linux
 
-### Configuration:
+## Configuration
 
 ```toml
 [[inputs.ipvs]]
   # no configuration
 ```
 
-### Permissions:
+## Permissions
 
 Assuming you installed the telegraf package via one of the published packages,
 the process will be running as the `telegraf` user. However, in order for this
@@ -20,9 +20,78 @@ plugin to communicate over netlink sockets it needs the telegraf process to be
 running as `root` (or some user with `CAP_NET_ADMIN` and `CAP_NET_RAW`). Be sure
 to ensure these permissions before running telegraf with this plugin included.
 
-### Example Output:
+## Metrics
+
+### Virtual Servers
+
+Metrics report for each `ipvs_virtual_server`:
+
+- `ipvs_virtual_server`
+  - tags:
+    - `sched` - the scheduler in use
+    - `netmask` - the mask used for determining affinity
+    - `address_family` - inet/inet6
+    - ONE of `address` + `port` + `protocol` *OR* `fwmark`
+  - fields:
+    - Connections
+    - PacketsIn
+    - PacketsOut
+    - BytesIn
+    - BytesOut
+    - CPS
+    - PPSIn
+    - PPSOut
+    - BPSIn
+    - BPSOut
+
+Each virtual server will contain tags identifying how it was configured, using
+one of `address` + `port` + `protocol` *OR* `fwmark`. This is how one would
+normally configure a virtual server using `ipvsadm`.
+
+### Real Servers
+
+Metrics reported for each `ipvs_real_server`:
+
+- `ipvs_real_server`
+  - tags:
+    - `address`
+    - `port`
+    - `address_family`
+    - ONE of `virtual_address` + `virtual_port` + `virtual_protocol` OR `virtual_fwmark`
+  - fields:
+    - ActiveConnections
+    - InactiveConnections
+    - Connections
+    - PacketsIn
+    - PacketsOut
+    - BytesIn
+    - BytesOut
+    - CPS
+    - PPSIn
+    - PPSOut
+    - BPSIn
+    - BPSOut
+
+Each real server can be identified as belonging to a virtual server using one of
+either `virtual_address + virtual_port + virtual_protocol` OR `virtual_fwmark`
+
+## Example Output
+
+### Virtual servers
+
+Example (when a virtual server is configured using `fwmark` and backed by 2 real servers):
+```
+ipvs_virtual_server,address=172.18.64.234,address_family=inet,netmask=32,port=9000,protocol=tcp,sched=rr bytes_in=0i,bytes_out=0i,pps_in=0i,pps_out=0i,cps=0i,connections=0i,pkts_in=0i,pkts_out=0i 1541019340000000000
+ipvs_real_server,address=172.18.64.220,address_family=inet,port=9000,virtual_address=172.18.64.234,virtual_port=9000,virtual_protocol=tcp active_connections=0i,inactive_connections=0i,pkts_in=0i,bytes_out=0i,pps_out=0i,connections=0i,pkts_out=0i,bytes_in=0i,pps_in=0i,cps=0i 1541019340000000000
+ipvs_real_server,address=172.18.64.219,address_family=inet,port=9000,virtual_address=172.18.64.234,virtual_port=9000,virtual_protocol=tcp active_connections=0i,inactive_connections=0i,pps_in=0i,pps_out=0i,connections=0i,pkts_in=0i,pkts_out=0i,bytes_in=0i,bytes_out=0i,cps=0i 1541019340000000000
 
 ```
-ipvs_virtual_server,address=172.18.64.234,address_family=inet,netmask=32,port=9000,protocol=tcp,sched=mh_418 bytes_out=0i,pps_in=0i,pps_out=0i,cps=0i,pkts_in=0i,pkts_out=0i,connections=0i,bytes_in=0i 1540407540000000000
+
-ipvs_virtual_server,address_family=inet,fwmark=47,netmask=32,sched=mh_418 connections=0i,pkts_in=0i,bytes_out=0i,pps_in=0i,pps_out=0i,pkts_out=0i,bytes_in=0i,cps=0i 1540407540000000000
+### Real servers
+
+Example (when a real server is configured using `proto+addr+port` and backed by 2 real servers):
+```
+ipvs_virtual_server,address_family=inet,fwmark=47,netmask=32,sched=rr cps=0i,connections=0i,pkts_in=0i,pkts_out=0i,bytes_in=0i,bytes_out=0i,pps_in=0i,pps_out=0i 1541019340000000000
+ipvs_real_server,address=172.18.64.220,address_family=inet,port=9000,virtual_fwmark=47 inactive_connections=0i,pkts_out=0i,bytes_out=0i,pps_in=0i,cps=0i,active_connections=0i,pkts_in=0i,bytes_in=0i,pps_out=0i,connections=0i 1541019340000000000
+ipvs_real_server,address=172.18.64.219,address_family=inet,port=9000,virtual_fwmark=47 cps=0i,active_connections=0i,inactive_connections=0i,connections=0i,pkts_in=0i,bytes_out=0i,pkts_out=0i,bytes_in=0i,pps_in=0i,pps_out=0i 1541019340000000000
 ```

plugins/inputs/ipvs/ipvs.go

@@ -5,6 +5,7 @@ package ipvs
 import (
 	"errors"
 	"fmt"
+	"log"
 	"math/bits"
 	"strconv"
 	"syscall"
@@ -57,6 +58,36 @@ func (i *IPVS) Gather(acc telegraf.Accumulator) error {
 			"cps":         s.Stats.CPS,
 		}
 		acc.AddGauge("ipvs_virtual_server", fields, serviceTags(s))
+
+		destinations, err := i.handle.GetDestinations(s)
+		if err != nil {
+			log.Println("E! Failed to list destinations for a virtual server")
+			continue // move on to the next virtual server
+		}
+
+		for _, d := range destinations {
+			fields := map[string]interface{}{
+				"active_connections":   d.ActiveConnections,
+				"inactive_connections": d.InactiveConnections,
+				"connections":          d.Stats.Connections,
+				"pkts_in":              d.Stats.PacketsIn,
+				"pkts_out":             d.Stats.PacketsOut,
+				"bytes_in":             d.Stats.BytesIn,
+				"bytes_out":            d.Stats.BytesOut,
+				"pps_in":               d.Stats.PPSIn,
+				"pps_out":              d.Stats.PPSOut,
+				"cps":                  d.Stats.CPS,
+			}
+			destTags := destinationTags(d)
+			if s.FWMark > 0 {
+				destTags["virtual_fwmark"] = strconv.Itoa(int(s.FWMark))
+			} else {
+				destTags["virtual_protocol"] = protocolToString(s.Protocol)
+				destTags["virtual_address"] = s.Address.String()
+				destTags["virtual_port"] = strconv.Itoa(int(s.Port))
+			}
+			acc.AddGauge("ipvs_real_server", fields, destTags)
+		}
 	}
 
 	return nil
@@ -66,7 +97,7 @@ func (i *IPVS) Gather(acc telegraf.Accumulator) error {
 func serviceTags(s *ipvs.Service) map[string]string {
 	ret := map[string]string{
 		"sched":          s.SchedName,
-		"netmask":        fmt.Sprintf("%d", bits.OnesCount32(s.Netmask)),
+		"netmask":        strconv.Itoa(bits.OnesCount32(s.Netmask)),
 		"address_family": addressFamilyToString(s.AddressFamily),
 	}
 	// Per the ipvsadm man page, a virtual service is defined "based on
@@ -81,6 +112,15 @@ func serviceTags(s *ipvs.Service) map[string]string {
 	return ret
 }
 
+// helper: given a Destination, return tags that identify it
+func destinationTags(d *ipvs.Destination) map[string]string {
+	return map[string]string{
+		"address":        d.Address.String(),
+		"port":           strconv.Itoa(int(d.Port)),
+		"address_family": addressFamilyToString(d.AddressFamily),
+	}
+}
+
 // helper: convert protocol uint16 to human readable string (if possible)
 func protocolToString(p uint16) string {
 	switch p {