Moved system package inputs out to top level (#4406)

plugins/inputs/net/NETSTAT_README.md

@@ -0,0 +1,57 @@
+# Netstat Input Plugin
+
+This plugin collects TCP connections state and UDP socket counts by using `lsof`.
+
+### Configuration:
+
+``` toml
+# Collect TCP connections state and UDP socket counts
+[[inputs.netstat]]
+  # no configuration
+```
+
+# Measurements:
+
+Supported TCP Connection states are follows.
+
+- established
+- syn_sent
+- syn_recv
+- fin_wait1
+- fin_wait2
+- time_wait
+- close
+- close_wait
+- last_ack
+- listen
+- closing
+- none
+
+### TCP Connection State measurements:
+
+Meta:
+- units: counts
+
+Measurement names:
+- tcp_established
+- tcp_syn_sent
+- tcp_syn_recv
+- tcp_fin_wait1
+- tcp_fin_wait2
+- tcp_time_wait
+- tcp_close
+- tcp_close_wait
+- tcp_last_ack
+- tcp_listen
+- tcp_closing
+- tcp_none
+
+If there are no connection on the state, the metric is not counted.
+
+### UDP socket counts measurements:
+
+Meta:
+- units: counts
+
+Measurement names:
+- udp_socket

plugins/inputs/net/NET_README.md

@@ -0,0 +1,73 @@
+# Net Input Plugin
+
+This plugin gathers metrics about network interface and protocol usage (Linux only).
+
+### Configuration:
+
+```toml
+# Gather metrics about network interfaces
+[[inputs.net]]
+  ## By default, telegraf gathers stats from any up interface (excluding loopback)
+  ## Setting interfaces will tell it to gather these explicit interfaces,
+  ## regardless of status. When specifying an interface, glob-style
+  ## patterns are also supported.
+  ##
+  # interfaces = ["eth*", "enp0s[0-1]", "lo"]
+  ##
+  ## On linux systems telegraf also collects protocol stats.
+  ## Setting ignore_protocol_stats to true will skip reporting of protocol metrics.
+  ##
+  # ignore_protocol_stats = false
+  ##
+```
+
+### Measurements & Fields:
+
+The fields from this plugin are gathered in the _net_ measurement.
+
+Fields (all platforms):
+
+* bytes_sent - The total number of bytes sent by the interface
+* bytes_recv - The total number of bytes received by the interface
+* packets_sent - The total number of packets sent by the interface
+* packets_recv - The total number of packets received by the interface
+* err_in - The total number of receive errors detected by the interface
+* err_out - The total number of transmit errors detected by the interface
+* drop_in - The total number of received packets dropped by the interface
+* drop_out - The total number of transmitted packets dropped by the interface
+
+Different platforms gather the data above with different mechanisms. Telegraf uses the ([gopsutil](https://github.com/shirou/gopsutil)) package, which under Linux reads the /proc/net/dev file.
+Under freebsd/openbsd and darwin the plugin uses netstat.
+
+Additionally, for the time being _only under Linux_, the plugin gathers system wide stats for different network protocols using /proc/net/snmp (tcp, udp, icmp, etc.).
+Explanation of the different metrics exposed by snmp is out of the scope of this document. The best way to find information would be tracing the constants in the Linux kernel source [here](http://lxr.free-electrons.com/source/net/ipv4/proc.c) and their usage. If /proc/net/snmp cannot be read for some reason, telegraf ignores the error silently.
+
+### Tags:
+
+* Net measurements have the following tags:
+    - interface (the interface from which metrics are gathered)
+
+Under Linux the system wide protocol metrics have the interface=all tag.
+
+### Sample Queries:
+
+You can use the following query to get the upload/download traffic rate per second for all interfaces in the last hour. The query uses the (derivative function)[https://docs.influxdata.com/influxdb/v1.2/query_language/functions#derivative] which calculates the rate of change between subsequent field values.
+
+```
+SELECT derivative(first(bytes_recv), 1s) as "download bytes/sec", derivative(first(bytes_sent), 1s) as "upload bytes/sec" FROM net WHERE time > now() - 1h AND interface != 'all' GROUP BY time(10s), interface fill(0);
+```
+
+### Example Output:
+
+```
+# All platforms
+$ ./telegraf --config telegraf.conf --input-filter net --test
+net,interface=eth0,host=HOST bytes_sent=451838509i,bytes_recv=3284081640i,packets_sent=2663590i,packets_recv=3585442i,err_in=0i,err_out=0i,drop_in=4i,drop_out=0i 1492834180000000000
+```
+
+```
+# Linux
+$ ./telegraf --config telegraf.conf --input-filter net --test
+net,interface=eth0,host=HOST bytes_sent=451838509i,bytes_recv=3284081640i,packets_sent=2663590i,packets_recv=3585442i,err_in=0i,err_out=0i,drop_in=4i,drop_out=0i 1492834180000000000
+net,interface=all,host=HOST ip_reasmfails=0i,icmp_insrcquenchs=0i,icmp_outtimestamps=0i,ip_inhdrerrors=0i,ip_inunknownprotos=0i,icmp_intimeexcds=10i,icmp_outaddrmasks=0i,icmp_indestunreachs=11005i,icmpmsg_outtype0=6i,tcp_retranssegs=14669i,udplite_outdatagrams=0i,ip_reasmtimeout=0i,ip_outnoroutes=2577i,ip_inaddrerrors=186i,icmp_outaddrmaskreps=0i,tcp_incsumerrors=0i,tcp_activeopens=55965i,ip_reasmoks=0i,icmp_inechos=6i,icmp_outdestunreachs=9417i,ip_reasmreqds=0i,icmp_outtimestampreps=0i,tcp_rtoalgorithm=1i,icmpmsg_intype3=11005i,icmpmsg_outtype69=129i,tcp_outsegs=2777459i,udplite_rcvbuferrors=0i,ip_fragoks=0i,icmp_inmsgs=13398i,icmp_outerrors=0i,tcp_outrsts=14951i,udplite_noports=0i,icmp_outmsgs=11517i,icmp_outechoreps=6i,icmpmsg_intype11=10i,icmp_inparmprobs=0i,ip_forwdatagrams=0i,icmp_inechoreps=1909i,icmp_outredirects=0i,icmp_intimestampreps=0i,icmpmsg_intype5=468i,tcp_rtomax=120000i,tcp_maxconn=-1i,ip_fragcreates=0i,ip_fragfails=0i,icmp_inredirects=468i,icmp_outtimeexcds=0i,icmp_outechos=1965i,icmp_inaddrmasks=0i,tcp_inerrs=389i,tcp_rtomin=200i,ip_defaultttl=64i,ip_outrequests=3366408i,ip_forwarding=2i,udp_incsumerrors=0i,udp_indatagrams=522136i,udplite_incsumerrors=0i,ip_outdiscards=871i,icmp_inerrors=958i,icmp_outsrcquenchs=0i,icmpmsg_intype0=1909i,tcp_insegs=3580226i,udp_outdatagrams=577265i,udp_rcvbuferrors=0i,udplite_sndbuferrors=0i,icmp_incsumerrors=0i,icmp_outparmprobs=0i,icmpmsg_outtype3=9417i,tcp_attemptfails=2652i,udplite_inerrors=0i,udplite_indatagrams=0i,ip_inreceives=4172969i,icmpmsg_outtype8=1965i,tcp_currestab=59i,udp_noports=5961i,ip_indelivers=4099279i,ip_indiscards=0i,tcp_estabresets=5818i,udp_sndbuferrors=3i,icmp_intimestamps=0i,icmpmsg_intype8=6i,udp_inerrors=0i,icmp_inaddrmaskreps=0i,tcp_passiveopens=452i 1492831540000000000
+``

plugins/inputs/net/net.go

@@ -0,0 +1,125 @@
+package net
+
+import (
+	"fmt"
+	"net"
+	"strings"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/filter"
+	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/plugins/inputs/system"
+)
+
+type NetIOStats struct {
+	filter filter.Filter
+	ps     system.PS
+
+	skipChecks          bool
+	IgnoreProtocolStats bool
+	Interfaces          []string
+}
+
+func (_ *NetIOStats) Description() string {
+	return "Read metrics about network interface usage"
+}
+
+var netSampleConfig = `
+  ## By default, telegraf gathers stats from any up interface (excluding loopback)
+  ## Setting interfaces will tell it to gather these explicit interfaces,
+  ## regardless of status.
+  ##
+  # interfaces = ["eth0"]
+  ##
+  ## On linux systems telegraf also collects protocol stats.
+  ## Setting ignore_protocol_stats to true will skip reporting of protocol metrics.
+  ##
+  # ignore_protocol_stats = false
+  ##
+`
+
+func (_ *NetIOStats) SampleConfig() string {
+	return netSampleConfig
+}
+
+func (s *NetIOStats) Gather(acc telegraf.Accumulator) error {
+	netio, err := s.ps.NetIO()
+	if err != nil {
+		return fmt.Errorf("error getting net io info: %s", err)
+	}
+
+	if s.filter == nil {
+		if s.filter, err = filter.Compile(s.Interfaces); err != nil {
+			return fmt.Errorf("error compiling filter: %s", err)
+		}
+	}
+
+	for _, io := range netio {
+		if len(s.Interfaces) != 0 {
+			var found bool
+
+			if s.filter.Match(io.Name) {
+				found = true
+			}
+
+			if !found {
+				continue
+			}
+		} else if !s.skipChecks {
+			iface, err := net.InterfaceByName(io.Name)
+			if err != nil {
+				continue
+			}
+
+			if iface.Flags&net.FlagLoopback == net.FlagLoopback {
+				continue
+			}
+
+			if iface.Flags&net.FlagUp == 0 {
+				continue
+			}
+		}
+
+		tags := map[string]string{
+			"interface": io.Name,
+		}
+
+		fields := map[string]interface{}{
+			"bytes_sent":   io.BytesSent,
+			"bytes_recv":   io.BytesRecv,
+			"packets_sent": io.PacketsSent,
+			"packets_recv": io.PacketsRecv,
+			"err_in":       io.Errin,
+			"err_out":      io.Errout,
+			"drop_in":      io.Dropin,
+			"drop_out":     io.Dropout,
+		}
+		acc.AddCounter("net", fields, tags)
+	}
+
+	// Get system wide stats for different network protocols
+	// (ignore these stats if the call fails)
+	if !s.IgnoreProtocolStats {
+		netprotos, _ := s.ps.NetProto()
+		fields := make(map[string]interface{})
+		for _, proto := range netprotos {
+			for stat, value := range proto.Stats {
+				name := fmt.Sprintf("%s_%s", strings.ToLower(proto.Protocol),
+					strings.ToLower(stat))
+				fields[name] = value
+			}
+		}
+		tags := map[string]string{
+			"interface": "all",
+		}
+		acc.AddFields("net", fields, tags)
+	}
+
+	return nil
+}
+
+func init() {
+	inputs.Add("net", func() telegraf.Input {
+		return &NetIOStats{ps: system.NewSystemPS()}
+	})
+}

plugins/inputs/net/net_test.go

@@ -0,0 +1,116 @@
+package net
+
+import (
+	"syscall"
+	"testing"
+
+	"github.com/influxdata/telegraf/plugins/inputs/system"
+	"github.com/influxdata/telegraf/testutil"
+	"github.com/shirou/gopsutil/net"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNetStats(t *testing.T) {
+	var mps system.MockPS
+	var err error
+	defer mps.AssertExpectations(t)
+	var acc testutil.Accumulator
+
+	netio := net.IOCountersStat{
+		Name:        "eth0",
+		BytesSent:   1123,
+		BytesRecv:   8734422,
+		PacketsSent: 781,
+		PacketsRecv: 23456,
+		Errin:       832,
+		Errout:      8,
+		Dropin:      7,
+		Dropout:     1,
+	}
+
+	mps.On("NetIO").Return([]net.IOCountersStat{netio}, nil)
+
+	netprotos := []net.ProtoCountersStat{
+		net.ProtoCountersStat{
+			Protocol: "Udp",
+			Stats: map[string]int64{
+				"InDatagrams": 4655,
+				"NoPorts":     892592,
+			},
+		},
+	}
+	mps.On("NetProto").Return(netprotos, nil)
+
+	netstats := []net.ConnectionStat{
+		net.ConnectionStat{
+			Type: syscall.SOCK_DGRAM,
+		},
+		net.ConnectionStat{
+			Status: "ESTABLISHED",
+		},
+		net.ConnectionStat{
+			Status: "ESTABLISHED",
+		},
+		net.ConnectionStat{
+			Status: "CLOSE",
+		},
+	}
+
+	mps.On("NetConnections").Return(netstats, nil)
+
+	err = (&NetIOStats{ps: &mps, skipChecks: true}).Gather(&acc)
+	require.NoError(t, err)
+
+	ntags := map[string]string{
+		"interface": "eth0",
+	}
+
+	fields1 := map[string]interface{}{
+		"bytes_sent":   uint64(1123),
+		"bytes_recv":   uint64(8734422),
+		"packets_sent": uint64(781),
+		"packets_recv": uint64(23456),
+		"err_in":       uint64(832),
+		"err_out":      uint64(8),
+		"drop_in":      uint64(7),
+		"drop_out":     uint64(1),
+	}
+	acc.AssertContainsTaggedFields(t, "net", fields1, ntags)
+
+	fields2 := map[string]interface{}{
+		"udp_noports":     int64(892592),
+		"udp_indatagrams": int64(4655),
+	}
+	ntags = map[string]string{
+		"interface": "all",
+	}
+	acc.AssertContainsTaggedFields(t, "net", fields2, ntags)
+
+	acc.Metrics = nil
+
+	err = (&NetStats{&mps}).Gather(&acc)
+	require.NoError(t, err)
+
+	fields3 := map[string]interface{}{
+		"tcp_established": 2,
+		"tcp_syn_sent":    0,
+		"tcp_syn_recv":    0,
+		"tcp_fin_wait1":   0,
+		"tcp_fin_wait2":   0,
+		"tcp_time_wait":   0,
+		"tcp_close":       1,
+		"tcp_close_wait":  0,
+		"tcp_last_ack":    0,
+		"tcp_listen":      0,
+		"tcp_closing":     0,
+		"tcp_none":        0,
+		"udp_socket":      1,
+	}
+	acc.AssertContainsTaggedFields(t, "netstat", fields3, make(map[string]string))
+
+	acc.Metrics = nil
+	err = (&NetIOStats{ps: &mps, IgnoreProtocolStats: true}).Gather(&acc)
+	require.NoError(t, err)
+
+	acc.AssertDoesNotContainsTaggedFields(t, "netstat", fields3, make(map[string]string))
+}

plugins/inputs/net/netstat.go

@@ -0,0 +1,72 @@
+package net
+
+import (
+	"fmt"
+	"syscall"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/plugins/inputs/system"
+)
+
+type NetStats struct {
+	ps system.PS
+}
+
+func (_ *NetStats) Description() string {
+	return "Read TCP metrics such as established, time wait and sockets counts."
+}
+
+var tcpstatSampleConfig = ""
+
+func (_ *NetStats) SampleConfig() string {
+	return tcpstatSampleConfig
+}
+
+func (s *NetStats) Gather(acc telegraf.Accumulator) error {
+	netconns, err := s.ps.NetConnections()
+	if err != nil {
+		return fmt.Errorf("error getting net connections info: %s", err)
+	}
+	counts := make(map[string]int)
+	counts["UDP"] = 0
+
+	// TODO: add family to tags or else
+	tags := map[string]string{}
+	for _, netcon := range netconns {
+		if netcon.Type == syscall.SOCK_DGRAM {
+			counts["UDP"] += 1
+			continue // UDP has no status
+		}
+		c, ok := counts[netcon.Status]
+		if !ok {
+			counts[netcon.Status] = 0
+		}
+		counts[netcon.Status] = c + 1
+	}
+
+	fields := map[string]interface{}{
+		"tcp_established": counts["ESTABLISHED"],
+		"tcp_syn_sent":    counts["SYN_SENT"],
+		"tcp_syn_recv":    counts["SYN_RECV"],
+		"tcp_fin_wait1":   counts["FIN_WAIT1"],
+		"tcp_fin_wait2":   counts["FIN_WAIT2"],
+		"tcp_time_wait":   counts["TIME_WAIT"],
+		"tcp_close":       counts["CLOSE"],
+		"tcp_close_wait":  counts["CLOSE_WAIT"],
+		"tcp_last_ack":    counts["LAST_ACK"],
+		"tcp_listen":      counts["LISTEN"],
+		"tcp_closing":     counts["CLOSING"],
+		"tcp_none":        counts["NONE"],
+		"udp_socket":      counts["UDP"],
+	}
+	acc.AddFields("netstat", fields, tags)
+
+	return nil
+}
+
+func init() {
+	inputs.Add("netstat", func() telegraf.Input {
+		return &NetStats{ps: system.NewSystemPS()}
+	})
+}