Implement telegraf collecting stats on itself

closes #1348

plugins/inputs/internal/README.md

@@ -0,0 +1,83 @@
+# Internal Input Plugin
+
+The `internal` plugin collects metrics about the telegraf agent itself.
+
+Note that some metrics are aggregates across all instances of one type of
+plugin.
+
+### Configuration:
+
+```toml
+# Collect statistics about itself
+[[inputs.internal]]
+  ## If true, collect telegraf memory stats.
+  # collect_memstats = true
+```
+
+### Measurements & Fields:
+
+memstats are taken from the Go runtime: https://golang.org/pkg/runtime/#MemStats
+
+- internal\_memstats
+    - alloc\_bytes
+    - frees
+    - heap\_alloc\_bytes
+    - heap\_idle\_bytes
+    - heap\_in\_use\_bytes
+    - heap\_objects\_bytes
+    - heap\_released\_bytes
+    - heap\_sys\_bytes
+    - mallocs
+    - num\_gc
+    - pointer\_lookups
+    - sys\_bytes
+    - total\_alloc\_bytes
+
+agent stats collect aggregate stats on all telegraf plugins.
+
+- internal\_agent
+    - gather\_errors
+    - metrics\_dropped
+    - metrics\_gathered
+    - metrics\_written
+
+internal\_gather stats collect aggregate stats on all input plugins
+that are of the same input type. They are tagged with `input=<plugin_name>`.
+
+- internal\_gather
+    - gather\_time\_ns
+    - metrics\_gathered
+
+internal\_write stats collect aggregate stats on all output plugins
+that are of the same input type. They are tagged with `output=<plugin_name>`.
+
+
+- internal\_write
+    - buffer\_limit
+    - buffer\_size
+    - metrics\_written
+    - metrics\_filtered
+    - write\_time\_ns
+
+internal\_\<plugin\_name\> are metrics which are defined on a per-plugin basis, and
+usually contain tags which differentiate each instance of a particular type of
+plugin.
+
+- internal\_\<plugin\_name\>
+    - individual plugin-specific fields, such as requests counts.
+
+### Tags:
+
+All measurements for specific plugins are tagged with information relevant
+to each particular plugin.
+
+### Example Output:
+
+```
+internal_memstats,host=tyrion alloc_bytes=4457408i,sys_bytes=10590456i,pointer_lookups=7i,mallocs=17642i,frees=7473i,heap_sys_bytes=6848512i,heap_idle_bytes=1368064i,heap_in_use_bytes=5480448i,heap_released_bytes=0i,total_alloc_bytes=6875560i,heap_alloc_bytes=4457408i,heap_objects_bytes=10169i,num_gc=2i 1480682800000000000
+internal_agent,host=tyrion metrics_written=18i,metrics_dropped=0i,metrics_gathered=19i,gather_errors=0i 1480682800000000000
+internal_write,output=file,host=tyrion buffer_limit=10000i,write_time_ns=636609i,metrics_written=18i,buffer_size=0i 1480682800000000000
+internal_gather,input=internal,host=tyrion metrics_gathered=19i,gather_time_ns=442114i 1480682800000000000
+internal_gather,input=http_listener,host=tyrion metrics_gathered=0i,gather_time_ns=167285i 1480682800000000000
+internal_http_listener,address=:8186,host=tyrion queries_received=0i,writes_received=0i,requests_received=0i,buffers_created=0i,requests_served=0i,pings_received=0i,bytes_received=0i,not_founds_served=0i,pings_served=0i,queries_served=0i,writes_served=0i 1480682800000000000
+```

plugins/inputs/internal/internal.go

@@ -0,0 +1,66 @@
+package internal
+
+import (
+	"runtime"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/selfstat"
+)
+
+type Self struct {
+	CollectMemstats bool
+}
+
+func NewSelf() telegraf.Input {
+	return &Self{
+		CollectMemstats: true,
+	}
+}
+
+var sampleConfig = `
+  ## If true, collect telegraf memory stats.
+  # collect_memstats = true
+`
+
+func (s *Self) Description() string {
+	return "Collect statistics about itself"
+}
+
+func (s *Self) SampleConfig() string {
+	return sampleConfig
+}
+
+func (s *Self) Gather(acc telegraf.Accumulator) error {
+	if s.CollectMemstats {
+		m := &runtime.MemStats{}
+		runtime.ReadMemStats(m)
+		fields := map[string]interface{}{
+			"alloc_bytes":       m.Alloc,      // bytes allocated and not yet freed
+			"total_alloc_bytes": m.TotalAlloc, // bytes allocated (even if freed)
+			"sys_bytes":         m.Sys,        // bytes obtained from system (sum of XxxSys below)
+			"pointer_lookups":   m.Lookups,    // number of pointer lookups
+			"mallocs":           m.Mallocs,    // number of mallocs
+			"frees":             m.Frees,      // number of frees
+			// Main allocation heap statistics.
+			"heap_alloc_bytes":    m.HeapAlloc,    // bytes allocated and not yet freed (same as Alloc above)
+			"heap_sys_bytes":      m.HeapSys,      // bytes obtained from system
+			"heap_idle_bytes":     m.HeapIdle,     // bytes in idle spans
+			"heap_in_use_bytes":   m.HeapInuse,    // bytes in non-idle span
+			"heap_released_bytes": m.HeapReleased, // bytes released to the OS
+			"heap_objects_bytes":  m.HeapObjects,  // total number of allocated objects
+			"num_gc":              m.NumGC,
+		}
+		acc.AddFields("internal_memstats", fields, map[string]string{})
+	}
+
+	for _, m := range selfstat.Metrics() {
+		acc.AddFields(m.Name(), m.Fields(), m.Tags(), m.Time())
+	}
+
+	return nil
+}
+
+func init() {
+	inputs.Add("internal", NewSelf)
+}

plugins/inputs/internal/internal_test.go

@@ -0,0 +1,62 @@
+package internal
+
+import (
+	"testing"
+
+	"github.com/influxdata/telegraf/selfstat"
+	"github.com/influxdata/telegraf/testutil"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSelfPlugin(t *testing.T) {
+	s := NewSelf()
+	acc := &testutil.Accumulator{}
+
+	s.Gather(acc)
+	assert.True(t, acc.HasMeasurement("internal_memstats"))
+
+	// test that a registered stat is incremented
+	stat := selfstat.Register("mytest", "test", map[string]string{"test": "foo"})
+	stat.Incr(1)
+	stat.Incr(2)
+	s.Gather(acc)
+	acc.AssertContainsTaggedFields(t, "internal_mytest",
+		map[string]interface{}{
+			"test": int64(3),
+		},
+		map[string]string{
+			"test": "foo",
+		},
+	)
+	acc.ClearMetrics()
+
+	// test that a registered stat is set properly
+	stat.Set(101)
+	s.Gather(acc)
+	acc.AssertContainsTaggedFields(t, "internal_mytest",
+		map[string]interface{}{
+			"test": int64(101),
+		},
+		map[string]string{
+			"test": "foo",
+		},
+	)
+	acc.ClearMetrics()
+
+	// test that regular and timing stats can share the same measurement, and
+	// that timings are set properly.
+	timing := selfstat.RegisterTiming("mytest", "test_ns", map[string]string{"test": "foo"})
+	timing.Incr(100)
+	timing.Incr(200)
+	s.Gather(acc)
+	acc.AssertContainsTaggedFields(t, "internal_mytest",
+		map[string]interface{}{
+			"test":    int64(101),
+			"test_ns": int64(150),
+		},
+		map[string]string{
+			"test": "foo",
+		},
+	)
+}