Support Processor & Aggregator Plugins

closes #1726

plugins/aggregators/all/all.go

@@ -0,0 +1,5 @@
+package all
+
+import (
+	_ "github.com/influxdata/telegraf/plugins/aggregators/minmax"
+)

plugins/aggregators/minmax/minmax.go

@@ -0,0 +1,192 @@
+package minmax
+
+import (
+	"sync"
+	"time"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal"
+	"github.com/influxdata/telegraf/plugins/aggregators"
+)
+
+type MinMax struct {
+	Period internal.Duration
+
+	// metrics waiting to be processed
+	metrics  chan telegraf.Metric
+	shutdown chan struct{}
+	wg       sync.WaitGroup
+
+	// caches for metric fields, names, and tags
+	fieldCache map[uint64]map[string]minmax
+	nameCache  map[uint64]string
+	tagCache   map[uint64]map[string]string
+
+	acc telegraf.Accumulator
+}
+
+type minmax struct {
+	min interface{}
+	max interface{}
+}
+
+var sampleConfig = `
+  ## TODO doc
+  period = "30s"
+`
+
+func (m *MinMax) SampleConfig() string {
+	return sampleConfig
+}
+
+func (m *MinMax) Description() string {
+	return "Keep the aggregate min/max of each metric passing through."
+}
+
+func (m *MinMax) Apply(in telegraf.Metric) {
+	m.metrics <- in
+}
+
+func (m *MinMax) apply(in telegraf.Metric) {
+	id := in.HashID()
+	if _, ok := m.nameCache[id]; !ok {
+		// hit an uncached metric, create caches for first time:
+		m.nameCache[id] = in.Name()
+		m.tagCache[id] = in.Tags()
+		m.fieldCache[id] = make(map[string]minmax)
+		for k, v := range in.Fields() {
+			m.fieldCache[id][k] = minmax{
+				min: v,
+				max: v,
+			}
+		}
+	} else {
+		for k, v := range in.Fields() {
+			cmpmin := compare(m.fieldCache[id][k].min, v)
+			cmpmax := compare(m.fieldCache[id][k].max, v)
+			if cmpmin == 1 {
+				tmp := m.fieldCache[id][k]
+				tmp.min = v
+				m.fieldCache[id][k] = tmp
+			}
+			if cmpmax == -1 {
+				tmp := m.fieldCache[id][k]
+				tmp.max = v
+				m.fieldCache[id][k] = tmp
+			}
+		}
+	}
+}
+
+func (m *MinMax) Start(acc telegraf.Accumulator) error {
+	m.metrics = make(chan telegraf.Metric, 10)
+	m.shutdown = make(chan struct{})
+	m.clearCache()
+	m.acc = acc
+	m.wg.Add(1)
+	if m.Period.Duration > 0 {
+		go m.periodHandler()
+	} else {
+		go m.continuousHandler()
+	}
+	return nil
+}
+
+func (m *MinMax) Stop() {
+	close(m.shutdown)
+	m.wg.Wait()
+}
+
+func (m *MinMax) addfields(id uint64) {
+	fields := map[string]interface{}{}
+	for k, v := range m.fieldCache[id] {
+		fields[k+"_min"] = v.min
+		fields[k+"_max"] = v.max
+	}
+	m.acc.AddFields(m.nameCache[id], fields, m.tagCache[id])
+}
+
+func (m *MinMax) clearCache() {
+	m.fieldCache = make(map[uint64]map[string]minmax)
+	m.nameCache = make(map[uint64]string)
+	m.tagCache = make(map[uint64]map[string]string)
+}
+
+// periodHandler only adds the aggregate metrics on the configured Period.
+//   thus if telegraf's collection interval is 10s, and period is 30s, there
+//   will only be one aggregate sent every 3 metrics.
+func (m *MinMax) periodHandler() {
+	// TODO make this sleep less of a hack!
+	time.Sleep(time.Millisecond * 200)
+	defer m.wg.Done()
+	ticker := time.NewTicker(m.Period.Duration)
+	defer ticker.Stop()
+	for {
+		select {
+		case in := <-m.metrics:
+			m.apply(in)
+		case <-m.shutdown:
+			if len(m.metrics) > 0 {
+				continue
+			}
+			return
+		case <-ticker.C:
+			for id, _ := range m.nameCache {
+				m.addfields(id)
+			}
+			m.clearCache()
+		}
+	}
+}
+
+// continuousHandler sends one metric for every metric that passes through it.
+func (m *MinMax) continuousHandler() {
+	defer m.wg.Done()
+	for {
+		select {
+		case in := <-m.metrics:
+			m.apply(in)
+			m.addfields(in.HashID())
+		case <-m.shutdown:
+			if len(m.metrics) > 0 {
+				continue
+			}
+			return
+		}
+	}
+}
+
+func compare(a, b interface{}) int {
+	switch at := a.(type) {
+	case int64:
+		if bt, ok := b.(int64); ok {
+			if at < bt {
+				return -1
+			} else if at > bt {
+				return 1
+			}
+			return 0
+		} else {
+			return 0
+		}
+	case float64:
+		if bt, ok := b.(float64); ok {
+			if at < bt {
+				return -1
+			} else if at > bt {
+				return 1
+			}
+			return 0
+		} else {
+			return 0
+		}
+	default:
+		return 0
+	}
+}
+
+func init() {
+	aggregators.Add("minmax", func() telegraf.Aggregator {
+		return &MinMax{}
+	})
+}

plugins/aggregators/minmax/minmax_test.go

@@ -0,0 +1,51 @@
+package minmax
+
+import (
+	"testing"
+	"time"
+
+	"github.com/influxdata/telegraf"
+)
+
+func BenchmarkApply(b *testing.B) {
+	minmax := MinMax{}
+	minmax.clearCache()
+
+	m1, _ := telegraf.NewMetric("m1",
+		map[string]string{"foo": "bar"},
+		map[string]interface{}{
+			"a": int64(1),
+			"b": int64(1),
+			"c": int64(1),
+			"d": int64(1),
+			"e": int64(1),
+			"f": float64(2),
+			"g": float64(2),
+			"h": float64(2),
+			"i": float64(2),
+			"j": float64(3),
+		},
+		time.Now(),
+	)
+	m2, _ := telegraf.NewMetric("m1",
+		map[string]string{"foo": "bar"},
+		map[string]interface{}{
+			"a": int64(3),
+			"b": int64(3),
+			"c": int64(3),
+			"d": int64(3),
+			"e": int64(3),
+			"f": float64(1),
+			"g": float64(1),
+			"h": float64(1),
+			"i": float64(1),
+			"j": float64(1),
+		},
+		time.Now(),
+	)
+
+	for n := 0; n < b.N; n++ {
+		minmax.apply(m1)
+		minmax.apply(m2)
+	}
+}

plugins/aggregators/registry.go

@@ -0,0 +1,11 @@
+package aggregators
+
+import "github.com/influxdata/telegraf"
+
+type Creator func() telegraf.Aggregator
+
+var Aggregators = map[string]Creator{}
+
+func Add(name string, creator Creator) {
+	Aggregators[name] = creator
+}

plugins/inputs/http_listener/http_listener_test.go

@@ -99,14 +99,14 @@ func TestWriteHTTPHighTraffic(t *testing.T) {
 	var wg sync.WaitGroup
 	for i := 0; i < 10; i++ {
 		wg.Add(1)
-		go func() {
+		go func(innerwg *sync.WaitGroup) {
+			defer innerwg.Done()
 			for i := 0; i < 500; i++ {
 				resp, err := http.Post("http://localhost:8286/write?db=mydb", "", bytes.NewBuffer([]byte(testMsgs)))
 				require.NoError(t, err)
 				require.EqualValues(t, 204, resp.StatusCode)
 			}
-			wg.Done()
-		}()
+		}(&wg)
 	}
 
 	wg.Wait()

plugins/inputs/postgresql_extensible/postgresql_extensible.go

@@ -29,6 +29,7 @@ type Postgresql struct {
 		Tagvalue    string
 		Measurement string
 	}
+	Debug bool
 }
 
 type query []struct {

plugins/processors/all/all.go

@@ -0,0 +1,5 @@
+package all
+
+import (
+	_ "github.com/influxdata/telegraf/plugins/processors/printer"
+)

plugins/processors/printer/printer.go

@@ -0,0 +1,35 @@
+package printer
+
+import (
+	"fmt"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/plugins/processors"
+)
+
+type Printer struct {
+}
+
+var sampleConfig = `
+`
+
+func (p *Printer) SampleConfig() string {
+	return sampleConfig
+}
+
+func (p *Printer) Description() string {
+	return "Print all metrics that pass through this filter."
+}
+
+func (p *Printer) Apply(in ...telegraf.Metric) []telegraf.Metric {
+	for _, metric := range in {
+		fmt.Println(metric.String())
+	}
+	return in
+}
+
+func init() {
+	processors.Add("printer", func() telegraf.Processor {
+		return &Printer{}
+	})
+}

plugins/processors/registry.go

@@ -0,0 +1,11 @@
+package processors
+
+import "github.com/influxdata/telegraf"
+
+type Creator func() telegraf.Processor
+
+var Processors = map[string]Creator{}
+
+func Add(name string, creator Creator) {
+	Processors[name] = creator
+}