Implement a per-output fixed size metric buffer

Also moved some objects out of config.go and put them in their own
package, internal/models

fixes #568
closes #285

internal/models/filter.go

@@ -0,0 +1,92 @@
+package models
+
+import (
+	"strings"
+
+	"github.com/influxdata/influxdb/client/v2"
+	"github.com/influxdata/telegraf/internal"
+)
+
+// TagFilter is the name of a tag, and the values on which to filter
+type TagFilter struct {
+	Name   string
+	Filter []string
+}
+
+// Filter containing drop/pass and tagdrop/tagpass rules
+type Filter struct {
+	Drop []string
+	Pass []string
+
+	TagDrop []TagFilter
+	TagPass []TagFilter
+
+	IsActive bool
+}
+
+func (f Filter) ShouldPointPass(point *client.Point) bool {
+	if f.ShouldPass(point.Name()) && f.ShouldTagsPass(point.Tags()) {
+		return true
+	}
+	return false
+}
+
+// ShouldPass returns true if the metric should pass, false if should drop
+// based on the drop/pass filter parameters
+func (f Filter) ShouldPass(key string) bool {
+	if f.Pass != nil {
+		for _, pat := range f.Pass {
+			// TODO remove HasPrefix check, leaving it for now for legacy support.
+			// Cam, 2015-12-07
+			if strings.HasPrefix(key, pat) || internal.Glob(pat, key) {
+				return true
+			}
+		}
+		return false
+	}
+
+	if f.Drop != nil {
+		for _, pat := range f.Drop {
+			// TODO remove HasPrefix check, leaving it for now for legacy support.
+			// Cam, 2015-12-07
+			if strings.HasPrefix(key, pat) || internal.Glob(pat, key) {
+				return false
+			}
+		}
+
+		return true
+	}
+	return true
+}
+
+// ShouldTagsPass returns true if the metric should pass, false if should drop
+// based on the tagdrop/tagpass filter parameters
+func (f Filter) ShouldTagsPass(tags map[string]string) bool {
+	if f.TagPass != nil {
+		for _, pat := range f.TagPass {
+			if tagval, ok := tags[pat.Name]; ok {
+				for _, filter := range pat.Filter {
+					if internal.Glob(filter, tagval) {
+						return true
+					}
+				}
+			}
+		}
+		return false
+	}
+
+	if f.TagDrop != nil {
+		for _, pat := range f.TagDrop {
+			if tagval, ok := tags[pat.Name]; ok {
+				for _, filter := range pat.Filter {
+					if internal.Glob(filter, tagval) {
+						return false
+					}
+				}
+			}
+		}
+		return true
+	}
+
+	return true
+}

internal/models/filter_test.go

@@ -0,0 +1,177 @@
+package models
+
+import (
+	"testing"
+)
+
+func TestFilter_Empty(t *testing.T) {
+	f := Filter{}
+
+	measurements := []string{
+		"foo",
+		"bar",
+		"barfoo",
+		"foo_bar",
+		"foo.bar",
+		"foo-bar",
+		"supercalifradjulisticexpialidocious",
+	}
+
+	for _, measurement := range measurements {
+		if !f.ShouldPass(measurement) {
+			t.Errorf("Expected measurement %s to pass", measurement)
+		}
+	}
+}
+
+func TestFilter_Pass(t *testing.T) {
+	f := Filter{
+		Pass: []string{"foo*", "cpu_usage_idle"},
+	}
+
+	passes := []string{
+		"foo",
+		"foo_bar",
+		"foo.bar",
+		"foo-bar",
+		"cpu_usage_idle",
+	}
+
+	drops := []string{
+		"bar",
+		"barfoo",
+		"bar_foo",
+		"cpu_usage_busy",
+	}
+
+	for _, measurement := range passes {
+		if !f.ShouldPass(measurement) {
+			t.Errorf("Expected measurement %s to pass", measurement)
+		}
+	}
+
+	for _, measurement := range drops {
+		if f.ShouldPass(measurement) {
+			t.Errorf("Expected measurement %s to drop", measurement)
+		}
+	}
+}
+
+func TestFilter_Drop(t *testing.T) {
+	f := Filter{
+		Drop: []string{"foo*", "cpu_usage_idle"},
+	}
+
+	drops := []string{
+		"foo",
+		"foo_bar",
+		"foo.bar",
+		"foo-bar",
+		"cpu_usage_idle",
+	}
+
+	passes := []string{
+		"bar",
+		"barfoo",
+		"bar_foo",
+		"cpu_usage_busy",
+	}
+
+	for _, measurement := range passes {
+		if !f.ShouldPass(measurement) {
+			t.Errorf("Expected measurement %s to pass", measurement)
+		}
+	}
+
+	for _, measurement := range drops {
+		if f.ShouldPass(measurement) {
+			t.Errorf("Expected measurement %s to drop", measurement)
+		}
+	}
+}
+
+func TestFilter_TagPass(t *testing.T) {
+	filters := []TagFilter{
+		TagFilter{
+			Name:   "cpu",
+			Filter: []string{"cpu-*"},
+		},
+		TagFilter{
+			Name:   "mem",
+			Filter: []string{"mem_free"},
+		}}
+	f := Filter{
+		TagPass: filters,
+	}
+
+	passes := []map[string]string{
+		{"cpu": "cpu-total"},
+		{"cpu": "cpu-0"},
+		{"cpu": "cpu-1"},
+		{"cpu": "cpu-2"},
+		{"mem": "mem_free"},
+	}
+
+	drops := []map[string]string{
+		{"cpu": "cputotal"},
+		{"cpu": "cpu0"},
+		{"cpu": "cpu1"},
+		{"cpu": "cpu2"},
+		{"mem": "mem_used"},
+	}
+
+	for _, tags := range passes {
+		if !f.ShouldTagsPass(tags) {
+			t.Errorf("Expected tags %v to pass", tags)
+		}
+	}
+
+	for _, tags := range drops {
+		if f.ShouldTagsPass(tags) {
+			t.Errorf("Expected tags %v to drop", tags)
+		}
+	}
+}
+
+func TestFilter_TagDrop(t *testing.T) {
+	filters := []TagFilter{
+		TagFilter{
+			Name:   "cpu",
+			Filter: []string{"cpu-*"},
+		},
+		TagFilter{
+			Name:   "mem",
+			Filter: []string{"mem_free"},
+		}}
+	f := Filter{
+		TagDrop: filters,
+	}
+
+	drops := []map[string]string{
+		{"cpu": "cpu-total"},
+		{"cpu": "cpu-0"},
+		{"cpu": "cpu-1"},
+		{"cpu": "cpu-2"},
+		{"mem": "mem_free"},
+	}
+
+	passes := []map[string]string{
+		{"cpu": "cputotal"},
+		{"cpu": "cpu0"},
+		{"cpu": "cpu1"},
+		{"cpu": "cpu2"},
+		{"mem": "mem_used"},
+	}
+
+	for _, tags := range passes {
+		if !f.ShouldTagsPass(tags) {
+			t.Errorf("Expected tags %v to pass", tags)
+		}
+	}
+
+	for _, tags := range drops {
+		if f.ShouldTagsPass(tags) {
+			t.Errorf("Expected tags %v to drop", tags)
+		}
+	}
+}

internal/models/running_input.go

@@ -0,0 +1,24 @@
+package models
+
+import (
+	"time"
+
+	"github.com/influxdata/telegraf/plugins/inputs"
+)
+
+type RunningInput struct {
+	Name   string
+	Input  inputs.Input
+	Config *InputConfig
+}
+
+// InputConfig containing a name, interval, and filter
+type InputConfig struct {
+	Name              string
+	NameOverride      string
+	MeasurementPrefix string
+	MeasurementSuffix string
+	Tags              map[string]string
+	Filter            Filter
+	Interval          time.Duration
+}

internal/models/running_output.go

@@ -0,0 +1,77 @@
+package models
+
+import (
+	"log"
+	"time"
+
+	"github.com/influxdata/telegraf/plugins/outputs"
+
+	"github.com/influxdata/influxdb/client/v2"
+)
+
+const DEFAULT_POINT_BUFFER_LIMIT = 10000
+
+type RunningOutput struct {
+	Name             string
+	Output           outputs.Output
+	Config           *OutputConfig
+	Quiet            bool
+	PointBufferLimit int
+
+	points           []*client.Point
+	overwriteCounter int
+}
+
+func NewRunningOutput(
+	name string,
+	output outputs.Output,
+	conf *OutputConfig,
+) *RunningOutput {
+	ro := &RunningOutput{
+		Name:             name,
+		points:           make([]*client.Point, 0),
+		Output:           output,
+		Config:           conf,
+		PointBufferLimit: DEFAULT_POINT_BUFFER_LIMIT,
+	}
+	return ro
+}
+
+func (ro *RunningOutput) AddPoint(point *client.Point) {
+	if ro.Config.Filter.IsActive {
+		if !ro.Config.Filter.ShouldPointPass(point) {
+			return
+		}
+	}
+
+	if len(ro.points) < ro.PointBufferLimit {
+		ro.points = append(ro.points, point)
+	} else {
+		if ro.overwriteCounter == len(ro.points) {
+			ro.overwriteCounter = 0
+		}
+		ro.points[ro.overwriteCounter] = point
+		ro.overwriteCounter++
+	}
+}
+
+func (ro *RunningOutput) Write() error {
+	start := time.Now()
+	err := ro.Output.Write(ro.points)
+	elapsed := time.Since(start)
+	if err == nil {
+		if !ro.Quiet {
+			log.Printf("Wrote %d metrics to output %s in %s\n",
+				len(ro.points), ro.Name, elapsed)
+		}
+		ro.points = make([]*client.Point, 0)
+		ro.overwriteCounter = 0
+	}
+	return err
+}
+
+// OutputConfig containing name and filter
+type OutputConfig struct {
+	Name   string
+	Filter Filter
+}