Flush based on buffer size rather than time

this includes:
- Add Accumulator to the Start() function of service inputs
- For message consumer plugins, use the Accumulator to constantly add
  metrics and make Gather a dummy function
- rework unit tests to match this new behavior.
- make "flush_buffer_when_full" a config option that defaults to true

closes #666

plugins/inputs/nats_consumer/nats_consumer_test.go

@@ -4,7 +4,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/plugins/parsers"
 	"github.com/influxdata/telegraf/testutil"
 	"github.com/nats-io/nats"
@@ -21,15 +20,13 @@ const (
 func newTestNatsConsumer() (*natsConsumer, chan *nats.Msg) {
 	in := make(chan *nats.Msg, metricBuffer)
 	n := &natsConsumer{
-		QueueGroup:   "test",
-		Subjects:     []string{"telegraf"},
-		Servers:      []string{"nats://localhost:4222"},
-		Secure:       false,
-		MetricBuffer: metricBuffer,
-		in:           in,
-		errs:         make(chan error, metricBuffer),
-		done:         make(chan struct{}),
-		metricC:      make(chan telegraf.Metric, metricBuffer),
+		QueueGroup: "test",
+		Subjects:   []string{"telegraf"},
+		Servers:    []string{"nats://localhost:4222"},
+		Secure:     false,
+		in:         in,
+		errs:       make(chan error, metricBuffer),
+		done:       make(chan struct{}),
 	}
 	return n, in
 }
@@ -37,66 +34,51 @@ func newTestNatsConsumer() (*natsConsumer, chan *nats.Msg) {
 // Test that the parser parses NATS messages into metrics
 func TestRunParser(t *testing.T) {
 	n, in := newTestNatsConsumer()
+	acc := testutil.Accumulator{}
+	n.acc = &acc
 	defer close(n.done)
 
 	n.parser, _ = parsers.NewInfluxParser()
 	go n.receiver()
 	in <- natsMsg(testMsg)
-	time.Sleep(time.Millisecond)
+	time.Sleep(time.Millisecond * 25)
 
-	if a := len(n.metricC); a != 1 {
-		t.Errorf("got %v, expected %v", a, 1)
+	if acc.NFields() != 1 {
+		t.Errorf("got %v, expected %v", acc.NFields(), 1)
 	}
 }
 
 // Test that the parser ignores invalid messages
 func TestRunParserInvalidMsg(t *testing.T) {
 	n, in := newTestNatsConsumer()
+	acc := testutil.Accumulator{}
+	n.acc = &acc
 	defer close(n.done)
 
 	n.parser, _ = parsers.NewInfluxParser()
 	go n.receiver()
 	in <- natsMsg(invalidMsg)
-	time.Sleep(time.Millisecond)
+	time.Sleep(time.Millisecond * 25)
 
-	if a := len(n.metricC); a != 0 {
-		t.Errorf("got %v, expected %v", a, 0)
-	}
-}
-
-// Test that metrics are dropped when we hit the buffer limit
-func TestRunParserRespectsBuffer(t *testing.T) {
-	n, in := newTestNatsConsumer()
-	defer close(n.done)
-
-	n.parser, _ = parsers.NewInfluxParser()
-	go n.receiver()
-	for i := 0; i < metricBuffer+1; i++ {
-		in <- natsMsg(testMsg)
-	}
-	time.Sleep(time.Millisecond)
-
-	if a := len(n.metricC); a != metricBuffer {
-		t.Errorf("got %v, expected %v", a, metricBuffer)
+	if acc.NFields() != 0 {
+		t.Errorf("got %v, expected %v", acc.NFields(), 0)
 	}
 }
 
 // Test that the parser parses line format messages into metrics
 func TestRunParserAndGather(t *testing.T) {
 	n, in := newTestNatsConsumer()
+	acc := testutil.Accumulator{}
+	n.acc = &acc
 	defer close(n.done)
 
 	n.parser, _ = parsers.NewInfluxParser()
 	go n.receiver()
 	in <- natsMsg(testMsg)
-	time.Sleep(time.Millisecond)
+	time.Sleep(time.Millisecond * 25)
 
-	acc := testutil.Accumulator{}
 	n.Gather(&acc)
 
-	if a := len(acc.Metrics); a != 1 {
-		t.Errorf("got %v, expected %v", a, 1)
-	}
 	acc.AssertContainsFields(t, "cpu_load_short",
 		map[string]interface{}{"value": float64(23422)})
 }
@@ -104,19 +86,17 @@ func TestRunParserAndGather(t *testing.T) {
 // Test that the parser parses graphite format messages into metrics
 func TestRunParserAndGatherGraphite(t *testing.T) {
 	n, in := newTestNatsConsumer()
+	acc := testutil.Accumulator{}
+	n.acc = &acc
 	defer close(n.done)
 
 	n.parser, _ = parsers.NewGraphiteParser("_", []string{}, nil)
 	go n.receiver()
 	in <- natsMsg(testMsgGraphite)
-	time.Sleep(time.Millisecond)
+	time.Sleep(time.Millisecond * 25)
 
-	acc := testutil.Accumulator{}
 	n.Gather(&acc)
 
-	if a := len(acc.Metrics); a != 1 {
-		t.Errorf("got %v, expected %v", a, 1)
-	}
 	acc.AssertContainsFields(t, "cpu_load_short_graphite",
 		map[string]interface{}{"value": float64(23422)})
 }
@@ -124,19 +104,17 @@ func TestRunParserAndGatherGraphite(t *testing.T) {
 // Test that the parser parses json format messages into metrics
 func TestRunParserAndGatherJSON(t *testing.T) {
 	n, in := newTestNatsConsumer()
+	acc := testutil.Accumulator{}
+	n.acc = &acc
 	defer close(n.done)
 
 	n.parser, _ = parsers.NewJSONParser("nats_json_test", []string{}, nil)
 	go n.receiver()
 	in <- natsMsg(testMsgJSON)
-	time.Sleep(time.Millisecond)
+	time.Sleep(time.Millisecond * 25)
 
-	acc := testutil.Accumulator{}
 	n.Gather(&acc)
 
-	if a := len(acc.Metrics); a != 1 {
-		t.Errorf("got %v, expected %v", a, 1)
-	}
 	acc.AssertContainsFields(t, "nats_json_test",
 		map[string]interface{}{
 			"a":   float64(5),