Flush based on buffer size rather than time

this includes:
- Add Accumulator to the Start() function of service inputs
- For message consumer plugins, use the Accumulator to constantly add
  metrics and make Gather a dummy function
- rework unit tests to match this new behavior.
- make "flush_buffer_when_full" a config option that defaults to true

closes #666

plugins/inputs/mqtt_consumer/mqtt_consumer.go

@@ -15,15 +15,17 @@ import (
 )
 
 type MQTTConsumer struct {
-	Servers      []string
-	Topics       []string
-	Username     string
-	Password     string
-	MetricBuffer int
-	QoS          int `toml:"qos"`
+	Servers  []string
+	Topics   []string
+	Username string
+	Password string
+	QoS      int `toml:"qos"`
 
 	parser parsers.Parser
 
+	// Legacy metric buffer support
+	MetricBuffer int
+
 	// Path to CA file
 	SSLCA string `toml:"ssl_ca"`
 	// Path to host cert file
@@ -35,13 +37,12 @@ type MQTTConsumer struct {
 
 	sync.Mutex
 	client *mqtt.Client
-	// channel for all incoming parsed mqtt metrics
-	metricC chan telegraf.Metric
-	// channel for the topics of all incoming metrics (for tagging metrics)
-	topicC chan string
 	// channel of all incoming raw mqtt messages
 	in   chan mqtt.Message
 	done chan struct{}
+
+	// keep the accumulator internally:
+	acc telegraf.Accumulator
 }
 
 var sampleConfig = `
@@ -56,9 +57,6 @@ var sampleConfig = `
     "sensors/#",
   ]
 
-  ### Maximum number of metrics to buffer between collection intervals
-  metric_buffer = 100000
-
   ### username and password to connect MQTT server.
   # username = "telegraf"
   # password = "metricsmetricsmetricsmetrics"
@@ -89,9 +87,11 @@ func (m *MQTTConsumer) SetParser(parser parsers.Parser) {
 	m.parser = parser
 }
 
-func (m *MQTTConsumer) Start() error {
+func (m *MQTTConsumer) Start(acc telegraf.Accumulator) error {
 	m.Lock()
 	defer m.Unlock()
+
+	m.acc = acc
 	if m.QoS > 2 || m.QoS < 0 {
 		return fmt.Errorf("MQTT Consumer, invalid QoS value: %d", m.QoS)
 	}
@@ -106,13 +106,8 @@ func (m *MQTTConsumer) Start() error {
 		return token.Error()
 	}
 
-	m.in = make(chan mqtt.Message, m.MetricBuffer)
+	m.in = make(chan mqtt.Message, 1000)
 	m.done = make(chan struct{})
-	if m.MetricBuffer == 0 {
-		m.MetricBuffer = 100000
-	}
-	m.metricC = make(chan telegraf.Metric, m.MetricBuffer)
-	m.topicC = make(chan string, m.MetricBuffer)
 
 	topics := make(map[string]byte)
 	for _, topic := range m.Topics {
@@ -145,13 +140,9 @@ func (m *MQTTConsumer) receiver() {
 			}
 
 			for _, metric := range metrics {
-				select {
-				case m.metricC <- metric:
-					m.topicC <- topic
-				default:
-					log.Printf("MQTT Consumer buffer is full, dropping a metric." +
-						" You may want to increase the metric_buffer setting")
-				}
+				tags := metric.Tags()
+				tags["topic"] = topic
+				m.acc.AddFields(metric.Name(), metric.Fields(), tags, metric.Time())
 			}
 		}
 	}
@@ -169,16 +160,6 @@ func (m *MQTTConsumer) Stop() {
 }
 
 func (m *MQTTConsumer) Gather(acc telegraf.Accumulator) error {
-	m.Lock()
-	defer m.Unlock()
-	nmetrics := len(m.metricC)
-	for i := 0; i < nmetrics; i++ {
-		metric := <-m.metricC
-		topic := <-m.topicC
-		tags := metric.Tags()
-		tags["topic"] = topic
-		acc.AddFields(metric.Name(), metric.Fields(), tags, metric.Time())
-	}
 	return nil
 }