Add new line protocol parser and serializer, influxdb output (#3924)

agent/accumulator_test.go

@@ -15,63 +15,36 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestAdd(t *testing.T) {
-	now := time.Now()
-	metrics := make(chan telegraf.Metric, 10)
-	defer close(metrics)
-	a := NewAccumulator(&TestMetricMaker{}, metrics)
-
-	a.AddFields("acctest",
-		map[string]interface{}{"value": float64(101)},
-		map[string]string{})
-	a.AddFields("acctest",
-		map[string]interface{}{"value": float64(101)},
-		map[string]string{"acc": "test"})
-	a.AddFields("acctest",
-		map[string]interface{}{"value": float64(101)},
-		map[string]string{"acc": "test"}, now)
-
-	testm := <-metrics
-	actual := testm.String()
-	assert.Contains(t, actual, "acctest value=101")
-
-	testm = <-metrics
-	actual = testm.String()
-	assert.Contains(t, actual, "acctest,acc=test value=101")
-
-	testm = <-metrics
-	actual = testm.String()
-	assert.Equal(t,
-		fmt.Sprintf("acctest,acc=test value=101 %d\n", now.UnixNano()),
-		actual)
-}
-
 func TestAddFields(t *testing.T) {
-	now := time.Now()
 	metrics := make(chan telegraf.Metric, 10)
 	defer close(metrics)
 	a := NewAccumulator(&TestMetricMaker{}, metrics)
 
+	tags := map[string]string{"foo": "bar"}
 	fields := map[string]interface{}{
 		"usage": float64(99),
 	}
-	a.AddFields("acctest", fields, map[string]string{})
-	a.AddGauge("acctest", fields, map[string]string{"acc": "test"})
-	a.AddCounter("acctest", fields, map[string]string{"acc": "test"}, now)
+	now := time.Now()
+	a.AddCounter("acctest", fields, tags, now)
 
 	testm := <-metrics
-	actual := testm.String()
-	assert.Contains(t, actual, "acctest usage=99")
 
-	testm = <-metrics
-	actual = testm.String()
-	assert.Contains(t, actual, "acctest,acc=test usage=99")
+	require.Equal(t, "acctest", testm.Name())
+	actual, ok := testm.GetField("usage")
 
-	testm = <-metrics
-	actual = testm.String()
-	assert.Equal(t,
-		fmt.Sprintf("acctest,acc=test usage=99 %d\n", now.UnixNano()),
-		actual)
+	require.True(t, ok)
+	require.Equal(t, float64(99), actual)
+
+	actual, ok = testm.GetTag("foo")
+	require.True(t, ok)
+	require.Equal(t, "bar", actual)
+
+	tm := testm.Time()
+	// okay if monotonic clock differs
+	require.True(t, now.Equal(tm))
+
+	tp := testm.Type()
+	require.Equal(t, telegraf.Counter, tp)
 }
 
 func TestAccAddError(t *testing.T) {
@@ -98,215 +71,61 @@ func TestAccAddError(t *testing.T) {
 	assert.Contains(t, string(errs[2]), "baz")
 }
 
-func TestAddNoIntervalWithPrecision(t *testing.T) {
-	now := time.Date(2006, time.February, 10, 12, 0, 0, 82912748, time.UTC)
-	metrics := make(chan telegraf.Metric, 10)
-	defer close(metrics)
-	a := NewAccumulator(&TestMetricMaker{}, metrics)
-	a.SetPrecision(0, time.Second)
+func TestSetPrecision(t *testing.T) {
+	tests := []struct {
+		name      string
+		unset     bool
+		precision time.Duration
+		interval  time.Duration
+		timestamp time.Time
+		expected  time.Time
+	}{
+		{
+			name:      "default precision is nanosecond",
+			unset:     true,
+			timestamp: time.Date(2006, time.February, 10, 12, 0, 0, 82912748, time.UTC),
+			expected:  time.Date(2006, time.February, 10, 12, 0, 0, 82912748, time.UTC),
+		},
+		{
+			name:      "second interval",
+			interval:  time.Second,
+			timestamp: time.Date(2006, time.February, 10, 12, 0, 0, 82912748, time.UTC),
+			expected:  time.Date(2006, time.February, 10, 12, 0, 0, 0, time.UTC),
+		},
+		{
+			name:      "microsecond interval",
+			interval:  time.Microsecond,
+			timestamp: time.Date(2006, time.February, 10, 12, 0, 0, 82912748, time.UTC),
+			expected:  time.Date(2006, time.February, 10, 12, 0, 0, 82913000, time.UTC),
+		},
+		{
+			name:      "2 second precision",
+			precision: 2 * time.Second,
+			timestamp: time.Date(2006, time.February, 10, 12, 0, 2, 4, time.UTC),
+			expected:  time.Date(2006, time.February, 10, 12, 0, 2, 0, time.UTC),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			metrics := make(chan telegraf.Metric, 10)
 
-	a.AddFields("acctest",
-		map[string]interface{}{"value": float64(101)},
-		map[string]string{})
-	a.AddFields("acctest",
-		map[string]interface{}{"value": float64(101)},
-		map[string]string{"acc": "test"})
-	a.AddFields("acctest",
-		map[string]interface{}{"value": float64(101)},
-		map[string]string{"acc": "test"}, now)
+			a := NewAccumulator(&TestMetricMaker{}, metrics)
+			if !tt.unset {
+				a.SetPrecision(tt.precision, tt.interval)
+			}
 
-	testm := <-metrics
-	actual := testm.String()
-	assert.Contains(t, actual, "acctest value=101")
+			a.AddFields("acctest",
+				map[string]interface{}{"value": float64(101)},
+				map[string]string{},
+				tt.timestamp,
+			)
 
-	testm = <-metrics
-	actual = testm.String()
-	assert.Contains(t, actual, "acctest,acc=test value=101")
+			testm := <-metrics
+			require.Equal(t, tt.expected, testm.Time())
 
-	testm = <-metrics
-	actual = testm.String()
-	assert.Equal(t,
-		fmt.Sprintf("acctest,acc=test value=101 %d\n", int64(1139572800000000000)),
-		actual)
-}
-
-func TestAddDisablePrecision(t *testing.T) {
-	now := time.Date(2006, time.February, 10, 12, 0, 0, 82912748, time.UTC)
-	metrics := make(chan telegraf.Metric, 10)
-	defer close(metrics)
-	a := NewAccumulator(&TestMetricMaker{}, metrics)
-
-	a.SetPrecision(time.Nanosecond, 0)
-	a.AddFields("acctest",
-		map[string]interface{}{"value": float64(101)},
-		map[string]string{})
-	a.AddFields("acctest",
-		map[string]interface{}{"value": float64(101)},
-		map[string]string{"acc": "test"})
-	a.AddFields("acctest",
-		map[string]interface{}{"value": float64(101)},
-		map[string]string{"acc": "test"}, now)
-
-	testm := <-metrics
-	actual := testm.String()
-	assert.Contains(t, actual, "acctest value=101")
-
-	testm = <-metrics
-	actual = testm.String()
-	assert.Contains(t, actual, "acctest,acc=test value=101")
-
-	testm = <-metrics
-	actual = testm.String()
-	assert.Equal(t,
-		fmt.Sprintf("acctest,acc=test value=101 %d\n", int64(1139572800082912748)),
-		actual)
-}
-
-func TestAddNoPrecisionWithInterval(t *testing.T) {
-	now := time.Date(2006, time.February, 10, 12, 0, 0, 82912748, time.UTC)
-	metrics := make(chan telegraf.Metric, 10)
-	defer close(metrics)
-	a := NewAccumulator(&TestMetricMaker{}, metrics)
-
-	a.SetPrecision(0, time.Second)
-	a.AddFields("acctest",
-		map[string]interface{}{"value": float64(101)},
-		map[string]string{})
-	a.AddFields("acctest",
-		map[string]interface{}{"value": float64(101)},
-		map[string]string{"acc": "test"})
-	a.AddFields("acctest",
-		map[string]interface{}{"value": float64(101)},
-		map[string]string{"acc": "test"}, now)
-
-	testm := <-metrics
-	actual := testm.String()
-	assert.Contains(t, actual, "acctest value=101")
-
-	testm = <-metrics
-	actual = testm.String()
-	assert.Contains(t, actual, "acctest,acc=test value=101")
-
-	testm = <-metrics
-	actual = testm.String()
-	assert.Equal(t,
-		fmt.Sprintf("acctest,acc=test value=101 %d\n", int64(1139572800000000000)),
-		actual)
-}
-
-func TestDifferentPrecisions(t *testing.T) {
-	now := time.Date(2006, time.February, 10, 12, 0, 0, 82912748, time.UTC)
-	metrics := make(chan telegraf.Metric, 10)
-	defer close(metrics)
-	a := NewAccumulator(&TestMetricMaker{}, metrics)
-
-	a.SetPrecision(0, time.Second)
-	a.AddFields("acctest",
-		map[string]interface{}{"value": float64(101)},
-		map[string]string{"acc": "test"}, now)
-	testm := <-metrics
-	actual := testm.String()
-	assert.Equal(t,
-		fmt.Sprintf("acctest,acc=test value=101 %d\n", int64(1139572800000000000)),
-		actual)
-
-	a.SetPrecision(0, time.Millisecond)
-	a.AddFields("acctest",
-		map[string]interface{}{"value": float64(101)},
-		map[string]string{"acc": "test"}, now)
-	testm = <-metrics
-	actual = testm.String()
-	assert.Equal(t,
-		fmt.Sprintf("acctest,acc=test value=101 %d\n", int64(1139572800083000000)),
-		actual)
-
-	a.SetPrecision(0, time.Microsecond)
-	a.AddFields("acctest",
-		map[string]interface{}{"value": float64(101)},
-		map[string]string{"acc": "test"}, now)
-	testm = <-metrics
-	actual = testm.String()
-	assert.Equal(t,
-		fmt.Sprintf("acctest,acc=test value=101 %d\n", int64(1139572800082913000)),
-		actual)
-
-	a.SetPrecision(0, time.Nanosecond)
-	a.AddFields("acctest",
-		map[string]interface{}{"value": float64(101)},
-		map[string]string{"acc": "test"}, now)
-	testm = <-metrics
-	actual = testm.String()
-	assert.Equal(t,
-		fmt.Sprintf("acctest,acc=test value=101 %d\n", int64(1139572800082912748)),
-		actual)
-}
-
-func TestAddGauge(t *testing.T) {
-	now := time.Now()
-	metrics := make(chan telegraf.Metric, 10)
-	defer close(metrics)
-	a := NewAccumulator(&TestMetricMaker{}, metrics)
-
-	a.AddGauge("acctest",
-		map[string]interface{}{"value": float64(101)},
-		map[string]string{})
-	a.AddGauge("acctest",
-		map[string]interface{}{"value": float64(101)},
-		map[string]string{"acc": "test"})
-	a.AddGauge("acctest",
-		map[string]interface{}{"value": float64(101)},
-		map[string]string{"acc": "test"}, now)
-
-	testm := <-metrics
-	actual := testm.String()
-	assert.Contains(t, actual, "acctest value=101")
-	assert.Equal(t, testm.Type(), telegraf.Gauge)
-
-	testm = <-metrics
-	actual = testm.String()
-	assert.Contains(t, actual, "acctest,acc=test value=101")
-	assert.Equal(t, testm.Type(), telegraf.Gauge)
-
-	testm = <-metrics
-	actual = testm.String()
-	assert.Equal(t,
-		fmt.Sprintf("acctest,acc=test value=101 %d\n", now.UnixNano()),
-		actual)
-	assert.Equal(t, testm.Type(), telegraf.Gauge)
-}
-
-func TestAddCounter(t *testing.T) {
-	now := time.Now()
-	metrics := make(chan telegraf.Metric, 10)
-	defer close(metrics)
-	a := NewAccumulator(&TestMetricMaker{}, metrics)
-
-	a.AddCounter("acctest",
-		map[string]interface{}{"value": float64(101)},
-		map[string]string{})
-	a.AddCounter("acctest",
-		map[string]interface{}{"value": float64(101)},
-		map[string]string{"acc": "test"})
-	a.AddCounter("acctest",
-		map[string]interface{}{"value": float64(101)},
-		map[string]string{"acc": "test"}, now)
-
-	testm := <-metrics
-	actual := testm.String()
-	assert.Contains(t, actual, "acctest value=101")
-	assert.Equal(t, testm.Type(), telegraf.Counter)
-
-	testm = <-metrics
-	actual = testm.String()
-	assert.Contains(t, actual, "acctest,acc=test value=101")
-	assert.Equal(t, testm.Type(), telegraf.Counter)
-
-	testm = <-metrics
-	actual = testm.String()
-	assert.Equal(t,
-		fmt.Sprintf("acctest,acc=test value=101 %d\n", now.UnixNano()),
-		actual)
-	assert.Equal(t, testm.Type(), telegraf.Counter)
+			close(metrics)
+		})
+	}
 }
 
 type TestMetricMaker struct {

agent/agent.go

@@ -271,11 +271,9 @@ func (a *Agent) flusher(shutdown chan struct{}, metricC chan telegraf.Metric, ag
 				// if dropOriginal is set to true, then we will only send this
 				// metric to the aggregators, not the outputs.
 				var dropOriginal bool
-				if !m.IsAggregate() {
-					for _, agg := range a.Config.Aggregators {
-						if ok := agg.Add(m.Copy()); ok {
-							dropOriginal = true
-						}
+				for _, agg := range a.Config.Aggregators {
+					if ok := agg.Add(m.Copy()); ok {
+						dropOriginal = true
 					}
 				}
 				if !dropOriginal {