telegraf/plugins/parsers/ltsv/parser_test.go

package ltsv

import (
	"bytes"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

const (
	validLTSV1 = "time:2016-03-06T09:24:12Z\tstr1:value1\tint1:23\tint2:34\tfloat1:1.23\tbool1:true\tbool2:false\tignore_field1:foo\ttag1:tval1\tignore_tag1:bar\ttag2:tval2"
)

var validLTSV2 = [][]byte{
	[]byte("time:2016-03-06T09:24:12.012+09:00\tstr1:value1\tint1:23\tint2:34\tfloat1:1.23\tbool1:true\tbool2:fal"),
	[]byte("se\tignore_field1:foo\ttag1:tval1\tignore_tag1:bar\ttag2:tval2\ntime:2016-03-06T09:24:12.125+09:00\ts"),
	// NOTE: validLTSV2[2] contains an empty line, and it is safely ignored.
	[]byte("tr1:value2\ntime:2016-03-06T09:24:13.000+09:00\tstr1:value3\n\ntime:2016-03-06T09:24:15.999+09:00\tst"),
	// NOTE: validLTSV2[3] does not end with a newline, so you need to call Parse(nil) to parse the rest of data.
	[]byte("r1:value4"),
	nil,
}

var validLTSV3 = []string{
	"time:2016-03-06T09:24:12.000000000+09:00\tint1:1\ttag1:tval1",
	"time:2016-03-06T09:24:12.000000000+09:00\tint1:2\ttag1:tval1",
	"time:2016-03-06T09:24:12.000000000+09:00\tint1:3\ttag1:tval1",
	"time:2016-03-06T09:24:12.000000002+09:00\tint1:4\ttag1:tval1",
}

func TestParseLineValidLTSV(t *testing.T) {
	parser := LTSVParser{
		MetricName:                    "ltsv_test",
		TimeLabel:                     "time",
		TimeFormat:                    "2006-01-02T15:04:05Z07:00",
		StrFieldLabels:                []string{"str1"},
		IntFieldLabels:                []string{"int1", "int2"},
		FloatFieldLabels:              []string{"float1"},
		BoolFieldLabels:               []string{"bool1", "bool2", "bool3", "bool4"},
		TagLabels:                     []string{"tag1", "tag2"},
		DuplicatePointsModifierMethod: "no_op",
		DefaultTags: map[string]string{
			"log_host": "log.example.com",
		},
	}
	metric, err := parser.ParseLine(validLTSV1)
	assert.NoError(t, err)
	assert.NotNil(t, metric)
	assert.Equal(t, "ltsv_test", metric.Name())

	fields := metric.Fields()
	assert.Equal(t, map[string]interface{}{
		"str1":   "value1",
		"int1":   int64(23),
		"int2":   int64(34),
		"float1": float64(1.23),
		"bool1":  true,
		"bool2":  false,
	}, fields)
	assert.NotContains(t, fields, "ignore_field1", "ignore_tag1")

	tags := metric.Tags()
	assert.Equal(t, map[string]string{
		"log_host": "log.example.com",
		"tag1":     "tval1",
		"tag2":     "tval2",
	}, tags)
	assert.NotContains(t, tags, "ignore_field1", "ignore_tag1")
}

func TestParseValidLTSV(t *testing.T) {
	parser := LTSVParser{
		MetricName:                    "ltsv_test",
		TimeLabel:                     "time",
		TimeFormat:                    "2006-01-02T15:04:05Z07:00",
		StrFieldLabels:                []string{"str1"},
		IntFieldLabels:                []string{"int1", "int2"},
		FloatFieldLabels:              []string{"float1"},
		BoolFieldLabels:               []string{"bool1", "bool2", "bool3", "bool4"},
		TagLabels:                     []string{"tag1", "tag2"},
		DuplicatePointsModifierMethod: "no_op",
		DefaultTags: map[string]string{
			"log_host": "log.example.com",
		},
	}
	metrics, err := parser.Parse(validLTSV2[0])
	assert.NoError(t, err)
	assert.Len(t, metrics, 0)

	metrics, err = parser.Parse(validLTSV2[1])
	assert.NoError(t, err)
	assert.Len(t, metrics, 1)
	assert.Equal(t, "ltsv_test", metrics[0].Name())

	fields := metrics[0].Fields()
	assert.Equal(t, map[string]interface{}{
		"str1":   "value1",
		"int1":   int64(23),
		"int2":   int64(34),
		"float1": float64(1.23),
		"bool1":  true,
		"bool2":  false,
	}, fields)
	assert.NotContains(t, fields, "ignore_field1", "ignore_tag1")

	tags := metrics[0].Tags()
	assert.Equal(t, map[string]string{
		"log_host": "log.example.com",
		"tag1":     "tval1",
		"tag2":     "tval2",
	}, tags)
	assert.NotContains(t, tags, "ignore_field1", "ignore_tag1")

	metrics, err = parser.Parse(validLTSV2[2])
	assert.NoError(t, err)
	assert.Len(t, metrics, 2)
	assert.Equal(t, "ltsv_test", metrics[0].Name())

	assert.Equal(t, map[string]interface{}{
		"str1": "value2",
	}, metrics[0].Fields())
	assert.Equal(t, map[string]string{
		"log_host": "log.example.com",
	}, metrics[0].Tags())

	assert.Equal(t, map[string]interface{}{
		"str1": "value3",
	}, metrics[1].Fields())
	assert.Equal(t, map[string]string{
		"log_host": "log.example.com",
	}, metrics[1].Tags())

	metrics, err = parser.Parse(validLTSV2[3])
	assert.NoError(t, err)
	assert.Len(t, metrics, 0)

	metrics, err = parser.Parse(validLTSV2[4])
	assert.NoError(t, err)
	assert.Len(t, metrics, 1)
	assert.Equal(t, "ltsv_test", metrics[0].Name())

	assert.Equal(t, map[string]interface{}{
		"str1": "value4",
	}, metrics[0].Fields())
	assert.Equal(t, map[string]string{
		"log_host": "log.example.com",
	}, metrics[0].Tags())
}

func TestAlwaysAddTagDuplicatePointModifier(t *testing.T) {
	parser := LTSVParser{
		MetricName:                     "ltsv_test",
		TimeLabel:                      "time",
		TimeFormat:                     "2006-01-02T15:04:05.000000000Z07:00",
		IntFieldLabels:                 []string{"int1"},
		TagLabels:                      []string{"tag1"},
		DuplicatePointsModifierMethod:  "add_uniq_tag",
		DuplicatePointsModifierUniqTag: "uniq",
		DefaultTags: map[string]string{
			"log_host": "log.example.com",
			"uniq":     "0",
		},
	}

	metric, err := parser.ParseLine(validLTSV3[0])
	assert.NoError(t, err)
	assert.NotNil(t, metric)
	assert.Equal(t, "ltsv_test", metric.Name())
	assert.Equal(t, map[string]interface{}{
		"int1": int64(1),
	}, metric.Fields())
	assert.Equal(t, map[string]string{
		"log_host": "log.example.com",
		"tag1":     "tval1",
		"uniq":     "0",
	}, metric.Tags())
	assert.Equal(t, "2016-03-06T09:24:12.000000000+09:00", metric.Time().Format(parser.TimeFormat))

	metric, err = parser.ParseLine(validLTSV3[1])
	assert.NoError(t, err)
	assert.NotNil(t, metric)
	assert.Equal(t, "ltsv_test", metric.Name())
	assert.Equal(t, map[string]interface{}{
		"int1": int64(2),
	}, metric.Fields())
	assert.Equal(t, map[string]string{
		"log_host": "log.example.com",
		"tag1":     "tval1",
		"uniq":     "1",
	}, metric.Tags())
	assert.Equal(t, "2016-03-06T09:24:12.000000000+09:00", metric.Time().Format(parser.TimeFormat))

	metric, err = parser.ParseLine(validLTSV3[2])
	assert.NoError(t, err)
	assert.NotNil(t, metric)
	assert.Equal(t, "ltsv_test", metric.Name())
	assert.Equal(t, map[string]interface{}{
		"int1": int64(3),
	}, metric.Fields())
	assert.Equal(t, map[string]string{
		"log_host": "log.example.com",
		"tag1":     "tval1",
		"uniq":     "2",
	}, metric.Tags())
	assert.Equal(t, "2016-03-06T09:24:12.000000000+09:00", metric.Time().Format(parser.TimeFormat))

	metric, err = parser.ParseLine(validLTSV3[3])
	assert.NoError(t, err)
	assert.NotNil(t, metric)
	assert.Equal(t, "ltsv_test", metric.Name())
	assert.Equal(t, map[string]interface{}{
		"int1": int64(4),
	}, metric.Fields())
	assert.Equal(t, map[string]string{
		"log_host": "log.example.com",
		"tag1":     "tval1",
		"uniq":     "0",
	}, metric.Tags())
	assert.Equal(t, "2016-03-06T09:24:12.000000002+09:00", metric.Time().Format(parser.TimeFormat))
}

func TestAddTagDuplicatePointModifier(t *testing.T) {
	parser := LTSVParser{
		MetricName:                     "ltsv_test",
		TimeLabel:                      "time",
		TimeFormat:                     "2006-01-02T15:04:05.000000000Z07:00",
		IntFieldLabels:                 []string{"int1"},
		TagLabels:                      []string{"tag1"},
		DuplicatePointsModifierMethod:  "add_uniq_tag",
		DuplicatePointsModifierUniqTag: "uniq",
		DefaultTags: map[string]string{
			"log_host": "log.example.com",
		},
	}

	metric, err := parser.ParseLine(validLTSV3[0])
	assert.NoError(t, err)
	assert.NotNil(t, metric)
	assert.Equal(t, "ltsv_test", metric.Name())
	assert.Equal(t, map[string]interface{}{
		"int1": int64(1),
	}, metric.Fields())
	assert.Equal(t, map[string]string{
		"log_host": "log.example.com",
		"tag1":     "tval1",
	}, metric.Tags())
	assert.Equal(t, "2016-03-06T09:24:12.000000000+09:00", metric.Time().Format(parser.TimeFormat))

	metric, err = parser.ParseLine(validLTSV3[1])
	assert.NoError(t, err)
	assert.NotNil(t, metric)
	assert.Equal(t, "ltsv_test", metric.Name())
	assert.Equal(t, map[string]interface{}{
		"int1": int64(2),
	}, metric.Fields())
	assert.Equal(t, map[string]string{
		"log_host": "log.example.com",
		"tag1":     "tval1",
		"uniq":     "1",
	}, metric.Tags())
	assert.Equal(t, "2016-03-06T09:24:12.000000000+09:00", metric.Time().Format(parser.TimeFormat))

	metric, err = parser.ParseLine(validLTSV3[2])
	assert.NoError(t, err)
	assert.NotNil(t, metric)
	assert.Equal(t, "ltsv_test", metric.Name())
	assert.Equal(t, map[string]interface{}{
		"int1": int64(3),
	}, metric.Fields())
	assert.Equal(t, map[string]string{
		"log_host": "log.example.com",
		"tag1":     "tval1",
		"uniq":     "2",
	}, metric.Tags())
	assert.Equal(t, "2016-03-06T09:24:12.000000000+09:00", metric.Time().Format(parser.TimeFormat))

	metric, err = parser.ParseLine(validLTSV3[3])
	assert.NoError(t, err)
	assert.NotNil(t, metric)
	assert.Equal(t, "ltsv_test", metric.Name())
	assert.Equal(t, map[string]interface{}{
		"int1": int64(4),
	}, metric.Fields())
	assert.Equal(t, map[string]string{
		"log_host": "log.example.com",
		"tag1":     "tval1",
	}, metric.Tags())
	assert.Equal(t, "2016-03-06T09:24:12.000000002+09:00", metric.Time().Format(parser.TimeFormat))
}

func TestIncTimeDuplicatePointModifier(t *testing.T) {
	parser := LTSVParser{
		MetricName:                       "ltsv_test",
		TimeLabel:                        "time",
		TimeFormat:                       "2006-01-02T15:04:05.000000000Z07:00",
		IntFieldLabels:                   []string{"int1"},
		TagLabels:                        []string{"tag1"},
		DuplicatePointsModifierMethod:    "increment_time",
		DuplicatePointsIncrementDuration: time.Nanosecond,
		DefaultTags: map[string]string{
			"log_host": "log.example.com",
		},
	}

	metric, err := parser.ParseLine(validLTSV3[0])
	assert.NoError(t, err)
	assert.NotNil(t, metric)
	assert.Equal(t, "ltsv_test", metric.Name())
	assert.Equal(t, map[string]interface{}{
		"int1": int64(1),
	}, metric.Fields())
	assert.Equal(t, map[string]string{
		"log_host": "log.example.com",
		"tag1":     "tval1",
	}, metric.Tags())
	assert.Equal(t, "2016-03-06T09:24:12.000000000+09:00", metric.Time().Format(parser.TimeFormat))

	metric, err = parser.ParseLine(validLTSV3[1])
	assert.NoError(t, err)
	assert.NotNil(t, metric)
	assert.Equal(t, "ltsv_test", metric.Name())
	assert.Equal(t, map[string]interface{}{
		"int1": int64(2),
	}, metric.Fields())
	assert.Equal(t, map[string]string{
		"log_host": "log.example.com",
		"tag1":     "tval1",
	}, metric.Tags())
	assert.Equal(t, "2016-03-06T09:24:12.000000001+09:00", metric.Time().Format(parser.TimeFormat))

	metric, err = parser.ParseLine(validLTSV3[2])
	assert.NoError(t, err)
	assert.NotNil(t, metric)
	assert.Equal(t, "ltsv_test", metric.Name())
	assert.Equal(t, map[string]interface{}{
		"int1": int64(3),
	}, metric.Fields())
	assert.Equal(t, map[string]string{
		"log_host": "log.example.com",
		"tag1":     "tval1",
	}, metric.Tags())
	assert.Equal(t, "2016-03-06T09:24:12.000000002+09:00", metric.Time().Format(parser.TimeFormat))

	metric, err = parser.ParseLine(validLTSV3[3])
	assert.NoError(t, err)
	assert.NotNil(t, metric)
	assert.Equal(t, "ltsv_test", metric.Name())
	assert.Equal(t, map[string]interface{}{
		"int1": int64(4),
	}, metric.Fields())
	assert.Equal(t, map[string]string{
		"log_host": "log.example.com",
		"tag1":     "tval1",
	}, metric.Tags())
	assert.Equal(t, "2016-03-06T09:24:12.000000003+09:00", metric.Time().Format(parser.TimeFormat))
}

func TestNoOpDuplicatePointModifier(t *testing.T) {
	parser := LTSVParser{
		MetricName:                    "ltsv_test",
		TimeLabel:                     "time",
		TimeFormat:                    "2006-01-02T15:04:05.000000000Z07:00",
		IntFieldLabels:                []string{"int1"},
		TagLabels:                     []string{"tag1"},
		DuplicatePointsModifierMethod: "no_op",
		DefaultTags: map[string]string{
			"log_host": "log.example.com",
		},
	}

	var buf bytes.Buffer
	for _, line := range validLTSV3 {
		buf.WriteString(line)
		buf.WriteByte(byte('\n'))
	}

	metrics, err := parser.Parse(buf.Bytes())
	assert.NoError(t, err)
	// NOTE: Even though 4 metrics are created here, 3 of these will be merged on
	// a InfluxDB database.
	assert.Len(t, metrics, 4)

	assert.Equal(t, "ltsv_test", metrics[0].Name())
	assert.Equal(t, map[string]interface{}{
		"int1": int64(1),
	}, metrics[0].Fields())
	assert.Equal(t, map[string]string{
		"log_host": "log.example.com",
		"tag1":     "tval1",
	}, metrics[0].Tags())
	assert.Equal(t, "2016-03-06T09:24:12.000000000+09:00", metrics[0].Time().Format(parser.TimeFormat))

	assert.Equal(t, "ltsv_test", metrics[1].Name())
	assert.Equal(t, map[string]interface{}{
		"int1": int64(2),
	}, metrics[1].Fields())
	assert.Equal(t, map[string]string{
		"log_host": "log.example.com",
		"tag1":     "tval1",
	}, metrics[1].Tags())
	assert.Equal(t, "2016-03-06T09:24:12.000000000+09:00", metrics[1].Time().Format(parser.TimeFormat))

	assert.Equal(t, "ltsv_test", metrics[2].Name())
	assert.Equal(t, map[string]interface{}{
		"int1": int64(3),
	}, metrics[2].Fields())
	assert.Equal(t, map[string]string{
		"log_host": "log.example.com",
		"tag1":     "tval1",
	}, metrics[2].Tags())
	assert.Equal(t, "2016-03-06T09:24:12.000000000+09:00", metrics[2].Time().Format(parser.TimeFormat))

	assert.Equal(t, "ltsv_test", metrics[3].Name())
	assert.Equal(t, map[string]interface{}{
		"int1": int64(4),
	}, metrics[3].Fields())
	assert.Equal(t, map[string]string{
		"log_host": "log.example.com",
		"tag1":     "tval1",
	}, metrics[3].Tags())
	assert.Equal(t, "2016-03-06T09:24:12.000000002+09:00", metrics[3].Time().Format(parser.TimeFormat))
}

func TestInvalidDuplicatePointsModifierMethod(t *testing.T) {
	parser := LTSVParser{
		DuplicatePointsModifierMethod: "",
	}
	metric, err := parser.ParseLine("")
	assert.Error(t, err)
	assert.Nil(t, metric)
}