From a0527db0370633ce93c5e3d91006a73e4642d3c5 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Tue, 5 Mar 2019 11:07:39 -0800 Subject: [PATCH] Allow grok to produce metrics with no fields (#5533) --- plugins/parsers/grok/parser.go | 7 +-- plugins/parsers/grok/parser_test.go | 87 ++++++++++++++++++----------- plugins/parsers/registry.go | 1 - 3 files changed, 55 insertions(+), 40 deletions(-) diff --git a/plugins/parsers/grok/parser.go b/plugins/parsers/grok/parser.go index eb1d1e71c..5984e288e 100644 --- a/plugins/parsers/grok/parser.go +++ b/plugins/parsers/grok/parser.go @@ -11,10 +11,9 @@ import ( "strings" "time" - "github.com/vjeantet/grok" - "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/metric" + "github.com/vjeantet/grok" ) var timeLayouts = map[string]string{ @@ -361,10 +360,6 @@ func (p *Parser) ParseLine(line string) (telegraf.Metric, error) { } } - if len(fields) == 0 { - return nil, fmt.Errorf("grok: must have one or more fields") - } - if p.UniqueTimestamp != "auto" { return metric.New(p.Measurement, tags, fields, timestamp) } diff --git a/plugins/parsers/grok/parser_test.go b/plugins/parsers/grok/parser_test.go index 22007971b..23af0af44 100644 --- a/plugins/parsers/grok/parser_test.go +++ b/plugins/parsers/grok/parser_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -571,61 +572,81 @@ func TestCompileErrors(t *testing.T) { assert.Error(t, p.Compile()) } -func TestParseErrors(t *testing.T) { - // Parse fails because the pattern doesn't exist +func TestParseErrors_MissingPattern(t *testing.T) { p := &Parser{ - Patterns: []string{"%{TEST_LOG_B}"}, + Measurement: "grok", + Patterns: []string{"%{TEST_LOG_B}"}, CustomPatterns: ` TEST_LOG_A %{HTTPDATE:ts:ts-httpd} %{WORD:myword:int} %{} `, } - assert.Error(t, p.Compile()) + require.Error(t, p.Compile()) _, err := p.ParseLine(`[04/Jun/2016:12:41:45 +0100] notnumber 200 192.168.1.1 5.432µs 101`) - assert.Error(t, err) + require.Error(t, err) +} - // Parse fails because myword is not an int - p = &Parser{ - Patterns: []string{"%{TEST_LOG_A}"}, +func TestParseErrors_WrongIntegerType(t *testing.T) { + p := &Parser{ + Measurement: "grok", + Patterns: []string{"%{TEST_LOG_A}"}, CustomPatterns: ` - TEST_LOG_A %{HTTPDATE:ts:ts-httpd} %{WORD:myword:int} + TEST_LOG_A %{NUMBER:ts:ts-epoch} %{WORD:myword:int} `, } - assert.NoError(t, p.Compile()) - _, err = p.ParseLine(`04/Jun/2016:12:41:45 +0100 notnumber`) - assert.Error(t, err) + require.NoError(t, p.Compile()) + m, err := p.ParseLine(`0 notnumber`) + require.NoError(t, err) + testutil.RequireMetricEqual(t, + m, + testutil.MustMetric("grok", map[string]string{}, map[string]interface{}{}, time.Unix(0, 0))) +} - // Parse fails because myword is not a float - p = &Parser{ - Patterns: []string{"%{TEST_LOG_A}"}, +func TestParseErrors_WrongFloatType(t *testing.T) { + p := &Parser{ + Measurement: "grok", + Patterns: []string{"%{TEST_LOG_A}"}, CustomPatterns: ` - TEST_LOG_A %{HTTPDATE:ts:ts-httpd} %{WORD:myword:float} + TEST_LOG_A %{NUMBER:ts:ts-epoch} %{WORD:myword:float} `, } - assert.NoError(t, p.Compile()) - _, err = p.ParseLine(`04/Jun/2016:12:41:45 +0100 notnumber`) - assert.Error(t, err) + require.NoError(t, p.Compile()) + m, err := p.ParseLine(`0 notnumber`) + require.NoError(t, err) + testutil.RequireMetricEqual(t, + m, + testutil.MustMetric("grok", map[string]string{}, map[string]interface{}{}, time.Unix(0, 0))) +} - // Parse fails because myword is not a duration - p = &Parser{ - Patterns: []string{"%{TEST_LOG_A}"}, +func TestParseErrors_WrongDurationType(t *testing.T) { + p := &Parser{ + Measurement: "grok", + Patterns: []string{"%{TEST_LOG_A}"}, CustomPatterns: ` - TEST_LOG_A %{HTTPDATE:ts:ts-httpd} %{WORD:myword:duration} + TEST_LOG_A %{NUMBER:ts:ts-epoch} %{WORD:myword:duration} `, } - assert.NoError(t, p.Compile()) - _, err = p.ParseLine(`04/Jun/2016:12:41:45 +0100 notnumber`) - assert.Error(t, err) + require.NoError(t, p.Compile()) + m, err := p.ParseLine(`0 notnumber`) + require.NoError(t, err) + testutil.RequireMetricEqual(t, + m, + testutil.MustMetric("grok", map[string]string{}, map[string]interface{}{}, time.Unix(0, 0))) +} - // Parse fails because the time layout is wrong. - p = &Parser{ - Patterns: []string{"%{TEST_LOG_A}"}, +func TestParseErrors_WrongTimeLayout(t *testing.T) { + p := &Parser{ + Measurement: "grok", + Patterns: []string{"%{TEST_LOG_A}"}, CustomPatterns: ` - TEST_LOG_A %{HTTPDATE:ts:ts-unix} %{WORD:myword:duration} + TEST_LOG_A %{NUMBER:ts:ts-epoch} %{WORD:myword:duration} `, } - assert.NoError(t, p.Compile()) - _, err = p.ParseLine(`04/Jun/2016:12:41:45 +0100 notnumber`) - assert.Error(t, err) + require.NoError(t, p.Compile()) + m, err := p.ParseLine(`0 notnumber`) + require.NoError(t, err) + testutil.RequireMetricEqual(t, + m, + testutil.MustMetric("grok", map[string]string{}, map[string]interface{}{}, time.Unix(0, 0))) } func TestTsModder(t *testing.T) { diff --git a/plugins/parsers/registry.go b/plugins/parsers/registry.go index ad54e35ad..e6e15469f 100644 --- a/plugins/parsers/registry.go +++ b/plugins/parsers/registry.go @@ -298,7 +298,6 @@ func newJSONParser( return parser } -//Deprecated: Use NewParser to get a JSONParser object func newGrokParser(metricName string, patterns []string, nPatterns []string, cPatterns string, cPatternFiles []string,