dom/telegraf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Support string field glob matching in json parser (#6102)

Browse Source
This commit is contained in:
Daniel Nelson
2019-07-15 16:48:19 -07:00
committed by GitHub
parent 96530c220f
commit 41176dd1f1
4 changed files with 245 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files

244
plugins/parsers/json/parser_test.go
View File

@@ -53,9 +53,10 @@ const validJSONArrayTags = `
`
func TestParseValidJSON(t *testing.T) {
parser := JSONParser{
parser, err := New(&Config{
MetricName: "json_test",
}
})
require.NoError(t, err)
// Most basic vanilla test
metrics, err := parser.Parse([]byte(validJSON))
@@ -102,9 +103,10 @@ func TestParseValidJSON(t *testing.T) {
}
func TestParseLineValidJSON(t *testing.T) {
parser := JSONParser{
parser, err := New(&Config{
MetricName: "json_test",
}
})
require.NoError(t, err)
// Most basic vanilla test
metric, err := parser.ParseLine(validJSON)
@@ -138,11 +140,12 @@ func TestParseLineValidJSON(t *testing.T) {
}
func TestParseInvalidJSON(t *testing.T) {
parser := JSONParser{
parser, err := New(&Config{
MetricName: "json_test",
}
})
require.NoError(t, err)
_, err := parser.Parse([]byte(invalidJSON))
_, err = parser.Parse([]byte(invalidJSON))
require.Error(t, err)
_, err = parser.Parse([]byte(invalidJSON2))
require.Error(t, err)
@@ -152,10 +155,12 @@ func TestParseInvalidJSON(t *testing.T) {
func TestParseWithTagKeys(t *testing.T) {
// Test that strings not matching tag keys are ignored
parser := JSONParser{
parser, err := New(&Config{
MetricName: "json_test",
TagKeys: []string{"wrongtagkey"},
}
})
require.NoError(t, err)
metrics, err := parser.Parse([]byte(validJSONTags))
require.NoError(t, err)
require.Len(t, metrics, 1)
@@ -167,10 +172,12 @@ func TestParseWithTagKeys(t *testing.T) {
require.Equal(t, map[string]string{}, metrics[0].Tags())
// Test that single tag key is found and applied
parser = JSONParser{
parser, err = New(&Config{
MetricName: "json_test",
TagKeys: []string{"mytag"},
}
})
require.NoError(t, err)
metrics, err = parser.Parse([]byte(validJSONTags))
require.NoError(t, err)
require.Len(t, metrics, 1)
@@ -184,10 +191,11 @@ func TestParseWithTagKeys(t *testing.T) {
}, metrics[0].Tags())
// Test that both tag keys are found and applied
parser = JSONParser{
parser, err = New(&Config{
MetricName: "json_test",
TagKeys: []string{"mytag", "othertag"},
}
})
require.NoError(t, err)
metrics, err = parser.Parse([]byte(validJSONTags))
require.NoError(t, err)
require.Len(t, metrics, 1)
@@ -204,10 +212,11 @@ func TestParseWithTagKeys(t *testing.T) {
func TestParseLineWithTagKeys(t *testing.T) {
// Test that strings not matching tag keys are ignored
parser := JSONParser{
parser, err := New(&Config{
MetricName: "json_test",
TagKeys: []string{"wrongtagkey"},
}
})
require.NoError(t, err)
metric, err := parser.ParseLine(validJSONTags)
require.NoError(t, err)
require.Equal(t, "json_test", metric.Name())
@@ -218,10 +227,12 @@ func TestParseLineWithTagKeys(t *testing.T) {
require.Equal(t, map[string]string{}, metric.Tags())
// Test that single tag key is found and applied
parser = JSONParser{
parser, err = New(&Config{
MetricName: "json_test",
TagKeys: []string{"mytag"},
}
})
require.NoError(t, err)
metric, err = parser.ParseLine(validJSONTags)
require.NoError(t, err)
require.Equal(t, "json_test", metric.Name())
@@ -234,10 +245,12 @@ func TestParseLineWithTagKeys(t *testing.T) {
}, metric.Tags())
// Test that both tag keys are found and applied
parser = JSONParser{
parser, err = New(&Config{
MetricName: "json_test",
TagKeys: []string{"mytag", "othertag"},
}
})
require.NoError(t, err)
metric, err = parser.ParseLine(validJSONTags)
require.NoError(t, err)
require.Equal(t, "json_test", metric.Name())
@@ -252,13 +265,14 @@ func TestParseLineWithTagKeys(t *testing.T) {
}
func TestParseValidJSONDefaultTags(t *testing.T) {
parser := JSONParser{
parser, err := New(&Config{
MetricName: "json_test",
TagKeys: []string{"mytag"},
DefaultTags: map[string]string{
"t4g": "default",
},
}
})
require.NoError(t, err)
// Most basic vanilla test
metrics, err := parser.Parse([]byte(validJSON))
@@ -288,13 +302,14 @@ func TestParseValidJSONDefaultTags(t *testing.T) {
// Test that default tags are overridden by tag keys
func TestParseValidJSONDefaultTagsOverride(t *testing.T) {
parser := JSONParser{
parser, err := New(&Config{
MetricName: "json_test",
TagKeys: []string{"mytag"},
DefaultTags: map[string]string{
"mytag": "default",
},
}
})
require.NoError(t, err)
// Most basic vanilla test
metrics, err := parser.Parse([]byte(validJSON))
@@ -323,9 +338,10 @@ func TestParseValidJSONDefaultTagsOverride(t *testing.T) {
// Test that json arrays can be parsed
func TestParseValidJSONArray(t *testing.T) {
parser := JSONParser{
parser, err := New(&Config{
MetricName: "json_array_test",
}
})
require.NoError(t, err)
// Most basic vanilla test
metrics, err := parser.Parse([]byte(validJSONArray))
@@ -358,10 +374,12 @@ func TestParseValidJSONArray(t *testing.T) {
func TestParseArrayWithTagKeys(t *testing.T) {
// Test that strings not matching tag keys are ignored
parser := JSONParser{
parser, err := New(&Config{
MetricName: "json_array_test",
TagKeys: []string{"wrongtagkey"},
}
})
require.NoError(t, err)
metrics, err := parser.Parse([]byte(validJSONArrayTags))
require.NoError(t, err)
require.Len(t, metrics, 2)
@@ -380,10 +398,12 @@ func TestParseArrayWithTagKeys(t *testing.T) {
require.Equal(t, map[string]string{}, metrics[1].Tags())
// Test that single tag key is found and applied
parser = JSONParser{
parser, err = New(&Config{
MetricName: "json_array_test",
TagKeys: []string{"mytag"},
}
})
require.NoError(t, err)
metrics, err = parser.Parse([]byte(validJSONArrayTags))
require.NoError(t, err)
require.Len(t, metrics, 2)
@@ -406,10 +426,12 @@ func TestParseArrayWithTagKeys(t *testing.T) {
}, metrics[1].Tags())
// Test that both tag keys are found and applied
parser = JSONParser{
parser, err = New(&Config{
MetricName: "json_array_test",
TagKeys: []string{"mytag", "othertag"},
}
})
require.NoError(t, err)
metrics, err = parser.Parse([]byte(validJSONArrayTags))
require.NoError(t, err)
require.Len(t, metrics, 2)
@@ -437,12 +459,13 @@ func TestParseArrayWithTagKeys(t *testing.T) {
var jsonBOM = []byte("\xef\xbb\xbf[{\"value\":17}]")
func TestHttpJsonBOM(t *testing.T) {
parser := JSONParser{
parser, err := New(&Config{
MetricName: "json_test",
}
})
require.NoError(t, err)
// Most basic vanilla test
_, err := parser.Parse(jsonBOM)
_, err = parser.Parse(jsonBOM)
require.NoError(t, err)
}
@@ -466,15 +489,16 @@ func TestJSONParseNestedArray(t *testing.T) {
}
}`
parser := JSONParser{
parser, err := New(&Config{
MetricName: "json_test",
TagKeys: []string{"total_devices", "total_threads", "shares_tester3_fun"},
}
})
require.NoError(t, err)
metrics, err := parser.Parse([]byte(testString))
log.Printf("m[0] name: %v, tags: %v, fields: %v", metrics[0].Name(), metrics[0].Tags(), metrics[0].Fields())
require.NoError(t, err)
require.Equal(t, len(parser.TagKeys), len(metrics[0].Tags()))
require.Equal(t, 3, len(metrics[0].Tags()))
}
func TestJSONQueryErrorOnArray(t *testing.T) {
@@ -494,13 +518,14 @@ func TestJSONQueryErrorOnArray(t *testing.T) {
}
}`
parser := JSONParser{
parser, err := New(&Config{
MetricName: "json_test",
TagKeys: []string{},
JSONQuery: "shares.myArr",
}
Query: "shares.myArr",
})
require.NoError(t, err)
_, err := parser.Parse([]byte(testString))
_, err = parser.Parse([]byte(testString))
require.Error(t, err)
}
@@ -527,11 +552,12 @@ func TestArrayOfObjects(t *testing.T) {
"more_stuff":"junk"
}`
parser := JSONParser{
parser, err := New(&Config{
MetricName: "json_test",
TagKeys: []string{"ice"},
JSONQuery: "meta.shares",
}
Query: "meta.shares",
})
require.NoError(t, err)
metrics, err := parser.Parse([]byte(testString))
require.NoError(t, err)
@@ -553,12 +579,13 @@ func TestUseCaseJSONQuery(t *testing.T) {
}
}`
parser := JSONParser{
parser, err := New(&Config{
MetricName: "json_test",
StringFields: []string{"last"},
TagKeys: []string{"first"},
JSONQuery: "obj.friends",
}
Query: "obj.friends",
})
require.NoError(t, err)
metrics, err := parser.Parse([]byte(testString))
require.NoError(t, err)
@@ -588,11 +615,12 @@ func TestTimeParser(t *testing.T) {
}
]`
parser := JSONParser{
MetricName: "json_test",
JSONTimeKey: "b_time",
JSONTimeFormat: "02 Jan 06 15:04 MST",
}
parser, err := New(&Config{
MetricName: "json_test",
TimeKey: "b_time",
TimeFormat: "02 Jan 06 15:04 MST",
})
require.NoError(t, err)
metrics, err := parser.Parse([]byte(testString))
require.NoError(t, err)
require.Equal(t, 2, len(metrics))
@@ -604,12 +632,13 @@ func TestTimeParserWithTimezone(t *testing.T) {
"time": "04 Jan 06 15:04"
}`
parser := JSONParser{
MetricName: "json_test",
JSONTimeKey: "time",
JSONTimeFormat: "02 Jan 06 15:04",
JSONTimezone: "America/New_York",
}
parser, err := New(&Config{
MetricName: "json_test",
TimeKey: "time",
TimeFormat: "02 Jan 06 15:04",
Timezone: "America/New_York",
})
require.NoError(t, err)
metrics, err := parser.Parse([]byte(testString))
require.NoError(t, err)
require.Equal(t, 1, len(metrics))
@@ -638,11 +667,13 @@ func TestUnixTimeParser(t *testing.T) {
}
]`
parser := JSONParser{
MetricName: "json_test",
JSONTimeKey: "b_time",
JSONTimeFormat: "unix",
}
parser, err := New(&Config{
MetricName: "json_test",
TimeKey: "b_time",
TimeFormat: "unix",
})
require.NoError(t, err)
metrics, err := parser.Parse([]byte(testString))
require.NoError(t, err)
require.Equal(t, 2, len(metrics))
@@ -671,11 +702,13 @@ func TestUnixMsTimeParser(t *testing.T) {
}
]`
parser := JSONParser{
MetricName: "json_test",
JSONTimeKey: "b_time",
JSONTimeFormat: "unix_ms",
}
parser, err := New(&Config{
MetricName: "json_test",
TimeKey: "b_time",
TimeFormat: "unix_ms",
})
require.NoError(t, err)
metrics, err := parser.Parse([]byte(testString))
require.NoError(t, err)
require.Equal(t, 2, len(metrics))
@@ -693,11 +726,12 @@ func TestTimeErrors(t *testing.T) {
"my_tag_2": "baz"
}`
parser := JSONParser{
MetricName: "json_test",
JSONTimeKey: "b_time",
JSONTimeFormat: "02 January 06 15:04 MST",
}
parser, err := New(&Config{
MetricName: "json_test",
TimeKey: "b_time",
TimeFormat: "02 January 06 15:04 MST",
})
require.NoError(t, err)
metrics, err := parser.Parse([]byte(testString))
require.Error(t, err)
@@ -712,11 +746,12 @@ func TestTimeErrors(t *testing.T) {
"my_tag_2": "baz"
}`
parser = JSONParser{
MetricName: "json_test",
JSONTimeKey: "b_time",
JSONTimeFormat: "02 January 06 15:04 MST",
}
parser, err = New(&Config{
MetricName: "json_test",
TimeKey: "b_time",
TimeFormat: "02 January 06 15:04 MST",
})
require.NoError(t, err)
metrics, err = parser.Parse([]byte(testString2))
log.Printf("err: %v", err)
@@ -736,9 +771,10 @@ func TestNameKey(t *testing.T) {
"my_tag_2": "baz"
}`
parser := JSONParser{
JSONNameKey: "b_c",
}
parser, err := New(&Config{
NameKey: "b_c",
})
require.NoError(t, err)
metrics, err := parser.Parse([]byte(testString))
require.NoError(t, err)
@@ -751,11 +787,12 @@ func TestTimeKeyDelete(t *testing.T) {
"value": 42
}`
parser := JSONParser{
MetricName: "json",
JSONTimeKey: "timestamp",
JSONTimeFormat: "unix",
}
parser, err := New(&Config{
MetricName: "json",
TimeKey: "timestamp",
TimeFormat: "unix",
})
require.NoError(t, err)
metrics, err := parser.Parse([]byte(data))
require.NoError(t, err)
@@ -768,3 +805,38 @@ func TestTimeKeyDelete(t *testing.T) {
testutil.RequireMetricsEqual(t, expected, metrics)
}
func TestStringFieldGlob(t *testing.T) {
data := `
{
"color": "red",
"status": "error",
"time": "1541183052"
}
`
parser, err := New(&Config{
MetricName: "json",
StringFields: []string{"*"},
TimeKey: "time",
TimeFormat: "unix",
})
require.NoError(t, err)
actual, err := parser.Parse([]byte(data))
require.NoError(t, err)
expected := []telegraf.Metric{
testutil.MustMetric(
"json",
map[string]string{},
map[string]interface{}{
"color": "red",
"status": "error",
},
time.Unix(1541183052, 0),
),
}
testutil.RequireMetricsEqual(t, expected, actual)
}