dom
/
telegraf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Remove trailing underscore trimming from json flattener (#6744)

This commit is contained in:
Daniel Nelson 2019-12-03 11:46:29 -08:00 committed by GitHub
parent 4feef67c21
commit 03de92b962
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 124 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
plugins/parsers/json/parser.go
View File

@ -7,7 +7,6 @@ import (
"fmt"
"log"
"strconv"
"strings"
"time"
"github.com/influxdata/telegraf"
@ -260,19 +259,27 @@ func (f *JSONFlattener) FullFlattenJSON(
if f.Fields == nil {
f.Fields = make(map[string]interface{})
}
fieldname = strings.Trim(fieldname, "_")
switch t := v.(type) {
case map[string]interface{}:
for k, v := range t {
err := f.FullFlattenJSON(fieldname+"_"+k+"_", v, convertString, convertBool)
fieldkey := k
if fieldname != "" {
fieldkey = fieldname + "_" + fieldkey
}
err := f.FullFlattenJSON(fieldkey, v, convertString, convertBool)
if err != nil {
return err
}
}
case []interface{}:
for i, v := range t {
k := strconv.Itoa(i)
err := f.FullFlattenJSON(fieldname+"_"+k+"_", v, convertString, convertBool)
fieldkey := strconv.Itoa(i)
if fieldname != "" {
fieldkey = fieldname + "_" + fieldkey
}
err := f.FullFlattenJSON(fieldkey, v, convertString, convertBool)
if err != nil {
return nil
}

210
plugins/parsers/json/parser_test.go
View File

@ -815,104 +815,6 @@ func TestNameKey(t *testing.T) {
require.Equal(t, "this is my name", metrics[0].Name())
}
func TestTimeKeyDelete(t *testing.T) {
data := `{
"timestamp": 1541183052,
"value": 42
}`
parser, err := New(&Config{
MetricName: "json",
TimeKey: "timestamp",
TimeFormat: "unix",
})
require.NoError(t, err)
metrics, err := parser.Parse([]byte(data))
require.NoError(t, err)
expected := []telegraf.Metric{
testutil.MustMetric("json",
map[string]string{},
map[string]interface{}{"value": 42.0},
time.Unix(1541183052, 0)),
}
testutil.RequireMetricsEqual(t, expected, metrics)
}
func TestStringFieldGlob(t *testing.T) {
data := `
{
"color": "red",
"status": "error",
"time": "1541183052"
}
`
parser, err := New(&Config{
MetricName: "json",
StringFields: []string{"*"},
TimeKey: "time",
TimeFormat: "unix",
})
require.NoError(t, err)
actual, err := parser.Parse([]byte(data))
require.NoError(t, err)
expected := []telegraf.Metric{
testutil.MustMetric(
"json",
map[string]string{},
map[string]interface{}{
"color": "red",
"status": "error",
},
time.Unix(1541183052, 0),
),
}
testutil.RequireMetricsEqual(t, expected, actual)
}
func TestParseEmptyArray(t *testing.T) {
data := `[]`
parser, err := New(&Config{})
require.NoError(t, err)
actual, err := parser.Parse([]byte(data))
require.NoError(t, err)
expected := []telegraf.Metric{}
testutil.RequireMetricsEqual(t, expected, actual)
}
func TestParseSimpleArray(t *testing.T) {
data := `[{"answer": 42}]`
parser, err := New(&Config{
MetricName: "json",
})
require.NoError(t, err)
actual, err := parser.Parse([]byte(data))
require.NoError(t, err)
expected := []telegraf.Metric{
testutil.MustMetric(
"json",
map[string]string{},
map[string]interface{}{
"answer": 42.0,
},
time.Unix(0, 0),
),
}
testutil.RequireMetricsEqual(t, expected, actual, testutil.IgnoreTime())
}
func TestParseArrayWithWrongType(t *testing.T) {
data := `[{"answer": 42}, 123]`
@ -922,3 +824,115 @@ func TestParseArrayWithWrongType(t *testing.T) {
_, err = parser.Parse([]byte(data))
require.Error(t, err)
}
func TestParse(t *testing.T) {
tests := []struct {
name string
config *Config
input []byte
expected []telegraf.Metric
}{
{
name: "tag keys with underscore issue 6705",
config: &Config{
MetricName: "json",
TagKeys: []string{"metric___name__"},
},
input: []byte(`{"metric": {"__name__": "howdy", "time_idle": 42}}`),
expected: []telegraf.Metric{
testutil.MustMetric(
"json",
map[string]string{
"metric___name__": "howdy",
},
map[string]interface{}{
"metric_time_idle": 42.0,
},
time.Unix(0, 0),
),
},
},
{
name: "parse empty array",
config: &Config{},
input: []byte(`[]`),
expected: []telegraf.Metric{},
},
{
name: "parse simple array",
config: &Config{
MetricName: "json",
},
input: []byte(`[{"answer": 42}]`),
expected: []telegraf.Metric{
testutil.MustMetric(
"json",
map[string]string{},
map[string]interface{}{
"answer": 42.0,
},
time.Unix(0, 0),
),
},
},
{
name: "string field glob",
config: &Config{
MetricName: "json",
StringFields: []string{"*"},
},
input: []byte(`
{
"color": "red",
"status": "error"
}
`),
expected: []telegraf.Metric{
testutil.MustMetric(
"json",
map[string]string{},
map[string]interface{}{
"color": "red",
"status": "error",
},
time.Unix(0, 0),
),
},
},
{
name: "time key is deleted from fields",
config: &Config{
MetricName: "json",
TimeKey: "timestamp",
TimeFormat: "unix",
},
input: []byte(`
{
"value": 42,
"timestamp": 1541183052
}
`),
expected: []telegraf.Metric{
testutil.MustMetric(
"json",
map[string]string{},
map[string]interface{}{
"value": 42.0,
},
time.Unix(1541183052, 0),
),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
parser, err := New(tt.config)
require.NoError(t, err)
actual, err := parser.Parse(tt.input)
require.NoError(t, err)
testutil.RequireMetricsEqual(t, tt.expected, actual, testutil.IgnoreTime())
})
}
}