Remove trailing underscore trimming from json flattener (#6744)

This commit is contained in:
Daniel Nelson 2019-12-03 11:46:29 -08:00 committed by GitHub
parent 4feef67c21
commit 03de92b962
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 124 additions and 103 deletions

View File

@ -7,7 +7,6 @@ import (
"fmt" "fmt"
"log" "log"
"strconv" "strconv"
"strings"
"time" "time"
"github.com/influxdata/telegraf" "github.com/influxdata/telegraf"
@ -260,19 +259,27 @@ func (f *JSONFlattener) FullFlattenJSON(
if f.Fields == nil { if f.Fields == nil {
f.Fields = make(map[string]interface{}) f.Fields = make(map[string]interface{})
} }
fieldname = strings.Trim(fieldname, "_")
switch t := v.(type) { switch t := v.(type) {
case map[string]interface{}: case map[string]interface{}:
for k, v := range t { for k, v := range t {
err := f.FullFlattenJSON(fieldname+"_"+k+"_", v, convertString, convertBool) fieldkey := k
if fieldname != "" {
fieldkey = fieldname + "_" + fieldkey
}
err := f.FullFlattenJSON(fieldkey, v, convertString, convertBool)
if err != nil { if err != nil {
return err return err
} }
} }
case []interface{}: case []interface{}:
for i, v := range t { for i, v := range t {
k := strconv.Itoa(i) fieldkey := strconv.Itoa(i)
err := f.FullFlattenJSON(fieldname+"_"+k+"_", v, convertString, convertBool) if fieldname != "" {
fieldkey = fieldname + "_" + fieldkey
}
err := f.FullFlattenJSON(fieldkey, v, convertString, convertBool)
if err != nil { if err != nil {
return nil return nil
} }

View File

@ -815,104 +815,6 @@ func TestNameKey(t *testing.T) {
require.Equal(t, "this is my name", metrics[0].Name()) require.Equal(t, "this is my name", metrics[0].Name())
} }
func TestTimeKeyDelete(t *testing.T) {
data := `{
"timestamp": 1541183052,
"value": 42
}`
parser, err := New(&Config{
MetricName: "json",
TimeKey: "timestamp",
TimeFormat: "unix",
})
require.NoError(t, err)
metrics, err := parser.Parse([]byte(data))
require.NoError(t, err)
expected := []telegraf.Metric{
testutil.MustMetric("json",
map[string]string{},
map[string]interface{}{"value": 42.0},
time.Unix(1541183052, 0)),
}
testutil.RequireMetricsEqual(t, expected, metrics)
}
func TestStringFieldGlob(t *testing.T) {
data := `
{
"color": "red",
"status": "error",
"time": "1541183052"
}
`
parser, err := New(&Config{
MetricName: "json",
StringFields: []string{"*"},
TimeKey: "time",
TimeFormat: "unix",
})
require.NoError(t, err)
actual, err := parser.Parse([]byte(data))
require.NoError(t, err)
expected := []telegraf.Metric{
testutil.MustMetric(
"json",
map[string]string{},
map[string]interface{}{
"color": "red",
"status": "error",
},
time.Unix(1541183052, 0),
),
}
testutil.RequireMetricsEqual(t, expected, actual)
}
func TestParseEmptyArray(t *testing.T) {
data := `[]`
parser, err := New(&Config{})
require.NoError(t, err)
actual, err := parser.Parse([]byte(data))
require.NoError(t, err)
expected := []telegraf.Metric{}
testutil.RequireMetricsEqual(t, expected, actual)
}
func TestParseSimpleArray(t *testing.T) {
data := `[{"answer": 42}]`
parser, err := New(&Config{
MetricName: "json",
})
require.NoError(t, err)
actual, err := parser.Parse([]byte(data))
require.NoError(t, err)
expected := []telegraf.Metric{
testutil.MustMetric(
"json",
map[string]string{},
map[string]interface{}{
"answer": 42.0,
},
time.Unix(0, 0),
),
}
testutil.RequireMetricsEqual(t, expected, actual, testutil.IgnoreTime())
}
func TestParseArrayWithWrongType(t *testing.T) { func TestParseArrayWithWrongType(t *testing.T) {
data := `[{"answer": 42}, 123]` data := `[{"answer": 42}, 123]`
@ -922,3 +824,115 @@ func TestParseArrayWithWrongType(t *testing.T) {
_, err = parser.Parse([]byte(data)) _, err = parser.Parse([]byte(data))
require.Error(t, err) require.Error(t, err)
} }
func TestParse(t *testing.T) {
tests := []struct {
name string
config *Config
input []byte
expected []telegraf.Metric
}{
{
name: "tag keys with underscore issue 6705",
config: &Config{
MetricName: "json",
TagKeys: []string{"metric___name__"},
},
input: []byte(`{"metric": {"__name__": "howdy", "time_idle": 42}}`),
expected: []telegraf.Metric{
testutil.MustMetric(
"json",
map[string]string{
"metric___name__": "howdy",
},
map[string]interface{}{
"metric_time_idle": 42.0,
},
time.Unix(0, 0),
),
},
},
{
name: "parse empty array",
config: &Config{},
input: []byte(`[]`),
expected: []telegraf.Metric{},
},
{
name: "parse simple array",
config: &Config{
MetricName: "json",
},
input: []byte(`[{"answer": 42}]`),
expected: []telegraf.Metric{
testutil.MustMetric(
"json",
map[string]string{},
map[string]interface{}{
"answer": 42.0,
},
time.Unix(0, 0),
),
},
},
{
name: "string field glob",
config: &Config{
MetricName: "json",
StringFields: []string{"*"},
},
input: []byte(`
{
"color": "red",
"status": "error"
}
`),
expected: []telegraf.Metric{
testutil.MustMetric(
"json",
map[string]string{},
map[string]interface{}{
"color": "red",
"status": "error",
},
time.Unix(0, 0),
),
},
},
{
name: "time key is deleted from fields",
config: &Config{
MetricName: "json",
TimeKey: "timestamp",
TimeFormat: "unix",
},
input: []byte(`
{
"value": 42,
"timestamp": 1541183052
}
`),
expected: []telegraf.Metric{
testutil.MustMetric(
"json",
map[string]string{},
map[string]interface{}{
"value": 42.0,
},
time.Unix(1541183052, 0),
),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
parser, err := New(tt.config)
require.NoError(t, err)
actual, err := parser.Parse(tt.input)
require.NoError(t, err)
testutil.RequireMetricsEqual(t, tt.expected, actual, testutil.IgnoreTime())
})
}
}