Implementing generic parser plugins and documentation
This constitutes a large change in how we will parse different data formats going forward (for the plugins that support it) This is working off @henrypfhu's changes.
This commit is contained in:
Cameron Sparr
109
plugins/parsers/json/parser.go
Normal file
109
plugins/parsers/json/parser.go
Normal file
@@ -0,0 +1,109 @@
|
||||
package json
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/influxdata/telegraf"
|
||||
)
|
||||
|
||||
type JSONParser struct {
|
||||
MetricName string
|
||||
TagKeys []string
|
||||
DefaultTags map[string]string
|
||||
}
|
||||
|
||||
func (p *JSONParser) Parse(buf []byte) ([]telegraf.Metric, error) {
|
||||
metrics := make([]telegraf.Metric, 0)
|
||||
|
||||
var jsonOut map[string]interface{}
|
||||
err := json.Unmarshal(buf, &jsonOut)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("unable to parse out as JSON, %s", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tags := make(map[string]string)
|
||||
for k, v := range p.DefaultTags {
|
||||
tags[k] = v
|
||||
}
|
||||
|
||||
for _, tag := range p.TagKeys {
|
||||
switch v := jsonOut[tag].(type) {
|
||||
case string:
|
||||
tags[tag] = v
|
||||
}
|
||||
delete(jsonOut, tag)
|
||||
}
|
||||
|
||||
f := JSONFlattener{}
|
||||
err = f.FlattenJSON("", jsonOut)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
metric, err := telegraf.NewMetric(p.MetricName, tags, f.Fields, time.Now().UTC())
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(metrics, metric), nil
|
||||
}
|
||||
|
||||
func (p *JSONParser) ParseLine(line string) (telegraf.Metric, error) {
|
||||
metrics, err := p.Parse([]byte(line + "\n"))
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(metrics) < 1 {
|
||||
return nil, fmt.Errorf("Can not parse the line: %s, for data format: influx ", line)
|
||||
}
|
||||
|
||||
return metrics[0], nil
|
||||
}
|
||||
|
||||
type JSONFlattener struct {
|
||||
Fields map[string]interface{}
|
||||
}
|
||||
|
||||
// FlattenJSON flattens nested maps/interfaces into a fields map
|
||||
func (f *JSONFlattener) FlattenJSON(
|
||||
fieldname string,
|
||||
v interface{},
|
||||
) error {
|
||||
if f.Fields == nil {
|
||||
f.Fields = make(map[string]interface{})
|
||||
}
|
||||
fieldname = strings.Trim(fieldname, "_")
|
||||
switch t := v.(type) {
|
||||
case map[string]interface{}:
|
||||
for k, v := range t {
|
||||
err := f.FlattenJSON(fieldname+"_"+k+"_", v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case []interface{}:
|
||||
for i, v := range t {
|
||||
k := strconv.Itoa(i)
|
||||
err := f.FlattenJSON(fieldname+"_"+k+"_", v)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
case float64:
|
||||
f.Fields[fieldname] = t
|
||||
case bool, string, nil:
|
||||
// ignored types
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("JSON Flattener: got unexpected type %T with value %v (%s)",
|
||||
t, t, fieldname)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
284
plugins/parsers/json/parser_test.go
Normal file
284
plugins/parsers/json/parser_test.go
Normal file
@@ -0,0 +1,284 @@
|
||||
package json
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
const (
|
||||
validJSON = "{\"a\": 5, \"b\": {\"c\": 6}}"
|
||||
validJSONNewline = "\n{\"d\": 7, \"b\": {\"d\": 8}}\n"
|
||||
invalidJSON = "I don't think this is JSON"
|
||||
invalidJSON2 = "{\"a\": 5, \"b\": \"c\": 6}}"
|
||||
)
|
||||
|
||||
const validJSONTags = `
|
||||
{
|
||||
"a": 5,
|
||||
"b": {
|
||||
"c": 6
|
||||
},
|
||||
"mytag": "foobar",
|
||||
"othertag": "baz"
|
||||
}
|
||||
`
|
||||
|
||||
func TestParseValidJSON(t *testing.T) {
|
||||
parser := JSONParser{
|
||||
MetricName: "json_test",
|
||||
}
|
||||
|
||||
// Most basic vanilla test
|
||||
metrics, err := parser.Parse([]byte(validJSON))
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, metrics, 1)
|
||||
assert.Equal(t, "json_test", metrics[0].Name())
|
||||
assert.Equal(t, map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
}, metrics[0].Fields())
|
||||
assert.Equal(t, map[string]string{}, metrics[0].Tags())
|
||||
|
||||
// Test that newlines are fine
|
||||
metrics, err = parser.Parse([]byte(validJSONNewline))
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, metrics, 1)
|
||||
assert.Equal(t, "json_test", metrics[0].Name())
|
||||
assert.Equal(t, map[string]interface{}{
|
||||
"d": float64(7),
|
||||
"b_d": float64(8),
|
||||
}, metrics[0].Fields())
|
||||
assert.Equal(t, map[string]string{}, metrics[0].Tags())
|
||||
|
||||
// Test that strings without TagKeys defined are ignored
|
||||
metrics, err = parser.Parse([]byte(validJSONTags))
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, metrics, 1)
|
||||
assert.Equal(t, "json_test", metrics[0].Name())
|
||||
assert.Equal(t, map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
}, metrics[0].Fields())
|
||||
assert.Equal(t, map[string]string{}, metrics[0].Tags())
|
||||
}
|
||||
|
||||
func TestParseLineValidJSON(t *testing.T) {
|
||||
parser := JSONParser{
|
||||
MetricName: "json_test",
|
||||
}
|
||||
|
||||
// Most basic vanilla test
|
||||
metric, err := parser.ParseLine(validJSON)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "json_test", metric.Name())
|
||||
assert.Equal(t, map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
}, metric.Fields())
|
||||
assert.Equal(t, map[string]string{}, metric.Tags())
|
||||
|
||||
// Test that newlines are fine
|
||||
metric, err = parser.ParseLine(validJSONNewline)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "json_test", metric.Name())
|
||||
assert.Equal(t, map[string]interface{}{
|
||||
"d": float64(7),
|
||||
"b_d": float64(8),
|
||||
}, metric.Fields())
|
||||
assert.Equal(t, map[string]string{}, metric.Tags())
|
||||
|
||||
// Test that strings without TagKeys defined are ignored
|
||||
metric, err = parser.ParseLine(validJSONTags)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "json_test", metric.Name())
|
||||
assert.Equal(t, map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
}, metric.Fields())
|
||||
assert.Equal(t, map[string]string{}, metric.Tags())
|
||||
}
|
||||
|
||||
func TestParseInvalidJSON(t *testing.T) {
|
||||
parser := JSONParser{
|
||||
MetricName: "json_test",
|
||||
}
|
||||
|
||||
_, err := parser.Parse([]byte(invalidJSON))
|
||||
assert.Error(t, err)
|
||||
_, err = parser.Parse([]byte(invalidJSON2))
|
||||
assert.Error(t, err)
|
||||
_, err = parser.ParseLine(invalidJSON)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestParseWithTagKeys(t *testing.T) {
|
||||
// Test that strings not matching tag keys are ignored
|
||||
parser := JSONParser{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"wrongtagkey"},
|
||||
}
|
||||
metrics, err := parser.Parse([]byte(validJSONTags))
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, metrics, 1)
|
||||
assert.Equal(t, "json_test", metrics[0].Name())
|
||||
assert.Equal(t, map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
}, metrics[0].Fields())
|
||||
assert.Equal(t, map[string]string{}, metrics[0].Tags())
|
||||
|
||||
// Test that single tag key is found and applied
|
||||
parser = JSONParser{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"mytag"},
|
||||
}
|
||||
metrics, err = parser.Parse([]byte(validJSONTags))
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, metrics, 1)
|
||||
assert.Equal(t, "json_test", metrics[0].Name())
|
||||
assert.Equal(t, map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
}, metrics[0].Fields())
|
||||
assert.Equal(t, map[string]string{
|
||||
"mytag": "foobar",
|
||||
}, metrics[0].Tags())
|
||||
|
||||
// Test that both tag keys are found and applied
|
||||
parser = JSONParser{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"mytag", "othertag"},
|
||||
}
|
||||
metrics, err = parser.Parse([]byte(validJSONTags))
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, metrics, 1)
|
||||
assert.Equal(t, "json_test", metrics[0].Name())
|
||||
assert.Equal(t, map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
}, metrics[0].Fields())
|
||||
assert.Equal(t, map[string]string{
|
||||
"mytag": "foobar",
|
||||
"othertag": "baz",
|
||||
}, metrics[0].Tags())
|
||||
}
|
||||
|
||||
func TestParseLineWithTagKeys(t *testing.T) {
|
||||
// Test that strings not matching tag keys are ignored
|
||||
parser := JSONParser{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"wrongtagkey"},
|
||||
}
|
||||
metric, err := parser.ParseLine(validJSONTags)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "json_test", metric.Name())
|
||||
assert.Equal(t, map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
}, metric.Fields())
|
||||
assert.Equal(t, map[string]string{}, metric.Tags())
|
||||
|
||||
// Test that single tag key is found and applied
|
||||
parser = JSONParser{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"mytag"},
|
||||
}
|
||||
metric, err = parser.ParseLine(validJSONTags)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "json_test", metric.Name())
|
||||
assert.Equal(t, map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
}, metric.Fields())
|
||||
assert.Equal(t, map[string]string{
|
||||
"mytag": "foobar",
|
||||
}, metric.Tags())
|
||||
|
||||
// Test that both tag keys are found and applied
|
||||
parser = JSONParser{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"mytag", "othertag"},
|
||||
}
|
||||
metric, err = parser.ParseLine(validJSONTags)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "json_test", metric.Name())
|
||||
assert.Equal(t, map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
}, metric.Fields())
|
||||
assert.Equal(t, map[string]string{
|
||||
"mytag": "foobar",
|
||||
"othertag": "baz",
|
||||
}, metric.Tags())
|
||||
}
|
||||
|
||||
func TestParseValidJSONDefaultTags(t *testing.T) {
|
||||
parser := JSONParser{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"mytag"},
|
||||
DefaultTags: map[string]string{
|
||||
"t4g": "default",
|
||||
},
|
||||
}
|
||||
|
||||
// Most basic vanilla test
|
||||
metrics, err := parser.Parse([]byte(validJSON))
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, metrics, 1)
|
||||
assert.Equal(t, "json_test", metrics[0].Name())
|
||||
assert.Equal(t, map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
}, metrics[0].Fields())
|
||||
assert.Equal(t, map[string]string{"t4g": "default"}, metrics[0].Tags())
|
||||
|
||||
// Test that tagkeys and default tags are applied
|
||||
metrics, err = parser.Parse([]byte(validJSONTags))
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, metrics, 1)
|
||||
assert.Equal(t, "json_test", metrics[0].Name())
|
||||
assert.Equal(t, map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
}, metrics[0].Fields())
|
||||
assert.Equal(t, map[string]string{
|
||||
"t4g": "default",
|
||||
"mytag": "foobar",
|
||||
}, metrics[0].Tags())
|
||||
}
|
||||
|
||||
// Test that default tags are overridden by tag keys
|
||||
func TestParseValidJSONDefaultTagsOverride(t *testing.T) {
|
||||
parser := JSONParser{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"mytag"},
|
||||
DefaultTags: map[string]string{
|
||||
"mytag": "default",
|
||||
},
|
||||
}
|
||||
|
||||
// Most basic vanilla test
|
||||
metrics, err := parser.Parse([]byte(validJSON))
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, metrics, 1)
|
||||
assert.Equal(t, "json_test", metrics[0].Name())
|
||||
assert.Equal(t, map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
}, metrics[0].Fields())
|
||||
assert.Equal(t, map[string]string{"mytag": "default"}, metrics[0].Tags())
|
||||
|
||||
// Test that tagkeys override default tags
|
||||
metrics, err = parser.Parse([]byte(validJSONTags))
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, metrics, 1)
|
||||
assert.Equal(t, "json_test", metrics[0].Name())
|
||||
assert.Equal(t, map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
}, metrics[0].Fields())
|
||||
assert.Equal(t, map[string]string{
|
||||
"mytag": "foobar",
|
||||
}, metrics[0].Tags())
|
||||
}
|
||||
Reference in New Issue
Block a user