dom/telegraf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add name, time, path and string field options to JSON parser (#4351)

Browse Source
This commit is contained in:
maxunt
2018-08-22 19:26:48 -07:00
committed by Daniel Nelson
parent d6d6539e26
commit 2729378b7f
15 changed files with 672 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
plugins/inputs/exec/exec_test.go
View File

@@ -93,7 +93,10 @@ func (r runnerMock) Run(e *Exec, command string, acc telegraf.Accumulator) ([]by
}
func TestExec(t *testing.T) {
parser, _ := parsers.NewJSONParser("exec", []string{}, nil)
parser, _ := parsers.NewParser(&parsers.Config{
DataFormat: "json",
MetricName: "exec",
})
e := &Exec{
runner: newRunnerMock([]byte(validJson), nil),
Commands: []string{"testcommand arg1"},
@@ -119,7 +122,10 @@ func TestExec(t *testing.T) {
}
func TestExecMalformed(t *testing.T) {
parser, _ := parsers.NewJSONParser("exec", []string{}, nil)
parser, _ := parsers.NewParser(&parsers.Config{
DataFormat: "json",
MetricName: "exec",
})
e := &Exec{
runner: newRunnerMock([]byte(malformedJson), nil),
Commands: []string{"badcommand arg1"},
@@ -132,7 +138,10 @@ func TestExecMalformed(t *testing.T) {
}
func TestCommandError(t *testing.T) {
parser, _ := parsers.NewJSONParser("exec", []string{}, nil)
parser, _ := parsers.NewParser(&parsers.Config{
DataFormat: "json",
MetricName: "exec",
})
e := &Exec{
runner: newRunnerMock(nil, fmt.Errorf("exit status code 1")),
Commands: []string{"badcommand"},

24
plugins/inputs/http/http_test.go
View File

@@ -26,7 +26,11 @@ func TestHTTPwithJSONFormat(t *testing.T) {
URLs: []string{url},
}
metricName := "metricName"
p, _ := parsers.NewJSONParser(metricName, nil, nil)
p, _ := parsers.NewParser(&parsers.Config{
DataFormat: "json",
MetricName: "metricName",
})
plugin.SetParser(p)
var acc testutil.Accumulator
@@ -63,8 +67,11 @@ func TestHTTPHeaders(t *testing.T) {
URLs: []string{url},
Headers: map[string]string{header: headerValue},
}
metricName := "metricName"
p, _ := parsers.NewJSONParser(metricName, nil, nil)
p, _ := parsers.NewParser(&parsers.Config{
DataFormat: "json",
MetricName: "metricName",
})
plugin.SetParser(p)
var acc testutil.Accumulator
@@ -83,7 +90,10 @@ func TestInvalidStatusCode(t *testing.T) {
}
metricName := "metricName"
p, _ := parsers.NewJSONParser(metricName, nil, nil)
p, _ := parsers.NewParser(&parsers.Config{
DataFormat: "json",
MetricName: metricName,
})
plugin.SetParser(p)
var acc testutil.Accumulator
@@ -105,8 +115,10 @@ func TestMethod(t *testing.T) {
Method: "POST",
}
metricName := "metricName"
p, _ := parsers.NewJSONParser(metricName, nil, nil)
p, _ := parsers.NewParser(&parsers.Config{
DataFormat: "json",
MetricName: "metricName",
})
plugin.SetParser(p)
var acc testutil.Accumulator

7
plugins/inputs/httpjson/httpjson.go
View File

@@ -181,7 +181,12 @@ func (h *HttpJson) gatherServer(
"server": serverURL,
}
parser, err := parsers.NewJSONParser(msrmnt_name, h.TagKeys, tags)
parser, err := parsers.NewParser(&parsers.Config{
DataFormat: "json",
MetricName: msrmnt_name,
TagKeys: h.TagKeys,
DefaultTags: tags,
})
if err != nil {
return err
}

5
plugins/inputs/kafka_consumer/kafka_consumer_test.go
View File

@@ -125,7 +125,10 @@ func TestRunParserAndGatherJSON(t *testing.T) {
k.acc = &acc
defer close(k.done)
k.parser, _ = parsers.NewJSONParser("kafka_json_test", []string{}, nil)
k.parser, _ = parsers.NewParser(&parsers.Config{
DataFormat: "json",
MetricName: "kafka_json_test",
})
go k.receiver()
in <- saramaMsg(testMsgJSON)
acc.Wait(1)

5
plugins/inputs/kafka_consumer_legacy/kafka_consumer_legacy_test.go
View File

@@ -125,7 +125,10 @@ func TestRunParserAndGatherJSON(t *testing.T) {
k.acc = &acc
defer close(k.done)
k.parser, _ = parsers.NewJSONParser("kafka_json_test", []string{}, nil)
k.parser, _ = parsers.NewParser(&parsers.Config{
DataFormat: "json",
MetricName: "kafka_json_test",
})
go k.receiver()
in <- saramaMsg(testMsgJSON)
acc.Wait(1)

5
plugins/inputs/mqtt_consumer/mqtt_consumer_test.go
View File

@@ -172,7 +172,10 @@ func TestRunParserAndGatherJSON(t *testing.T) {
n.acc = &acc
defer close(n.done)
n.parser, _ = parsers.NewJSONParser("nats_json_test", []string{}, nil)
n.parser, _ = parsers.NewParser(&parsers.Config{
DataFormat: "json",
MetricName: "nats_json_test",
})
go n.receiver()
in <- mqttMsg(testMsgJSON)

5
plugins/inputs/nats_consumer/nats_consumer_test.go
View File

@@ -108,7 +108,10 @@ func TestRunParserAndGatherJSON(t *testing.T) {
n.acc = &acc
defer close(n.done)
n.parser, _ = parsers.NewJSONParser("nats_json_test", []string{}, nil)
n.parser, _ = parsers.NewParser(&parsers.Config{
DataFormat: "json",
MetricName: "nats_json_test",
})
n.wg.Add(1)
go n.receiver()
in <- natsMsg(testMsgJSON)

5
plugins/inputs/tcp_listener/tcp_listener_test.go
View File

@@ -300,7 +300,10 @@ func TestRunParserJSONMsg(t *testing.T) {
listener.acc = &acc
defer close(listener.done)
listener.parser, _ = parsers.NewJSONParser("udp_json_test", []string{}, nil)
listener.parser, _ = parsers.NewParser(&parsers.Config{
DataFormat: "json",
MetricName: "udp_json_test",
})
listener.wg.Add(1)
go listener.tcpParser()

5
plugins/inputs/udp_listener/udp_listener_test.go
View File

@@ -193,7 +193,10 @@ func TestRunParserJSONMsg(t *testing.T) {
listener.acc = &acc
defer close(listener.done)
listener.parser, _ = parsers.NewJSONParser("udp_json_test", []string{}, nil)
listener.parser, _ = parsers.NewParser(&parsers.Config{
DataFormat: "json",
MetricName: "udp_json_test",
})
listener.wg.Add(1)
go listener.udpParser()

75
plugins/parsers/json/parser.go
View File

@@ -11,6 +11,7 @@ import (
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/metric"
"github.com/tidwall/gjson"
)
var (
@@ -18,9 +19,14 @@ var (
)
type JSONParser struct {
MetricName string
TagKeys []string
DefaultTags map[string]string
MetricName string
TagKeys []string
StringFields []string
JSONNameKey string
JSONQuery string
JSONTimeKey string
JSONTimeFormat string
DefaultTags map[string]string
}
func (p *JSONParser) parseArray(buf []byte) ([]telegraf.Metric, error) {
@@ -34,6 +40,9 @@ func (p *JSONParser) parseArray(buf []byte) ([]telegraf.Metric, error) {
}
for _, item := range jsonOut {
metrics, err = p.parseObject(metrics, item)
if err != nil {
return nil, err
}
}
return metrics, nil
}
@@ -51,10 +60,42 @@ func (p *JSONParser) parseObject(metrics []telegraf.Metric, jsonOut map[string]i
return nil, err
}
//checks if json_name_key is set
if p.JSONNameKey != "" {
p.MetricName = f.Fields[p.JSONNameKey].(string)
}
//if time key is specified, set it to nTime
nTime := time.Now().UTC()
if p.JSONTimeKey != "" {
if p.JSONTimeFormat == "" {
err := fmt.Errorf("use of 'json_time_key' requires 'json_time_format'")
return nil, err
}
if f.Fields[p.JSONTimeKey] == nil {
err := fmt.Errorf("JSON time key could not be found")
return nil, err
}
timeStr, ok := f.Fields[p.JSONTimeKey].(string)
if !ok {
err := fmt.Errorf("time: %v could not be converted to string", f.Fields[p.JSONTimeKey])
return nil, err
}
nTime, err = time.Parse(p.JSONTimeFormat, timeStr)
if err != nil {
return nil, err
}
//if the year is 0, set to current year
if nTime.Year() == 0 {
nTime = nTime.AddDate(time.Now().Year(), 0, 0)
}
}
tags, nFields := p.switchFieldToTag(tags, f.Fields)
metric, err := metric.New(p.MetricName, tags, nFields, time.Now().UTC())
metric, err := metric.New(p.MetricName, tags, nFields, nTime)
if err != nil {
return nil, err
}
@@ -88,6 +129,17 @@ func (p *JSONParser) switchFieldToTag(tags map[string]string, fields map[string]
//remove any additional string/bool values from fields
for k := range fields {
//check if field is in StringFields
sField := false
for _, v := range p.StringFields {
if v == k {
sField = true
}
}
if sField {
continue
}
switch fields[k].(type) {
case string:
delete(fields, k)
@@ -99,6 +151,15 @@ func (p *JSONParser) switchFieldToTag(tags map[string]string, fields map[string]
}
func (p *JSONParser) Parse(buf []byte) ([]telegraf.Metric, error) {
if p.JSONQuery != "" {
result := gjson.GetBytes(buf, p.JSONQuery)
buf = []byte(result.Raw)
if !result.IsArray() && !result.IsObject() {
err := fmt.Errorf("E! Query path must lead to a JSON object or array of objects, but lead to: %v", result.Type)
return nil, err
}
}
buf = bytes.TrimSpace(buf)
buf = bytes.TrimPrefix(buf, utf8BOM)
if len(buf) == 0 {
@@ -126,7 +187,7 @@ func (p *JSONParser) ParseLine(line string) (telegraf.Metric, error) {
}
if len(metrics) < 1 {
return nil, fmt.Errorf("Can not parse the line: %s, for data format: influx ", line)
return nil, fmt.Errorf("can not parse the line: %s, for data format: json ", line)
}
return metrics[0], nil

449
plugins/parsers/json/parser_test.go
View File

@@ -1,9 +1,10 @@
package json
import (
"fmt"
"log"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -55,46 +56,46 @@ func TestParseValidJSON(t *testing.T) {
// Most basic vanilla test
metrics, err := parser.Parse([]byte(validJSON))
assert.NoError(t, err)
assert.Len(t, metrics, 1)
assert.Equal(t, "json_test", metrics[0].Name())
assert.Equal(t, map[string]interface{}{
require.NoError(t, err)
require.Len(t, metrics, 1)
require.Equal(t, "json_test", metrics[0].Name())
require.Equal(t, map[string]interface{}{
"a": float64(5),
"b_c": float64(6),
}, metrics[0].Fields())
assert.Equal(t, map[string]string{}, metrics[0].Tags())
require.Equal(t, map[string]string{}, metrics[0].Tags())
// Test that newlines are fine
metrics, err = parser.Parse([]byte(validJSONNewline))
assert.NoError(t, err)
assert.Len(t, metrics, 1)
assert.Equal(t, "json_test", metrics[0].Name())
assert.Equal(t, map[string]interface{}{
require.NoError(t, err)
require.Len(t, metrics, 1)
require.Equal(t, "json_test", metrics[0].Name())
require.Equal(t, map[string]interface{}{
"d": float64(7),
"b_d": float64(8),
}, metrics[0].Fields())
assert.Equal(t, map[string]string{}, metrics[0].Tags())
require.Equal(t, map[string]string{}, metrics[0].Tags())
// Test that strings without TagKeys defined are ignored
metrics, err = parser.Parse([]byte(validJSONTags))
assert.NoError(t, err)
assert.Len(t, metrics, 1)
assert.Equal(t, "json_test", metrics[0].Name())
assert.Equal(t, map[string]interface{}{
require.NoError(t, err)
require.Len(t, metrics, 1)
require.Equal(t, "json_test", metrics[0].Name())
require.Equal(t, map[string]interface{}{
"a": float64(5),
"b_c": float64(6),
}, metrics[0].Fields())
assert.Equal(t, map[string]string{}, metrics[0].Tags())
require.Equal(t, map[string]string{}, metrics[0].Tags())
// Test that whitespace only will parse as an empty list of metrics
metrics, err = parser.Parse([]byte("\n\t"))
assert.NoError(t, err)
assert.Len(t, metrics, 0)
require.NoError(t, err)
require.Len(t, metrics, 0)
// Test that an empty string will parse as an empty list of metrics
metrics, err = parser.Parse([]byte(""))
assert.NoError(t, err)
assert.Len(t, metrics, 0)
require.NoError(t, err)
require.Len(t, metrics, 0)
}
func TestParseLineValidJSON(t *testing.T) {
@@ -104,33 +105,33 @@ func TestParseLineValidJSON(t *testing.T) {
// Most basic vanilla test
metric, err := parser.ParseLine(validJSON)
assert.NoError(t, err)
assert.Equal(t, "json_test", metric.Name())
assert.Equal(t, map[string]interface{}{
require.NoError(t, err)
require.Equal(t, "json_test", metric.Name())
require.Equal(t, map[string]interface{}{
"a": float64(5),
"b_c": float64(6),
}, metric.Fields())
assert.Equal(t, map[string]string{}, metric.Tags())
require.Equal(t, map[string]string{}, metric.Tags())
// Test that newlines are fine
metric, err = parser.ParseLine(validJSONNewline)
assert.NoError(t, err)
assert.Equal(t, "json_test", metric.Name())
assert.Equal(t, map[string]interface{}{
require.NoError(t, err)
require.Equal(t, "json_test", metric.Name())
require.Equal(t, map[string]interface{}{
"d": float64(7),
"b_d": float64(8),
}, metric.Fields())
assert.Equal(t, map[string]string{}, metric.Tags())
require.Equal(t, map[string]string{}, metric.Tags())
// Test that strings without TagKeys defined are ignored
metric, err = parser.ParseLine(validJSONTags)
assert.NoError(t, err)
assert.Equal(t, "json_test", metric.Name())
assert.Equal(t, map[string]interface{}{
require.NoError(t, err)
require.Equal(t, "json_test", metric.Name())
require.Equal(t, map[string]interface{}{
"a": float64(5),
"b_c": float64(6),
}, metric.Fields())
assert.Equal(t, map[string]string{}, metric.Tags())
require.Equal(t, map[string]string{}, metric.Tags())
}
func TestParseInvalidJSON(t *testing.T) {
@@ -139,11 +140,11 @@ func TestParseInvalidJSON(t *testing.T) {
}
_, err := parser.Parse([]byte(invalidJSON))
assert.Error(t, err)
require.Error(t, err)
_, err = parser.Parse([]byte(invalidJSON2))
assert.Error(t, err)
require.Error(t, err)
_, err = parser.ParseLine(invalidJSON)
assert.Error(t, err)
require.Error(t, err)
}
func TestParseWithTagKeys(t *testing.T) {
@@ -153,14 +154,14 @@ func TestParseWithTagKeys(t *testing.T) {
TagKeys: []string{"wrongtagkey"},
}
metrics, err := parser.Parse([]byte(validJSONTags))
assert.NoError(t, err)
assert.Len(t, metrics, 1)
assert.Equal(t, "json_test", metrics[0].Name())
assert.Equal(t, map[string]interface{}{
require.NoError(t, err)
require.Len(t, metrics, 1)
require.Equal(t, "json_test", metrics[0].Name())
require.Equal(t, map[string]interface{}{
"a": float64(5),
"b_c": float64(6),
}, metrics[0].Fields())
assert.Equal(t, map[string]string{}, metrics[0].Tags())
require.Equal(t, map[string]string{}, metrics[0].Tags())
// Test that single tag key is found and applied
parser = JSONParser{
@@ -168,14 +169,14 @@ func TestParseWithTagKeys(t *testing.T) {
TagKeys: []string{"mytag"},
}
metrics, err = parser.Parse([]byte(validJSONTags))
assert.NoError(t, err)
assert.Len(t, metrics, 1)
assert.Equal(t, "json_test", metrics[0].Name())
assert.Equal(t, map[string]interface{}{
require.NoError(t, err)
require.Len(t, metrics, 1)
require.Equal(t, "json_test", metrics[0].Name())
require.Equal(t, map[string]interface{}{
"a": float64(5),
"b_c": float64(6),
}, metrics[0].Fields())
assert.Equal(t, map[string]string{
require.Equal(t, map[string]string{
"mytag": "foobar",
}, metrics[0].Tags())
@@ -185,14 +186,14 @@ func TestParseWithTagKeys(t *testing.T) {
TagKeys: []string{"mytag", "othertag"},
}
metrics, err = parser.Parse([]byte(validJSONTags))
assert.NoError(t, err)
assert.Len(t, metrics, 1)
assert.Equal(t, "json_test", metrics[0].Name())
assert.Equal(t, map[string]interface{}{
require.NoError(t, err)
require.Len(t, metrics, 1)
require.Equal(t, "json_test", metrics[0].Name())
require.Equal(t, map[string]interface{}{
"a": float64(5),
"b_c": float64(6),
}, metrics[0].Fields())
assert.Equal(t, map[string]string{
require.Equal(t, map[string]string{
"mytag": "foobar",
"othertag": "baz",
}, metrics[0].Tags())
@@ -205,13 +206,13 @@ func TestParseLineWithTagKeys(t *testing.T) {
TagKeys: []string{"wrongtagkey"},
}
metric, err := parser.ParseLine(validJSONTags)
assert.NoError(t, err)
assert.Equal(t, "json_test", metric.Name())
assert.Equal(t, map[string]interface{}{
require.NoError(t, err)
require.Equal(t, "json_test", metric.Name())
require.Equal(t, map[string]interface{}{
"a": float64(5),
"b_c": float64(6),
}, metric.Fields())
assert.Equal(t, map[string]string{}, metric.Tags())
require.Equal(t, map[string]string{}, metric.Tags())
// Test that single tag key is found and applied
parser = JSONParser{
@@ -219,13 +220,13 @@ func TestParseLineWithTagKeys(t *testing.T) {
TagKeys: []string{"mytag"},
}
metric, err = parser.ParseLine(validJSONTags)
assert.NoError(t, err)
assert.Equal(t, "json_test", metric.Name())
assert.Equal(t, map[string]interface{}{
require.NoError(t, err)
require.Equal(t, "json_test", metric.Name())
require.Equal(t, map[string]interface{}{
"a": float64(5),
"b_c": float64(6),
}, metric.Fields())
assert.Equal(t, map[string]string{
require.Equal(t, map[string]string{
"mytag": "foobar",
}, metric.Tags())
@@ -235,13 +236,13 @@ func TestParseLineWithTagKeys(t *testing.T) {
TagKeys: []string{"mytag", "othertag"},
}
metric, err = parser.ParseLine(validJSONTags)
assert.NoError(t, err)
assert.Equal(t, "json_test", metric.Name())
assert.Equal(t, map[string]interface{}{
require.NoError(t, err)
require.Equal(t, "json_test", metric.Name())
require.Equal(t, map[string]interface{}{
"a": float64(5),
"b_c": float64(6),
}, metric.Fields())
assert.Equal(t, map[string]string{
require.Equal(t, map[string]string{
"mytag": "foobar",
"othertag": "baz",
}, metric.Tags())
@@ -258,25 +259,25 @@ func TestParseValidJSONDefaultTags(t *testing.T) {
// Most basic vanilla test
metrics, err := parser.Parse([]byte(validJSON))
assert.NoError(t, err)
assert.Len(t, metrics, 1)
assert.Equal(t, "json_test", metrics[0].Name())
assert.Equal(t, map[string]interface{}{
require.NoError(t, err)
require.Len(t, metrics, 1)
require.Equal(t, "json_test", metrics[0].Name())
require.Equal(t, map[string]interface{}{
"a": float64(5),
"b_c": float64(6),
}, metrics[0].Fields())
assert.Equal(t, map[string]string{"t4g": "default"}, metrics[0].Tags())
require.Equal(t, map[string]string{"t4g": "default"}, metrics[0].Tags())
// Test that tagkeys and default tags are applied
metrics, err = parser.Parse([]byte(validJSONTags))
assert.NoError(t, err)
assert.Len(t, metrics, 1)
assert.Equal(t, "json_test", metrics[0].Name())
assert.Equal(t, map[string]interface{}{
require.NoError(t, err)
require.Len(t, metrics, 1)
require.Equal(t, "json_test", metrics[0].Name())
require.Equal(t, map[string]interface{}{
"a": float64(5),
"b_c": float64(6),
}, metrics[0].Fields())
assert.Equal(t, map[string]string{
require.Equal(t, map[string]string{
"t4g": "default",
"mytag": "foobar",
}, metrics[0].Tags())
@@ -294,25 +295,25 @@ func TestParseValidJSONDefaultTagsOverride(t *testing.T) {
// Most basic vanilla test
metrics, err := parser.Parse([]byte(validJSON))
assert.NoError(t, err)
assert.Len(t, metrics, 1)
assert.Equal(t, "json_test", metrics[0].Name())
assert.Equal(t, map[string]interface{}{
require.NoError(t, err)
require.Len(t, metrics, 1)
require.Equal(t, "json_test", metrics[0].Name())
require.Equal(t, map[string]interface{}{
"a": float64(5),
"b_c": float64(6),
}, metrics[0].Fields())
assert.Equal(t, map[string]string{"mytag": "default"}, metrics[0].Tags())
require.Equal(t, map[string]string{"mytag": "default"}, metrics[0].Tags())
// Test that tagkeys override default tags
metrics, err = parser.Parse([]byte(validJSONTags))
assert.NoError(t, err)
assert.Len(t, metrics, 1)
assert.Equal(t, "json_test", metrics[0].Name())
assert.Equal(t, map[string]interface{}{
require.NoError(t, err)
require.Len(t, metrics, 1)
require.Equal(t, "json_test", metrics[0].Name())
require.Equal(t, map[string]interface{}{
"a": float64(5),
"b_c": float64(6),
}, metrics[0].Fields())
assert.Equal(t, map[string]string{
require.Equal(t, map[string]string{
"mytag": "foobar",
}, metrics[0].Tags())
}
@@ -325,31 +326,31 @@ func TestParseValidJSONArray(t *testing.T) {
// Most basic vanilla test
metrics, err := parser.Parse([]byte(validJSONArray))
assert.NoError(t, err)
assert.Len(t, metrics, 1)
assert.Equal(t, "json_array_test", metrics[0].Name())
assert.Equal(t, map[string]interface{}{
require.NoError(t, err)
require.Len(t, metrics, 1)
require.Equal(t, "json_array_test", metrics[0].Name())
require.Equal(t, map[string]interface{}{
"a": float64(5),
"b_c": float64(6),
}, metrics[0].Fields())
assert.Equal(t, map[string]string{}, metrics[0].Tags())
require.Equal(t, map[string]string{}, metrics[0].Tags())
// Basic multiple datapoints
metrics, err = parser.Parse([]byte(validJSONArrayMultiple))
assert.NoError(t, err)
assert.Len(t, metrics, 2)
assert.Equal(t, "json_array_test", metrics[0].Name())
assert.Equal(t, map[string]interface{}{
require.NoError(t, err)
require.Len(t, metrics, 2)
require.Equal(t, "json_array_test", metrics[0].Name())
require.Equal(t, map[string]interface{}{
"a": float64(5),
"b_c": float64(6),
}, metrics[0].Fields())
assert.Equal(t, map[string]string{}, metrics[1].Tags())
assert.Equal(t, "json_array_test", metrics[1].Name())
assert.Equal(t, map[string]interface{}{
require.Equal(t, map[string]string{}, metrics[1].Tags())
require.Equal(t, "json_array_test", metrics[1].Name())
require.Equal(t, map[string]interface{}{
"a": float64(7),
"b_c": float64(8),
}, metrics[1].Fields())
assert.Equal(t, map[string]string{}, metrics[1].Tags())
require.Equal(t, map[string]string{}, metrics[1].Tags())
}
func TestParseArrayWithTagKeys(t *testing.T) {
@@ -359,21 +360,21 @@ func TestParseArrayWithTagKeys(t *testing.T) {
TagKeys: []string{"wrongtagkey"},
}
metrics, err := parser.Parse([]byte(validJSONArrayTags))
assert.NoError(t, err)
assert.Len(t, metrics, 2)
assert.Equal(t, "json_array_test", metrics[0].Name())
assert.Equal(t, map[string]interface{}{
require.NoError(t, err)
require.Len(t, metrics, 2)
require.Equal(t, "json_array_test", metrics[0].Name())
require.Equal(t, map[string]interface{}{
"a": float64(5),
"b_c": float64(6),
}, metrics[0].Fields())
assert.Equal(t, map[string]string{}, metrics[0].Tags())
require.Equal(t, map[string]string{}, metrics[0].Tags())
assert.Equal(t, "json_array_test", metrics[1].Name())
assert.Equal(t, map[string]interface{}{
require.Equal(t, "json_array_test", metrics[1].Name())
require.Equal(t, map[string]interface{}{
"a": float64(7),
"b_c": float64(8),
}, metrics[1].Fields())
assert.Equal(t, map[string]string{}, metrics[1].Tags())
require.Equal(t, map[string]string{}, metrics[1].Tags())
// Test that single tag key is found and applied
parser = JSONParser{
@@ -381,23 +382,23 @@ func TestParseArrayWithTagKeys(t *testing.T) {
TagKeys: []string{"mytag"},
}
metrics, err = parser.Parse([]byte(validJSONArrayTags))
assert.NoError(t, err)
assert.Len(t, metrics, 2)
assert.Equal(t, "json_array_test", metrics[0].Name())
assert.Equal(t, map[string]interface{}{
require.NoError(t, err)
require.Len(t, metrics, 2)
require.Equal(t, "json_array_test", metrics[0].Name())
require.Equal(t, map[string]interface{}{
"a": float64(5),
"b_c": float64(6),
}, metrics[0].Fields())
assert.Equal(t, map[string]string{
require.Equal(t, map[string]string{
"mytag": "foo",
}, metrics[0].Tags())
assert.Equal(t, "json_array_test", metrics[1].Name())
assert.Equal(t, map[string]interface{}{
require.Equal(t, "json_array_test", metrics[1].Name())
require.Equal(t, map[string]interface{}{
"a": float64(7),
"b_c": float64(8),
}, metrics[1].Fields())
assert.Equal(t, map[string]string{
require.Equal(t, map[string]string{
"mytag": "bar",
}, metrics[1].Tags())
@@ -407,24 +408,24 @@ func TestParseArrayWithTagKeys(t *testing.T) {
TagKeys: []string{"mytag", "othertag"},
}
metrics, err = parser.Parse([]byte(validJSONArrayTags))
assert.NoError(t, err)
assert.Len(t, metrics, 2)
assert.Equal(t, "json_array_test", metrics[0].Name())
assert.Equal(t, map[string]interface{}{
require.NoError(t, err)
require.Len(t, metrics, 2)
require.Equal(t, "json_array_test", metrics[0].Name())
require.Equal(t, map[string]interface{}{
"a": float64(5),
"b_c": float64(6),
}, metrics[0].Fields())
assert.Equal(t, map[string]string{
require.Equal(t, map[string]string{
"mytag": "foo",
"othertag": "baz",
}, metrics[0].Tags())
assert.Equal(t, "json_array_test", metrics[1].Name())
assert.Equal(t, map[string]interface{}{
require.Equal(t, "json_array_test", metrics[1].Name())
require.Equal(t, map[string]interface{}{
"a": float64(7),
"b_c": float64(8),
}, metrics[1].Fields())
assert.Equal(t, map[string]string{
require.Equal(t, map[string]string{
"mytag": "bar",
"othertag": "baz",
}, metrics[1].Tags())
@@ -439,7 +440,7 @@ func TestHttpJsonBOM(t *testing.T) {
// Most basic vanilla test
_, err := parser.Parse(jsonBOM)
assert.NoError(t, err)
require.NoError(t, err)
}
//for testing issue #4260
@@ -448,22 +449,212 @@ func TestJSONParseNestedArray(t *testing.T) {
"total_devices": 5,
"total_threads": 10,
"shares": {
"total": 5,
"accepted": 5,
"rejected": 0,
"avg_find_time": 4,
"tester": "work",
"tester2": "don't want this",
"tester3": 7.93
"total": 5,
"accepted": 5,
"rejected": 0,
"avg_find_time": 4,
"tester": "work",
"tester2": "don't want this",
"tester3": {
"hello":"sup",
"fun":"money",
"break":9
}
}
}`
parser := JSONParser{
MetricName: "json_test",
TagKeys: []string{"total_devices", "total_threads", "shares_tester", "shares_tester3"},
TagKeys: []string{"total_devices", "total_threads", "shares_tester3_fun"},
}
metrics, err := parser.Parse([]byte(testString))
log.Printf("m[0] name: %v, tags: %v, fields: %v", metrics[0].Name(), metrics[0].Tags(), metrics[0].Fields())
require.NoError(t, err)
require.Equal(t, len(parser.TagKeys), len(metrics[0].Tags()))
}
func TestJSONQueryErrorOnArray(t *testing.T) {
testString := `{
"total_devices": 5,
"total_threads": 10,
"shares": {
"total": 5,
"accepted": 6,
"test_string": "don't want this",
"test_obj": {
"hello":"sup",
"fun":"money",
"break":9
},
"myArr":[4,5,6]
}
}`
parser := JSONParser{
MetricName: "json_test",
TagKeys: []string{},
JSONQuery: "shares.myArr",
}
_, err := parser.Parse([]byte(testString))
require.Error(t, err)
}
func TestArrayOfObjects(t *testing.T) {
testString := `{
"meta": {
"info":9,
"shares": [{
"channel": 6,
"time": 1130,
"ice":"man"
},
{
"channel": 5,
"time": 1030,
"ice":"bucket"
},
{
"channel": 10,
"time": 330,
"ice":"cream"
}]
},
"more_stuff":"junk"
}`
parser := JSONParser{
MetricName: "json_test",
TagKeys: []string{"ice"},
JSONQuery: "meta.shares",
}
metrics, err := parser.Parse([]byte(testString))
require.NoError(t, err)
require.Equal(t, len(parser.TagKeys), len(metrics[0].Tags()))
require.Equal(t, 3, len(metrics))
}
func TestUseCaseJSONQuery(t *testing.T) {
testString := `{
"obj": {
"name": {"first": "Tom", "last": "Anderson"},
"age":37,
"children": ["Sara","Alex","Jack"],
"fav.movie": "Deer Hunter",
"friends": [
{"first": "Dale", "last": "Murphy", "age": 44},
{"first": "Roger", "last": "Craig", "age": 68},
{"first": "Jane", "last": "Murphy", "age": 47}
]
}
}`
parser := JSONParser{
MetricName: "json_test",
StringFields: []string{"last"},
TagKeys: []string{"first"},
JSONQuery: "obj.friends",
}
metrics, err := parser.Parse([]byte(testString))
require.NoError(t, err)
require.Equal(t, 3, len(metrics))
require.Equal(t, metrics[0].Fields()["last"], "Murphy")
}
func TestTimeParser(t *testing.T) {
testString := `[
{
"a": 5,
"b": {
"c": 6,
"time":"04 Jan 06 15:04 MST"
},
"my_tag_1": "foo",
"my_tag_2": "baz"
},
{
"a": 7,
"b": {
"c": 8,
"time":"11 Jan 07 15:04 MST"
},
"my_tag_1": "bar",
"my_tag_2": "baz"
}
]`
parser := JSONParser{
MetricName: "json_test",
JSONTimeKey: "b_time",
JSONTimeFormat: "02 Jan 06 15:04 MST",
}
metrics, err := parser.Parse([]byte(testString))
require.NoError(t, err)
require.Equal(t, 2, len(metrics))
require.Equal(t, false, metrics[0].Time() == metrics[1].Time())
}
func TestTimeErrors(t *testing.T) {
testString := `{
"a": 5,
"b": {
"c": 6,
"time":"04 Jan 06 15:04 MST"
},
"my_tag_1": "foo",
"my_tag_2": "baz"
}`
parser := JSONParser{
MetricName: "json_test",
JSONTimeKey: "b_time",
JSONTimeFormat: "02 January 06 15:04 MST",
}
metrics, err := parser.Parse([]byte(testString))
require.Error(t, err)
require.Equal(t, 0, len(metrics))
testString2 := `{
"a": 5,
"b": {
"c": 6
},
"my_tag_1": "foo",
"my_tag_2": "baz"
}`
parser = JSONParser{
MetricName: "json_test",
JSONTimeKey: "b_time",
JSONTimeFormat: "02 January 06 15:04 MST",
}
metrics, err = parser.Parse([]byte(testString2))
log.Printf("err: %v", err)
require.Error(t, err)
require.Equal(t, 0, len(metrics))
require.Equal(t, fmt.Errorf("JSON time key could not be found"), err)
}
func TestNameKey(t *testing.T) {
testString := `{
"a": 5,
"b": {
"c": "this is my name",
"time":"04 Jan 06 15:04 MST"
},
"my_tag_1": "foo",
"my_tag_2": "baz"
}`
parser := JSONParser{
JSONNameKey: "b_c",
}
metrics, err := parser.Parse([]byte(testString))
require.NoError(t, err)
require.Equal(t, "this is my name", metrics[0].Name())
}

47
plugins/parsers/registry.go
View File

@@ -59,9 +59,22 @@ type Config struct {
// TagKeys only apply to JSON data
TagKeys []string
// FieldKeys only apply to JSON
JSONStringFields []string
JSONNameKey string
// MetricName applies to JSON & value. This will be the name of the measurement.
MetricName string
// holds a gjson path for json parser
JSONQuery string
// key of time
JSONTimeKey string
// time format
JSONTimeFormat string
// Authentication file for collectd
CollectdAuthFile string
// One of none (default), sign, or encrypt
@@ -108,8 +121,14 @@ func NewParser(config *Config) (Parser, error) {
var parser Parser
switch config.DataFormat {
case "json":
parser, err = NewJSONParser(config.MetricName,
config.TagKeys, config.DefaultTags)
parser = newJSONParser(config.MetricName,
config.TagKeys,
config.JSONNameKey,
config.JSONStringFields,
config.JSONQuery,
config.JSONTimeKey,
config.JSONTimeFormat,
config.DefaultTags)
case "value":
parser, err = NewValueParser(config.MetricName,
config.DataType, config.DefaultTags)
@@ -151,6 +170,30 @@ func NewParser(config *Config) (Parser, error) {
return parser, err
}
func newJSONParser(
metricName string,
tagKeys []string,
jsonNameKey string,
stringFields []string,
jsonQuery string,
timeKey string,
timeFormat string,
defaultTags map[string]string,
) Parser {
parser := &json.JSONParser{
MetricName: metricName,
TagKeys: tagKeys,
StringFields: stringFields,
JSONNameKey: jsonNameKey,
JSONQuery: jsonQuery,
JSONTimeKey: timeKey,
JSONTimeFormat: timeFormat,
DefaultTags: defaultTags,
}
return parser
}
//Deprecated: Use NewParser to get a JSONParser object
func newGrokParser(metricName string,
patterns []string,
nPatterns []string,