telegraf/plugins/parsers/json/parser.go

327 lines
8.0 KiB
Go

package json
import (
"bytes"
"encoding/json"
"fmt"
"log"
"strconv"
"strings"
"time"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/metric"
"github.com/pkg/errors"
"github.com/tidwall/gjson"
"math"
"regexp"
)
var (
utf8BOM = []byte("\xef\xbb\xbf")
)
type JSONParser struct {
MetricName string
TagKeys []string
StringFields []string
JSONNameKey string
JSONQuery string
JSONTimeKey string
JSONTimeFormat string
DefaultTags map[string]string
}
func (p *JSONParser) parseArray(buf []byte) ([]telegraf.Metric, error) {
metrics := make([]telegraf.Metric, 0)
var jsonOut []map[string]interface{}
err := json.Unmarshal(buf, &jsonOut)
if err != nil {
err = fmt.Errorf("unable to parse out as JSON Array, %s", err)
return nil, err
}
for _, item := range jsonOut {
metrics, err = p.parseObject(metrics, item)
if err != nil {
return nil, err
}
}
return metrics, nil
}
// format = "unix": epoch is assumed to be in seconds and can come as number or string. Can have a decimal part.
// format = "unix_ms": epoch is assumed to be in milliseconds and can come as number or string. Cannot have a decimal part.
func parseUnixTimestamp(jsonValue interface{}, format string) (time.Time, error) {
timeInt, timeFractional := int64(0), int64(0)
timeEpochStr, ok := jsonValue.(string)
var err error
if !ok {
timeEpochFloat, ok := jsonValue.(float64)
if !ok {
err := fmt.Errorf("time: %v could not be converted to string nor float64", jsonValue)
return time.Time{}, err
}
intPart, frac := math.Modf(timeEpochFloat)
timeInt, timeFractional = int64(intPart), int64(frac*1e9)
} else {
splitted := regexp.MustCompile("[.,]").Split(timeEpochStr, 2)
timeInt, err = strconv.ParseInt(splitted[0], 10, 64)
if err != nil {
return time.Time{}, err
}
if len(splitted) == 2 {
if len(splitted[1]) > 9 {
splitted[1] = splitted[1][:9] //truncates decimal part to nanoseconds precision
}
nanosecStr := splitted[1] + strings.Repeat("0", 9-len(splitted[1])) //adds 0's to the right to obtain a valid number of nanoseconds
timeFractional, err = strconv.ParseInt(nanosecStr, 10, 64)
if err != nil {
return time.Time{}, err
}
}
}
if strings.EqualFold(format, "unix") {
return time.Unix(timeInt, timeFractional).UTC(), nil
} else if strings.EqualFold(format, "unix_ms") {
return time.Unix(timeInt/1000, (timeInt%1000)*1e6).UTC(), nil
} else {
return time.Time{}, errors.New("Invalid unix format")
}
}
func (p *JSONParser) parseObject(metrics []telegraf.Metric, jsonOut map[string]interface{}) ([]telegraf.Metric, error) {
tags := make(map[string]string)
for k, v := range p.DefaultTags {
tags[k] = v
}
f := JSONFlattener{}
err := f.FullFlattenJSON("", jsonOut, true, true)
if err != nil {
return nil, err
}
//checks if json_name_key is set
if p.JSONNameKey != "" {
p.MetricName = f.Fields[p.JSONNameKey].(string)
}
//if time key is specified, set it to nTime
nTime := time.Now().UTC()
if p.JSONTimeKey != "" {
if p.JSONTimeFormat == "" {
err := fmt.Errorf("use of 'json_time_key' requires 'json_time_format'")
return nil, err
}
if f.Fields[p.JSONTimeKey] == nil {
err := fmt.Errorf("JSON time key could not be found")
return nil, err
}
if strings.EqualFold(p.JSONTimeFormat, "unix") || strings.EqualFold(p.JSONTimeFormat, "unix_ms") {
nTime, err = parseUnixTimestamp(f.Fields[p.JSONTimeKey], p.JSONTimeFormat)
if err != nil {
return nil, err
}
} else {
timeStr, ok := f.Fields[p.JSONTimeKey].(string)
if !ok {
err := fmt.Errorf("time: %v could not be converted to string", f.Fields[p.JSONTimeKey])
return nil, err
}
nTime, err = time.Parse(p.JSONTimeFormat, timeStr)
if err != nil {
return nil, err
}
}
//if the year is 0, set to current year
if nTime.Year() == 0 {
nTime = nTime.AddDate(time.Now().Year(), 0, 0)
}
}
tags, nFields := p.switchFieldToTag(tags, f.Fields)
metric, err := metric.New(p.MetricName, tags, nFields, nTime)
if err != nil {
return nil, err
}
return append(metrics, metric), nil
}
//will take in field map with strings and bools,
//search for TagKeys that match fieldnames and add them to tags
//will delete any strings/bools that shouldn't be fields
//assumes that any non-numeric values in TagKeys should be displayed as tags
func (p *JSONParser) switchFieldToTag(tags map[string]string, fields map[string]interface{}) (map[string]string, map[string]interface{}) {
for _, name := range p.TagKeys {
//switch any fields in tagkeys into tags
if fields[name] == nil {
continue
}
switch value := fields[name].(type) {
case string:
tags[name] = value
delete(fields, name)
case bool:
tags[name] = strconv.FormatBool(value)
delete(fields, name)
case float64:
tags[name] = strconv.FormatFloat(value, 'f', -1, 64)
delete(fields, name)
default:
log.Printf("E! [parsers.json] Unrecognized type %T", value)
}
}
//remove any additional string/bool values from fields
for k := range fields {
//check if field is in StringFields
sField := false
for _, v := range p.StringFields {
if v == k {
sField = true
}
}
if sField {
continue
}
switch fields[k].(type) {
case string:
delete(fields, k)
case bool:
delete(fields, k)
}
}
return tags, fields
}
func (p *JSONParser) Parse(buf []byte) ([]telegraf.Metric, error) {
if p.JSONQuery != "" {
result := gjson.GetBytes(buf, p.JSONQuery)
buf = []byte(result.Raw)
if !result.IsArray() && !result.IsObject() {
err := fmt.Errorf("E! Query path must lead to a JSON object or array of objects, but lead to: %v", result.Type)
return nil, err
}
}
buf = bytes.TrimSpace(buf)
buf = bytes.TrimPrefix(buf, utf8BOM)
if len(buf) == 0 {
return make([]telegraf.Metric, 0), nil
}
if !isarray(buf) {
metrics := make([]telegraf.Metric, 0)
var jsonOut map[string]interface{}
err := json.Unmarshal(buf, &jsonOut)
if err != nil {
err = fmt.Errorf("unable to parse out as JSON, %s", err)
return nil, err
}
return p.parseObject(metrics, jsonOut)
}
return p.parseArray(buf)
}
func (p *JSONParser) ParseLine(line string) (telegraf.Metric, error) {
metrics, err := p.Parse([]byte(line + "\n"))
if err != nil {
return nil, err
}
if len(metrics) < 1 {
return nil, fmt.Errorf("can not parse the line: %s, for data format: json ", line)
}
return metrics[0], nil
}
func (p *JSONParser) SetDefaultTags(tags map[string]string) {
p.DefaultTags = tags
}
type JSONFlattener struct {
Fields map[string]interface{}
}
// FlattenJSON flattens nested maps/interfaces into a fields map (ignoring bools and string)
func (f *JSONFlattener) FlattenJSON(
fieldname string,
v interface{}) error {
if f.Fields == nil {
f.Fields = make(map[string]interface{})
}
return f.FullFlattenJSON(fieldname, v, false, false)
}
// FullFlattenJSON flattens nested maps/interfaces into a fields map (including bools and string)
func (f *JSONFlattener) FullFlattenJSON(
fieldname string,
v interface{},
convertString bool,
convertBool bool,
) error {
if f.Fields == nil {
f.Fields = make(map[string]interface{})
}
fieldname = strings.Trim(fieldname, "_")
switch t := v.(type) {
case map[string]interface{}:
for k, v := range t {
err := f.FullFlattenJSON(fieldname+"_"+k+"_", v, convertString, convertBool)
if err != nil {
return err
}
}
case []interface{}:
for i, v := range t {
k := strconv.Itoa(i)
err := f.FullFlattenJSON(fieldname+"_"+k+"_", v, convertString, convertBool)
if err != nil {
return nil
}
}
case float64:
f.Fields[fieldname] = t
case string:
if convertString {
f.Fields[fieldname] = v.(string)
} else {
return nil
}
case bool:
if convertBool {
f.Fields[fieldname] = v.(bool)
} else {
return nil
}
case nil:
return nil
default:
return fmt.Errorf("JSON Flattener: got unexpected type %T with value %v (%s)",
t, t, fieldname)
}
return nil
}
func isarray(buf []byte) bool {
ia := bytes.IndexByte(buf, '[')
ib := bytes.IndexByte(buf, '{')
if ia > -1 && ia < ib {
return true
} else {
return false
}
}