package json import ( "bytes" "encoding/json" "fmt" "log" "strconv" "strings" "time" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/metric" "github.com/pkg/errors" "github.com/tidwall/gjson" "math" "regexp" ) var ( utf8BOM = []byte("\xef\xbb\xbf") ) type JSONParser struct { MetricName string TagKeys []string StringFields []string JSONNameKey string JSONQuery string JSONTimeKey string JSONTimeFormat string DefaultTags map[string]string } func (p *JSONParser) parseArray(buf []byte) ([]telegraf.Metric, error) { metrics := make([]telegraf.Metric, 0) var jsonOut []map[string]interface{} err := json.Unmarshal(buf, &jsonOut) if err != nil { err = fmt.Errorf("unable to parse out as JSON Array, %s", err) return nil, err } for _, item := range jsonOut { metrics, err = p.parseObject(metrics, item) if err != nil { return nil, err } } return metrics, nil } // format = "unix": epoch is assumed to be in seconds and can come as number or string. Can have a decimal part. // format = "unix_ms": epoch is assumed to be in milliseconds and can come as number or string. Cannot have a decimal part. func parseUnixTimestamp(jsonValue interface{}, format string) (time.Time, error) { timeInt, timeFractional := int64(0), int64(0) timeEpochStr, ok := jsonValue.(string) var err error if !ok { timeEpochFloat, ok := jsonValue.(float64) if !ok { err := fmt.Errorf("time: %v could not be converted to string nor float64", jsonValue) return time.Time{}, err } intPart, frac := math.Modf(timeEpochFloat) timeInt, timeFractional = int64(intPart), int64(frac*1e9) } else { splitted := regexp.MustCompile("[.,]").Split(timeEpochStr, 2) timeInt, err = strconv.ParseInt(splitted[0], 10, 64) if err != nil { return time.Time{}, err } if len(splitted) == 2 { if len(splitted[1]) > 9 { splitted[1] = splitted[1][:9] //truncates decimal part to nanoseconds precision } nanosecStr := splitted[1] + strings.Repeat("0", 9-len(splitted[1])) //adds 0's to the right to obtain a valid number of nanoseconds timeFractional, err = strconv.ParseInt(nanosecStr, 10, 64) if err != nil { return time.Time{}, err } } } if strings.EqualFold(format, "unix") { return time.Unix(timeInt, timeFractional).UTC(), nil } else if strings.EqualFold(format, "unix_ms") { return time.Unix(timeInt/1000, (timeInt%1000)*1e6).UTC(), nil } else { return time.Time{}, errors.New("Invalid unix format") } } func (p *JSONParser) parseObject(metrics []telegraf.Metric, jsonOut map[string]interface{}) ([]telegraf.Metric, error) { tags := make(map[string]string) for k, v := range p.DefaultTags { tags[k] = v } f := JSONFlattener{} err := f.FullFlattenJSON("", jsonOut, true, true) if err != nil { return nil, err } //checks if json_name_key is set if p.JSONNameKey != "" { p.MetricName = f.Fields[p.JSONNameKey].(string) } //if time key is specified, set it to nTime nTime := time.Now().UTC() if p.JSONTimeKey != "" { if p.JSONTimeFormat == "" { err := fmt.Errorf("use of 'json_time_key' requires 'json_time_format'") return nil, err } if f.Fields[p.JSONTimeKey] == nil { err := fmt.Errorf("JSON time key could not be found") return nil, err } if strings.EqualFold(p.JSONTimeFormat, "unix") || strings.EqualFold(p.JSONTimeFormat, "unix_ms") { nTime, err = parseUnixTimestamp(f.Fields[p.JSONTimeKey], p.JSONTimeFormat) if err != nil { return nil, err } } else { timeStr, ok := f.Fields[p.JSONTimeKey].(string) if !ok { err := fmt.Errorf("time: %v could not be converted to string", f.Fields[p.JSONTimeKey]) return nil, err } nTime, err = time.Parse(p.JSONTimeFormat, timeStr) if err != nil { return nil, err } } //if the year is 0, set to current year if nTime.Year() == 0 { nTime = nTime.AddDate(time.Now().Year(), 0, 0) } } tags, nFields := p.switchFieldToTag(tags, f.Fields) metric, err := metric.New(p.MetricName, tags, nFields, nTime) if err != nil { return nil, err } return append(metrics, metric), nil } //will take in field map with strings and bools, //search for TagKeys that match fieldnames and add them to tags //will delete any strings/bools that shouldn't be fields //assumes that any non-numeric values in TagKeys should be displayed as tags func (p *JSONParser) switchFieldToTag(tags map[string]string, fields map[string]interface{}) (map[string]string, map[string]interface{}) { for _, name := range p.TagKeys { //switch any fields in tagkeys into tags if fields[name] == nil { continue } switch value := fields[name].(type) { case string: tags[name] = value delete(fields, name) case bool: tags[name] = strconv.FormatBool(value) delete(fields, name) case float64: tags[name] = strconv.FormatFloat(value, 'f', -1, 64) delete(fields, name) default: log.Printf("E! [parsers.json] Unrecognized type %T", value) } } //remove any additional string/bool values from fields for k := range fields { //check if field is in StringFields sField := false for _, v := range p.StringFields { if v == k { sField = true } } if sField { continue } switch fields[k].(type) { case string: delete(fields, k) case bool: delete(fields, k) } } return tags, fields } func (p *JSONParser) Parse(buf []byte) ([]telegraf.Metric, error) { if p.JSONQuery != "" { result := gjson.GetBytes(buf, p.JSONQuery) buf = []byte(result.Raw) if !result.IsArray() && !result.IsObject() { err := fmt.Errorf("E! Query path must lead to a JSON object or array of objects, but lead to: %v", result.Type) return nil, err } } buf = bytes.TrimSpace(buf) buf = bytes.TrimPrefix(buf, utf8BOM) if len(buf) == 0 { return make([]telegraf.Metric, 0), nil } if !isarray(buf) { metrics := make([]telegraf.Metric, 0) var jsonOut map[string]interface{} err := json.Unmarshal(buf, &jsonOut) if err != nil { err = fmt.Errorf("unable to parse out as JSON, %s", err) return nil, err } return p.parseObject(metrics, jsonOut) } return p.parseArray(buf) } func (p *JSONParser) ParseLine(line string) (telegraf.Metric, error) { metrics, err := p.Parse([]byte(line + "\n")) if err != nil { return nil, err } if len(metrics) < 1 { return nil, fmt.Errorf("can not parse the line: %s, for data format: json ", line) } return metrics[0], nil } func (p *JSONParser) SetDefaultTags(tags map[string]string) { p.DefaultTags = tags } type JSONFlattener struct { Fields map[string]interface{} } // FlattenJSON flattens nested maps/interfaces into a fields map (ignoring bools and string) func (f *JSONFlattener) FlattenJSON( fieldname string, v interface{}) error { if f.Fields == nil { f.Fields = make(map[string]interface{}) } return f.FullFlattenJSON(fieldname, v, false, false) } // FullFlattenJSON flattens nested maps/interfaces into a fields map (including bools and string) func (f *JSONFlattener) FullFlattenJSON( fieldname string, v interface{}, convertString bool, convertBool bool, ) error { if f.Fields == nil { f.Fields = make(map[string]interface{}) } fieldname = strings.Trim(fieldname, "_") switch t := v.(type) { case map[string]interface{}: for k, v := range t { err := f.FullFlattenJSON(fieldname+"_"+k+"_", v, convertString, convertBool) if err != nil { return err } } case []interface{}: for i, v := range t { k := strconv.Itoa(i) err := f.FullFlattenJSON(fieldname+"_"+k+"_", v, convertString, convertBool) if err != nil { return nil } } case float64: f.Fields[fieldname] = t case string: if convertString { f.Fields[fieldname] = v.(string) } else { return nil } case bool: if convertBool { f.Fields[fieldname] = v.(bool) } else { return nil } case nil: return nil default: return fmt.Errorf("JSON Flattener: got unexpected type %T with value %v (%s)", t, t, fieldname) } return nil } func isarray(buf []byte) bool { ia := bytes.IndexByte(buf, '[') ib := bytes.IndexByte(buf, '{') if ia > -1 && ia < ib { return true } else { return false } }