config file change

Makefile

@@ -54,11 +54,11 @@ fmtcheck:
 	@echo '[INFO] done.'
 
 test-windows:
-	go test ./plugins/inputs/ping/...
+	go test -short ./plugins/inputs/ping/...
-	go test ./plugins/inputs/win_perf_counters/...
+	go test -short ./plugins/inputs/win_perf_counters/...
-	go test ./plugins/inputs/win_services/...
+	go test -short ./plugins/inputs/win_services/...
-	go test ./plugins/inputs/procstat/...
+	go test -short ./plugins/inputs/procstat/...
-	go test ./plugins/inputs/ntpq/...
+	go test -short ./plugins/inputs/ntpq/...
 
 # vet runs the Go source code static analysis tool `vet` to find
 # any common errors.
@@ -92,4 +92,4 @@ docker-image:
 plugins/parsers/influx/machine.go: plugins/parsers/influx/machine.go.rl
 	ragel -Z -G2 $^ -o $@
 
-.PHONY: deps telegraf install test test-windows lint vet test-all package clean docker-image fmtcheck uint64
+.PHONY: deps telegraf install test test-windows lint vet test-all package clean docker-image fmtcheck uint64

docs/DATA_FORMATS_INPUT.md

@@ -4,13 +4,12 @@ Telegraf is able to parse the following input data formats into metrics:
 
 1. [InfluxDB Line Protocol](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#influx)
 1. [JSON](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#json)
+1. [GJSON](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#gjson)
 1. [Graphite](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#graphite)
 1. [Value](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#value), ie: 45 or "booyah"
 1. [Nagios](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#nagios) (exec input only)
 1. [Collectd](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#collectd)
 1. [Dropwizard](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#dropwizard)
-1. [Grok](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#grok)
-
 
 Telegraf metrics, like InfluxDB
 [points](https://docs.influxdata.com/influxdb/v0.10/write_protocols/line/),
@@ -207,6 +206,69 @@ exec_mycollector,my_tag_1=foo,my_tag_2=baz a=5,b_c=6
 exec_mycollector,my_tag_1=bar,my_tag_2=baz a=7,b_c=8
 ```
 
+# GJSON:
+GJSON also parses JSON data, but uses paths to name and identify fields of your choosing.
+
+The GJSON parser supports 5 different configuration fields for json objects:
+
+1.'gjson_tag_paths'
+2.'gjson_string_paths'
+3.'gjson_int_paths'
+4.'gjson_float_paths'
+5.'gjson_bool_paths'
+
+Each field is a map type that will map a field_name to a field_path.  Path syntax is described below.
+Path maps should be configured as:
+`toml gjson_tag_paths = {"field_name" = "field.path", "field_name2" = "field.path2"}`
+
+Any paths specified in gjson_tag_paths will be converted to strings and stored as tags.
+Any paths otherwise specified will be their marked type and stored as fields.
+
+#### GJSON Configuration:
+Paths are a series of keys seperated by a dot, ie "obj.sub_obj".
+Paths should not lead to an JSON array, but a single object.  
+An error message will be thrown if a path describes an array.
+Further reading for path syntax can be found here: https://github.com/tidwall/gjson
+
+As an example, if you had the json:
+
+```json
+{
+  "name": {"first": "Tom", "last": "Anderson"},
+  "age":37,
+  "children": ["Sara","Alex","Jack"],
+  "fav.movie": "Deer Hunter",
+  "friends": [
+    {"first": "Dale", "last": "Murphy", "age": 44},
+    {"first": "Roger", "last": "Craig", "age": 68},
+    {"first": "Jane", "last": "Murphy", "age": 47}
+  ]
+}
+```
+with the config:
+
+```toml
+[[inputs.exec]]
+  ## Commands array
+  commands = ["/usr/bin/mycollector --foo=bar"]
+
+  ## Data format to consume.
+  ## Each data format has its own unique set of configuration options, read
+  ## more about them here:
+  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
+  data_format = "gjson"
+
+  name_override = "gjson_sample"
+
+  gjson_tag_paths = {"first_name_tag" = "name.first"}
+  gjson_string_paths = {"last_name" = "name.last"}
+  gjson_int_paths = {"age" = "age", "Janes_age" = "friends.2.age"}
+```
+
+would output the metric:
+`gjson_sample, first_name_tag=Tom last_name=Anderson,age=37,Janes_age=47`
+
+
 # Value:
 
 The "value" data format translates single values into Telegraf metrics. This
@@ -653,37 +715,5 @@ For more information about the dropwizard json format see
   # [inputs.exec.dropwizard_tag_paths]
   #   tag1 = "tags.tag1"
   #   tag2 = "tags.tag2"
-```
 
-#### Grok
+```
-Parse logstash-style "grok" patterns:
-```toml
-  [inputs.reader]
-    ## This is a list of patterns to check the given log file(s) for.
-    ## Note that adding patterns here increases processing time. The most
-    ## efficient configuration is to have one pattern per logparser.
-    ## Other common built-in patterns are:
-    ##   %{COMMON_LOG_FORMAT}   (plain apache & nginx access logs)
-    ##   %{COMBINED_LOG_FORMAT} (access logs + referrer & agent)
-    patterns = ["%{COMBINED_LOG_FORMAT}"]
-
-    ## Name of the outputted measurement name.
-    name_override = "apache_access_log"
-
-    ## Full path(s) to custom pattern files.
-    custom_pattern_files = []
-
-    ## Custom patterns can also be defined here. Put one pattern per line.
-    custom_patterns = '''
-
-    ## Timezone allows you to provide an override for timestamps that
-    ## don't already include an offset
-    ## e.g. 04/06/2016 12:41:45 data one two 5.43µs
-    ##
-    ## Default: "" which renders UTC
-    ## Options are as follows:
-    ##   1. Local             -- interpret based on machine localtime
-    ##   2. "Canada/Eastern"  -- Unix TZ values like those found in https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
-    ##   3. UTC               -- or blank/unspecified, will return timestamp in UTC
-    timezone = "Canada/Eastern"
-```

internal/config/config.go

@@ -211,12 +211,16 @@ var header = `# Telegraf Configuration
 # Environment variables can be used anywhere in this config file, simply prepend
 # them with $. For strings the variable must be within quotes (ie, "$STR_VAR"),
 # for numbers and booleans they should be plain (ie, $INT_VAR, $BOOL_VAR)
+
+
 # Global tags can be specified here in key="value" format.
 [global_tags]
   # dc = "us-east-1" # will tag all metrics with dc=us-east-1
   # rack = "1a"
   ## Environment variables can be used as tags, and throughout the config file
   # user = "$USER"
+
+
 # Configuration for telegraf agent
 [agent]
   ## Default data collection interval for all inputs
@@ -224,20 +228,24 @@ var header = `# Telegraf Configuration
   ## Rounds collection interval to 'interval'
   ## ie, if interval="10s" then always collect on :00, :10, :20, etc.
   round_interval = true
+
   ## Telegraf will send metrics to outputs in batches of at most
   ## metric_batch_size metrics.
   ## This controls the size of writes that Telegraf sends to output plugins.
   metric_batch_size = 1000
+
   ## For failed writes, telegraf will cache metric_buffer_limit metrics for each
   ## output, and will flush this buffer on a successful write. Oldest metrics
   ## are dropped first when this buffer fills.
   ## This buffer only fills when writes fail to output plugin(s).
   metric_buffer_limit = 10000
+
   ## Collection jitter is used to jitter the collection by a random amount.
   ## Each plugin will sleep for a random time within jitter before collecting.
   ## This can be used to avoid many plugins querying things like sysfs at the
   ## same time, which can have a measurable effect on the system.
   collection_jitter = "0s"
+
   ## Default flushing interval for all outputs. You shouldn't set this below
   ## interval. Maximum flush_interval will be flush_interval + flush_jitter
   flush_interval = "10s"
@@ -245,6 +253,7 @@ var header = `# Telegraf Configuration
   ## large write spikes for users running a large number of telegraf instances.
   ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
   flush_jitter = "0s"
+
   ## By default or when set to "0s", precision will be set to the same
   ## timestamp order as the collection interval, with the maximum being 1s.
   ##   ie, when interval = "10s", precision will be "1s"
@@ -253,6 +262,7 @@ var header = `# Telegraf Configuration
   ## service input to set the timestamp at the appropriate precision.
   ## Valid time units are "ns", "us" (or "µs"), "ms", "s".
   precision = ""
+
   ## Logging configuration:
   ## Run telegraf with debug log messages.
   debug = false
@@ -260,34 +270,41 @@ var header = `# Telegraf Configuration
   quiet = false
   ## Specify the log file name. The empty string means to log to stderr.
   logfile = ""
+
   ## Override default hostname, if empty use os.Hostname()
   hostname = ""
   ## If set to true, do no set the "host" tag in the telegraf agent.
   omit_hostname = false
+
+
 ###############################################################################
 #                            OUTPUT PLUGINS                                   #
 ###############################################################################
 `
 
 var processorHeader = `
+
 ###############################################################################
 #                            PROCESSOR PLUGINS                                #
 ###############################################################################
 `
 
 var aggregatorHeader = `
+
 ###############################################################################
 #                            AGGREGATOR PLUGINS                               #
 ###############################################################################
 `
 
 var inputHeader = `
+
 ###############################################################################
 #                            INPUT PLUGINS                                    #
 ###############################################################################
 `
 
 var serviceInputHeader = `
+
 ###############################################################################
 #                            SERVICE INPUT PLUGINS                            #
 ###############################################################################
@@ -1321,55 +1338,67 @@ func buildParser(name string, tbl *ast.Table) (parsers.Parser, error) {
 		}
 	}
 
-	//for grok data_format
+	c.GJSONTagPaths = make(map[string]string)
-	if node, ok := tbl.Fields["named_patterns"]; ok {
+	if node, ok := tbl.Fields["gjson_tag_paths"]; ok {
-		if kv, ok := node.(*ast.KeyValue); ok {
+		if subtbl, ok := node.(*ast.Table); ok {
-			if ary, ok := kv.Value.(*ast.Array); ok {
+			for name, val := range subtbl.Fields {
-				for _, elem := range ary.Value {
+				if kv, ok := val.(*ast.KeyValue); ok {
-					if str, ok := elem.(*ast.String); ok {
+					if str, ok := kv.Value.(*ast.String); ok {
-						c.NamedPatterns = append(c.NamedPatterns, str.Value)
+						c.GJSONTagPaths[name] = str.Value
 					}
 				}
 			}
 		}
 	}
 
-	if node, ok := tbl.Fields["patterns"]; ok {
+	c.GJSONBoolPaths = make(map[string]string)
-		if kv, ok := node.(*ast.KeyValue); ok {
+	if node, ok := tbl.Fields["gjson_bool_paths"]; ok {
-			if ary, ok := kv.Value.(*ast.Array); ok {
+		if subtbl, ok := node.(*ast.Table); ok {
-				for _, elem := range ary.Value {
+			for name, val := range subtbl.Fields {
-					if str, ok := elem.(*ast.String); ok {
+				if kv, ok := val.(*ast.KeyValue); ok {
-						c.Patterns = append(c.Patterns, str.Value)
+					if str, ok := kv.Value.(*ast.String); ok {
+						c.GJSONBoolPaths[name] = str.Value
 					}
 				}
 			}
 		}
 	}
 
-	if node, ok := tbl.Fields["custom_patterns"]; ok {
+	c.GJSONFloatPaths = make(map[string]string)
-		if kv, ok := node.(*ast.KeyValue); ok {
+	if node, ok := tbl.Fields["gjson_float_paths"]; ok {
-			if str, ok := kv.Value.(*ast.String); ok {
+		if subtbl, ok := node.(*ast.Table); ok {
-				c.CustomPatterns = str.Value
+			for name, val := range subtbl.Fields {
-			}
+				if kv, ok := val.(*ast.KeyValue); ok {
-		}
+					if str, ok := kv.Value.(*ast.String); ok {
-	}
+						c.GJSONFloatPaths[name] = str.Value
-
-	if node, ok := tbl.Fields["custom_pattern_files"]; ok {
-		if kv, ok := node.(*ast.KeyValue); ok {
-			if ary, ok := kv.Value.(*ast.Array); ok {
-				for _, elem := range ary.Value {
-					if str, ok := elem.(*ast.String); ok {
-						c.CustomPatternFiles = append(c.CustomPatternFiles, str.Value)
 					}
 				}
 			}
 		}
 	}
 
-	if node, ok := tbl.Fields["timezone"]; ok {
+	c.GJSONStringPaths = make(map[string]string)
-		if kv, ok := node.(*ast.KeyValue); ok {
+	if node, ok := tbl.Fields["gjson_string_paths"]; ok {
-			if str, ok := kv.Value.(*ast.String); ok {
+		if subtbl, ok := node.(*ast.Table); ok {
-				c.TimeZone = str.Value
+			for name, val := range subtbl.Fields {
+				if kv, ok := val.(*ast.KeyValue); ok {
+					if str, ok := kv.Value.(*ast.String); ok {
+						c.GJSONStringPaths[name] = str.Value
+					}
+				}
+			}
+		}
+	}
+
+	c.GJSONIntPaths = make(map[string]string)
+	if node, ok := tbl.Fields["gjson_int_paths"]; ok {
+		if subtbl, ok := node.(*ast.Table); ok {
+			for name, val := range subtbl.Fields {
+				if kv, ok := val.(*ast.KeyValue); ok {
+					if str, ok := kv.Value.(*ast.String); ok {
+						c.GJSONIntPaths[name] = str.Value
+					}
+				}
 			}
 		}
 	}
@@ -1389,11 +1418,11 @@ func buildParser(name string, tbl *ast.Table) (parsers.Parser, error) {
 	delete(tbl.Fields, "dropwizard_time_format")
 	delete(tbl.Fields, "dropwizard_tags_path")
 	delete(tbl.Fields, "dropwizard_tag_paths")
-	delete(tbl.Fields, "named_patterns")
+	delete(tbl.Fields, "gjson_tag_paths")
-	delete(tbl.Fields, "patterns")
+	delete(tbl.Fields, "gjson_bool_paths")
-	delete(tbl.Fields, "custom_patterns")
+	delete(tbl.Fields, "gjson_float_paths")
-	delete(tbl.Fields, "custom_pattern_files")
+	delete(tbl.Fields, "gjson_string_paths")
-	delete(tbl.Fields, "timezone")
+	delete(tbl.Fields, "gjson_int_paths")
 
 	return parsers.NewParser(c)
 }

plugins/inputs/all/all.go

@@ -85,7 +85,6 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/puppetagent"
 	_ "github.com/influxdata/telegraf/plugins/inputs/rabbitmq"
 	_ "github.com/influxdata/telegraf/plugins/inputs/raindrops"
-	_ "github.com/influxdata/telegraf/plugins/inputs/reader"
 	_ "github.com/influxdata/telegraf/plugins/inputs/redis"
 	_ "github.com/influxdata/telegraf/plugins/inputs/rethinkdb"
 	_ "github.com/influxdata/telegraf/plugins/inputs/riak"

plugins/inputs/reader/README.md

@@ -1,23 +0,0 @@
-# Reader Input Plugin
-
-The `reader` plugin reads and parses files every interval.  Reader will always begin at the top of each file.
-
-Reader supports all data_format formats
-
-### Configuration
-
-```toml
-## Files to parse each interval.
-## These accept standard unix glob matching rules, but with the addition of
-## ** as a "super asterisk". ie:
-##   /var/log/**.log     -> recursively find all .log files in /var/log
-##   /var/log/*/*.log    -> find all .log files with a parent dir in /var/log
-##   /var/log/apache.log -> only tail the apache log file
-files = ["/var/log/apache/access.log"]
-
-## The dataformat to be read from files
-## Each data format has its own unique set of configuration options, read
-## more about them here:
-## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
-data_format = ""
-```

plugins/inputs/reader/dev/docker-compose.yml

@@ -1,14 +0,0 @@
-version: '3'
-
-services:
-  telegraf:
-      image: glinton/scratch
-      volumes:
-        - ./telegraf.conf:/telegraf.conf
-        - ../../../../telegraf:/telegraf
-        - ./json_a.log:/var/log/test.log
-      entrypoint:
-        - /telegraf
-        - --config
-        - /telegraf.conf
-        

plugins/inputs/reader/dev/json_a.log

@@ -1,15 +0,0 @@
-{
-		"parent": {
-			"child": 3.0,
-			"ignored_child": "hi"
-		},
-		"ignored_null": null,
-		"integer": 4,
-		"list": [3, 4],
-		"ignored_parent": {
-			"another_ignored_null": null,
-			"ignored_string": "hello, world!"
-		},
-		"another_list": [4]
-	}
-	

plugins/inputs/reader/dev/telegraf.conf

@@ -1,8 +0,0 @@
-[[inputs.reader]]
-  files = ["/var/log/test.log"]
-  data_format = "json"
-  name_override = "json_reader"
-
-[[outputs.file]]
-  files = ["stdout"]
-  

plugins/inputs/reader/reader.go

@@ -1,102 +0,0 @@
-package reader
-
-import (
-	"io/ioutil"
-	"log"
-
-	"github.com/influxdata/telegraf"
-	"github.com/influxdata/telegraf/internal/globpath"
-	"github.com/influxdata/telegraf/plugins/inputs"
-	"github.com/influxdata/telegraf/plugins/parsers"
-)
-
-type Reader struct {
-	Filepaths     []string `toml:"files"`
-	FromBeginning bool
-	parser        parsers.Parser
-
-	Filenames []string
-}
-
-const sampleConfig = `## Files to parse each interval.
-## These accept standard unix glob matching rules, but with the addition of
-## ** as a "super asterisk". ie:
-##   /var/log/**.log     -> recursively find all .log files in /var/log
-##   /var/log/*/*.log    -> find all .log files with a parent dir in /var/log
-##   /var/log/apache.log -> only tail the apache log file
-files = ["/var/log/apache/access.log"]
-
-## The dataformat to be read from files
-## Each data format has its own unique set of configuration options, read
-## more about them here:
-## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
-data_format = ""
-`
-
-// SampleConfig returns the default configuration of the Input
-func (r *Reader) SampleConfig() string {
-	return sampleConfig
-}
-
-func (r *Reader) Description() string {
-	return "reload and gather from file[s] on telegraf's interval"
-}
-
-func (r *Reader) Gather(acc telegraf.Accumulator) error {
-	r.refreshFilePaths()
-	for _, k := range r.Filenames {
-		metrics, err := r.readMetric(k)
-		if err != nil {
-			return err
-		}
-
-		for i, m := range metrics {
-
-			//error if m is nil
-			if m == nil {
-				log.Printf("E! Metric could not be parsed from: %v, on line %v", k, i)
-				continue
-			}
-			acc.AddFields(m.Name(), m.Fields(), m.Tags())
-		}
-	}
-	return nil
-}
-
-func (r *Reader) SetParser(p parsers.Parser) {
-	r.parser = p
-}
-
-func (r *Reader) refreshFilePaths() {
-	var allFiles []string
-	for _, filepath := range r.Filepaths {
-		g, err := globpath.Compile(filepath)
-		if err != nil {
-			log.Printf("E! Error Glob %s failed to compile, %s", filepath, err)
-			continue
-		}
-		files := g.Match()
-
-		for k := range files {
-			allFiles = append(allFiles, k)
-		}
-	}
-
-	r.Filenames = allFiles
-}
-
-//requires that Parser has been compiled
-func (r *Reader) readMetric(filename string) ([]telegraf.Metric, error) {
-	fileContents, err := ioutil.ReadFile(filename)
-	if err != nil {
-		log.Printf("E! File could not be opened: %v", filename)
-	}
-	return r.parser.Parse(fileContents)
-
-}
-
-func init() {
-	inputs.Add("reader", func() telegraf.Input {
-		return &Reader{}
-	})
-}

plugins/inputs/reader/reader_test.go

@@ -1,64 +0,0 @@
-package reader
-
-import (
-	"runtime"
-	"strings"
-	"testing"
-
-	"github.com/influxdata/telegraf/plugins/parsers"
-	"github.com/influxdata/telegraf/testutil"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestRefreshFilePaths(t *testing.T) {
-	testDir := getPluginDir()
-	r := Reader{
-		Filepaths: []string{testDir + "/logparser/grok/testdata/**.log"},
-	}
-
-	r.refreshFilePaths()
-	assert.Equal(t, len(r.Filenames), 2)
-}
-func TestJSONParserCompile(t *testing.T) {
-	testDir := getPluginDir()
-	var acc testutil.Accumulator
-	r := Reader{
-		Filepaths: []string{testDir + "/reader/testfiles/json_a.log"},
-	}
-	parserConfig := parsers.Config{
-		DataFormat: "json",
-		TagKeys:    []string{"parent_ignored_child"},
-	}
-	nParser, err := parsers.NewParser(&parserConfig)
-	r.parser = nParser
-	assert.NoError(t, err)
-
-	r.Gather(&acc)
-	assert.Equal(t, map[string]string{"parent_ignored_child": "hi"}, acc.Metrics[0].Tags)
-	assert.Equal(t, 5, len(acc.Metrics[0].Fields))
-}
-
-func TestGrokParser(t *testing.T) {
-	testDir := getPluginDir()
-	var acc testutil.Accumulator
-	r := Reader{
-		Filepaths: []string{testDir + "/reader/testfiles/grok_a.log"},
-	}
-
-	parserConfig := parsers.Config{
-		DataFormat: "grok",
-		Patterns:   []string{"%{COMMON_LOG_FORMAT}"},
-	}
-
-	nParser, err := parsers.NewParser(&parserConfig)
-	r.parser = nParser
-	assert.NoError(t, err)
-
-	err = r.Gather(&acc)
-	assert.Equal(t, 2, len(acc.Metrics))
-}
-
-func getPluginDir() string {
-	_, filename, _, _ := runtime.Caller(1)
-	return strings.Replace(filename, "/reader/reader_test.go", "", 1)
-}

plugins/inputs/reader/testfiles/grok_a.log

@@ -1,2 +0,0 @@
-127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326
-128.0.0.1 user-identifier tony [10/Oct/2000:13:55:36 -0800] "GET /apache_pb.gif HTTP/1.0" 300 45

plugins/inputs/reader/testfiles/json_a.log

@@ -1,14 +0,0 @@
-{
-		"parent": {
-			"child": 3.0,
-			"ignored_child": "hi"
-		},
-		"ignored_null": null,
-		"integer": 4,
-		"list": [3, 4],
-		"ignored_parent": {
-			"another_ignored_null": null,
-			"ignored_string": "hello, world!"
-		},
-		"another_list": [4]
-	}

plugins/parsers/gjson/parser.go

@@ -0,0 +1,96 @@
+package gjson
+
+import (
+	"log"
+	"time"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/metric"
+	"github.com/tidwall/gjson"
+)
+
+type JSONPath struct {
+	MetricName  string
+	TagPath     map[string]string
+	FloatPath   map[string]string
+	IntPath     map[string]string
+	StrPath     map[string]string
+	BoolPath    map[string]string
+	DefaultTags map[string]string
+}
+
+func (j *JSONPath) Parse(buf []byte) ([]telegraf.Metric, error) {
+	tags := make(map[string]string)
+	for k, v := range j.DefaultTags {
+		tags[k] = v
+	}
+	fields := make(map[string]interface{})
+	metrics := make([]telegraf.Metric, 0)
+
+	for k, v := range j.TagPath {
+		c := gjson.GetBytes(buf, v)
+		if c.IsArray() {
+			log.Printf("E! GJSON cannot assign array to field on path: %v", v)
+			continue
+		}
+		tags[k] = c.String()
+	}
+
+	for k, v := range j.FloatPath {
+		c := gjson.GetBytes(buf, v)
+		if c.IsArray() {
+			log.Printf("E! GJSON cannot assign array to field on path: %v", v)
+			continue
+		}
+		fields[k] = c.Float()
+	}
+
+	for k, v := range j.IntPath {
+		c := gjson.GetBytes(buf, v)
+		if c.IsArray() {
+			log.Printf("E! GJSON cannot assign array to field on path: %v", v)
+			continue
+		}
+		fields[k] = c.Int()
+	}
+
+	for k, v := range j.BoolPath {
+		c := gjson.GetBytes(buf, v)
+		if c.IsArray() {
+			log.Printf("E! GJSON cannot assign array to field on path: %v", v)
+			continue
+		}
+		if c.String() == "true" {
+			fields[k] = true
+		} else if c.String() == "false" {
+			fields[k] = false
+		} else {
+			log.Printf("E! Cannot decode: %v as bool", c.String())
+		}
+	}
+
+	for k, v := range j.StrPath {
+		c := gjson.GetBytes(buf, v)
+		if c.IsArray() {
+			log.Printf("E! GJSON cannot assign array to field on path: %v", v)
+			continue
+		}
+		fields[k] = c.String()
+	}
+
+	m, err := metric.New(j.MetricName, tags, fields, time.Now())
+	if err != nil {
+		return nil, err
+	}
+	metrics = append(metrics, m)
+	return metrics, nil
+}
+
+func (j *JSONPath) ParseLine(str string) (telegraf.Metric, error) {
+	m, err := j.Parse([]byte(str))
+	return m[0], err
+}
+
+func (j *JSONPath) SetDefaultTags(tags map[string]string) {
+	j.DefaultTags = tags
+}

plugins/parsers/gjson/parser_test.go

@@ -0,0 +1,72 @@
+package gjson
+
+import (
+	"log"
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseJsonPath(t *testing.T) {
+	testString := `{
+		"total_devices": 5,
+		"total_threads": 10,
+		"shares": {
+			"total": 5,
+			"accepted": 5,
+			"rejected": 0,
+			"avg_find_time": 4,
+			"tester": "work",
+			"tester2": true,
+			"tester3": {
+				"hello":"sup",
+				"fun":"money",
+				"break":9
+			}
+		}
+	}`
+
+	jsonParser := JSONPath{
+		MetricName: "jsonpather",
+		TagPath:    map[string]string{"hello": "shares.tester3.hello"},
+		BoolPath:   map[string]string{"bool": "shares.tester2"},
+	}
+
+	metrics, err := jsonParser.Parse([]byte(testString))
+	assert.NoError(t, err)
+	log.Printf("m[0] name: %v, tags: %v, fields: %v", metrics[0].Name(), metrics[0].Tags(), metrics[0].Fields())
+
+}
+
+func TestTagTypes(t *testing.T) {
+	testString := `{
+		"total_devices": 5,
+		"total_threads": 10,
+		"shares": {
+			"total": 5,
+			"accepted": 5,
+			"rejected": 0,
+			"my_bool": true,
+			"tester": "work",
+			"tester2": {
+				"hello":"sup",
+				"fun":true,
+				"break":9.97
+			}
+		}
+	}`
+
+	r := JSONPath{
+		TagPath:   map[string]string{"int1": "total_devices", "my_bool": "shares.my_bool"},
+		FloatPath: map[string]string{"total": "shares.total"},
+		BoolPath:  map[string]string{"fun": "shares.tester2.fun"},
+		StrPath:   map[string]string{"hello": "shares.tester2.hello"},
+		IntPath:   map[string]string{"accepted": "shares.accepted"},
+	}
+
+	metrics, err := r.Parse([]byte(testString))
+	log.Printf("m[0] name: %v, tags: %v, fields: %v", metrics[0].Name(), metrics[0].Tags(), metrics[0].Fields())
+	assert.NoError(t, err)
+	assert.Equal(t, true, reflect.DeepEqual(map[string]interface{}{"total": 5.0, "fun": true, "hello": "sup", "accepted": int64(5)}, metrics[0].Fields()))
+}

plugins/parsers/grok/influx-patterns

@@ -1,73 +0,0 @@
-# Captures are a slightly modified version of logstash "grok" patterns, with
-#  the format %{<capture syntax>[:<semantic name>][:<modifier>]}
-# By default all named captures are converted into string fields.
-# Modifiers can be used to convert captures to other types or tags.
-# Timestamp modifiers can be used to convert captures to the timestamp of the
-#  parsed metric.
-
-# View logstash grok pattern docs here:
-#   https://www.elastic.co/guide/en/logstash/current/plugins-filters-grok.html
-# All default logstash patterns are supported, these can be viewed here:
-#   https://github.com/logstash-plugins/logstash-patterns-core/blob/master/patterns/grok-patterns
-
-# Available modifiers:
-#   string   (default if nothing is specified)
-#   int
-#   float
-#   duration (ie, 5.23ms gets converted to int nanoseconds)
-#   tag      (converts the field into a tag)
-#   drop     (drops the field completely)
-# Timestamp modifiers:
-#   ts-ansic         ("Mon Jan _2 15:04:05 2006")
-#   ts-unix          ("Mon Jan _2 15:04:05 MST 2006")
-#   ts-ruby          ("Mon Jan 02 15:04:05 -0700 2006")
-#   ts-rfc822        ("02 Jan 06 15:04 MST")
-#   ts-rfc822z       ("02 Jan 06 15:04 -0700")
-#   ts-rfc850        ("Monday, 02-Jan-06 15:04:05 MST")
-#   ts-rfc1123       ("Mon, 02 Jan 2006 15:04:05 MST")
-#   ts-rfc1123z      ("Mon, 02 Jan 2006 15:04:05 -0700")
-#   ts-rfc3339       ("2006-01-02T15:04:05Z07:00")
-#   ts-rfc3339nano   ("2006-01-02T15:04:05.999999999Z07:00")
-#   ts-httpd         ("02/Jan/2006:15:04:05 -0700")
-#   ts-epoch         (seconds since unix epoch)
-#   ts-epochnano     (nanoseconds since unix epoch)
-#   ts-"CUSTOM"
-# CUSTOM time layouts must be within quotes and be the representation of the
-# "reference time", which is Mon Jan 2 15:04:05 -0700 MST 2006
-# See https://golang.org/pkg/time/#Parse for more details.
-
-# Example log file pattern, example log looks like this:
-#   [04/Jun/2016:12:41:45 +0100] 1.25 200 192.168.1.1 5.432µs
-# Breakdown of the DURATION pattern below:
-#   NUMBER  is a builtin logstash grok pattern matching float & int numbers.
-#   [nuµm]? is a regex specifying 0 or 1 of the characters within brackets.
-#   s       is also regex, this pattern must end in "s".
-# so DURATION will match something like '5.324ms' or '6.1µs' or '10s'
-DURATION %{NUMBER}[nuµm]?s
-RESPONSE_CODE %{NUMBER:response_code:tag}
-RESPONSE_TIME %{DURATION:response_time_ns:duration}
-EXAMPLE_LOG \[%{HTTPDATE:ts:ts-httpd}\] %{NUMBER:myfloat:float} %{RESPONSE_CODE} %{IPORHOST:clientip} %{RESPONSE_TIME}
-
-# Wider-ranging username matching vs. logstash built-in %{USER}
-NGUSERNAME [a-zA-Z0-9\.\@\-\+_%]+
-NGUSER %{NGUSERNAME}
-# Wider-ranging client IP matching
-CLIENT (?:%{IPORHOST}|%{HOSTPORT}|::1)
-
-##
-## COMMON LOG PATTERNS
-##
-
-# apache & nginx logs, this is also known as the "common log format"
-#   see https://en.wikipedia.org/wiki/Common_Log_Format
-COMMON_LOG_FORMAT %{CLIENT:client_ip} %{NOTSPACE:ident} %{NOTSPACE:auth} \[%{HTTPDATE:ts:ts-httpd}\] "(?:%{WORD:verb:tag} %{NOTSPACE:request}(?: HTTP/%{NUMBER:http_version:float})?|%{DATA})" %{NUMBER:resp_code:tag} (?:%{NUMBER:resp_bytes:int}|-)
-
-# Combined log format is the same as the common log format but with the addition
-# of two quoted strings at the end for "referrer" and "agent"
-#   See Examples at http://httpd.apache.org/docs/current/mod/mod_log_config.html
-COMBINED_LOG_FORMAT %{COMMON_LOG_FORMAT} %{QS:referrer} %{QS:agent}
-
-# HTTPD log formats
-HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel:tag}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg}
-HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel:tag}\] \[pid %{POSINT:pid:int}:tid %{NUMBER:tid:int}\]( \(%{POSINT:proxy_errorcode:int}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}
-HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}

plugins/parsers/grok/influx_patterns.go

@@ -1,78 +0,0 @@
-package grok
-
-// DEFAULT_PATTERNS SHOULD BE KEPT IN-SYNC WITH patterns/influx-patterns
-const DEFAULT_PATTERNS = `
-# Captures are a slightly modified version of logstash "grok" patterns, with
-#  the format %{<capture syntax>[:<semantic name>][:<modifier>]}
-# By default all named captures are converted into string fields.
-# Modifiers can be used to convert captures to other types or tags.
-# Timestamp modifiers can be used to convert captures to the timestamp of the
-#  parsed metric.
-
-# View logstash grok pattern docs here:
-#   https://www.elastic.co/guide/en/logstash/current/plugins-filters-grok.html
-# All default logstash patterns are supported, these can be viewed here:
-#   https://github.com/logstash-plugins/logstash-patterns-core/blob/master/patterns/grok-patterns
-
-# Available modifiers:
-#   string   (default if nothing is specified)
-#   int
-#   float
-#   duration (ie, 5.23ms gets converted to int nanoseconds)
-#   tag      (converts the field into a tag)
-#   drop     (drops the field completely)
-# Timestamp modifiers:
-#   ts-ansic         ("Mon Jan _2 15:04:05 2006")
-#   ts-unix          ("Mon Jan _2 15:04:05 MST 2006")
-#   ts-ruby          ("Mon Jan 02 15:04:05 -0700 2006")
-#   ts-rfc822        ("02 Jan 06 15:04 MST")
-#   ts-rfc822z       ("02 Jan 06 15:04 -0700")
-#   ts-rfc850        ("Monday, 02-Jan-06 15:04:05 MST")
-#   ts-rfc1123       ("Mon, 02 Jan 2006 15:04:05 MST")
-#   ts-rfc1123z      ("Mon, 02 Jan 2006 15:04:05 -0700")
-#   ts-rfc3339       ("2006-01-02T15:04:05Z07:00")
-#   ts-rfc3339nano   ("2006-01-02T15:04:05.999999999Z07:00")
-#   ts-httpd         ("02/Jan/2006:15:04:05 -0700")
-#   ts-epoch         (seconds since unix epoch)
-#   ts-epochnano     (nanoseconds since unix epoch)
-#   ts-"CUSTOM"
-# CUSTOM time layouts must be within quotes and be the representation of the
-# "reference time", which is Mon Jan 2 15:04:05 -0700 MST 2006
-# See https://golang.org/pkg/time/#Parse for more details.
-
-# Example log file pattern, example log looks like this:
-#   [04/Jun/2016:12:41:45 +0100] 1.25 200 192.168.1.1 5.432µs
-# Breakdown of the DURATION pattern below:
-#   NUMBER  is a builtin logstash grok pattern matching float & int numbers.
-#   [nuµm]? is a regex specifying 0 or 1 of the characters within brackets.
-#   s       is also regex, this pattern must end in "s".
-# so DURATION will match something like '5.324ms' or '6.1µs' or '10s'
-DURATION %{NUMBER}[nuµm]?s
-RESPONSE_CODE %{NUMBER:response_code:tag}
-RESPONSE_TIME %{DURATION:response_time_ns:duration}
-EXAMPLE_LOG \[%{HTTPDATE:ts:ts-httpd}\] %{NUMBER:myfloat:float} %{RESPONSE_CODE} %{IPORHOST:clientip} %{RESPONSE_TIME}
-
-# Wider-ranging username matching vs. logstash built-in %{USER}
-NGUSERNAME [a-zA-Z0-9\.\@\-\+_%]+
-NGUSER %{NGUSERNAME}
-# Wider-ranging client IP matching
-CLIENT (?:%{IPV6}|%{IPV4}|%{HOSTNAME}|%{HOSTPORT})
-
-##
-## COMMON LOG PATTERNS
-##
-
-# apache & nginx logs, this is also known as the "common log format"
-#   see https://en.wikipedia.org/wiki/Common_Log_Format
-COMMON_LOG_FORMAT %{CLIENT:client_ip} %{NOTSPACE:ident} %{NOTSPACE:auth} \[%{HTTPDATE:ts:ts-httpd}\] "(?:%{WORD:verb:tag} %{NOTSPACE:request}(?: HTTP/%{NUMBER:http_version:float})?|%{DATA})" %{NUMBER:resp_code:tag} (?:%{NUMBER:resp_bytes:int}|-)
-
-# Combined log format is the same as the common log format but with the addition
-# of two quoted strings at the end for "referrer" and "agent"
-#   See Examples at http://httpd.apache.org/docs/current/mod/mod_log_config.html
-COMBINED_LOG_FORMAT %{COMMON_LOG_FORMAT} %{QS:referrer} %{QS:agent}
-
-# HTTPD log formats
-HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel:tag}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg}
-HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel:tag}\] \[pid %{POSINT:pid:int}:tid %{NUMBER:tid:int}\]( \(%{POSINT:proxy_errorcode:int}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}
-HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}
-`

plugins/parsers/grok/parser.go

@@ -1,527 +0,0 @@
-package grok
-
-import (
-	"bufio"
-	"fmt"
-	"log"
-	"os"
-	"regexp"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/vjeantet/grok"
-
-	"github.com/influxdata/telegraf"
-	"github.com/influxdata/telegraf/metric"
-)
-
-var timeLayouts = map[string]string{
-	"ts-ansic":       "Mon Jan _2 15:04:05 2006",
-	"ts-unix":        "Mon Jan _2 15:04:05 MST 2006",
-	"ts-ruby":        "Mon Jan 02 15:04:05 -0700 2006",
-	"ts-rfc822":      "02 Jan 06 15:04 MST",
-	"ts-rfc822z":     "02 Jan 06 15:04 -0700", // RFC822 with numeric zone
-	"ts-rfc850":      "Monday, 02-Jan-06 15:04:05 MST",
-	"ts-rfc1123":     "Mon, 02 Jan 2006 15:04:05 MST",
-	"ts-rfc1123z":    "Mon, 02 Jan 2006 15:04:05 -0700", // RFC1123 with numeric zone
-	"ts-rfc3339":     "2006-01-02T15:04:05Z07:00",
-	"ts-rfc3339nano": "2006-01-02T15:04:05.999999999Z07:00",
-	"ts-httpd":       "02/Jan/2006:15:04:05 -0700",
-	// These three are not exactly "layouts", but they are special cases that
-	// will get handled in the ParseLine function.
-	"ts-epoch":     "EPOCH",
-	"ts-epochnano": "EPOCH_NANO",
-	"ts-syslog":    "SYSLOG_TIMESTAMP",
-	"ts":           "GENERIC_TIMESTAMP", // try parsing all known timestamp layouts.
-}
-
-const (
-	INT               = "int"
-	TAG               = "tag"
-	FLOAT             = "float"
-	STRING            = "string"
-	DURATION          = "duration"
-	DROP              = "drop"
-	EPOCH             = "EPOCH"
-	EPOCH_NANO        = "EPOCH_NANO"
-	SYSLOG_TIMESTAMP  = "SYSLOG_TIMESTAMP"
-	GENERIC_TIMESTAMP = "GENERIC_TIMESTAMP"
-)
-
-var (
-	// matches named captures that contain a modifier.
-	//   ie,
-	//     %{NUMBER:bytes:int}
-	//     %{IPORHOST:clientip:tag}
-	//     %{HTTPDATE:ts1:ts-http}
-	//     %{HTTPDATE:ts2:ts-"02 Jan 06 15:04"}
-	modifierRe = regexp.MustCompile(`%{\w+:(\w+):(ts-".+"|t?s?-?\w+)}`)
-	// matches a plain pattern name. ie, %{NUMBER}
-	patternOnlyRe = regexp.MustCompile(`%{(\w+)}`)
-)
-
-// Parser is the primary struct to handle and grok-patterns defined in the config toml
-type Parser struct {
-	Patterns []string
-	// namedPatterns is a list of internally-assigned names to the patterns
-	// specified by the user in Patterns.
-	// They will look like:
-	//   GROK_INTERNAL_PATTERN_0, GROK_INTERNAL_PATTERN_1, etc.
-	NamedPatterns      []string
-	CustomPatterns     string
-	CustomPatternFiles []string
-	Measurement        string
-
-	// Timezone is an optional component to help render log dates to
-	// your chosen zone.
-	// Default: "" which renders UTC
-	// Options are as follows:
-	// 1. Local             -- interpret based on machine localtime
-	// 2. "America/Chicago" -- Unix TZ values like those found in https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
-	// 3. UTC               -- or blank/unspecified, will return timestamp in UTC
-	Timezone string
-	loc      *time.Location
-
-	// typeMap is a map of patterns -> capture name -> modifier,
-	//   ie, {
-	//          "%{TESTLOG}":
-	//             {
-	//                "bytes": "int",
-	//                "clientip": "tag"
-	//             }
-	//       }
-	typeMap map[string]map[string]string
-	// tsMap is a map of patterns -> capture name -> timestamp layout.
-	//   ie, {
-	//          "%{TESTLOG}":
-	//             {
-	//                "httptime": "02/Jan/2006:15:04:05 -0700"
-	//             }
-	//       }
-	tsMap map[string]map[string]string
-	// patterns is a map of all of the parsed patterns from CustomPatterns
-	// and CustomPatternFiles.
-	//   ie, {
-	//          "DURATION":      "%{NUMBER}[nuµm]?s"
-	//          "RESPONSE_CODE": "%{NUMBER:rc:tag}"
-	//       }
-	patterns map[string]string
-	// foundTsLayouts is a slice of timestamp patterns that have been found
-	// in the log lines. This slice gets updated if the user uses the generic
-	// 'ts' modifier for timestamps. This slice is checked first for matches,
-	// so that previously-matched layouts get priority over all other timestamp
-	// layouts.
-	foundTsLayouts []string
-
-	timeFunc func() time.Time
-	g        *grok.Grok
-	tsModder *tsModder
-}
-
-// Compile is a bound method to Parser which will process the options for our parser
-func (p *Parser) Compile() error {
-	p.typeMap = make(map[string]map[string]string)
-	p.tsMap = make(map[string]map[string]string)
-	p.patterns = make(map[string]string)
-	p.tsModder = &tsModder{}
-	var err error
-	p.g, err = grok.NewWithConfig(&grok.Config{NamedCapturesOnly: true})
-	if err != nil {
-		return err
-	}
-
-	// Give Patterns fake names so that they can be treated as named
-	// "custom patterns"
-	p.NamedPatterns = make([]string, 0, len(p.Patterns))
-	for i, pattern := range p.Patterns {
-		pattern = strings.TrimSpace(pattern)
-		if pattern == "" {
-			continue
-		}
-		name := fmt.Sprintf("GROK_INTERNAL_PATTERN_%d", i)
-		p.CustomPatterns += "\n" + name + " " + pattern + "\n"
-		p.NamedPatterns = append(p.NamedPatterns, "%{"+name+"}")
-	}
-
-	if len(p.NamedPatterns) == 0 {
-		return fmt.Errorf("pattern required")
-	}
-
-	// Combine user-supplied CustomPatterns with DEFAULT_PATTERNS and parse
-	// them together as the same type of pattern.
-	p.CustomPatterns = DEFAULT_PATTERNS + p.CustomPatterns
-	if len(p.CustomPatterns) != 0 {
-		scanner := bufio.NewScanner(strings.NewReader(p.CustomPatterns))
-		p.addCustomPatterns(scanner)
-	}
-
-	// Parse any custom pattern files supplied.
-	for _, filename := range p.CustomPatternFiles {
-		file, fileErr := os.Open(filename)
-		if fileErr != nil {
-			return fileErr
-		}
-
-		scanner := bufio.NewScanner(bufio.NewReader(file))
-		p.addCustomPatterns(scanner)
-	}
-
-	if p.Measurement == "" {
-		p.Measurement = "logparser_grok"
-	}
-
-	p.loc, err = time.LoadLocation(p.Timezone)
-	if err != nil {
-		log.Printf("W! improper timezone supplied (%s), setting loc to UTC", p.Timezone)
-		p.loc, _ = time.LoadLocation("UTC")
-	}
-
-	if p.timeFunc == nil {
-		p.timeFunc = time.Now
-	}
-
-	return p.compileCustomPatterns()
-}
-
-// ParseLine is the primary function to process individual lines, returning the metrics
-func (p *Parser) ParseLine(line string) (telegraf.Metric, error) {
-	var err error
-	// values are the parsed fields from the log line
-	var values map[string]string
-	// the matching pattern string
-	var patternName string
-	for _, pattern := range p.NamedPatterns {
-		if values, err = p.g.Parse(pattern, line); err != nil {
-			return nil, err
-		}
-		if len(values) != 0 {
-			patternName = pattern
-			break
-		}
-	}
-
-	if len(values) == 0 {
-		log.Printf("D! Grok no match found for: %q", line)
-		return nil, nil
-	}
-
-	fields := make(map[string]interface{})
-	tags := make(map[string]string)
-	timestamp := time.Now()
-	for k, v := range values {
-		if k == "" || v == "" {
-			continue
-		}
-
-		// t is the modifier of the field
-		var t string
-		// check if pattern has some modifiers
-		if types, ok := p.typeMap[patternName]; ok {
-			t = types[k]
-		}
-		// if we didn't find a modifier, check if we have a timestamp layout
-		if t == "" {
-			if ts, ok := p.tsMap[patternName]; ok {
-				// check if the modifier is a timestamp layout
-				if layout, ok := ts[k]; ok {
-					t = layout
-				}
-			}
-		}
-		// if we didn't find a type OR timestamp modifier, assume string
-		if t == "" {
-			t = STRING
-		}
-
-		switch t {
-		case INT:
-			iv, err := strconv.ParseInt(v, 10, 64)
-			if err != nil {
-				log.Printf("E! Error parsing %s to int: %s", v, err)
-			} else {
-				fields[k] = iv
-			}
-		case FLOAT:
-			fv, err := strconv.ParseFloat(v, 64)
-			if err != nil {
-				log.Printf("E! Error parsing %s to float: %s", v, err)
-			} else {
-				fields[k] = fv
-			}
-		case DURATION:
-			d, err := time.ParseDuration(v)
-			if err != nil {
-				log.Printf("E! Error parsing %s to duration: %s", v, err)
-			} else {
-				fields[k] = int64(d)
-			}
-		case TAG:
-			tags[k] = v
-		case STRING:
-			fields[k] = strings.Trim(v, `"`)
-		case EPOCH:
-			parts := strings.SplitN(v, ".", 2)
-			if len(parts) == 0 {
-				log.Printf("E! Error parsing %s to timestamp: %s", v, err)
-				break
-			}
-
-			sec, err := strconv.ParseInt(parts[0], 10, 64)
-			if err != nil {
-				log.Printf("E! Error parsing %s to timestamp: %s", v, err)
-				break
-			}
-			ts := time.Unix(sec, 0)
-
-			if len(parts) == 2 {
-				padded := fmt.Sprintf("%-9s", parts[1])
-				nsString := strings.Replace(padded[:9], " ", "0", -1)
-				nanosec, err := strconv.ParseInt(nsString, 10, 64)
-				if err != nil {
-					log.Printf("E! Error parsing %s to timestamp: %s", v, err)
-					break
-				}
-				ts = ts.Add(time.Duration(nanosec) * time.Nanosecond)
-			}
-			timestamp = ts
-		case EPOCH_NANO:
-			iv, err := strconv.ParseInt(v, 10, 64)
-			if err != nil {
-				log.Printf("E! Error parsing %s to int: %s", v, err)
-			} else {
-				timestamp = time.Unix(0, iv)
-			}
-		case SYSLOG_TIMESTAMP:
-			ts, err := time.ParseInLocation("Jan 02 15:04:05", v, p.loc)
-			if err == nil {
-				if ts.Year() == 0 {
-					ts = ts.AddDate(timestamp.Year(), 0, 0)
-				}
-				timestamp = ts
-			} else {
-				log.Printf("E! Error parsing %s to time layout [%s]: %s", v, t, err)
-			}
-		case GENERIC_TIMESTAMP:
-			var foundTs bool
-			// first try timestamp layouts that we've already found
-			for _, layout := range p.foundTsLayouts {
-				ts, err := time.ParseInLocation(layout, v, p.loc)
-				if err == nil {
-					timestamp = ts
-					foundTs = true
-					break
-				}
-			}
-			// if we haven't found a timestamp layout yet, try all timestamp
-			// layouts.
-			if !foundTs {
-				for _, layout := range timeLayouts {
-					ts, err := time.ParseInLocation(layout, v, p.loc)
-					if err == nil {
-						timestamp = ts
-						foundTs = true
-						p.foundTsLayouts = append(p.foundTsLayouts, layout)
-						break
-					}
-				}
-			}
-			// if we still haven't found a timestamp layout, log it and we will
-			// just use time.Now()
-			if !foundTs {
-				log.Printf("E! Error parsing timestamp [%s], could not find any "+
-					"suitable time layouts.", v)
-			}
-		case DROP:
-		// goodbye!
-		default:
-			ts, err := time.ParseInLocation(t, v, p.loc)
-			if err == nil {
-				timestamp = ts
-			} else {
-				log.Printf("E! Error parsing %s to time layout [%s]: %s", v, t, err)
-			}
-		}
-	}
-
-	if len(fields) == 0 {
-		return nil, fmt.Errorf("logparser_grok: must have one or more fields")
-	}
-
-	return metric.New(p.Measurement, tags, fields, p.tsModder.tsMod(timestamp))
-}
-
-func (p *Parser) Parse(buf []byte) ([]telegraf.Metric, error) {
-	lines := strings.Split(string(buf), "\n")
-	var metrics []telegraf.Metric
-
-	for _, line := range lines {
-		m, err := p.ParseLine(line)
-		if err != nil {
-			return nil, err
-		}
-		metrics = append(metrics, m)
-	}
-
-	return metrics, nil
-}
-
-func (p *Parser) SetDefaultTags(tags map[string]string) {
-	//needs implementation
-}
-
-func (p *Parser) addCustomPatterns(scanner *bufio.Scanner) {
-	for scanner.Scan() {
-		line := strings.TrimSpace(scanner.Text())
-		if len(line) > 0 && line[0] != '#' {
-			names := strings.SplitN(line, " ", 2)
-			p.patterns[names[0]] = names[1]
-		}
-	}
-}
-
-func (p *Parser) compileCustomPatterns() error {
-	var err error
-	// check if the pattern contains a subpattern that is already defined
-	// replace it with the subpattern for modifier inheritance.
-	for i := 0; i < 2; i++ {
-		for name, pattern := range p.patterns {
-			subNames := patternOnlyRe.FindAllStringSubmatch(pattern, -1)
-			for _, subName := range subNames {
-				if subPattern, ok := p.patterns[subName[1]]; ok {
-					pattern = strings.Replace(pattern, subName[0], subPattern, 1)
-				}
-			}
-			p.patterns[name] = pattern
-		}
-	}
-
-	// check if pattern contains modifiers. Parse them out if it does.
-	for name, pattern := range p.patterns {
-		if modifierRe.MatchString(pattern) {
-			// this pattern has modifiers, so parse out the modifiers
-			pattern, err = p.parseTypedCaptures(name, pattern)
-			if err != nil {
-				return err
-			}
-			p.patterns[name] = pattern
-		}
-	}
-
-	return p.g.AddPatternsFromMap(p.patterns)
-}
-
-// parseTypedCaptures parses the capture modifiers, and then deletes the
-// modifier from the line so that it is a valid "grok" pattern again.
-//   ie,
-//     %{NUMBER:bytes:int}      => %{NUMBER:bytes}      (stores %{NUMBER}->bytes->int)
-//     %{IPORHOST:clientip:tag} => %{IPORHOST:clientip} (stores %{IPORHOST}->clientip->tag)
-func (p *Parser) parseTypedCaptures(name, pattern string) (string, error) {
-	matches := modifierRe.FindAllStringSubmatch(pattern, -1)
-
-	// grab the name of the capture pattern
-	patternName := "%{" + name + "}"
-	// create type map for this pattern
-	p.typeMap[patternName] = make(map[string]string)
-	p.tsMap[patternName] = make(map[string]string)
-
-	// boolean to verify that each pattern only has a single ts- data type.
-	hasTimestamp := false
-	for _, match := range matches {
-		// regex capture 1 is the name of the capture
-		// regex capture 2 is the modifier of the capture
-		if strings.HasPrefix(match[2], "ts") {
-			if hasTimestamp {
-				return pattern, fmt.Errorf("logparser pattern compile error: "+
-					"Each pattern is allowed only one named "+
-					"timestamp data type. pattern: %s", pattern)
-			}
-			if layout, ok := timeLayouts[match[2]]; ok {
-				// built-in time format
-				p.tsMap[patternName][match[1]] = layout
-			} else {
-				// custom time format
-				p.tsMap[patternName][match[1]] = strings.TrimSuffix(strings.TrimPrefix(match[2], `ts-"`), `"`)
-			}
-			hasTimestamp = true
-		} else {
-			p.typeMap[patternName][match[1]] = match[2]
-		}
-
-		// the modifier is not a valid part of a "grok" pattern, so remove it
-		// from the pattern.
-		pattern = strings.Replace(pattern, ":"+match[2]+"}", "}", 1)
-	}
-
-	return pattern, nil
-}
-
-// tsModder is a struct for incrementing identical timestamps of log lines
-// so that we don't push identical metrics that will get overwritten.
-type tsModder struct {
-	dupe     time.Time
-	last     time.Time
-	incr     time.Duration
-	incrn    time.Duration
-	rollover time.Duration
-}
-
-// tsMod increments the given timestamp one unit more from the previous
-// duplicate timestamp.
-// the increment unit is determined as the next smallest time unit below the
-// most significant time unit of ts.
-//   ie, if the input is at ms precision, it will increment it 1µs.
-func (t *tsModder) tsMod(ts time.Time) time.Time {
-	defer func() { t.last = ts }()
-	// don't mod the time if we don't need to
-	if t.last.IsZero() || ts.IsZero() {
-		t.incrn = 0
-		t.rollover = 0
-		return ts
-	}
-	if !ts.Equal(t.last) && !ts.Equal(t.dupe) {
-		t.incr = 0
-		t.incrn = 0
-		t.rollover = 0
-		return ts
-	}
-
-	if ts.Equal(t.last) {
-		t.dupe = ts
-	}
-
-	if ts.Equal(t.dupe) && t.incr == time.Duration(0) {
-		tsNano := ts.UnixNano()
-
-		d := int64(10)
-		counter := 1
-		for {
-			a := tsNano % d
-			if a > 0 {
-				break
-			}
-			d = d * 10
-			counter++
-		}
-
-		switch {
-		case counter <= 6:
-			t.incr = time.Nanosecond
-		case counter <= 9:
-			t.incr = time.Microsecond
-		case counter > 9:
-			t.incr = time.Millisecond
-		}
-	}
-
-	t.incrn++
-	if t.incrn == 999 && t.incr > time.Nanosecond {
-		t.rollover = t.incr * t.incrn
-		t.incrn = 1
-		t.incr = t.incr / 1000
-		if t.incr < time.Nanosecond {
-			t.incr = time.Nanosecond
-		}
-	}
-	return ts.Add(t.incr*t.incrn + t.rollover)
-}

plugins/parsers/grok/parser_test.go

@@ -1,19 +0,0 @@
-package grok
-
-import (
-	"log"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestGrokParse(t *testing.T) {
-	parser := Parser{
-		Measurement: "t_met",
-		Patterns:    []string{"%{COMMON_LOG_FORMAT}"},
-	}
-	parser.Compile()
-	metrics, err := parser.Parse([]byte(`127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326`))
-	log.Printf("metric_tags: %v, metric_fields: %v", metrics[0].Tags(), metrics[0].Fields())
-	assert.NoError(t, err)
-}

plugins/parsers/registry.go

@@ -7,8 +7,8 @@ import (
 
 	"github.com/influxdata/telegraf/plugins/parsers/collectd"
 	"github.com/influxdata/telegraf/plugins/parsers/dropwizard"
+	"github.com/influxdata/telegraf/plugins/parsers/gjson"
 	"github.com/influxdata/telegraf/plugins/parsers/graphite"
-	"github.com/influxdata/telegraf/plugins/parsers/grok"
 	"github.com/influxdata/telegraf/plugins/parsers/influx"
 	"github.com/influxdata/telegraf/plugins/parsers/json"
 	"github.com/influxdata/telegraf/plugins/parsers/nagios"
@@ -89,12 +89,12 @@ type Config struct {
 	// used if TagsPath is empty or doesn't return any tags
 	DropwizardTagPathsMap map[string]string
 
-	//grok patterns
+	//for gjson format
-	Patterns           []string
+	GJSONTagPaths    map[string]string
-	NamedPatterns      []string
+	GJSONBoolPaths   map[string]string
-	CustomPatterns     string
+	GJSONFloatPaths  map[string]string
-	CustomPatternFiles []string
+	GJSONStringPaths map[string]string
-	TimeZone           string
+	GJSONIntPaths    map[string]string
 }
 
 // NewParser returns a Parser interface based on the given config.
@@ -129,36 +129,34 @@ func NewParser(config *Config) (Parser, error) {
 			config.Separator,
 			config.Templates)
 
-	case "grok":
+	case "gjson":
-		parser, err = NewGrokParser(
+		parser, err = newGJSONParser(config.MetricName,
-			config.MetricName,
+			config.GJSONTagPaths,
-			config.Patterns,
+			config.GJSONStringPaths,
-			config.NamedPatterns,
+			config.GJSONBoolPaths,
-			config.CustomPatterns,
+			config.GJSONFloatPaths,
-			config.CustomPatternFiles,
+			config.GJSONIntPaths)
-			config.TimeZone)
 	default:
 		err = fmt.Errorf("Invalid data format: %s", config.DataFormat)
 	}
 	return parser, err
 }
 
-func NewGrokParser(metricName string,
+func newGJSONParser(metricName string,
-	patterns []string,
+	tagPaths map[string]string,
-	nPatterns []string,
+	strPaths map[string]string,
-	cPatterns string,
+	boolPaths map[string]string,
-	cPatternFiles []string, tZone string) (Parser, error) {
+	floatPaths map[string]string,
-	parser := grok.Parser{
+	intPaths map[string]string) (Parser, error) {
-		Measurement:        metricName,
+	parser := &gjson.JSONPath{
-		Patterns:           patterns,
+		MetricName: metricName,
-		NamedPatterns:      nPatterns,
+		TagPath:    tagPaths,
-		CustomPatterns:     cPatterns,
+		StrPath:    strPaths,
-		CustomPatternFiles: cPatternFiles,
+		BoolPath:   boolPaths,
-		Timezone:           tZone,
+		FloatPath:  floatPaths,
+		IntPath:    intPaths,
 	}
-
+	return parser, nil
-	parser.Compile()
-	return &parser, nil
 }
 
 func NewJSONParser(

telegraf.conf

@@ -1,104 +0,0 @@
-
-# Global tags can be specified here in key="value" format.
-[global_tags]
-  # dc = "us-east-1" # will tag all metrics with dc=us-east-1
-  # rack = "1a"
-  ## Environment variables can be used as tags, and throughout the config file
-  # user = "$USER"
-
-
-# Configuration for telegraf agent
-[agent]
-  ## Default data collection interval for all inputs
-  interval = "10s"
-  ## Rounds collection interval to 'interval'
-  ## ie, if interval="10s" then always collect on :00, :10, :20, etc.
-  round_interval = true
-
-  ## Telegraf will send metrics to outputs in batches of at most
-  ## metric_batch_size metrics.
-  ## This controls the size of writes that Telegraf sends to output plugins.
-  metric_batch_size = 1000
-
-  ## For failed writes, telegraf will cache metric_buffer_limit metrics for each
-  ## output, and will flush this buffer on a successful write. Oldest metrics
-  ## are dropped first when this buffer fills.
-  ## This buffer only fills when writes fail to output plugin(s).
-  metric_buffer_limit = 10000
-
-  ## Collection jitter is used to jitter the collection by a random amount.
-  ## Each plugin will sleep for a random time within jitter before collecting.
-  ## This can be used to avoid many plugins querying things like sysfs at the
-  ## same time, which can have a measurable effect on the system.
-  collection_jitter = "0s"
-
-  ## Default flushing interval for all outputs. You shouldn't set this below
-  ## interval. Maximum flush_interval will be flush_interval + flush_jitter
-  flush_interval = "10s"
-  ## Jitter the flush interval by a random amount. This is primarily to avoid
-  ## large write spikes for users running a large number of telegraf instances.
-  ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
-  flush_jitter = "0s"
-
-  ## By default or when set to "0s", precision will be set to the same
-  ## timestamp order as the collection interval, with the maximum being 1s.
-  ##   ie, when interval = "10s", precision will be "1s"
-  ##       when interval = "250ms", precision will be "1ms"
-  ## Precision will NOT be used for service inputs. It is up to each individual
-  ## service input to set the timestamp at the appropriate precision.
-  ## Valid time units are "ns", "us" (or "µs"), "ms", "s".
-  precision = ""
-
-  ## Logging configuration:
-  ## Run telegraf with debug log messages.
-  debug = false
-  ## Run telegraf in quiet mode (error log messages only).
-  quiet = false
-  ## Specify the log file name. The empty string means to log to stderr.
-  logfile = ""
-
-  ## Override default hostname, if empty use os.Hostname()
-  hostname = ""
-  ## If set to true, do no set the "host" tag in the telegraf agent.
-  omit_hostname = false
-
-# # reload and gather from file[s] on telegraf's interval
-[[inputs.reader]]
-# ## These accept standard unix glob matching rules, but with the addition of
-# ## ** as a "super asterisk". ie:
-# ##   /var/log/**.log     -> recursively find all .log files in /var/log
-# ##   /var/log/*/*.log    -> find all .log files with a parent dir in /var/log
-# ##   /var/log/apache.log -> only tail the apache log file
-files = ["/Users/maxu/go/src/github.com/influxdata/telegraf/plugins/inputs/logparser/grok/testdata/**.log"]
-#
-# ## The dataformat to be read from files
-# ## Each data format has its own unique set of configuration options, read
-# ## more about them here:
-# ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
-data_format = "grok"
-#
-
-patterns = ["%{TEST_LOG_B}","%{TEST_LOG_A}"]
-#
-#   ## Name of the outputted measurement name.
-name_override = "grok_reader"
-#
-#   ## Full path(s) to custom pattern files.
-custom_pattern_files = ["/Users/maxu/go/src/github.com/influxdata/telegraf/plugins/inputs/logparser/grok/testdata/test-patterns"]
-#
-#   ## Custom patterns can also be defined here. Put one pattern per line.
-#   custom_patterns = '''
-#   '''
-#
-#   ## Timezone allows you to provide an override for timestamps that
-#   ## don't already include an offset
-#   ## e.g. 04/06/2016 12:41:45 data one two 5.43µs
-#   ##
-#   ## Default: "" which renders UTC
-#   ## Options are as follows:
-#   ##   1. Local             -- interpret based on machine localtime
-#   ##   2. "Canada/Eastern"  -- Unix TZ values like those found in https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
-#   ##   3. UTC               -- or blank/unspecified, will return timestamp in UTC
-#   timezone = "Canada/Eastern"
-
-