Add option to disable timestamp adjustment in grok parser (#5488)

internal/config/config.go

@@ -1508,6 +1508,14 @@ func getParserConfig(name string, tbl *ast.Table) (*parsers.Config, error) {
 		}
 	}
 
+	if node, ok := tbl.Fields["grok_unique_timestamp"]; ok {
+		if kv, ok := node.(*ast.KeyValue); ok {
+			if str, ok := kv.Value.(*ast.String); ok {
+				c.GrokUniqueTimestamp = str.Value
+			}
+		}
+	}
+
 	//for csv parser
 	if node, ok := tbl.Fields["csv_column_names"]; ok {
 		if kv, ok := node.(*ast.KeyValue); ok {
@@ -1661,6 +1669,7 @@ func getParserConfig(name string, tbl *ast.Table) (*parsers.Config, error) {
 	delete(tbl.Fields, "grok_custom_patterns")
 	delete(tbl.Fields, "grok_custom_pattern_files")
 	delete(tbl.Fields, "grok_timezone")
+	delete(tbl.Fields, "grok_unique_timestamp")
 	delete(tbl.Fields, "csv_column_names")
 	delete(tbl.Fields, "csv_column_types")
 	delete(tbl.Fields, "csv_comment")

plugins/parsers/grok/README.md

@@ -110,6 +110,9 @@ you will find the https://grokdebug.herokuapp.com application quite useful!
   ##   2. "Canada/Eastern"  -- Unix TZ values like those found in https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
   ##   3. UTC               -- or blank/unspecified, will return timestamp in UTC
   grok_timezone = "Canada/Eastern"
+
+  ## When grok_unique_timestamp is set to "disable", timestamp will not incremented if there is a duplicate. Default is "auto"
+  # grok_unique_timestamp = "auto"
 ```
 
 #### Timestamp Examples

plugins/parsers/grok/parser.go

@@ -86,6 +86,9 @@ type Parser struct {
 	Timezone string
 	loc      *time.Location
 
+	// UniqueTimestamp when set to "disable", timestamp will not incremented if there is a duplicate.
+	UniqueTimestamp string
+
 	// typeMap is a map of patterns -> capture name -> modifier,
 	//   ie, {
 	//          "%{TESTLOG}":
@@ -134,6 +137,10 @@ func (p *Parser) Compile() error {
 		return err
 	}
 
+	if p.UniqueTimestamp == "" {
+		p.UniqueTimestamp = "auto"
+	}
+
 	// Give Patterns fake names so that they can be treated as named
 	// "custom patterns"
 	p.NamedPatterns = make([]string, 0, len(p.Patterns))
@@ -358,6 +365,10 @@ func (p *Parser) ParseLine(line string) (telegraf.Metric, error) {
 		return nil, fmt.Errorf("grok: must have one or more fields")
 	}
 
+	if p.UniqueTimestamp != "auto" {
+		return metric.New(p.Measurement, tags, fields, timestamp)
+	}
+
 	return metric.New(p.Measurement, tags, fields, p.tsModder.tsMod(timestamp))
 }
 

plugins/parsers/registry.go

@@ -126,6 +126,7 @@ type Config struct {
 	GrokCustomPatterns     string   `toml:"grok_custom_patterns"`
 	GrokCustomPatternFiles []string `toml:"grok_custom_pattern_files"`
 	GrokTimezone           string   `toml:"grok_timezone"`
+	GrokUniqueTimestamp    string   `toml:"grok_unique_timestamp"`
 
 	//csv configuration
 	CSVColumnNames       []string `toml:"csv_column_names"`
@@ -189,7 +190,8 @@ func NewParser(config *Config) (Parser, error) {
 			config.GrokNamedPatterns,
 			config.GrokCustomPatterns,
 			config.GrokCustomPatternFiles,
-			config.GrokTimezone)
+			config.GrokTimezone,
+			config.GrokUniqueTimestamp)
 	case "csv":
 		parser, err = newCSVParser(config.MetricName,
 			config.CSVHeaderRowCount,
@@ -298,10 +300,9 @@ func newJSONParser(
 
 //Deprecated: Use NewParser to get a JSONParser object
 func newGrokParser(metricName string,
-	patterns []string,
-	nPatterns []string,
-	cPatterns string,
-	cPatternFiles []string, tZone string) (Parser, error) {
+	patterns []string, nPatterns []string,
+	cPatterns string, cPatternFiles []string,
+	tZone string, uniqueTimestamp string) (Parser, error) {
 	parser := grok.Parser{
 		Measurement:        metricName,
 		Patterns:           patterns,
@@ -309,6 +310,7 @@ func newGrokParser(metricName string,
 		CustomPatterns:     cPatterns,
 		CustomPatternFiles: cPatternFiles,
 		Timezone:           tZone,
+		UniqueTimestamp:    uniqueTimestamp,
 	}
 
 	err := parser.Compile()