Add field creation to date processor and integer unix time support (#7464)

plugins/processors/date/README.md

@@ -16,15 +16,23 @@ A few example usecases include:
   ## New tag to create
   tag_key = "month"
 
+  ## New field to create (cannot set both field_key and tag_key)
+  # field_key = "month"
+
   ## Date format string, must be a representation of the Go "reference time"
   ## which is "Mon Jan 2 15:04:05 -0700 MST 2006".
   date_format = "Jan"
 
+  ## If destination is a field, date format can also be one of
+  ## "unix", "unix_ms", "unix_us", or "unix_ns", which will insert an integer field.
+  # date_format = "unix"
+
   ## Offset duration added to the date string when writing the new tag.
   # date_offset = "0s"
 
-  ## Timezone to use when generating the date.  This can be set to one of
+  ## Timezone to use when creating the tag or field using a reference time
-  ## "Local", "UTC", or to a location name in the IANA Time Zone database.
+  ## string.  This can be set to one of "UTC", "Local", or to a location name
+  ## in the IANA Time Zone database.
   ##   example: timezone = "America/Los_Angeles"
   # timezone = "UTC"
 ```

plugins/processors/date/date.go

@@ -1,6 +1,7 @@
 package date
 
 import (
+	"errors"
 	"time"
 
 	"github.com/influxdata/telegraf"
@@ -9,26 +10,35 @@ import (
 )
 
 const sampleConfig = `
-  ## New tag to create
+	## New tag to create
-  tag_key = "month"
+	tag_key = "month"
 
-  ## Date format string, must be a representation of the Go "reference time"
+	## New field to create (cannot set both field_key and tag_key)
-  ## which is "Mon Jan 2 15:04:05 -0700 MST 2006".
+	# field_key = "month"
-  date_format = "Jan"
 
-  ## Offset duration added to the date string when writing the new tag.
+	## Date format string, must be a representation of the Go "reference time"
-  # date_offset = "0s"
+	## which is "Mon Jan 2 15:04:05 -0700 MST 2006".
+	date_format = "Jan"
 
-  ## Timezone to use when creating the tag.  This can be set to one of
+	## If destination is a field, date format can also be one of
-  ## "UTC", "Local", or to a location name in the IANA Time Zone database.
+	## "unix", "unix_ms", "unix_us", or "unix_ns", which will insert an integer field.
-  ##   example: timezone = "America/Los_Angeles"
+	# date_format = "unix"
-  # timezone = "UTC"
+
+	## Offset duration added to the date string when writing the new tag.
+	# date_offset = "0s"
+
+	## Timezone to use when creating the tag or field using a reference time
+	## string.  This can be set to one of "UTC", "Local", or to a location name
+	## in the IANA Time Zone database.
+	##   example: timezone = "America/Los_Angeles"
+	# timezone = "UTC"
 `
 
 const defaultTimezone = "UTC"
 
 type Date struct {
 	TagKey     string            `toml:"tag_key"`
+	FieldKey   string            `toml:"field_key"`
 	DateFormat string            `toml:"date_format"`
 	DateOffset internal.Duration `toml:"date_offset"`
 	Timezone   string            `toml:"timezone"`
@@ -45,6 +55,13 @@ func (d *Date) Description() string {
 }
 
 func (d *Date) Init() error {
+	// Check either TagKey or FieldKey specified
+	if len(d.FieldKey) > 0 && len(d.TagKey) > 0 {
+		return errors.New("Only one of field_key or tag_key can be specified")
+	} else if len(d.FieldKey) == 0 && len(d.TagKey) == 0 {
+		return errors.New("One of field_key or tag_key must be specified")
+	}
+
 	var err error
 	// LoadLocation returns UTC if timezone is the empty string.
 	d.location, err = time.LoadLocation(d.Timezone)
@@ -54,7 +71,22 @@ func (d *Date) Init() error {
 func (d *Date) Apply(in ...telegraf.Metric) []telegraf.Metric {
 	for _, point := range in {
 		tm := point.Time().In(d.location).Add(d.DateOffset.Duration)
-		point.AddTag(d.TagKey, tm.Format(d.DateFormat))
+		if len(d.TagKey) > 0 {
+			point.AddTag(d.TagKey, tm.Format(d.DateFormat))
+		} else if len(d.FieldKey) > 0 {
+			switch d.DateFormat {
+			case "unix":
+				point.AddField(d.FieldKey, tm.Unix())
+			case "unix_ms":
+				point.AddField(d.FieldKey, tm.UnixNano()/1000000)
+			case "unix_us":
+				point.AddField(d.FieldKey, tm.UnixNano()/1000)
+			case "unix_ns":
+				point.AddField(d.FieldKey, tm.UnixNano())
+			default:
+				point.AddField(d.FieldKey, tm.Format(d.DateFormat))
+			}
+		}
 	}
 
 	return in

plugins/processors/date/date_test.go

@@ -23,6 +23,22 @@ func MustMetric(name string, tags map[string]string, fields map[string]interface
 	return m
 }
 
+func TestTagAndField(t *testing.T) {
+	dateFormatTagAndField := Date{
+		TagKey:   "month",
+		FieldKey: "month",
+	}
+	err := dateFormatTagAndField.Init()
+	require.Error(t, err)
+
+}
+
+func TestNoOutputSpecified(t *testing.T) {
+	dateFormatNoOutput := Date{}
+	err := dateFormatNoOutput.Init()
+	require.Error(t, err)
+}
+
 func TestMonthTag(t *testing.T) {
 	dateFormatMonth := Date{
 		TagKey:     "month",
@@ -43,25 +59,25 @@ func TestMonthTag(t *testing.T) {
 	assert.Equal(t, map[string]string{"month": month}, monthApply[2].Tags(), "should add tag 'month'")
 }
 
-func TestYearTag(t *testing.T) {
+func TestMonthField(t *testing.T) {
-	dateFormatYear := Date{
+	dateFormatMonth := Date{
-		TagKey:     "year",
+		FieldKey:   "month",
-		DateFormat: "2006",
+		DateFormat: "Jan",
 	}
 
-	err := dateFormatYear.Init()
+	err := dateFormatMonth.Init()
 	require.NoError(t, err)
 
 	currentTime := time.Now()
-	year := currentTime.Format("2006")
+	month := currentTime.Format("Jan")
 
-	m4 := MustMetric("foo", nil, nil, currentTime)
+	m1 := MustMetric("foo", nil, nil, currentTime)
-	m5 := MustMetric("bar", nil, nil, currentTime)
+	m2 := MustMetric("bar", nil, nil, currentTime)
-	m6 := MustMetric("baz", nil, nil, currentTime)
+	m3 := MustMetric("baz", nil, nil, currentTime)
-	yearApply := dateFormatYear.Apply(m4, m5, m6)
+	monthApply := dateFormatMonth.Apply(m1, m2, m3)
-	assert.Equal(t, map[string]string{"year": year}, yearApply[0].Tags(), "should add tag 'year'")
+	assert.Equal(t, map[string]interface{}{"month": month}, monthApply[0].Fields(), "should add field 'month'")
-	assert.Equal(t, map[string]string{"year": year}, yearApply[1].Tags(), "should add tag 'year'")
+	assert.Equal(t, map[string]interface{}{"month": month}, monthApply[1].Fields(), "should add field 'month'")
-	assert.Equal(t, map[string]string{"year": year}, yearApply[2].Tags(), "should add tag 'year'")
+	assert.Equal(t, map[string]interface{}{"month": month}, monthApply[2].Fields(), "should add field 'month'")
 }
 
 func TestOldDateTag(t *testing.T) {
@@ -78,6 +94,74 @@ func TestOldDateTag(t *testing.T) {
 	assert.Equal(t, map[string]string{"year": "1993"}, customDateApply[0].Tags(), "should add tag 'year'")
 }
 
+func TestFieldUnix(t *testing.T) {
+	dateFormatUnix := Date{
+		FieldKey:   "unix",
+		DateFormat: "unix",
+	}
+
+	err := dateFormatUnix.Init()
+	require.NoError(t, err)
+
+	currentTime := time.Now()
+	unixTime := currentTime.Unix()
+
+	m8 := MustMetric("foo", nil, nil, currentTime)
+	unixApply := dateFormatUnix.Apply(m8)
+	assert.Equal(t, map[string]interface{}{"unix": unixTime}, unixApply[0].Fields(), "should add unix time in s as field 'unix'")
+}
+
+func TestFieldUnixNano(t *testing.T) {
+	dateFormatUnixNano := Date{
+		FieldKey:   "unix_ns",
+		DateFormat: "unix_ns",
+	}
+
+	err := dateFormatUnixNano.Init()
+	require.NoError(t, err)
+
+	currentTime := time.Now()
+	unixNanoTime := currentTime.UnixNano()
+
+	m9 := MustMetric("foo", nil, nil, currentTime)
+	unixNanoApply := dateFormatUnixNano.Apply(m9)
+	assert.Equal(t, map[string]interface{}{"unix_ns": unixNanoTime}, unixNanoApply[0].Fields(), "should add unix time in ns as field 'unix_ns'")
+}
+
+func TestFieldUnixMillis(t *testing.T) {
+	dateFormatUnixMillis := Date{
+		FieldKey:   "unix_ms",
+		DateFormat: "unix_ms",
+	}
+
+	err := dateFormatUnixMillis.Init()
+	require.NoError(t, err)
+
+	currentTime := time.Now()
+	unixMillisTime := currentTime.UnixNano() / 1000000
+
+	m10 := MustMetric("foo", nil, nil, currentTime)
+	unixMillisApply := dateFormatUnixMillis.Apply(m10)
+	assert.Equal(t, map[string]interface{}{"unix_ms": unixMillisTime}, unixMillisApply[0].Fields(), "should add unix time in ms as field 'unix_ms'")
+}
+
+func TestFieldUnixMicros(t *testing.T) {
+	dateFormatUnixMicros := Date{
+		FieldKey:   "unix_us",
+		DateFormat: "unix_us",
+	}
+
+	err := dateFormatUnixMicros.Init()
+	require.NoError(t, err)
+
+	currentTime := time.Now()
+	unixMicrosTime := currentTime.UnixNano() / 1000
+
+	m11 := MustMetric("foo", nil, nil, currentTime)
+	unixMicrosApply := dateFormatUnixMicros.Apply(m11)
+	assert.Equal(t, map[string]interface{}{"unix_us": unixMicrosTime}, unixMicrosApply[0].Fields(), "should add unix time in us as field 'unix_us'")
+}
+
 func TestDateOffset(t *testing.T) {
 	plugin := &Date{
 		TagKey:     "hour",