From f076b6c115e4cd94d38e2dbe8b7b32b68deb07ee Mon Sep 17 00:00:00 2001 From: Rich Y Date: Fri, 8 May 2020 01:19:03 +0100 Subject: [PATCH] Add field creation to date processor and integer unix time support (#7464) --- plugins/processors/date/README.md | 12 ++- plugins/processors/date/date.go | 56 +++++++++++--- plugins/processors/date/date_test.go | 110 +++++++++++++++++++++++---- 3 files changed, 151 insertions(+), 27 deletions(-) diff --git a/plugins/processors/date/README.md b/plugins/processors/date/README.md index b04964b4a..215cd83e3 100644 --- a/plugins/processors/date/README.md +++ b/plugins/processors/date/README.md @@ -16,15 +16,23 @@ A few example usecases include: ## New tag to create tag_key = "month" + ## New field to create (cannot set both field_key and tag_key) + # field_key = "month" + ## Date format string, must be a representation of the Go "reference time" ## which is "Mon Jan 2 15:04:05 -0700 MST 2006". date_format = "Jan" + ## If destination is a field, date format can also be one of + ## "unix", "unix_ms", "unix_us", or "unix_ns", which will insert an integer field. + # date_format = "unix" + ## Offset duration added to the date string when writing the new tag. # date_offset = "0s" - ## Timezone to use when generating the date. This can be set to one of - ## "Local", "UTC", or to a location name in the IANA Time Zone database. + ## Timezone to use when creating the tag or field using a reference time + ## string. This can be set to one of "UTC", "Local", or to a location name + ## in the IANA Time Zone database. ## example: timezone = "America/Los_Angeles" # timezone = "UTC" ``` diff --git a/plugins/processors/date/date.go b/plugins/processors/date/date.go index c8007323f..ef8609811 100644 --- a/plugins/processors/date/date.go +++ b/plugins/processors/date/date.go @@ -1,6 +1,7 @@ package date import ( + "errors" "time" "github.com/influxdata/telegraf" @@ -9,26 +10,35 @@ import ( ) const sampleConfig = ` - ## New tag to create - tag_key = "month" + ## New tag to create + tag_key = "month" - ## Date format string, must be a representation of the Go "reference time" - ## which is "Mon Jan 2 15:04:05 -0700 MST 2006". - date_format = "Jan" + ## New field to create (cannot set both field_key and tag_key) + # field_key = "month" - ## Offset duration added to the date string when writing the new tag. - # date_offset = "0s" + ## Date format string, must be a representation of the Go "reference time" + ## which is "Mon Jan 2 15:04:05 -0700 MST 2006". + date_format = "Jan" - ## Timezone to use when creating the tag. This can be set to one of - ## "UTC", "Local", or to a location name in the IANA Time Zone database. - ## example: timezone = "America/Los_Angeles" - # timezone = "UTC" + ## If destination is a field, date format can also be one of + ## "unix", "unix_ms", "unix_us", or "unix_ns", which will insert an integer field. + # date_format = "unix" + + ## Offset duration added to the date string when writing the new tag. + # date_offset = "0s" + + ## Timezone to use when creating the tag or field using a reference time + ## string. This can be set to one of "UTC", "Local", or to a location name + ## in the IANA Time Zone database. + ## example: timezone = "America/Los_Angeles" + # timezone = "UTC" ` const defaultTimezone = "UTC" type Date struct { TagKey string `toml:"tag_key"` + FieldKey string `toml:"field_key"` DateFormat string `toml:"date_format"` DateOffset internal.Duration `toml:"date_offset"` Timezone string `toml:"timezone"` @@ -45,6 +55,13 @@ func (d *Date) Description() string { } func (d *Date) Init() error { + // Check either TagKey or FieldKey specified + if len(d.FieldKey) > 0 && len(d.TagKey) > 0 { + return errors.New("Only one of field_key or tag_key can be specified") + } else if len(d.FieldKey) == 0 && len(d.TagKey) == 0 { + return errors.New("One of field_key or tag_key must be specified") + } + var err error // LoadLocation returns UTC if timezone is the empty string. d.location, err = time.LoadLocation(d.Timezone) @@ -54,7 +71,22 @@ func (d *Date) Init() error { func (d *Date) Apply(in ...telegraf.Metric) []telegraf.Metric { for _, point := range in { tm := point.Time().In(d.location).Add(d.DateOffset.Duration) - point.AddTag(d.TagKey, tm.Format(d.DateFormat)) + if len(d.TagKey) > 0 { + point.AddTag(d.TagKey, tm.Format(d.DateFormat)) + } else if len(d.FieldKey) > 0 { + switch d.DateFormat { + case "unix": + point.AddField(d.FieldKey, tm.Unix()) + case "unix_ms": + point.AddField(d.FieldKey, tm.UnixNano()/1000000) + case "unix_us": + point.AddField(d.FieldKey, tm.UnixNano()/1000) + case "unix_ns": + point.AddField(d.FieldKey, tm.UnixNano()) + default: + point.AddField(d.FieldKey, tm.Format(d.DateFormat)) + } + } } return in diff --git a/plugins/processors/date/date_test.go b/plugins/processors/date/date_test.go index d97cc2a9c..42e094c93 100644 --- a/plugins/processors/date/date_test.go +++ b/plugins/processors/date/date_test.go @@ -23,6 +23,22 @@ func MustMetric(name string, tags map[string]string, fields map[string]interface return m } +func TestTagAndField(t *testing.T) { + dateFormatTagAndField := Date{ + TagKey: "month", + FieldKey: "month", + } + err := dateFormatTagAndField.Init() + require.Error(t, err) + +} + +func TestNoOutputSpecified(t *testing.T) { + dateFormatNoOutput := Date{} + err := dateFormatNoOutput.Init() + require.Error(t, err) +} + func TestMonthTag(t *testing.T) { dateFormatMonth := Date{ TagKey: "month", @@ -43,25 +59,25 @@ func TestMonthTag(t *testing.T) { assert.Equal(t, map[string]string{"month": month}, monthApply[2].Tags(), "should add tag 'month'") } -func TestYearTag(t *testing.T) { - dateFormatYear := Date{ - TagKey: "year", - DateFormat: "2006", +func TestMonthField(t *testing.T) { + dateFormatMonth := Date{ + FieldKey: "month", + DateFormat: "Jan", } - err := dateFormatYear.Init() + err := dateFormatMonth.Init() require.NoError(t, err) currentTime := time.Now() - year := currentTime.Format("2006") + month := currentTime.Format("Jan") - m4 := MustMetric("foo", nil, nil, currentTime) - m5 := MustMetric("bar", nil, nil, currentTime) - m6 := MustMetric("baz", nil, nil, currentTime) - yearApply := dateFormatYear.Apply(m4, m5, m6) - assert.Equal(t, map[string]string{"year": year}, yearApply[0].Tags(), "should add tag 'year'") - assert.Equal(t, map[string]string{"year": year}, yearApply[1].Tags(), "should add tag 'year'") - assert.Equal(t, map[string]string{"year": year}, yearApply[2].Tags(), "should add tag 'year'") + m1 := MustMetric("foo", nil, nil, currentTime) + m2 := MustMetric("bar", nil, nil, currentTime) + m3 := MustMetric("baz", nil, nil, currentTime) + monthApply := dateFormatMonth.Apply(m1, m2, m3) + assert.Equal(t, map[string]interface{}{"month": month}, monthApply[0].Fields(), "should add field 'month'") + assert.Equal(t, map[string]interface{}{"month": month}, monthApply[1].Fields(), "should add field 'month'") + assert.Equal(t, map[string]interface{}{"month": month}, monthApply[2].Fields(), "should add field 'month'") } func TestOldDateTag(t *testing.T) { @@ -78,6 +94,74 @@ func TestOldDateTag(t *testing.T) { assert.Equal(t, map[string]string{"year": "1993"}, customDateApply[0].Tags(), "should add tag 'year'") } +func TestFieldUnix(t *testing.T) { + dateFormatUnix := Date{ + FieldKey: "unix", + DateFormat: "unix", + } + + err := dateFormatUnix.Init() + require.NoError(t, err) + + currentTime := time.Now() + unixTime := currentTime.Unix() + + m8 := MustMetric("foo", nil, nil, currentTime) + unixApply := dateFormatUnix.Apply(m8) + assert.Equal(t, map[string]interface{}{"unix": unixTime}, unixApply[0].Fields(), "should add unix time in s as field 'unix'") +} + +func TestFieldUnixNano(t *testing.T) { + dateFormatUnixNano := Date{ + FieldKey: "unix_ns", + DateFormat: "unix_ns", + } + + err := dateFormatUnixNano.Init() + require.NoError(t, err) + + currentTime := time.Now() + unixNanoTime := currentTime.UnixNano() + + m9 := MustMetric("foo", nil, nil, currentTime) + unixNanoApply := dateFormatUnixNano.Apply(m9) + assert.Equal(t, map[string]interface{}{"unix_ns": unixNanoTime}, unixNanoApply[0].Fields(), "should add unix time in ns as field 'unix_ns'") +} + +func TestFieldUnixMillis(t *testing.T) { + dateFormatUnixMillis := Date{ + FieldKey: "unix_ms", + DateFormat: "unix_ms", + } + + err := dateFormatUnixMillis.Init() + require.NoError(t, err) + + currentTime := time.Now() + unixMillisTime := currentTime.UnixNano() / 1000000 + + m10 := MustMetric("foo", nil, nil, currentTime) + unixMillisApply := dateFormatUnixMillis.Apply(m10) + assert.Equal(t, map[string]interface{}{"unix_ms": unixMillisTime}, unixMillisApply[0].Fields(), "should add unix time in ms as field 'unix_ms'") +} + +func TestFieldUnixMicros(t *testing.T) { + dateFormatUnixMicros := Date{ + FieldKey: "unix_us", + DateFormat: "unix_us", + } + + err := dateFormatUnixMicros.Init() + require.NoError(t, err) + + currentTime := time.Now() + unixMicrosTime := currentTime.UnixNano() / 1000 + + m11 := MustMetric("foo", nil, nil, currentTime) + unixMicrosApply := dateFormatUnixMicros.Apply(m11) + assert.Equal(t, map[string]interface{}{"unix_us": unixMicrosTime}, unixMicrosApply[0].Fields(), "should add unix time in us as field 'unix_us'") +} + func TestDateOffset(t *testing.T) { plugin := &Date{ TagKey: "hour",