dom
/
telegraf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add field creation to date processor and integer unix time support (#7464)

This commit is contained in:
Rich Y 2020-05-08 01:19:03 +01:00 committed by GitHub
parent f14b3759d4
commit f076b6c115
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 151 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
plugins/processors/date/README.md
View File

@ -16,15 +16,23 @@ A few example usecases include:
## New tag to create
tag_key = "month"
## New field to create (cannot set both field_key and tag_key)
# field_key = "month"
## Date format string, must be a representation of the Go "reference time"
## which is "Mon Jan 2 15:04:05 -0700 MST 2006".
date_format = "Jan"
## If destination is a field, date format can also be one of
## "unix", "unix_ms", "unix_us", or "unix_ns", which will insert an integer field.
# date_format = "unix"
## Offset duration added to the date string when writing the new tag.
# date_offset = "0s"
## Timezone to use when generating the date. This can be set to one of
## "Local", "UTC", or to a location name in the IANA Time Zone database.
## Timezone to use when creating the tag or field using a reference time
## string. This can be set to one of "UTC", "Local", or to a location name
## in the IANA Time Zone database.
## example: timezone = "America/Los_Angeles"
# timezone = "UTC"
```

56
plugins/processors/date/date.go
View File

@ -1,6 +1,7 @@
package date
import (
"errors"
"time"
"github.com/influxdata/telegraf"
@ -9,26 +10,35 @@ import (
)
const sampleConfig = `
## New tag to create
tag_key = "month"
## New tag to create
tag_key = "month"
## Date format string, must be a representation of the Go "reference time"
## which is "Mon Jan 2 15:04:05 -0700 MST 2006".
date_format = "Jan"
## New field to create (cannot set both field_key and tag_key)
# field_key = "month"
## Offset duration added to the date string when writing the new tag.
# date_offset = "0s"
## Date format string, must be a representation of the Go "reference time"
## which is "Mon Jan 2 15:04:05 -0700 MST 2006".
date_format = "Jan"
## Timezone to use when creating the tag. This can be set to one of
## "UTC", "Local", or to a location name in the IANA Time Zone database.
## example: timezone = "America/Los_Angeles"
# timezone = "UTC"
## If destination is a field, date format can also be one of
## "unix", "unix_ms", "unix_us", or "unix_ns", which will insert an integer field.
# date_format = "unix"
## Offset duration added to the date string when writing the new tag.
# date_offset = "0s"
## Timezone to use when creating the tag or field using a reference time
## string. This can be set to one of "UTC", "Local", or to a location name
## in the IANA Time Zone database.
## example: timezone = "America/Los_Angeles"
# timezone = "UTC"
`
const defaultTimezone = "UTC"
type Date struct {
TagKey string `toml:"tag_key"`
FieldKey string `toml:"field_key"`
DateFormat string `toml:"date_format"`
DateOffset internal.Duration `toml:"date_offset"`
Timezone string `toml:"timezone"`
@ -45,6 +55,13 @@ func (d *Date) Description() string {
}
func (d *Date) Init() error {
// Check either TagKey or FieldKey specified
if len(d.FieldKey) > 0 && len(d.TagKey) > 0 {
return errors.New("Only one of field_key or tag_key can be specified")
} else if len(d.FieldKey) == 0 && len(d.TagKey) == 0 {
return errors.New("One of field_key or tag_key must be specified")
}
var err error
// LoadLocation returns UTC if timezone is the empty string.
d.location, err = time.LoadLocation(d.Timezone)
@ -54,7 +71,22 @@ func (d *Date) Init() error {
func (d *Date) Apply(in ...telegraf.Metric) []telegraf.Metric {
for _, point := range in {
tm := point.Time().In(d.location).Add(d.DateOffset.Duration)
point.AddTag(d.TagKey, tm.Format(d.DateFormat))
if len(d.TagKey) > 0 {
point.AddTag(d.TagKey, tm.Format(d.DateFormat))
} else if len(d.FieldKey) > 0 {
switch d.DateFormat {
case "unix":
point.AddField(d.FieldKey, tm.Unix())
case "unix_ms":
point.AddField(d.FieldKey, tm.UnixNano()/1000000)
case "unix_us":
point.AddField(d.FieldKey, tm.UnixNano()/1000)
case "unix_ns":
point.AddField(d.FieldKey, tm.UnixNano())
default:
point.AddField(d.FieldKey, tm.Format(d.DateFormat))
}
}
}
return in

110
plugins/processors/date/date_test.go
View File

@ -23,6 +23,22 @@ func MustMetric(name string, tags map[string]string, fields map[string]interface
return m
}
func TestTagAndField(t *testing.T) {
dateFormatTagAndField := Date{
TagKey: "month",
FieldKey: "month",
}
err := dateFormatTagAndField.Init()
require.Error(t, err)
}
func TestNoOutputSpecified(t *testing.T) {
dateFormatNoOutput := Date{}
err := dateFormatNoOutput.Init()
require.Error(t, err)
}
func TestMonthTag(t *testing.T) {
dateFormatMonth := Date{
TagKey: "month",
@ -43,25 +59,25 @@ func TestMonthTag(t *testing.T) {
assert.Equal(t, map[string]string{"month": month}, monthApply[2].Tags(), "should add tag 'month'")
}
func TestYearTag(t *testing.T) {
dateFormatYear := Date{
TagKey: "year",
DateFormat: "2006",
func TestMonthField(t *testing.T) {
dateFormatMonth := Date{
FieldKey: "month",
DateFormat: "Jan",
}
err := dateFormatYear.Init()
err := dateFormatMonth.Init()
require.NoError(t, err)
currentTime := time.Now()
year := currentTime.Format("2006")
month := currentTime.Format("Jan")
m4 := MustMetric("foo", nil, nil, currentTime)
m5 := MustMetric("bar", nil, nil, currentTime)
m6 := MustMetric("baz", nil, nil, currentTime)
yearApply := dateFormatYear.Apply(m4, m5, m6)
assert.Equal(t, map[string]string{"year": year}, yearApply[0].Tags(), "should add tag 'year'")
assert.Equal(t, map[string]string{"year": year}, yearApply[1].Tags(), "should add tag 'year'")
assert.Equal(t, map[string]string{"year": year}, yearApply[2].Tags(), "should add tag 'year'")
m1 := MustMetric("foo", nil, nil, currentTime)
m2 := MustMetric("bar", nil, nil, currentTime)
m3 := MustMetric("baz", nil, nil, currentTime)
monthApply := dateFormatMonth.Apply(m1, m2, m3)
assert.Equal(t, map[string]interface{}{"month": month}, monthApply[0].Fields(), "should add field 'month'")
assert.Equal(t, map[string]interface{}{"month": month}, monthApply[1].Fields(), "should add field 'month'")
assert.Equal(t, map[string]interface{}{"month": month}, monthApply[2].Fields(), "should add field 'month'")
}
func TestOldDateTag(t *testing.T) {
@ -78,6 +94,74 @@ func TestOldDateTag(t *testing.T) {
assert.Equal(t, map[string]string{"year": "1993"}, customDateApply[0].Tags(), "should add tag 'year'")
}
func TestFieldUnix(t *testing.T) {
dateFormatUnix := Date{
FieldKey: "unix",
DateFormat: "unix",
}
err := dateFormatUnix.Init()
require.NoError(t, err)
currentTime := time.Now()
unixTime := currentTime.Unix()
m8 := MustMetric("foo", nil, nil, currentTime)
unixApply := dateFormatUnix.Apply(m8)
assert.Equal(t, map[string]interface{}{"unix": unixTime}, unixApply[0].Fields(), "should add unix time in s as field 'unix'")
}
func TestFieldUnixNano(t *testing.T) {
dateFormatUnixNano := Date{
FieldKey: "unix_ns",
DateFormat: "unix_ns",
}
err := dateFormatUnixNano.Init()
require.NoError(t, err)
currentTime := time.Now()
unixNanoTime := currentTime.UnixNano()
m9 := MustMetric("foo", nil, nil, currentTime)
unixNanoApply := dateFormatUnixNano.Apply(m9)
assert.Equal(t, map[string]interface{}{"unix_ns": unixNanoTime}, unixNanoApply[0].Fields(), "should add unix time in ns as field 'unix_ns'")
}
func TestFieldUnixMillis(t *testing.T) {
dateFormatUnixMillis := Date{
FieldKey: "unix_ms",
DateFormat: "unix_ms",
}
err := dateFormatUnixMillis.Init()
require.NoError(t, err)
currentTime := time.Now()
unixMillisTime := currentTime.UnixNano() / 1000000
m10 := MustMetric("foo", nil, nil, currentTime)
unixMillisApply := dateFormatUnixMillis.Apply(m10)
assert.Equal(t, map[string]interface{}{"unix_ms": unixMillisTime}, unixMillisApply[0].Fields(), "should add unix time in ms as field 'unix_ms'")
}
func TestFieldUnixMicros(t *testing.T) {
dateFormatUnixMicros := Date{
FieldKey: "unix_us",
DateFormat: "unix_us",
}
err := dateFormatUnixMicros.Init()
require.NoError(t, err)
currentTime := time.Now()
unixMicrosTime := currentTime.UnixNano() / 1000
m11 := MustMetric("foo", nil, nil, currentTime)
unixMicrosApply := dateFormatUnixMicros.Apply(m11)
assert.Equal(t, map[string]interface{}{"unix_us": unixMicrosTime}, unixMicrosApply[0].Fields(), "should add unix time in us as field 'unix_us'")
}
func TestDateOffset(t *testing.T) {
plugin := &Date{
TagKey: "hour",