From 1886676e14a1a26a4a6aa0de4b4968e256d6f023 Mon Sep 17 00:00:00 2001 From: Douglas Drinka Date: Mon, 25 Feb 2019 12:30:33 -0700 Subject: [PATCH] Support configuring a default timezone in JSON parser (#5472) --- internal/config/config.go | 9 +++++++++ internal/internal.go | 13 ++++++++++-- internal/internal_test.go | 31 +++++++++++++++++++++++++++++ plugins/parsers/json/README.md | 20 ++++++++++++++++++- plugins/parsers/json/parser.go | 3 ++- plugins/parsers/json/parser_test.go | 17 ++++++++++++++++ plugins/parsers/registry.go | 6 ++++++ 7 files changed, 95 insertions(+), 4 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 504d8501c..4388d658d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1362,6 +1362,14 @@ func getParserConfig(name string, tbl *ast.Table) (*parsers.Config, error) { } } + if node, ok := tbl.Fields["json_timezone"]; ok { + if kv, ok := node.(*ast.KeyValue); ok { + if str, ok := kv.Value.(*ast.String); ok { + c.JSONTimezone = str.Value + } + } + } + if node, ok := tbl.Fields["data_type"]; ok { if kv, ok := node.(*ast.KeyValue); ok { if str, ok := kv.Value.(*ast.String); ok { @@ -1637,6 +1645,7 @@ func getParserConfig(name string, tbl *ast.Table) (*parsers.Config, error) { delete(tbl.Fields, "json_string_fields") delete(tbl.Fields, "json_time_format") delete(tbl.Fields, "json_time_key") + delete(tbl.Fields, "json_timezone") delete(tbl.Fields, "data_type") delete(tbl.Fields, "collectd_auth_file") delete(tbl.Fields, "collectd_security_level") diff --git a/internal/internal.go b/internal/internal.go index 368bc8bcf..b373c9c35 100644 --- a/internal/internal.go +++ b/internal/internal.go @@ -333,13 +333,18 @@ func CompressWithGzip(data io.Reader) (io.Reader, error) { return pipeReader, err } +// ParseTimestamp with no location provided parses a timestamp value as UTC +func ParseTimestamp(timestamp interface{}, format string) (time.Time, error) { + return ParseTimestampWithLocation(timestamp, format, "UTC") +} + // ParseTimestamp parses a timestamp value as a unix epoch of various precision. // // format = "unix": epoch is assumed to be in seconds and can come as number or string. Can have a decimal part. // format = "unix_ms": epoch is assumed to be in milliseconds and can come as number or string. Cannot have a decimal part. // format = "unix_us": epoch is assumed to be in microseconds and can come as number or string. Cannot have a decimal part. // format = "unix_ns": epoch is assumed to be in nanoseconds and can come as number or string. Cannot have a decimal part. -func ParseTimestamp(timestamp interface{}, format string) (time.Time, error) { +func ParseTimestampWithLocation(timestamp interface{}, format string, location string) (time.Time, error) { timeInt, timeFractional := int64(0), int64(0) timeEpochStr, ok := timestamp.(string) var err error @@ -355,7 +360,11 @@ func ParseTimestamp(timestamp interface{}, format string) (time.Time, error) { splitted := regexp.MustCompile("[.,]").Split(timeEpochStr, 2) timeInt, err = strconv.ParseInt(splitted[0], 10, 64) if err != nil { - return time.Parse(format, timeEpochStr) + loc, err := time.LoadLocation(location) + if err != nil { + return time.Time{}, fmt.Errorf("location: %s could not be loaded as a location", location) + } + return time.ParseInLocation(format, timeEpochStr, loc) } if len(splitted) == 2 { diff --git a/internal/internal_test.go b/internal/internal_test.go index 46b1b5962..681e1f808 100644 --- a/internal/internal_test.go +++ b/internal/internal_test.go @@ -270,3 +270,34 @@ func TestAlignDuration(t *testing.T) { }) } } + +func TestParseTimestamp(t *testing.T) { + time, err := ParseTimestamp("2019-02-20 21:50:34.029665", "2006-01-02 15:04:05.000000") + assert.Nil(t, err) + assert.EqualValues(t, int64(1550699434029665000), time.UnixNano()) + + time, err = ParseTimestamp("2019-02-20 21:50:34.029665-04:00", "2006-01-02 15:04:05.000000-07:00") + assert.Nil(t, err) + assert.EqualValues(t, int64(1550713834029665000), time.UnixNano()) + + time, err = ParseTimestamp("2019-02-20 21:50:34.029665", "2006-01-02 15:04:05.000000-06:00") + assert.NotNil(t, err) +} + +func TestParseTimestampWithLocation(t *testing.T) { + time, err := ParseTimestampWithLocation("2019-02-20 21:50:34.029665", "2006-01-02 15:04:05.000000", "UTC") + assert.Nil(t, err) + assert.EqualValues(t, int64(1550699434029665000), time.UnixNano()) + + time, err = ParseTimestampWithLocation("2019-02-20 21:50:34.029665", "2006-01-02 15:04:05.000000", "America/New_York") + assert.Nil(t, err) + assert.EqualValues(t, int64(1550717434029665000), time.UnixNano()) + + //Provided location is ignored if an offset is successfully parsed + time, err = ParseTimestampWithLocation("2019-02-20 21:50:34.029665-07:00", "2006-01-02 15:04:05.000000-07:00", "America/New_York") + assert.Nil(t, err) + assert.EqualValues(t, int64(1550724634029665000), time.UnixNano()) + + time, err = ParseTimestampWithLocation("2019-02-20 21:50:34.029665", "2006-01-02 15:04:05.000000", "InvalidTimeZone") + assert.NotNil(t, err) +} diff --git a/plugins/parsers/json/README.md b/plugins/parsers/json/README.md index 8b73b7214..60e1f3f9e 100644 --- a/plugins/parsers/json/README.md +++ b/plugins/parsers/json/README.md @@ -49,9 +49,21 @@ ignored unless specified in the `tag_key` or `json_string_fields` options. ## https://golang.org/pkg/time/#Time.Format ## ex: json_time_format = "Mon Jan 2 15:04:05 -0700 MST 2006" ## json_time_format = "2006-01-02T15:04:05Z07:00" + ## json_time_format = "01/02/2006 15:04:05" ## json_time_format = "unix" ## json_time_format = "unix_ms" json_time_format = "" + + ## Timezone allows you to provide an override for timestamps that + ## don't already include an offset + ## e.g. 04/06/2016 12:41:45 + ## + ## Default: "" which renders UTC + ## Options are as follows: + ## 1. Local -- interpret based on machine localtime + ## 2. "America/New_York" -- Unix TZ values like those found in https://en.wikipedia.org/wiki/List_of_tz_database_time_zones + ## 3. UTC -- or blank/unspecified, will return timestamp in UTC + json_timezone = "" ``` #### json_query @@ -62,7 +74,7 @@ query should contain a JSON object or an array of objects. Consult the GJSON [path syntax][gjson syntax] for details and examples. -#### json_time_key, json_time_format +#### json_time_key, json_time_format, json_timezone By default the current time will be used for all created metrics, to set the time using the JSON document you can use the `json_time_key` and @@ -77,6 +89,12 @@ the Go "reference time" which is defined to be the specific time: Consult the Go [time][time parse] package for details and additional examples on how to set the time format. +When parsing times that don't include a timezone specifier, times are assumed +to be UTC. To default to another timezone, or to local time, specify the +`json_timezone` option. This option should be set to a +[Unix TZ value](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones), +such as `America/New_York`, to `Local` to utilize the system timezone, or to `UTC`. + ### Examples #### Basic Parsing diff --git a/plugins/parsers/json/parser.go b/plugins/parsers/json/parser.go index 2f939a84f..ebe31fd23 100644 --- a/plugins/parsers/json/parser.go +++ b/plugins/parsers/json/parser.go @@ -28,6 +28,7 @@ type JSONParser struct { JSONQuery string JSONTimeKey string JSONTimeFormat string + JSONTimezone string DefaultTags map[string]string } @@ -82,7 +83,7 @@ func (p *JSONParser) parseObject(metrics []telegraf.Metric, jsonOut map[string]i return nil, err } - nTime, err = internal.ParseTimestamp(f.Fields[p.JSONTimeKey], p.JSONTimeFormat) + nTime, err = internal.ParseTimestampWithLocation(f.Fields[p.JSONTimeKey], p.JSONTimeFormat, p.JSONTimezone) if err != nil { return nil, err } diff --git a/plugins/parsers/json/parser_test.go b/plugins/parsers/json/parser_test.go index 382afcd35..2db9ad78f 100644 --- a/plugins/parsers/json/parser_test.go +++ b/plugins/parsers/json/parser_test.go @@ -599,6 +599,23 @@ func TestTimeParser(t *testing.T) { require.Equal(t, false, metrics[0].Time() == metrics[1].Time()) } +func TestTimeParserWithTimezone(t *testing.T) { + testString := `{ + "time": "04 Jan 06 15:04" + }` + + parser := JSONParser{ + MetricName: "json_test", + JSONTimeKey: "time", + JSONTimeFormat: "02 Jan 06 15:04", + JSONTimezone: "America/New_York", + } + metrics, err := parser.Parse([]byte(testString)) + require.NoError(t, err) + require.Equal(t, 1, len(metrics)) + require.EqualValues(t, int64(1136405040000000000), metrics[0].Time().UnixNano()) +} + func TestUnixTimeParser(t *testing.T) { testString := `[ { diff --git a/plugins/parsers/registry.go b/plugins/parsers/registry.go index c6ef8ae1e..ffa7d142f 100644 --- a/plugins/parsers/registry.go +++ b/plugins/parsers/registry.go @@ -85,6 +85,9 @@ type Config struct { // time format JSONTimeFormat string `toml:"json_time_format"` + // default timezone + JSONTimezone string `toml:"json_timezone"` + // Authentication file for collectd CollectdAuthFile string `toml:"collectd_auth_file"` // One of none (default), sign, or encrypt @@ -152,6 +155,7 @@ func NewParser(config *Config) (Parser, error) { config.JSONQuery, config.JSONTimeKey, config.JSONTimeFormat, + config.JSONTimezone, config.DefaultTags) case "value": parser, err = NewValueParser(config.MetricName, @@ -275,6 +279,7 @@ func newJSONParser( jsonQuery string, timeKey string, timeFormat string, + timezone string, defaultTags map[string]string, ) Parser { parser := &json.JSONParser{ @@ -285,6 +290,7 @@ func newJSONParser( JSONQuery: jsonQuery, JSONTimeKey: timeKey, JSONTimeFormat: timeFormat, + JSONTimezone: timezone, DefaultTags: defaultTags, } return parser