From a26aaa5e0360505a5b77607fc0419b00e90c5a7f Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Wed, 28 Nov 2018 16:19:53 -0800 Subject: [PATCH] Add support for unix_ms timestamps in csv parser. --- plugins/parsers/csv/README.md | 8 ++++---- plugins/parsers/csv/parser.go | 16 +++++++++++++--- plugins/parsers/csv/parser_test.go | 18 ++++++++++++++++++ 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/plugins/parsers/csv/README.md b/plugins/parsers/csv/README.md index f2cf34c69..488846b5e 100644 --- a/plugins/parsers/csv/README.md +++ b/plugins/parsers/csv/README.md @@ -73,10 +73,10 @@ time using the JSON document you can use the `csv_timestamp_column` and `csv_timestamp_format` options together to set the time to a value in the parsed document. -The `csv_timestamp_column` option specifies the column name containing the -time value and `csv_timestamp_format` must be set to a Go "reference time" -which is defined to be the specific time: `Mon Jan 2 15:04:05 MST 2006`, -it can also be `unix` (for epoch in ms format like 1257894000 ) +The `csv_timestamp_column` option specifies the key containing the time value and +`csv_timestamp_format` must be set to `unix`, `unix_ms`, or a format string in +using the Go "reference time" which is defined to be the **specific time**: +`Mon Jan 2 15:04:05 MST 2006`. Consult the Go [time][time parse] package for details and additional examples on how to set the time format. diff --git a/plugins/parsers/csv/parser.go b/plugins/parsers/csv/parser.go index e1bbdbbbb..9401c1dd1 100644 --- a/plugins/parsers/csv/parser.go +++ b/plugins/parsers/csv/parser.go @@ -219,9 +219,12 @@ outer: return m, nil } -// ParseTimestamp return a timestamp, if there is no timestamp on the csv it will be the current timestamp, else it will try to parse the time according to the format -// if the format is "unix" it tries to parse assuming that on the csv it will find an epoch in ms. -func parseTimestamp(timeFunc func() time.Time, recordFields map[string]interface{}, timestampColumn, timestampFormat string) (metricTime time.Time, err error) { +// ParseTimestamp return a timestamp, if there is no timestamp on the csv it +// will be the current timestamp, else it will try to parse the time according +// to the format. +func parseTimestamp(timeFunc func() time.Time, recordFields map[string]interface{}, + timestampColumn, timestampFormat string, +) (metricTime time.Time, err error) { metricTime = timeFunc() if timestampColumn != "" { @@ -243,6 +246,13 @@ func parseTimestamp(timeFunc func() time.Time, recordFields map[string]interface return } metricTime = time.Unix(unixTime, 0) + case "unix_ms": + var unixTime int64 + unixTime, err = strconv.ParseInt(tStr, 10, 64) + if err != nil { + return + } + metricTime = time.Unix(unixTime/1000, (unixTime%1000)*1e6) default: metricTime, err = time.Parse(timestampFormat, tStr) if err != nil { diff --git a/plugins/parsers/csv/parser_test.go b/plugins/parsers/csv/parser_test.go index 97da69cd2..93ae6bcdd 100644 --- a/plugins/parsers/csv/parser_test.go +++ b/plugins/parsers/csv/parser_test.go @@ -106,6 +106,24 @@ func TestTimestampUnixFormat(t *testing.T) { require.Equal(t, metrics[1].Time().UnixNano(), int64(1257609906000000000)) } +func TestTimestampUnixMSFormat(t *testing.T) { + p := Parser{ + HeaderRowCount: 1, + ColumnNames: []string{"first", "second", "third"}, + MeasurementColumn: "third", + TimestampColumn: "first", + TimestampFormat: "unix_ms", + TimeFunc: DefaultTime, + } + testCSV := `line1,line2,line3 +1243094706123,70,test_name +1257609906123,80,test_name2` + metrics, err := p.Parse([]byte(testCSV)) + require.NoError(t, err) + require.Equal(t, metrics[0].Time().UnixNano(), int64(1243094706123000000)) + require.Equal(t, metrics[1].Time().UnixNano(), int64(1257609906123000000)) +} + func TestQuotedCharacter(t *testing.T) { p := Parser{ HeaderRowCount: 1,