Add support for unix_ms timestamps in csv parser.

This commit is contained in:
Daniel Nelson 2018-11-28 16:19:53 -08:00
parent f9113b63b7
commit a26aaa5e03
No known key found for this signature in database
GPG Key ID: CAAD59C9444F6155
3 changed files with 35 additions and 7 deletions

View File

@ -73,10 +73,10 @@ time using the JSON document you can use the `csv_timestamp_column` and
`csv_timestamp_format` options together to set the time to a value in the parsed `csv_timestamp_format` options together to set the time to a value in the parsed
document. document.
The `csv_timestamp_column` option specifies the column name containing the The `csv_timestamp_column` option specifies the key containing the time value and
time value and `csv_timestamp_format` must be set to a Go "reference time" `csv_timestamp_format` must be set to `unix`, `unix_ms`, or a format string in
which is defined to be the specific time: `Mon Jan 2 15:04:05 MST 2006`, using the Go "reference time" which is defined to be the **specific time**:
it can also be `unix` (for epoch in ms format like 1257894000 ) `Mon Jan 2 15:04:05 MST 2006`.
Consult the Go [time][time parse] package for details and additional examples Consult the Go [time][time parse] package for details and additional examples
on how to set the time format. on how to set the time format.

View File

@ -219,9 +219,12 @@ outer:
return m, nil return m, nil
} }
// ParseTimestamp return a timestamp, if there is no timestamp on the csv it will be the current timestamp, else it will try to parse the time according to the format // ParseTimestamp return a timestamp, if there is no timestamp on the csv it
// if the format is "unix" it tries to parse assuming that on the csv it will find an epoch in ms. // will be the current timestamp, else it will try to parse the time according
func parseTimestamp(timeFunc func() time.Time, recordFields map[string]interface{}, timestampColumn, timestampFormat string) (metricTime time.Time, err error) { // to the format.
func parseTimestamp(timeFunc func() time.Time, recordFields map[string]interface{},
timestampColumn, timestampFormat string,
) (metricTime time.Time, err error) {
metricTime = timeFunc() metricTime = timeFunc()
if timestampColumn != "" { if timestampColumn != "" {
@ -243,6 +246,13 @@ func parseTimestamp(timeFunc func() time.Time, recordFields map[string]interface
return return
} }
metricTime = time.Unix(unixTime, 0) metricTime = time.Unix(unixTime, 0)
case "unix_ms":
var unixTime int64
unixTime, err = strconv.ParseInt(tStr, 10, 64)
if err != nil {
return
}
metricTime = time.Unix(unixTime/1000, (unixTime%1000)*1e6)
default: default:
metricTime, err = time.Parse(timestampFormat, tStr) metricTime, err = time.Parse(timestampFormat, tStr)
if err != nil { if err != nil {

View File

@ -106,6 +106,24 @@ func TestTimestampUnixFormat(t *testing.T) {
require.Equal(t, metrics[1].Time().UnixNano(), int64(1257609906000000000)) require.Equal(t, metrics[1].Time().UnixNano(), int64(1257609906000000000))
} }
func TestTimestampUnixMSFormat(t *testing.T) {
p := Parser{
HeaderRowCount: 1,
ColumnNames: []string{"first", "second", "third"},
MeasurementColumn: "third",
TimestampColumn: "first",
TimestampFormat: "unix_ms",
TimeFunc: DefaultTime,
}
testCSV := `line1,line2,line3
1243094706123,70,test_name
1257609906123,80,test_name2`
metrics, err := p.Parse([]byte(testCSV))
require.NoError(t, err)
require.Equal(t, metrics[0].Time().UnixNano(), int64(1243094706123000000))
require.Equal(t, metrics[1].Time().UnixNano(), int64(1257609906123000000))
}
func TestQuotedCharacter(t *testing.T) { func TestQuotedCharacter(t *testing.T) {
p := Parser{ p := Parser{
HeaderRowCount: 1, HeaderRowCount: 1,