Add support for unix_ms timestamps in csv parser.
This commit is contained in:
parent
f9113b63b7
commit
a26aaa5e03
|
@ -73,10 +73,10 @@ time using the JSON document you can use the `csv_timestamp_column` and
|
||||||
`csv_timestamp_format` options together to set the time to a value in the parsed
|
`csv_timestamp_format` options together to set the time to a value in the parsed
|
||||||
document.
|
document.
|
||||||
|
|
||||||
The `csv_timestamp_column` option specifies the column name containing the
|
The `csv_timestamp_column` option specifies the key containing the time value and
|
||||||
time value and `csv_timestamp_format` must be set to a Go "reference time"
|
`csv_timestamp_format` must be set to `unix`, `unix_ms`, or a format string in
|
||||||
which is defined to be the specific time: `Mon Jan 2 15:04:05 MST 2006`,
|
using the Go "reference time" which is defined to be the **specific time**:
|
||||||
it can also be `unix` (for epoch in ms format like 1257894000 )
|
`Mon Jan 2 15:04:05 MST 2006`.
|
||||||
|
|
||||||
Consult the Go [time][time parse] package for details and additional examples
|
Consult the Go [time][time parse] package for details and additional examples
|
||||||
on how to set the time format.
|
on how to set the time format.
|
||||||
|
|
|
@ -219,9 +219,12 @@ outer:
|
||||||
return m, nil
|
return m, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseTimestamp return a timestamp, if there is no timestamp on the csv it will be the current timestamp, else it will try to parse the time according to the format
|
// ParseTimestamp return a timestamp, if there is no timestamp on the csv it
|
||||||
// if the format is "unix" it tries to parse assuming that on the csv it will find an epoch in ms.
|
// will be the current timestamp, else it will try to parse the time according
|
||||||
func parseTimestamp(timeFunc func() time.Time, recordFields map[string]interface{}, timestampColumn, timestampFormat string) (metricTime time.Time, err error) {
|
// to the format.
|
||||||
|
func parseTimestamp(timeFunc func() time.Time, recordFields map[string]interface{},
|
||||||
|
timestampColumn, timestampFormat string,
|
||||||
|
) (metricTime time.Time, err error) {
|
||||||
metricTime = timeFunc()
|
metricTime = timeFunc()
|
||||||
|
|
||||||
if timestampColumn != "" {
|
if timestampColumn != "" {
|
||||||
|
@ -243,6 +246,13 @@ func parseTimestamp(timeFunc func() time.Time, recordFields map[string]interface
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
metricTime = time.Unix(unixTime, 0)
|
metricTime = time.Unix(unixTime, 0)
|
||||||
|
case "unix_ms":
|
||||||
|
var unixTime int64
|
||||||
|
unixTime, err = strconv.ParseInt(tStr, 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
metricTime = time.Unix(unixTime/1000, (unixTime%1000)*1e6)
|
||||||
default:
|
default:
|
||||||
metricTime, err = time.Parse(timestampFormat, tStr)
|
metricTime, err = time.Parse(timestampFormat, tStr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -106,6 +106,24 @@ func TestTimestampUnixFormat(t *testing.T) {
|
||||||
require.Equal(t, metrics[1].Time().UnixNano(), int64(1257609906000000000))
|
require.Equal(t, metrics[1].Time().UnixNano(), int64(1257609906000000000))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestTimestampUnixMSFormat(t *testing.T) {
|
||||||
|
p := Parser{
|
||||||
|
HeaderRowCount: 1,
|
||||||
|
ColumnNames: []string{"first", "second", "third"},
|
||||||
|
MeasurementColumn: "third",
|
||||||
|
TimestampColumn: "first",
|
||||||
|
TimestampFormat: "unix_ms",
|
||||||
|
TimeFunc: DefaultTime,
|
||||||
|
}
|
||||||
|
testCSV := `line1,line2,line3
|
||||||
|
1243094706123,70,test_name
|
||||||
|
1257609906123,80,test_name2`
|
||||||
|
metrics, err := p.Parse([]byte(testCSV))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, metrics[0].Time().UnixNano(), int64(1243094706123000000))
|
||||||
|
require.Equal(t, metrics[1].Time().UnixNano(), int64(1257609906123000000))
|
||||||
|
}
|
||||||
|
|
||||||
func TestQuotedCharacter(t *testing.T) {
|
func TestQuotedCharacter(t *testing.T) {
|
||||||
p := Parser{
|
p := Parser{
|
||||||
HeaderRowCount: 1,
|
HeaderRowCount: 1,
|
||||||
|
|
Loading…
Reference in New Issue