dom
/
telegraf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add index by week number to Elasticsearch output (#3490)

This commit is contained in:
Leandro Piccilli 2017-11-20 23:22:29 +01:00 committed by Daniel Nelson
parent 527892eef8
commit ca2c1e75c7
3 changed files with 19 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
plugins/outputs/elasticsearch/README.md
View File

@ -172,6 +172,7 @@ This plugin will format the events in the following way:
# %m - month (01..12) # %m - month (01..12)
# %d - day of month (e.g., 01) # %d - day of month (e.g., 01)
# %H - hour (00..23) # %H - hour (00..23)
# %V - week of the year (ISO week) (01..53)
index_name = "telegraf-%Y.%m.%d" # required. index_name = "telegraf-%Y.%m.%d" # required.
## Optional SSL Config ## Optional SSL Config
@ -220,6 +221,6 @@ Integer values collected that are bigger than 2^63 and smaller than 1e21 (or in
```{"error":{"root_cause":[{"type":"mapper_parsing_exception","reason":"failed to parse"}],"type":"mapper_parsing_exception","reason":"failed to parse","caused_by":{"type":"illegal_state_exception","reason":"No matching token for number_type [BIG_INTEGER]"}},"status":400}``` ```{"error":{"root_cause":[{"type":"mapper_parsing_exception","reason":"failed to parse"}],"type":"mapper_parsing_exception","reason":"failed to parse","caused_by":{"type":"illegal_state_exception","reason":"No matching token for number_type [BIG_INTEGER]"}},"status":400}```
The correct field mapping will be created on the telegraf index as soon as a supported JSON value is received by Elasticsearch, and subsequent insertions will work because the field mapping will already exist. The correct field mapping will be created on the telegraf index as soon as a supported JSON value is received by Elasticsearch, and subsequent insertions will work because the field mapping will already exist.
This issue is caused by the way Elasticsearch tries to detect integer fields, and by how golang encodes numbers in JSON. There is no clear workaround for this at the moment. This issue is caused by the way Elasticsearch tries to detect integer fields, and by how golang encodes numbers in JSON. There is no clear workaround for this at the moment.

16
plugins/outputs/elasticsearch/elasticsearch.go
View File

@ -3,15 +3,16 @@ package elasticsearch
import ( import (
"context" "context"
"fmt" "fmt"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/plugins/outputs"
"gopkg.in/olivere/elastic.v5"
"log" "log"
"net/http" "net/http"
"strconv" "strconv"
"strings" "strings"
"time" "time"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/plugins/outputs"
"gopkg.in/olivere/elastic.v5"
) )
type Elasticsearch struct { type Elasticsearch struct {
@ -58,6 +59,7 @@ var sampleConfig = `
# %m - month (01..12) # %m - month (01..12)
# %d - day of month (e.g., 01) # %d - day of month (e.g., 01)
# %H - hour (00..23) # %H - hour (00..23)
# %V - week of the year (ISO week) (01..53)
index_name = "telegraf-%Y.%m.%d" # required. index_name = "telegraf-%Y.%m.%d" # required.
## Optional SSL Config ## Optional SSL Config
@ -301,6 +303,7 @@ func (a *Elasticsearch) GetIndexName(indexName string, eventTime time.Time) stri
"%m", eventTime.UTC().Format("01"), "%m", eventTime.UTC().Format("01"),
"%d", eventTime.UTC().Format("02"), "%d", eventTime.UTC().Format("02"),
"%H", eventTime.UTC().Format("15"), "%H", eventTime.UTC().Format("15"),
"%V", getISOWeek(eventTime.UTC()),
) )
indexName = dateReplacer.Replace(indexName) indexName = dateReplacer.Replace(indexName)
@ -310,6 +313,11 @@ func (a *Elasticsearch) GetIndexName(indexName string, eventTime time.Time) stri
} }
func getISOWeek(eventTime time.Time) string {
_, week := eventTime.ISOWeek()
return strconv.Itoa(week)
}
func (a *Elasticsearch) SampleConfig() string { func (a *Elasticsearch) SampleConfig() string {
return sampleConfig return sampleConfig
} }

5
plugins/outputs/elasticsearch/elasticsearch_test.go
View File

@ -120,6 +120,11 @@ func TestGetIndexName(t *testing.T) {
"indexname-%y-%m", "indexname-%y-%m",
"indexname-14-12", "indexname-14-12",
}, },
{
time.Date(2014, 12, 01, 23, 30, 00, 00, time.UTC),
"indexname-%Y-%V",
"indexname-2014-49",
},
} }
for _, test := range tests { for _, test := range tests {
indexName := e.GetIndexName(test.IndexName, test.EventTime) indexName := e.GetIndexName(test.IndexName, test.EventTime)