From ca2c1e75c791f14fd3e4f7c3421559eff5f49553 Mon Sep 17 00:00:00 2001 From: Leandro Piccilli Date: Mon, 20 Nov 2017 23:22:29 +0100 Subject: [PATCH] Add index by week number to Elasticsearch output (#3490) --- plugins/outputs/elasticsearch/README.md | 3 ++- plugins/outputs/elasticsearch/elasticsearch.go | 16 ++++++++++++---- .../outputs/elasticsearch/elasticsearch_test.go | 5 +++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/plugins/outputs/elasticsearch/README.md b/plugins/outputs/elasticsearch/README.md index b69631ba8..d2c84a8d3 100644 --- a/plugins/outputs/elasticsearch/README.md +++ b/plugins/outputs/elasticsearch/README.md @@ -172,6 +172,7 @@ This plugin will format the events in the following way: # %m - month (01..12) # %d - day of month (e.g., 01) # %H - hour (00..23) + # %V - week of the year (ISO week) (01..53) index_name = "telegraf-%Y.%m.%d" # required. ## Optional SSL Config @@ -220,6 +221,6 @@ Integer values collected that are bigger than 2^63 and smaller than 1e21 (or in ```{"error":{"root_cause":[{"type":"mapper_parsing_exception","reason":"failed to parse"}],"type":"mapper_parsing_exception","reason":"failed to parse","caused_by":{"type":"illegal_state_exception","reason":"No matching token for number_type [BIG_INTEGER]"}},"status":400}``` -The correct field mapping will be created on the telegraf index as soon as a supported JSON value is received by Elasticsearch, and subsequent insertions will work because the field mapping will already exist. +The correct field mapping will be created on the telegraf index as soon as a supported JSON value is received by Elasticsearch, and subsequent insertions will work because the field mapping will already exist. This issue is caused by the way Elasticsearch tries to detect integer fields, and by how golang encodes numbers in JSON. There is no clear workaround for this at the moment. \ No newline at end of file diff --git a/plugins/outputs/elasticsearch/elasticsearch.go b/plugins/outputs/elasticsearch/elasticsearch.go index 31a702e55..a9fd5a491 100644 --- a/plugins/outputs/elasticsearch/elasticsearch.go +++ b/plugins/outputs/elasticsearch/elasticsearch.go @@ -3,15 +3,16 @@ package elasticsearch import ( "context" "fmt" - "github.com/influxdata/telegraf" - "github.com/influxdata/telegraf/internal" - "github.com/influxdata/telegraf/plugins/outputs" - "gopkg.in/olivere/elastic.v5" "log" "net/http" "strconv" "strings" "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/plugins/outputs" + "gopkg.in/olivere/elastic.v5" ) type Elasticsearch struct { @@ -58,6 +59,7 @@ var sampleConfig = ` # %m - month (01..12) # %d - day of month (e.g., 01) # %H - hour (00..23) + # %V - week of the year (ISO week) (01..53) index_name = "telegraf-%Y.%m.%d" # required. ## Optional SSL Config @@ -301,6 +303,7 @@ func (a *Elasticsearch) GetIndexName(indexName string, eventTime time.Time) stri "%m", eventTime.UTC().Format("01"), "%d", eventTime.UTC().Format("02"), "%H", eventTime.UTC().Format("15"), + "%V", getISOWeek(eventTime.UTC()), ) indexName = dateReplacer.Replace(indexName) @@ -310,6 +313,11 @@ func (a *Elasticsearch) GetIndexName(indexName string, eventTime time.Time) stri } +func getISOWeek(eventTime time.Time) string { + _, week := eventTime.ISOWeek() + return strconv.Itoa(week) +} + func (a *Elasticsearch) SampleConfig() string { return sampleConfig } diff --git a/plugins/outputs/elasticsearch/elasticsearch_test.go b/plugins/outputs/elasticsearch/elasticsearch_test.go index 9000676d9..dadd94da9 100644 --- a/plugins/outputs/elasticsearch/elasticsearch_test.go +++ b/plugins/outputs/elasticsearch/elasticsearch_test.go @@ -120,6 +120,11 @@ func TestGetIndexName(t *testing.T) { "indexname-%y-%m", "indexname-14-12", }, + { + time.Date(2014, 12, 01, 23, 30, 00, 00, time.UTC), + "indexname-%Y-%V", + "indexname-2014-49", + }, } for _, test := range tests { indexName := e.GetIndexName(test.IndexName, test.EventTime)