dom
/
telegraf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Use add time for prometheus expiration calculation (#7056)

This commit is contained in:
Daniel Nelson 2020-02-24 15:53:16 -08:00 committed by GitHub
parent e9e4f2c354
commit b5e0577d6b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 266 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
plugins/outputs/prometheus_client/v2/collector.go
View File

@ -83,7 +83,7 @@ func (c *Collector) Add(metrics []telegraf.Metric) error {
defer c.Unlock() defer c.Unlock()
for _, metric := range metrics { for _, metric := range metrics {
c.coll.Add(metric) c.coll.Add(metric, time.Now())
} }
// Expire metrics, doing this on Add ensure metrics are removed even if no // Expire metrics, doing this on Add ensure metrics are removed even if no

14
plugins/serializers/prometheus/collection.go
View File

@ -14,6 +14,8 @@ import (
const helpString = "Telegraf collected metric" const helpString = "Telegraf collected metric"
type TimeFunc func() time.Time
type MetricFamily struct { type MetricFamily struct {
Name string Name string
Type telegraf.ValueType Type telegraf.ValueType
@ -22,6 +24,7 @@ type MetricFamily struct {
type Metric struct { type Metric struct {
Labels []LabelPair Labels []LabelPair
Time time.Time Time time.Time
AddTime time.Time
Scaler *Scaler Scaler *Scaler
Histogram *Histogram Histogram *Histogram
Summary *Summary Summary *Summary
@ -97,14 +100,14 @@ type Entry struct {
} }
type Collection struct { type Collection struct {
config FormatConfig
Entries map[MetricFamily]Entry Entries map[MetricFamily]Entry
config FormatConfig
} }
func NewCollection(config FormatConfig) *Collection { func NewCollection(config FormatConfig) *Collection {
cache := &Collection{ cache := &Collection{
config: config,
Entries: make(map[MetricFamily]Entry), Entries: make(map[MetricFamily]Entry),
config: config,
} }
return cache return cache
} }
@ -177,7 +180,7 @@ func (c *Collection) createLabels(metric telegraf.Metric) []LabelPair {
return labels return labels
} }
func (c *Collection) Add(metric telegraf.Metric) { func (c *Collection) Add(metric telegraf.Metric, now time.Time) {
labels := c.createLabels(metric) labels := c.createLabels(metric)
for _, field := range metric.FieldList() { for _, field := range metric.FieldList() {
metricName := MetricName(metric.Name(), field.Key, metric.Type()) metricName := MetricName(metric.Name(), field.Key, metric.Type())
@ -227,6 +230,7 @@ func (c *Collection) Add(metric telegraf.Metric) {
m = &Metric{ m = &Metric{
Labels: labels, Labels: labels,
Time: metric.Time(), Time: metric.Time(),
AddTime: now,
Scaler: &Scaler{Value: value}, Scaler: &Scaler{Value: value},
} }
@ -236,6 +240,7 @@ func (c *Collection) Add(metric telegraf.Metric) {
m = &Metric{ m = &Metric{
Labels: labels, Labels: labels,
Time: metric.Time(), Time: metric.Time(),
AddTime: now,
Histogram: &Histogram{}, Histogram: &Histogram{},
} }
} }
@ -283,6 +288,7 @@ func (c *Collection) Add(metric telegraf.Metric) {
m = &Metric{ m = &Metric{
Labels: labels, Labels: labels,
Time: metric.Time(), Time: metric.Time(),
AddTime: now,
Summary: &Summary{}, Summary: &Summary{},
} }
} }
@ -331,7 +337,7 @@ func (c *Collection) Expire(now time.Time, age time.Duration) {
expireTime := now.Add(-age) expireTime := now.Add(-age)
for _, entry := range c.Entries { for _, entry := range c.Entries {
for key, metric := range entry.Metrics { for key, metric := range entry.Metrics {
if metric.Time.Before(expireTime) { if metric.AddTime.Before(expireTime) {
delete(entry.Metrics, key) delete(entry.Metrics, key)
if len(entry.Metrics) == 0 { if len(entry.Metrics) == 0 {
delete(c.Entries, entry.Family) delete(c.Entries, entry.Family)

136
plugins/serializers/prometheus/collection_test.go
View File

@ -12,20 +12,26 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
type Input struct {
metric telegraf.Metric
addtime time.Time
}
func TestCollectionExpire(t *testing.T) { func TestCollectionExpire(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
now time.Time now time.Time
age time.Duration age time.Duration
metrics []telegraf.Metric input []Input
expected []*dto.MetricFamily expected []*dto.MetricFamily
}{ }{
{ {
name: "not expired", name: "not expired",
now: time.Unix(1, 0), now: time.Unix(1, 0),
age: 10 * time.Second, age: 10 * time.Second,
metrics: []telegraf.Metric{ input: []Input{
testutil.MustMetric( {
metric: testutil.MustMetric(
"cpu", "cpu",
map[string]string{}, map[string]string{},
map[string]interface{}{ map[string]interface{}{
@ -33,6 +39,8 @@ func TestCollectionExpire(t *testing.T) {
}, },
time.Unix(0, 0), time.Unix(0, 0),
), ),
addtime: time.Unix(0, 0),
},
}, },
expected: []*dto.MetricFamily{ expected: []*dto.MetricFamily{
{ {
@ -52,8 +60,9 @@ func TestCollectionExpire(t *testing.T) {
name: "update metric expiration", name: "update metric expiration",
now: time.Unix(20, 0), now: time.Unix(20, 0),
age: 10 * time.Second, age: 10 * time.Second,
metrics: []telegraf.Metric{ input: []Input{
testutil.MustMetric( {
metric: testutil.MustMetric(
"cpu", "cpu",
map[string]string{}, map[string]string{},
map[string]interface{}{ map[string]interface{}{
@ -61,7 +70,10 @@ func TestCollectionExpire(t *testing.T) {
}, },
time.Unix(0, 0), time.Unix(0, 0),
), ),
testutil.MustMetric( addtime: time.Unix(0, 0),
},
{
metric: testutil.MustMetric(
"cpu", "cpu",
map[string]string{}, map[string]string{},
map[string]interface{}{ map[string]interface{}{
@ -69,6 +81,8 @@ func TestCollectionExpire(t *testing.T) {
}, },
time.Unix(12, 0), time.Unix(12, 0),
), ),
addtime: time.Unix(12, 0),
},
}, },
expected: []*dto.MetricFamily{ expected: []*dto.MetricFamily{
{ {
@ -88,8 +102,9 @@ func TestCollectionExpire(t *testing.T) {
name: "update metric expiration descending order", name: "update metric expiration descending order",
now: time.Unix(20, 0), now: time.Unix(20, 0),
age: 10 * time.Second, age: 10 * time.Second,
metrics: []telegraf.Metric{ input: []Input{
testutil.MustMetric( {
metric: testutil.MustMetric(
"cpu", "cpu",
map[string]string{}, map[string]string{},
map[string]interface{}{ map[string]interface{}{
@ -97,7 +112,9 @@ func TestCollectionExpire(t *testing.T) {
}, },
time.Unix(12, 0), time.Unix(12, 0),
), ),
testutil.MustMetric( addtime: time.Unix(12, 0),
}, {
metric: testutil.MustMetric(
"cpu", "cpu",
map[string]string{}, map[string]string{},
map[string]interface{}{ map[string]interface{}{
@ -105,6 +122,8 @@ func TestCollectionExpire(t *testing.T) {
}, },
time.Unix(0, 0), time.Unix(0, 0),
), ),
addtime: time.Unix(0, 0),
},
}, },
expected: []*dto.MetricFamily{ expected: []*dto.MetricFamily{
{ {
@ -124,8 +143,9 @@ func TestCollectionExpire(t *testing.T) {
name: "expired single metric in metric family", name: "expired single metric in metric family",
now: time.Unix(20, 0), now: time.Unix(20, 0),
age: 10 * time.Second, age: 10 * time.Second,
metrics: []telegraf.Metric{ input: []Input{
testutil.MustMetric( {
metric: testutil.MustMetric(
"cpu", "cpu",
map[string]string{}, map[string]string{},
map[string]interface{}{ map[string]interface{}{
@ -133,6 +153,8 @@ func TestCollectionExpire(t *testing.T) {
}, },
time.Unix(0, 0), time.Unix(0, 0),
), ),
addtime: time.Unix(0, 0),
},
}, },
expected: []*dto.MetricFamily{}, expected: []*dto.MetricFamily{},
}, },
@ -140,8 +162,9 @@ func TestCollectionExpire(t *testing.T) {
name: "expired one metric in metric family", name: "expired one metric in metric family",
now: time.Unix(20, 0), now: time.Unix(20, 0),
age: 10 * time.Second, age: 10 * time.Second,
metrics: []telegraf.Metric{ input: []Input{
testutil.MustMetric( {
metric: testutil.MustMetric(
"cpu", "cpu",
map[string]string{}, map[string]string{},
map[string]interface{}{ map[string]interface{}{
@ -149,7 +172,9 @@ func TestCollectionExpire(t *testing.T) {
}, },
time.Unix(0, 0), time.Unix(0, 0),
), ),
testutil.MustMetric( addtime: time.Unix(0, 0),
}, {
metric: testutil.MustMetric(
"cpu", "cpu",
map[string]string{}, map[string]string{},
map[string]interface{}{ map[string]interface{}{
@ -157,6 +182,8 @@ func TestCollectionExpire(t *testing.T) {
}, },
time.Unix(15, 0), time.Unix(15, 0),
), ),
addtime: time.Unix(15, 0),
},
}, },
expected: []*dto.MetricFamily{ expected: []*dto.MetricFamily{
{ {
@ -176,8 +203,9 @@ func TestCollectionExpire(t *testing.T) {
name: "histogram bucket updates", name: "histogram bucket updates",
now: time.Unix(0, 0), now: time.Unix(0, 0),
age: 10 * time.Second, age: 10 * time.Second,
metrics: []telegraf.Metric{ input: []Input{
testutil.MustMetric( {
metric: testutil.MustMetric(
"prometheus", "prometheus",
map[string]string{}, map[string]string{},
map[string]interface{}{ map[string]interface{}{
@ -187,7 +215,9 @@ func TestCollectionExpire(t *testing.T) {
time.Unix(0, 0), time.Unix(0, 0),
telegraf.Histogram, telegraf.Histogram,
), ),
testutil.MustMetric( addtime: time.Unix(0, 0),
}, {
metric: testutil.MustMetric(
"prometheus", "prometheus",
map[string]string{"le": "0.05"}, map[string]string{"le": "0.05"},
map[string]interface{}{ map[string]interface{}{
@ -196,7 +226,9 @@ func TestCollectionExpire(t *testing.T) {
time.Unix(0, 0), time.Unix(0, 0),
telegraf.Histogram, telegraf.Histogram,
), ),
testutil.MustMetric( addtime: time.Unix(0, 0),
}, {
metric: testutil.MustMetric(
"prometheus", "prometheus",
map[string]string{"le": "+Inf"}, map[string]string{"le": "+Inf"},
map[string]interface{}{ map[string]interface{}{
@ -205,8 +237,10 @@ func TestCollectionExpire(t *testing.T) {
time.Unix(0, 0), time.Unix(0, 0),
telegraf.Histogram, telegraf.Histogram,
), ),
addtime: time.Unix(0, 0),
}, {
// Next interval // Next interval
testutil.MustMetric( metric: testutil.MustMetric(
"prometheus", "prometheus",
map[string]string{}, map[string]string{},
map[string]interface{}{ map[string]interface{}{
@ -216,7 +250,9 @@ func TestCollectionExpire(t *testing.T) {
time.Unix(0, 0), time.Unix(0, 0),
telegraf.Histogram, telegraf.Histogram,
), ),
testutil.MustMetric( addtime: time.Unix(0, 0),
}, {
metric: testutil.MustMetric(
"prometheus", "prometheus",
map[string]string{"le": "0.05"}, map[string]string{"le": "0.05"},
map[string]interface{}{ map[string]interface{}{
@ -225,7 +261,9 @@ func TestCollectionExpire(t *testing.T) {
time.Unix(0, 0), time.Unix(0, 0),
telegraf.Histogram, telegraf.Histogram,
), ),
testutil.MustMetric( addtime: time.Unix(0, 0),
}, {
metric: testutil.MustMetric(
"prometheus", "prometheus",
map[string]string{"le": "+Inf"}, map[string]string{"le": "+Inf"},
map[string]interface{}{ map[string]interface{}{
@ -234,6 +272,8 @@ func TestCollectionExpire(t *testing.T) {
time.Unix(0, 0), time.Unix(0, 0),
telegraf.Histogram, telegraf.Histogram,
), ),
addtime: time.Unix(0, 0),
},
}, },
expected: []*dto.MetricFamily{ expected: []*dto.MetricFamily{
{ {
@ -266,8 +306,9 @@ func TestCollectionExpire(t *testing.T) {
name: "summary quantile updates", name: "summary quantile updates",
now: time.Unix(0, 0), now: time.Unix(0, 0),
age: 10 * time.Second, age: 10 * time.Second,
metrics: []telegraf.Metric{ input: []Input{
testutil.MustMetric( {
metric: testutil.MustMetric(
"prometheus", "prometheus",
map[string]string{}, map[string]string{},
map[string]interface{}{ map[string]interface{}{
@ -277,7 +318,9 @@ func TestCollectionExpire(t *testing.T) {
time.Unix(0, 0), time.Unix(0, 0),
telegraf.Summary, telegraf.Summary,
), ),
testutil.MustMetric( addtime: time.Unix(0, 0),
}, {
metric: testutil.MustMetric(
"prometheus", "prometheus",
map[string]string{"quantile": "0.01"}, map[string]string{"quantile": "0.01"},
map[string]interface{}{ map[string]interface{}{
@ -286,8 +329,10 @@ func TestCollectionExpire(t *testing.T) {
time.Unix(0, 0), time.Unix(0, 0),
telegraf.Summary, telegraf.Summary,
), ),
addtime: time.Unix(0, 0),
}, {
// Updated Summary // Updated Summary
testutil.MustMetric( metric: testutil.MustMetric(
"prometheus", "prometheus",
map[string]string{}, map[string]string{},
map[string]interface{}{ map[string]interface{}{
@ -297,7 +342,9 @@ func TestCollectionExpire(t *testing.T) {
time.Unix(0, 0), time.Unix(0, 0),
telegraf.Summary, telegraf.Summary,
), ),
testutil.MustMetric( addtime: time.Unix(0, 0),
}, {
metric: testutil.MustMetric(
"prometheus", "prometheus",
map[string]string{"quantile": "0.01"}, map[string]string{"quantile": "0.01"},
map[string]interface{}{ map[string]interface{}{
@ -306,6 +353,8 @@ func TestCollectionExpire(t *testing.T) {
time.Unix(0, 0), time.Unix(0, 0),
telegraf.Summary, telegraf.Summary,
), ),
addtime: time.Unix(0, 0),
},
}, },
expected: []*dto.MetricFamily{ expected: []*dto.MetricFamily{
{ {
@ -330,12 +379,43 @@ func TestCollectionExpire(t *testing.T) {
}, },
}, },
}, },
{
name: "expire based on add time",
now: time.Unix(20, 0),
age: 10 * time.Second,
input: []Input{
{
metric: testutil.MustMetric(
"cpu",
map[string]string{},
map[string]interface{}{
"time_idle": 42.0,
},
time.Unix(0, 0),
),
addtime: time.Unix(15, 0),
},
},
expected: []*dto.MetricFamily{
{
Name: proto.String("cpu_time_idle"),
Help: proto.String(helpString),
Type: dto.MetricType_UNTYPED.Enum(),
Metric: []*dto.Metric{
{
Label: []*dto.LabelPair{},
Untyped: &dto.Untyped{Value: proto.Float64(42.0)},
},
},
},
},
},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
c := NewCollection(FormatConfig{}) c := NewCollection(FormatConfig{})
for _, metric := range tt.metrics { for _, item := range tt.input {
c.Add(metric) c.Add(item.metric, item.addtime)
} }
c.Expire(tt.now, tt.age) c.Expire(tt.now, tt.age)

3
plugins/serializers/prometheus/prometheus.go
View File

@ -2,6 +2,7 @@ package prometheus
import ( import (
"bytes" "bytes"
"time"
"github.com/influxdata/telegraf" "github.com/influxdata/telegraf"
"github.com/prometheus/common/expfmt" "github.com/prometheus/common/expfmt"
@ -53,7 +54,7 @@ func (s *Serializer) Serialize(metric telegraf.Metric) ([]byte, error) {
func (s *Serializer) SerializeBatch(metrics []telegraf.Metric) ([]byte, error) { func (s *Serializer) SerializeBatch(metrics []telegraf.Metric) ([]byte, error) {
coll := NewCollection(s.config) coll := NewCollection(s.config)
for _, metric := range metrics { for _, metric := range metrics {
coll.Add(metric) coll.Add(metric, time.Now())
} }
var buf bytes.Buffer var buf bytes.Buffer