From e34c52402f8e4ceac7dced9271b0fe1d5c653761 Mon Sep 17 00:00:00 2001 From: Brian Brazil Date: Sat, 20 Jun 2015 13:32:32 +0100 Subject: [PATCH 1/2] Improve test infrastructure --- plugins/mysql/mysql_test.go | 2 +- plugins/postgresql/postgresql_test.go | 2 +- plugins/system/system_test.go | 4 +++- testutil/accumulator.go | 25 +++++++++++-------------- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/plugins/mysql/mysql_test.go b/plugins/mysql/mysql_test.go index 33643861a..76cb28b0d 100644 --- a/plugins/mysql/mysql_test.go +++ b/plugins/mysql/mysql_test.go @@ -39,7 +39,7 @@ func TestMysqlGeneratesMetrics(t *testing.T) { var count int for _, p := range acc.Points { - if strings.HasPrefix(p.Name, prefix.prefix) { + if strings.HasPrefix(p.Measurement, prefix.prefix) { count++ } } diff --git a/plugins/postgresql/postgresql_test.go b/plugins/postgresql/postgresql_test.go index b11200e9f..91776d619 100644 --- a/plugins/postgresql/postgresql_test.go +++ b/plugins/postgresql/postgresql_test.go @@ -91,7 +91,7 @@ func TestPostgresqlDefaultsToAllDatabases(t *testing.T) { var found bool for _, pnt := range acc.Points { - if pnt.Name == "xact_commit" { + if pnt.Measurement == "xact_commit" { if pnt.Tags["db"] == "postgres" { found = true break diff --git a/plugins/system/system_test.go b/plugins/system/system_test.go index b8e0e169c..1c40b7f97 100644 --- a/plugins/system/system_test.go +++ b/plugins/system/system_test.go @@ -272,7 +272,9 @@ func TestSystemStats_GenerateStats(t *testing.T) { require.NoError(t, err) dockertags := map[string]string{ - "id": "blah", + "name": "blah", + "id": "", + "command": "", } assert.True(t, acc.CheckTaggedValue("user", 3.1, dockertags)) diff --git a/testutil/accumulator.go b/testutil/accumulator.go index 645366fd5..6a7ad99f0 100644 --- a/testutil/accumulator.go +++ b/testutil/accumulator.go @@ -2,6 +2,7 @@ package testutil import ( "fmt" + "reflect" "time" ) @@ -18,6 +19,9 @@ type Accumulator struct { } func (a *Accumulator) Add(measurement string, value interface{}, tags map[string]string) { + if tags == nil { + tags = map[string]string{} + } a.Points = append( a.Points, &Point{ @@ -70,30 +74,23 @@ func (a *Accumulator) CheckTaggedValue(measurement string, val interface{}, tags } func (a *Accumulator) ValidateTaggedValue(measurement string, val interface{}, tags map[string]string) error { + if tags == nil { + tags = map[string]string{} + } for _, p := range a.Points { - var found bool - - if p.Tags == nil && tags == nil { - found = true - } else { - for k, v := range p.Tags { - if tags[k] == v { - found = true - break - } - } + if !reflect.DeepEqual(tags, p.Tags) { + continue } - if found && p.Measurement == measurement { + if p.Measurement == measurement { if p.Value != val { return fmt.Errorf("%v (%T) != %v (%T)", p.Value, p.Value, val, val) } - return nil } } - return fmt.Errorf("unknown value %s with tags %v", measurement, tags) + return fmt.Errorf("unknown measurement %s with tags %v", measurement, tags) } func (a *Accumulator) ValidateValue(measurement string, val interface{}) error { From 5390a8ea71ca3055507cd78bdf1b1db895626f37 Mon Sep 17 00:00:00 2001 From: Brian Brazil Date: Sat, 20 Jun 2015 13:38:01 +0100 Subject: [PATCH 2/2] Add Prometheus plugin. This allows pulling Prometheus metrics from any client library or exporter over HTTP. --- README.md | 1 + plugins/all/all.go | 1 + plugins/prometheus/prometheus.go | 105 ++++++++++++++++++++++++++ plugins/prometheus/prometheus_test.go | 56 ++++++++++++++ 4 files changed, 163 insertions(+) create mode 100644 plugins/prometheus/prometheus.go create mode 100644 plugins/prometheus/prometheus_test.go diff --git a/README.md b/README.md index d72701aaf..8b457abb4 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ Telegraf currently has support for collecting metrics from: * System (memory, CPU, network, etc.) * Docker * MySQL +* Prometheus (client libraries and exporters) * PostgreSQL * Redis diff --git a/plugins/all/all.go b/plugins/all/all.go index 71381aa87..23a3bc4c7 100644 --- a/plugins/all/all.go +++ b/plugins/all/all.go @@ -4,6 +4,7 @@ import ( _ "github.com/influxdb/telegraf/plugins/memcached" _ "github.com/influxdb/telegraf/plugins/mysql" _ "github.com/influxdb/telegraf/plugins/postgresql" + _ "github.com/influxdb/telegraf/plugins/prometheus" _ "github.com/influxdb/telegraf/plugins/redis" _ "github.com/influxdb/telegraf/plugins/system" ) diff --git a/plugins/prometheus/prometheus.go b/plugins/prometheus/prometheus.go new file mode 100644 index 000000000..4029e9932 --- /dev/null +++ b/plugins/prometheus/prometheus.go @@ -0,0 +1,105 @@ +package prometheus + +import ( + "errors" + "fmt" + "net/http" + "sync" + "time" + + "github.com/influxdb/telegraf/plugins" + "github.com/prometheus/client_golang/extraction" + "github.com/prometheus/client_golang/model" +) + +type Prometheus struct { + Urls []string +} + +var sampleConfig = ` +# An array of urls to scrape metrics from. +urls = ["http://localhost:9100/metrics"]` + +func (r *Prometheus) SampleConfig() string { + return sampleConfig +} + +func (r *Prometheus) Description() string { + return "Read metrics from one or many prometheus clients" +} + +var ErrProtocolError = errors.New("prometheus protocol error") + +// Reads stats from all configured servers accumulates stats. +// Returns one of the errors encountered while gather stats (if any). +func (g *Prometheus) Gather(acc plugins.Accumulator) error { + var wg sync.WaitGroup + + var outerr error + + for _, serv := range g.Urls { + wg.Add(1) + go func(serv string) { + defer wg.Done() + outerr = g.gatherURL(serv, acc) + }(serv) + } + + wg.Wait() + + return outerr +} + +func (g *Prometheus) gatherURL(url string, acc plugins.Accumulator) error { + resp, err := http.Get(url) + if err != nil { + return fmt.Errorf("error making HTTP request to %s: %s", url, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("%s returned HTTP status %s", url, resp.Status) + } + processor, err := extraction.ProcessorForRequestHeader(resp.Header) + if err != nil { + return fmt.Errorf("error getting extractor for %s: %s", url, err) + } + + ingestor := &Ingester{ + acc: acc, + } + + options := &extraction.ProcessOptions{ + Timestamp: model.TimestampFromTime(time.Now()), + } + + err = processor.ProcessSingle(resp.Body, ingestor, options) + if err != nil { + return fmt.Errorf("error getting processing samples for %s: %s", url, err) + } + return nil +} + +type Ingester struct { + acc plugins.Accumulator +} + +// Ingest implements an extraction.Ingester. +func (i *Ingester) Ingest(samples model.Samples) error { + for _, sample := range samples { + tags := map[string]string{} + for key, value := range sample.Metric { + if key == model.MetricNameLabel { + continue + } + tags[string(key)] = string(value) + } + i.acc.Add(string(sample.Metric[model.MetricNameLabel]), float64(sample.Value), tags) + } + return nil +} + +func init() { + plugins.Add("prometheus", func() plugins.Plugin { + return &Prometheus{} + }) +} diff --git a/plugins/prometheus/prometheus_test.go b/plugins/prometheus/prometheus_test.go new file mode 100644 index 000000000..be2b190e7 --- /dev/null +++ b/plugins/prometheus/prometheus_test.go @@ -0,0 +1,56 @@ +package prometheus + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/influxdb/telegraf/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + + +const sampleTextFormat = `# HELP go_gc_duration_seconds A summary of the GC invocation durations. +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{quantile="0"} 0.00010425500000000001 +go_gc_duration_seconds{quantile="0.25"} 0.000139108 +go_gc_duration_seconds{quantile="0.5"} 0.00015749400000000002 +go_gc_duration_seconds{quantile="0.75"} 0.000331463 +go_gc_duration_seconds{quantile="1"} 0.000667154 +go_gc_duration_seconds_sum 0.0018183950000000002 +go_gc_duration_seconds_count 7 +# HELP go_goroutines Number of goroutines that currently exist. +# TYPE go_goroutines gauge +go_goroutines 15 +` + +func TestPrometheusGeneratesMetrics(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, sampleTextFormat) + })) + defer ts.Close() + + p := &Prometheus{ + Urls: []string{ts.URL}, + } + + var acc testutil.Accumulator + + err := p.Gather(&acc) + require.NoError(t, err) + + expected := []struct { + name string + value float64 + tags map[string]string + }{ + {"go_gc_duration_seconds_count", 7, map[string]string{}}, + {"go_goroutines", 15, map[string]string{}}, + } + + for _, e := range expected { + assert.NoError(t, acc.ValidateValue(e.name, e.value)) + } +}