From 0f1b4e06f519e206cbc9607017a2cffb5873b108 Mon Sep 17 00:00:00 2001 From: Nikhil Bafna Date: Sat, 2 Apr 2016 10:13:21 +0530 Subject: [PATCH 01/55] Update README.md Fix redis input plugin name in configuration example --- plugins/inputs/redis/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/redis/README.md b/plugins/inputs/redis/README.md index d7d98ccc9..1cbaea0ca 100644 --- a/plugins/inputs/redis/README.md +++ b/plugins/inputs/redis/README.md @@ -62,7 +62,7 @@ Using this configuration: ``` -[[inputs.nginx]] +[[inputs.redis]] ## specify servers via a url matching: ## [protocol://][:password]@address[:port] ## e.g. From 357849c34846f0c347e6048f4372ed1f39014d06 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Sat, 2 Apr 2016 13:35:40 -0600 Subject: [PATCH 02/55] Godeps: update wvanbergen/kafka dependency see https://github.com/wvanbergen/kafka/pull/87 fixes #805 --- CHANGELOG.md | 1 + Godeps | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 305fa6d03..9546aba14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ - [#841](https://github.com/influxdata/telegraf/issues/841): Fix memcached unix socket panic. - [#873](https://github.com/influxdata/telegraf/issues/873): Fix SNMP plugin sometimes not returning metrics. Thanks @titiliambert! - [#934](https://github.com/influxdata/telegraf/pull/934): phpfpm: Fix fcgi uri path. Thanks @rudenkovk! +- [#805](https://github.com/influxdata/telegraf/issues/805): Kafka consumer stops gathering after i/o timeout. ## v0.11.1 [2016-03-17] diff --git a/Godeps b/Godeps index 2fc53d8c5..255b95ab5 100644 --- a/Godeps +++ b/Godeps @@ -41,7 +41,7 @@ github.com/shirou/gopsutil 1f32ce1bb380845be7f5d174ac641a2c592c0c42 github.com/soniah/gosnmp b1b4f885b12c5dcbd021c5cee1c904110de6db7d github.com/streadway/amqp b4f3ceab0337f013208d31348b578d83c0064744 github.com/stretchr/testify 1f4a1643a57e798696635ea4c126e9127adb7d3c -github.com/wvanbergen/kafka 1a8639a45164fcc245d5c7b4bd3ccfbd1a0ffbf3 +github.com/wvanbergen/kafka 46f9a1cf3f670edec492029fadded9c2d9e18866 github.com/wvanbergen/kazoo-go 0f768712ae6f76454f987c3356177e138df258f8 github.com/zensqlmonitor/go-mssqldb ffe5510c6fa5e15e6d983210ab501c815b56b363 golang.org/x/crypto 5dc8cb4b8a8eb076cbb5a06bc3b8682c15bdbbd3 From 8509101b830ff5a521dd5b99ae1a568edf3a04a3 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Mon, 4 Apr 2016 15:16:58 -0600 Subject: [PATCH 03/55] drop cpu_time_* from procstat by default closes #963 --- plugins/inputs/procstat/procstat.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/inputs/procstat/procstat.go b/plugins/inputs/procstat/procstat.go index e5ae207fe..a0e63fd6f 100644 --- a/plugins/inputs/procstat/procstat.go +++ b/plugins/inputs/procstat/procstat.go @@ -43,6 +43,8 @@ var sampleConfig = ` ## Field name prefix prefix = "" + ## comment this out if you want raw cpu_time stats + fielddrop = ["cpu_time_*"] ` func (_ *Procstat) SampleConfig() string { From 5fe8903fd2c229260e140af99d91255531b4f5ef Mon Sep 17 00:00:00 2001 From: Pierre Fersing Date: Mon, 4 Apr 2016 11:59:28 +0200 Subject: [PATCH 04/55] Use timeout smaller than 10 seconds closes #959 --- plugins/inputs/mongodb/mongodb.go | 2 +- plugins/inputs/mongodb/mongodb_test.go | 2 +- plugins/inputs/prometheus/prometheus.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/inputs/mongodb/mongodb.go b/plugins/inputs/mongodb/mongodb.go index 3be04477b..381814531 100644 --- a/plugins/inputs/mongodb/mongodb.go +++ b/plugins/inputs/mongodb/mongodb.go @@ -103,7 +103,7 @@ func (m *MongoDB) gatherServer(server *Server, acc telegraf.Accumulator) error { dialAddrs[0], err.Error()) } dialInfo.Direct = true - dialInfo.Timeout = time.Duration(10) * time.Second + dialInfo.Timeout = 5 * time.Second if m.Ssl.Enabled { tlsConfig := &tls.Config{} diff --git a/plugins/inputs/mongodb/mongodb_test.go b/plugins/inputs/mongodb/mongodb_test.go index 174128d19..73e68ed37 100644 --- a/plugins/inputs/mongodb/mongodb_test.go +++ b/plugins/inputs/mongodb/mongodb_test.go @@ -43,7 +43,7 @@ func testSetup(m *testing.M) { log.Fatalf("Unable to parse URL (%s), %s\n", dialAddrs[0], err.Error()) } dialInfo.Direct = true - dialInfo.Timeout = time.Duration(10) * time.Second + dialInfo.Timeout = 5 * time.Second sess, err := mgo.DialWithInfo(dialInfo) if err != nil { log.Fatalf("Unable to connect to MongoDB, %s\n", err.Error()) diff --git a/plugins/inputs/prometheus/prometheus.go b/plugins/inputs/prometheus/prometheus.go index 460a79faf..1c60a363e 100644 --- a/plugins/inputs/prometheus/prometheus.go +++ b/plugins/inputs/prometheus/prometheus.go @@ -80,10 +80,10 @@ func (p *Prometheus) gatherURL(url string, acc telegraf.Accumulator) error { var rt http.RoundTripper = &http.Transport{ Dial: (&net.Dialer{ - Timeout: 10 * time.Second, + Timeout: 5 * time.Second, KeepAlive: 30 * time.Second, }).Dial, - TLSHandshakeTimeout: 10 * time.Second, + TLSHandshakeTimeout: 5 * time.Second, TLSClientConfig: &tls.Config{ InsecureSkipVerify: p.InsecureSkipVerify, }, From d9bb1ceaeca516a1eb60e86297af5622b86e9491 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Mon, 4 Apr 2016 16:12:50 -0600 Subject: [PATCH 05/55] Changelog update --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9546aba14..46895cb05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ - [#873](https://github.com/influxdata/telegraf/issues/873): Fix SNMP plugin sometimes not returning metrics. Thanks @titiliambert! - [#934](https://github.com/influxdata/telegraf/pull/934): phpfpm: Fix fcgi uri path. Thanks @rudenkovk! - [#805](https://github.com/influxdata/telegraf/issues/805): Kafka consumer stops gathering after i/o timeout. +- [#959](https://github.com/influxdata/telegraf/pull/959): reduce mongodb & prometheus collection timeouts. Thanks @PierreF! ## v0.11.1 [2016-03-17] From d2d91e713a0727ce0ba7cdbe8027c723aecb3d9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florent=20Rami=C3=A8re?= Date: Mon, 4 Apr 2016 23:57:52 +0200 Subject: [PATCH 06/55] Add plugin links closes #964 --- README.md | 140 +++++++++++++++++++++++++++--------------------------- 1 file changed, 70 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index 000b27620..65d2d3226 100644 --- a/README.md +++ b/README.md @@ -156,55 +156,55 @@ more information on each, please look at the directory of the same name in Currently implemented sources: -* aerospike -* apache -* bcache -* couchbase -* couchdb -* disque -* dns query time -* docker -* dovecot -* elasticsearch -* exec (generic executable plugin, support JSON, influx, graphite and nagios) -* haproxy -* httpjson (generic JSON-emitting http service plugin) -* influxdb -* ipmi_sensor -* jolokia -* leofs -* lustre2 -* mailchimp -* memcached -* mesos -* mongodb -* mysql -* net_response -* nginx -* nsq -* ntpq -* phpfpm -* phusion passenger -* ping -* postgresql -* postgresql_extensible -* powerdns -* procstat -* prometheus -* puppetagent -* rabbitmq -* raindrops -* redis -* rethinkdb -* riak -* sensors (only available if built from source) -* snmp -* sql server (microsoft) -* twemproxy -* zfs -* zookeeper -* win_perf_counters (windows performance counters) -* system +* [aerospike](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/aerospike) +* [apache](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/apache) +* [bcache](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/bcache) +* [couchbase](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/couchbase) +* [couchdb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/couchdb) +* [disque](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/disque) +* [dns query time](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/dns_query) +* [docker](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/docker) +* [dovecot](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/dovecot) +* [elasticsearch](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/elasticsearch) +* [exec](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/exec ) (generic executable plugin, support JSON, influx, graphite and nagios) +* [haproxy](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/haproxy) +* [httpjson ](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/httpjson ) (generic JSON-emitting http service plugin) +* [influxdb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/influxdb) +* [ipmi_sensor](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/ipmi_sensor) +* [jolokia](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/jolokia) +* [leofs](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/leofs) +* [lustre2](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/lustre2) +* [mailchimp](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mailchimp) +* [memcached](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/memcached) +* [mesos](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mesos) +* [mongodb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mongodb) +* [mysql](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mysql) +* [net_response](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/net_response) +* [nginx](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nginx) +* [nsq](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nsq) +* [ntpq](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/ntpq) +* [phpfpm](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/phpfpm) +* [phusion passenger](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/passenger) +* [ping](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/ping) +* [postgresql](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/postgresql) +* [postgresql_extensible](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/postgresql_extensible) +* [powerdns](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/powerdns) +* [procstat](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/procstat) +* [prometheus](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/prometheus) +* [puppetagent](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/puppetagent) +* [rabbitmq](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/rabbitmq) +* [raindrops](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/raindrops) +* [redis](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/redis) +* [rethinkdb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/rethinkdb) +* [riak](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/riak) +* [sensors ](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/sensors) (only available if built from source) +* [snmp](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/snmp) +* [sql server](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/sqlserver) (microsoft) +* [twemproxy](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/twemproxy) +* [zfs](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/zfs) +* [zookeeper](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/zookeeper) +* [win_perf_counters ](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/win_perf_counters) (windows performance counters) +* [system](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/system) * cpu * mem * net @@ -217,33 +217,33 @@ Currently implemented sources: Telegraf can also collect metrics via the following service plugins: -* statsd -* udp_listener -* tcp_listener -* mqtt_consumer -* kafka_consumer -* nats_consumer -* github_webhooks +* [statsd](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/statsd) +* [udp_listener](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/udp_listener) +* [tcp_listener](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/tcp_listener) +* [mqtt_consumer](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mqtt_consumer) +* [kafka_consumer](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/kafka_consumer) +* [nats_consumer](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nats_consumer) +* [github_webhooks](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/github_webhooks) We'll be adding support for many more over the coming months. Read on if you want to add support for another service or third-party API. ## Supported Output Plugins -* influxdb -* amon -* amqp -* aws kinesis -* aws cloudwatch -* datadog -* graphite -* kafka -* librato -* mqtt -* nsq -* opentsdb -* prometheus -* riemann +* [influxdb](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/influxdb) +* [amon](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/amon) +* [amqp](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/amqp) +* [aws kinesis](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/kinesis) +* [aws cloudwatch](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/cloudwatch) +* [datadog](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/datadog) +* [graphite](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/graphite) +* [kafka](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/kafka) +* [librato](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/librato) +* [mqtt](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/mqtt) +* [nsq](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/nsq) +* [opentsdb](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/opentsdb) +* [prometheus](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/prometheus_client) +* [riemann](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/riemann) ## Contributing From a4a140bfadb667d92c6dfad3334b97d0a1427adb Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Mon, 4 Apr 2016 16:30:24 -0600 Subject: [PATCH 07/55] etc/telegraf.conf update for procstat change --- etc/telegraf.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etc/telegraf.conf b/etc/telegraf.conf index 633483e22..694bd6564 100644 --- a/etc/telegraf.conf +++ b/etc/telegraf.conf @@ -894,6 +894,8 @@ # # ## Field name prefix # prefix = "" +# ## comment this out if you want raw cpu_time stats +# fielddrop = ["cpu_time_*"] # # Read metrics from one or many prometheus clients From 70ef61ac6df9e7da234dbbe9ca488f1c2d770b2b Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Mon, 4 Apr 2016 16:34:41 -0600 Subject: [PATCH 08/55] Release 0.12 --- CHANGELOG.md | 2 +- README.md | 112 +++++++++++++++++++++++++-------------------------- 2 files changed, 57 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46895cb05..754701e44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.12.0 [unreleased] +## v0.12.0 [2016-04-05] ### Features - [#951](https://github.com/influxdata/telegraf/pull/951): Parse environment variables in the config file. diff --git a/README.md b/README.md index 65d2d3226..9813ca6d4 100644 --- a/README.md +++ b/README.md @@ -20,12 +20,12 @@ new plugins. ### Linux deb and rpm Packages: Latest: -* http://get.influxdb.org/telegraf/telegraf_0.11.1-1_amd64.deb -* http://get.influxdb.org/telegraf/telegraf-0.11.1-1.x86_64.rpm +* http://get.influxdb.org/telegraf/telegraf_0.12.0-1_amd64.deb +* http://get.influxdb.org/telegraf/telegraf-0.12.0-1.x86_64.rpm Latest (arm): -* http://get.influxdb.org/telegraf/telegraf_0.11.1-1_armhf.deb -* http://get.influxdb.org/telegraf/telegraf-0.11.1-1.armhf.rpm +* http://get.influxdb.org/telegraf/telegraf_0.12.0-1_armhf.deb +* http://get.influxdb.org/telegraf/telegraf-0.12.0-1.armhf.rpm ##### Package Instructions: @@ -46,28 +46,28 @@ to use this repo to install & update telegraf. ### Linux tarballs: Latest: -* http://get.influxdb.org/telegraf/telegraf-0.11.1-1_linux_amd64.tar.gz -* http://get.influxdb.org/telegraf/telegraf-0.11.1-1_linux_i386.tar.gz -* http://get.influxdb.org/telegraf/telegraf-0.11.1-1_linux_armhf.tar.gz +* http://get.influxdb.org/telegraf/telegraf-0.12.0-1_linux_amd64.tar.gz +* http://get.influxdb.org/telegraf/telegraf-0.12.0-1_linux_i386.tar.gz +* http://get.influxdb.org/telegraf/telegraf-0.12.0-1_linux_armhf.tar.gz ##### tarball Instructions: To install the full directory structure with config file, run: ``` -sudo tar -C / -zxvf ./telegraf-0.11.1-1_linux_amd64.tar.gz +sudo tar -C / -zxvf ./telegraf-0.12.0-1_linux_amd64.tar.gz ``` To extract only the binary, run: ``` -tar -zxvf telegraf-0.11.1-1_linux_amd64.tar.gz --strip-components=3 ./usr/bin/telegraf +tar -zxvf telegraf-0.12.0-1_linux_amd64.tar.gz --strip-components=3 ./usr/bin/telegraf ``` ### FreeBSD tarball: Latest: -* http://get.influxdb.org/telegraf/telegraf-0.11.1-1_freebsd_amd64.tar.gz +* http://get.influxdb.org/telegraf/telegraf-0.12.0-1_freebsd_amd64.tar.gz ##### tarball Instructions: @@ -87,8 +87,8 @@ brew install telegraf ### Windows Binaries (EXPERIMENTAL) Latest: -* http://get.influxdb.org/telegraf/telegraf-0.11.1-1_windows_amd64.zip -* http://get.influxdb.org/telegraf/telegraf-0.11.1-1_windows_i386.zip +* http://get.influxdb.org/telegraf/telegraf-0.12.0-1_windows_amd64.zip +* http://get.influxdb.org/telegraf/telegraf-0.12.0-1_windows_i386.zip ### From Source: @@ -156,55 +156,55 @@ more information on each, please look at the directory of the same name in Currently implemented sources: -* [aerospike](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/aerospike) -* [apache](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/apache) -* [bcache](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/bcache) -* [couchbase](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/couchbase) -* [couchdb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/couchdb) -* [disque](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/disque) -* [dns query time](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/dns_query) -* [docker](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/docker) -* [dovecot](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/dovecot) -* [elasticsearch](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/elasticsearch) +* [aerospike](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/aerospike) +* [apache](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/apache) +* [bcache](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/bcache) +* [couchbase](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/couchbase) +* [couchdb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/couchdb) +* [disque](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/disque) +* [dns query time](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/dns_query) +* [docker](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/docker) +* [dovecot](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/dovecot) +* [elasticsearch](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/elasticsearch) * [exec](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/exec ) (generic executable plugin, support JSON, influx, graphite and nagios) -* [haproxy](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/haproxy) +* [haproxy](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/haproxy) * [httpjson ](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/httpjson ) (generic JSON-emitting http service plugin) -* [influxdb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/influxdb) -* [ipmi_sensor](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/ipmi_sensor) -* [jolokia](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/jolokia) -* [leofs](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/leofs) -* [lustre2](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/lustre2) -* [mailchimp](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mailchimp) -* [memcached](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/memcached) -* [mesos](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mesos) -* [mongodb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mongodb) -* [mysql](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mysql) -* [net_response](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/net_response) -* [nginx](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nginx) -* [nsq](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nsq) -* [ntpq](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/ntpq) -* [phpfpm](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/phpfpm) -* [phusion passenger](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/passenger) -* [ping](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/ping) -* [postgresql](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/postgresql) -* [postgresql_extensible](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/postgresql_extensible) -* [powerdns](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/powerdns) -* [procstat](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/procstat) -* [prometheus](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/prometheus) -* [puppetagent](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/puppetagent) -* [rabbitmq](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/rabbitmq) -* [raindrops](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/raindrops) -* [redis](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/redis) -* [rethinkdb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/rethinkdb) -* [riak](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/riak) +* [influxdb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/influxdb) +* [ipmi_sensor](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/ipmi_sensor) +* [jolokia](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/jolokia) +* [leofs](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/leofs) +* [lustre2](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/lustre2) +* [mailchimp](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mailchimp) +* [memcached](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/memcached) +* [mesos](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mesos) +* [mongodb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mongodb) +* [mysql](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mysql) +* [net_response](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/net_response) +* [nginx](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nginx) +* [nsq](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nsq) +* [ntpq](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/ntpq) +* [phpfpm](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/phpfpm) +* [phusion passenger](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/passenger) +* [ping](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/ping) +* [postgresql](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/postgresql) +* [postgresql_extensible](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/postgresql_extensible) +* [powerdns](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/powerdns) +* [procstat](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/procstat) +* [prometheus](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/prometheus) +* [puppetagent](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/puppetagent) +* [rabbitmq](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/rabbitmq) +* [raindrops](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/raindrops) +* [redis](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/redis) +* [rethinkdb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/rethinkdb) +* [riak](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/riak) * [sensors ](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/sensors) (only available if built from source) -* [snmp](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/snmp) +* [snmp](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/snmp) * [sql server](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/sqlserver) (microsoft) -* [twemproxy](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/twemproxy) -* [zfs](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/zfs) -* [zookeeper](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/zookeeper) +* [twemproxy](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/twemproxy) +* [zfs](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/zfs) +* [zookeeper](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/zookeeper) * [win_perf_counters ](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/win_perf_counters) (windows performance counters) -* [system](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/system) +* [system](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/system) * cpu * mem * net From d871e9aee7d8828c6b11ab61fb76a30a7eda6f71 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Mon, 4 Apr 2016 17:43:53 -0600 Subject: [PATCH 09/55] Dummy kernel plugin added for consistent config generation --- etc/telegraf.conf | 5 +++++ plugins/inputs/system/kernel_notlinux.go | 27 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 plugins/inputs/system/kernel_notlinux.go diff --git a/etc/telegraf.conf b/etc/telegraf.conf index 694bd6564..3d65aaf62 100644 --- a/etc/telegraf.conf +++ b/etc/telegraf.conf @@ -378,6 +378,11 @@ # skip_serial_number = true +# Get kernel statistics from /proc/stat +[[inputs.kernel]] + # no configuration + + # Read metrics about memory usage [[inputs.mem]] # no configuration diff --git a/plugins/inputs/system/kernel_notlinux.go b/plugins/inputs/system/kernel_notlinux.go new file mode 100644 index 000000000..9053b5c04 --- /dev/null +++ b/plugins/inputs/system/kernel_notlinux.go @@ -0,0 +1,27 @@ +// +build !linux + +package system + +import ( + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" +) + +type Kernel struct { +} + +func (k *Kernel) Description() string { + return "Get kernel statistics from /proc/stat" +} + +func (k *Kernel) SampleConfig() string { return "" } + +func (k *Kernel) Gather(acc telegraf.Accumulator) error { + return nil +} + +func init() { + inputs.Add("kernel", func() telegraf.Input { + return &Kernel{} + }) +} From 863cbe512d63b9da0001190eb0c9d39ea0c6b24d Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Tue, 5 Apr 2016 10:21:57 -0600 Subject: [PATCH 10/55] processes plugin: fix case where there are spaces in cmd name fixes #968 --- CHANGELOG.md | 7 ++++++ plugins/inputs/system/processes.go | 13 ++++++++--- plugins/inputs/system/processes_test.go | 30 +++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 754701e44..82a224f21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## v0.12.1 [unreleased] + +### Features + +### Bugfixes +- [#968](https://github.com/influxdata/telegraf/issues/968): Processes plugin gets unknown state when spaces are in (command name) + ## v0.12.0 [2016-04-05] ### Features diff --git a/plugins/inputs/system/processes.go b/plugins/inputs/system/processes.go index aae0e6ba4..8c50a4ebd 100644 --- a/plugins/inputs/system/processes.go +++ b/plugins/inputs/system/processes.go @@ -144,11 +144,18 @@ func (p *Processes) gatherFromProc(fields map[string]interface{}) error { continue } + // Parse out data after () + i := bytes.LastIndex(data, []byte(")")) + if i == -1 { + continue + } + data = data[i+2:] + stats := bytes.Fields(data) if len(stats) < 3 { return fmt.Errorf("Something is terribly wrong with %s", statFile) } - switch stats[2][0] { + switch stats[0][0] { case 'R': fields["running"] = fields["running"].(int64) + int64(1) case 'S': @@ -163,11 +170,11 @@ func (p *Processes) gatherFromProc(fields map[string]interface{}) error { fields["paging"] = fields["paging"].(int64) + int64(1) default: log.Printf("processes: Unknown state [ %s ] in file %s", - string(stats[2][0]), statFile) + string(stats[0][0]), statFile) } fields["total"] = fields["total"].(int64) + int64(1) - threads, err := strconv.Atoi(string(stats[19])) + threads, err := strconv.Atoi(string(stats[17])) if err != nil { log.Printf("processes: Error parsing thread count: %s", err) continue diff --git a/plugins/inputs/system/processes_test.go b/plugins/inputs/system/processes_test.go index de9b6aa5b..eef52cd67 100644 --- a/plugins/inputs/system/processes_test.go +++ b/plugins/inputs/system/processes_test.go @@ -82,6 +82,28 @@ func TestFromProcFiles(t *testing.T) { acc.AssertContainsTaggedFields(t, "processes", fields, map[string]string{}) } +func TestFromProcFilesWithSpaceInCmd(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("This test only runs on linux") + } + tester := tester{} + processes := &Processes{ + readProcFile: tester.testProcFile2, + forceProc: true, + } + + var acc testutil.Accumulator + err := processes.Gather(&acc) + require.NoError(t, err) + + fields := getEmptyFields() + fields["sleeping"] = tester.calls + fields["total_threads"] = tester.calls * 2 + fields["total"] = tester.calls + + acc.AssertContainsTaggedFields(t, "processes", fields, map[string]string{}) +} + func testExecPS() ([]byte, error) { return []byte(testPSOut), nil } @@ -96,6 +118,11 @@ func (t *tester) testProcFile(_ string) ([]byte, error) { return []byte(fmt.Sprintf(testProcStat, "S", "2")), nil } +func (t *tester) testProcFile2(_ string) ([]byte, error) { + t.calls++ + return []byte(fmt.Sprintf(testProcStat2, "S", "2")), nil +} + func testExecPSError() ([]byte, error) { return []byte(testPSOut), fmt.Errorf("ERROR!") } @@ -149,3 +176,6 @@ S+ const testProcStat = `10 (rcuob/0) %s 2 0 0 0 -1 2129984 0 0 0 0 0 0 0 0 20 0 %s 0 11 0 0 18446744073709551615 0 0 0 0 0 0 0 2147483647 0 18446744073709551615 0 0 17 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ` + +const testProcStat2 = `10 (rcuob 0) %s 2 0 0 0 -1 2129984 0 0 0 0 0 0 0 0 20 0 %s 0 11 0 0 18446744073709551615 0 0 0 0 0 0 0 2147483647 0 18446744073709551615 0 0 17 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +` From bcf1fc658dbbac8f86b5d457cbfe93a225b3c88a Mon Sep 17 00:00:00 2001 From: Armin Wolfermann Date: Tue, 5 Apr 2016 15:24:24 +0200 Subject: [PATCH 11/55] ipmi_sensors: Allow : in password closes #969 --- CHANGELOG.md | 1 + plugins/inputs/ipmi_sensor/connection.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82a224f21..b7b327fde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Bugfixes - [#968](https://github.com/influxdata/telegraf/issues/968): Processes plugin gets unknown state when spaces are in (command name) +- [#969](https://github.com/influxdata/telegraf/issues/968): ipmi_sensors: allow : in password. Thanks @awaw! ## v0.12.0 [2016-04-05] diff --git a/plugins/inputs/ipmi_sensor/connection.go b/plugins/inputs/ipmi_sensor/connection.go index 3f4461438..1e9bfbdcb 100644 --- a/plugins/inputs/ipmi_sensor/connection.go +++ b/plugins/inputs/ipmi_sensor/connection.go @@ -28,7 +28,7 @@ func NewConnection(server string) *Connection { if inx1 > 0 { security := server[0:inx1] connstr = server[inx1+1 : len(server)] - up := strings.Split(security, ":") + up := strings.SplitN(security, ":", 2) conn.Username = up[0] conn.Password = up[1] } From 73bd98df577ebce5685930787191eac35dc2f9cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martti=20Rannanj=C3=A4rvi?= Date: Tue, 5 Apr 2016 16:56:00 +0300 Subject: [PATCH 12/55] dovecot: remove extra newline from stats query Extra newline in the stats query is interpreted as an empty query which is an error for dovecot. closes #972 --- CHANGELOG.md | 3 ++- plugins/inputs/dovecot/dovecot.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7b327fde..a742541ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ ### Bugfixes - [#968](https://github.com/influxdata/telegraf/issues/968): Processes plugin gets unknown state when spaces are in (command name) -- [#969](https://github.com/influxdata/telegraf/issues/968): ipmi_sensors: allow : in password. Thanks @awaw! +- [#969](https://github.com/influxdata/telegraf/pull/969): ipmi_sensors: allow : in password. Thanks @awaw! +- [#972](https://github.com/influxdata/telegraf/pull/972): dovecot: remove extra newline in dovecot command. Thanks @mrannanj! ## v0.12.0 [2016-04-05] diff --git a/plugins/inputs/dovecot/dovecot.go b/plugins/inputs/dovecot/dovecot.go index 3a6607da9..bf1b20269 100644 --- a/plugins/inputs/dovecot/dovecot.go +++ b/plugins/inputs/dovecot/dovecot.go @@ -85,7 +85,7 @@ func (d *Dovecot) gatherServer(addr string, acc telegraf.Accumulator, doms map[s // Extend connection c.SetDeadline(time.Now().Add(defaultTimeout)) - c.Write([]byte("EXPORT\tdomain\n\n")) + c.Write([]byte("EXPORT\tdomain\n")) var buf bytes.Buffer io.Copy(&buf, c) // buf := bufio.NewReader(c) From 03f2a35b31aeb2a3bc39b21c49dccc43ca7a0090 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Tue, 5 Apr 2016 13:54:02 -0600 Subject: [PATCH 13/55] Update jolokia plugin readme --- plugins/inputs/jolokia/README.md | 62 ++++++++++++-------------------- 1 file changed, 23 insertions(+), 39 deletions(-) diff --git a/plugins/inputs/jolokia/README.md b/plugins/inputs/jolokia/README.md index bda0c5f93..3a528b33f 100644 --- a/plugins/inputs/jolokia/README.md +++ b/plugins/inputs/jolokia/README.md @@ -1,16 +1,28 @@ # Telegraf plugin: Jolokia -#### Plugin arguments: -- **context** string: Context root used of jolokia url -- **servers** []Server: List of servers - + **name** string: Server's logical name - + **host** string: Server's ip address or hostname - + **port** string: Server's listening port -- **metrics** []Metric - + **name** string: Name of the measure - + **jmx** string: Jmx path that identifies mbeans attributes - + **pass** []string: Attributes to retain when collecting values - + **drop** []string: Attributes to drop when collecting values +#### Configuration + +```toml +[[inputs.jolokia]] + ## This is the context root used to compose the jolokia url + context = "/jolokia/read" + + ## List of servers exposing jolokia read service + [[inputs.jolokia.servers]] + name = "stable" + host = "192.168.103.2" + port = "8180" + # username = "myuser" + # password = "mypassword" + + ## List of metrics collected on above servers + ## Each metric consists in a name, a jmx path and either + ## a pass or drop slice attribute. + ## This collect all heap memory usage metrics. + [[inputs.jolokia.metrics]] + name = "heap_memory_usage" + jmx = "/java.lang:type=Memory/HeapMemoryUsage" +``` #### Description @@ -21,31 +33,3 @@ See: https://jolokia.org/ # Measurements: Jolokia plugin produces one measure for each metric configured, adding Server's `name`, `host` and `port` as tags. - -Given a configuration like: - -```ini -[jolokia] - -[[jolokia.servers]] - name = "as-service-1" - host = "127.0.0.1" - port = "8080" - -[[jolokia.servers]] - name = "as-service-2" - host = "127.0.0.1" - port = "8180" - -[[jolokia.metrics]] - name = "heap_memory_usage" - jmx = "/java.lang:type=Memory/HeapMemoryUsage" - pass = ["used", "max"] -``` - -The collected metrics will be: - -``` -jolokia_heap_memory_usage name=as-service-1,host=127.0.0.1,port=8080 used=xxx,max=yyy -jolokia_heap_memory_usage name=as-service-2,host=127.0.0.1,port=8180 used=vvv,max=zzz -``` From 4dd364e1c3d764e07f4f25c9e2cd92c5c32897e1 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Tue, 5 Apr 2016 14:42:20 -0600 Subject: [PATCH 14/55] Update all readme instances of data formats --- CONTRIBUTING.md | 4 ++-- docs/DATA_FORMATS_INPUT.md | 16 ++++++++------ docs/DATA_FORMATS_OUTPUT.md | 12 ++++++---- plugins/inputs/exec/README.md | 29 ++++++++++--------------- plugins/inputs/kafka_consumer/README.md | 3 ++- plugins/inputs/mqtt_consumer/README.md | 2 +- plugins/inputs/nats_consumer/README.md | 3 ++- plugins/inputs/tcp_listener/README.md | 3 ++- plugins/inputs/udp_listener/README.md | 3 ++- 9 files changed, 40 insertions(+), 35 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 68c9da478..3997a448e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -114,7 +114,7 @@ creating the `Parser` object. You should also add the following to your SampleConfig() return: ```toml - ## Data format to consume. This can be "json", "influx" or "graphite" + ## Data format to consume. ## Each data format has it's own unique set of configuration options, read ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md @@ -244,7 +244,7 @@ instantiating and creating the `Serializer` object. You should also add the following to your SampleConfig() return: ```toml - ## Data format to output. This can be "influx" or "graphite" + ## Data format to output. ## Each data format has it's own unique set of configuration options, read ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md diff --git a/docs/DATA_FORMATS_INPUT.md b/docs/DATA_FORMATS_INPUT.md index 589db53a3..3f970ec38 100644 --- a/docs/DATA_FORMATS_INPUT.md +++ b/docs/DATA_FORMATS_INPUT.md @@ -5,7 +5,8 @@ Telegraf is able to parse the following input data formats into metrics: 1. InfluxDB Line Protocol 1. JSON 1. Graphite -1. Value, ie 45 or "booyah" +1. Value, ie: 45 or "booyah" +1. Nagios (exec input only) Telegraf metrics, like InfluxDB [points](https://docs.influxdata.com/influxdb/v0.10/write_protocols/line/), @@ -38,7 +39,7 @@ example, in the exec plugin: ## measurement name suffix (for separating different commands) name_suffix = "_mycollector" - ## Data format to consume. This can be "json", "influx" or "graphite" + ## Data format to consume. ## Each data format has it's own unique set of configuration options, read ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md @@ -65,7 +66,7 @@ metrics are parsed directly into Telegraf metrics. ## measurement name suffix (for separating different commands) name_suffix = "_mycollector" - ## Data format to consume. This can be "json", "influx" or "graphite" + ## Data format to consume. ## Each data format has it's own unique set of configuration options, read ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md @@ -110,7 +111,7 @@ For example, if you had this configuration: ## measurement name suffix (for separating different commands) name_suffix = "_mycollector" - ## Data format to consume. This can be "json", "influx" or "graphite" + ## Data format to consume. ## Each data format has it's own unique set of configuration options, read ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md @@ -165,7 +166,7 @@ plugin. ## override the default metric name of "exec" name_override = "entropy_available" - ## Data format to consume. This can be "json", "value", influx" or "graphite" + ## Data format to consume. ## Each data format has it's own unique set of configuration options, read ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md @@ -301,7 +302,8 @@ There are many more options available, ## measurement name suffix (for separating different commands) name_suffix = "_mycollector" - ## Data format to consume. This can be "json", "influx" or "graphite" (line-protocol) + ## Data format to consume. + (line-protocol) ## Each data format has it's own unique set of configuration options, read ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md @@ -344,7 +346,7 @@ Note: Nagios Input Data Formats is only supported in `exec` input plugin. ## measurement name suffix (for separating different commands) name_suffix = "_mycollector" - ## Data format to consume. This can be "json", "influx", "graphite" or "nagios" + ## Data format to consume. ## Each data format has it's own unique set of configuration options, read ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md diff --git a/docs/DATA_FORMATS_OUTPUT.md b/docs/DATA_FORMATS_OUTPUT.md index a75816a71..28f8cd6c3 100644 --- a/docs/DATA_FORMATS_OUTPUT.md +++ b/docs/DATA_FORMATS_OUTPUT.md @@ -29,7 +29,8 @@ config option, for example, in the `file` output plugin: ## Files to write to, "stdout" is a specially handled file. files = ["stdout"] - ## Data format to output. This can be "influx" or "graphite" + ## Data format to output. + ## Each data format has it's own unique set of configuration options, read ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md @@ -53,7 +54,8 @@ metrics are serialized directly into InfluxDB line-protocol. ## Files to write to, "stdout" is a specially handled file. files = ["stdout", "/tmp/metrics.out"] - ## Data format to output. This can be "influx", "json" or "graphite" + ## Data format to output. + ## Each data format has it's own unique set of configuration options, read ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md @@ -87,7 +89,8 @@ tars.cpu-total.us-east-1.cpu.usage_idle 98.09 1455320690 ## Files to write to, "stdout" is a specially handled file. files = ["stdout", "/tmp/metrics.out"] - ## Data format to output. This can be "influx", "json" or "graphite" + ## Data format to output. + ## Each data format has it's own unique set of configuration options, read ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md @@ -123,7 +126,8 @@ The Json data format serialized Telegraf metrics in json format. The format is: ## Files to write to, "stdout" is a specially handled file. files = ["stdout", "/tmp/metrics.out"] - ## Data format to output. This can be "influx", "json" or "graphite" + ## Data format to output. + ## Each data format has it's own unique set of configuration options, read ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md diff --git a/plugins/inputs/exec/README.md b/plugins/inputs/exec/README.md index 730da1fd5..9912c4a48 100644 --- a/plugins/inputs/exec/README.md +++ b/plugins/inputs/exec/README.md @@ -2,18 +2,11 @@ Please also see: [Telegraf Input Data Formats](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md) -The exec input plugin can execute arbitrary commands which output: - -* JSON [javascript object notation](http://www.json.org/) -* InfluxDB [line-protocol](https://docs.influxdata.com/influxdb/v0.10/write_protocols/line/) -* Graphite [graphite-protocol](http://graphite.readthedocs.org/en/latest/feeding-carbon.html) - - ### Example 1 - JSON #### Configuration -In this example a script called ```/tmp/test.sh``` and a script called ```/tmp/test2.sh``` +In this example a script called ```/tmp/test.sh``` and a script called ```/tmp/test2.sh``` are configured for ```[[inputs.exec]]``` in JSON format. ``` @@ -22,7 +15,8 @@ are configured for ```[[inputs.exec]]``` in JSON format. # Shell/commands array commands = ["/tmp/test.sh", "/tmp/test2.sh"] - # Data format to consume. This can be "json", "influx" or "graphite" (line-protocol) + # Data format to consume. + (line-protocol) # NOTE json only reads numerical measurements, strings and booleans are ignored. data_format = "json" @@ -81,7 +75,7 @@ and strings will be ignored. ### Example 2 - Influx Line-Protocol In this example an application called ```/usr/bin/line_protocol_collector``` -and a script called ```/tmp/test2.sh``` are configured for ```[[inputs.exec]]``` +and a script called ```/tmp/test2.sh``` are configured for ```[[inputs.exec]]``` in influx line-protocol format. #### Configuration @@ -94,7 +88,7 @@ in influx line-protocol format. # command = "/usr/bin/line_protocol_collector" commands = ["/usr/bin/line_protocol_collector","/tmp/test2.sh"] - # Data format to consume. This can be "json" or "influx" (line-protocol) + # Data format to consume. # NOTE json only reads numerical measurements, strings and booleans are ignored. data_format = "influx" ``` @@ -113,7 +107,7 @@ cpu,cpu=cpu6,host=foo,datacenter=us-east usage_idle=99,usage_busy=1 You will get data in InfluxDB exactly as it is defined above, tags are cpu=cpuN, host=foo, and datacenter=us-east with fields usage_idle -and usage_busy. They will receive a timestamp at collection time. +and usage_busy. They will receive a timestamp at collection time. Each line must end in \n, just as the Influx line protocol does. @@ -121,8 +115,8 @@ Each line must end in \n, just as the Influx line protocol does. We can also change the data_format to "graphite" to use the metrics collecting scripts such as (compatible with graphite): -* Nagios [Mertics Plugins] (https://exchange.nagios.org/directory/Plugins) -* Sensu [Mertics Plugins] (https://github.com/sensu-plugins) +* Nagios [Metrics Plugins](https://exchange.nagios.org/directory/Plugins) +* Sensu [Metrics Plugins](https://github.com/sensu-plugins) In this example a script called /tmp/test.sh and a script called /tmp/test2.sh are configured for [[inputs.exec]] in graphite format. @@ -133,7 +127,8 @@ In this example a script called /tmp/test.sh and a script called /tmp/test2.sh a # Shell/commands array commands = ["/tmp/test.sh","/tmp/test2.sh"] - # Data format to consume. This can be "json", "influx" or "graphite" (line-protocol) + # Data format to consume. + (line-protocol) # NOTE json only reads numerical measurements, strings and booleans are ignored. data_format = "graphite" @@ -186,5 +181,5 @@ sensu.metric.net.server0.eth0.rx_dropped 0 1444234982 The templates configuration will be used to parse the graphite metrics to support influxdb/opentsdb tagging store engines. -More detail information about templates, please refer to [The graphite Input] (https://github.com/influxdata/influxdb/blob/master/services/graphite/README.md) - +More detail information about templates, please refer to [The graphite Input](https://github.com/influxdata/influxdb/blob/master/services/graphite/README.md) + diff --git a/plugins/inputs/kafka_consumer/README.md b/plugins/inputs/kafka_consumer/README.md index 885c67a28..f5f6a359e 100644 --- a/plugins/inputs/kafka_consumer/README.md +++ b/plugins/inputs/kafka_consumer/README.md @@ -22,7 +22,8 @@ from the same topic in parallel. ## Offset (must be either "oldest" or "newest") offset = "oldest" - ## Data format to consume. This can be "json", "influx" or "graphite" + ## Data format to consume. + ## Each data format has it's own unique set of configuration options, read ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md diff --git a/plugins/inputs/mqtt_consumer/README.md b/plugins/inputs/mqtt_consumer/README.md index 787494975..d5518b632 100644 --- a/plugins/inputs/mqtt_consumer/README.md +++ b/plugins/inputs/mqtt_consumer/README.md @@ -35,7 +35,7 @@ The plugin expects messages in the ## Use SSL but skip chain & host verification # insecure_skip_verify = false - ## Data format to consume. This can be "json", "influx" or "graphite" + ## Data format to consume. ## Each data format has it's own unique set of configuration options, read ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md diff --git a/plugins/inputs/nats_consumer/README.md b/plugins/inputs/nats_consumer/README.md index 90563ff55..31d947588 100644 --- a/plugins/inputs/nats_consumer/README.md +++ b/plugins/inputs/nats_consumer/README.md @@ -23,7 +23,8 @@ from a NATS cluster in parallel. ## Maximum number of metrics to buffer between collection intervals metric_buffer = 100000 - ## Data format to consume. This can be "json", "influx" or "graphite" + ## Data format to consume. + ## Each data format has it's own unique set of configuration options, read ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md diff --git a/plugins/inputs/tcp_listener/README.md b/plugins/inputs/tcp_listener/README.md index 63a7dea3c..d2dfeb575 100644 --- a/plugins/inputs/tcp_listener/README.md +++ b/plugins/inputs/tcp_listener/README.md @@ -22,7 +22,8 @@ This is a sample configuration for the plugin. ## Maximum number of concurrent TCP connections to allow max_tcp_connections = 250 - ## Data format to consume. This can be "json", "influx" or "graphite" + ## Data format to consume. + ## Each data format has it's own unique set of configuration options, read ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md diff --git a/plugins/inputs/udp_listener/README.md b/plugins/inputs/udp_listener/README.md index 724ae43ae..1dd03a2a7 100644 --- a/plugins/inputs/udp_listener/README.md +++ b/plugins/inputs/udp_listener/README.md @@ -23,7 +23,8 @@ This is a sample configuration for the plugin. ## usually 1500 bytes. udp_packet_size = 1500 - ## Data format to consume. This can be "json", "influx" or "graphite" + ## Data format to consume. + ## Each data format has it's own unique set of configuration options, read ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md From 40f2dd8c6c33541e52705b41cec163d3b508fef7 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Tue, 5 Apr 2016 15:22:58 -0600 Subject: [PATCH 15/55] Readme fixup for exec plugin --- plugins/inputs/exec/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/inputs/exec/README.md b/plugins/inputs/exec/README.md index 9912c4a48..549d54d68 100644 --- a/plugins/inputs/exec/README.md +++ b/plugins/inputs/exec/README.md @@ -128,7 +128,6 @@ In this example a script called /tmp/test.sh and a script called /tmp/test2.sh a commands = ["/tmp/test.sh","/tmp/test2.sh"] # Data format to consume. - (line-protocol) # NOTE json only reads numerical measurements, strings and booleans are ignored. data_format = "graphite" From 7e97787d9d7b1209ce21f8372aecf0aaab045607 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Tue, 5 Apr 2016 16:17:45 -0600 Subject: [PATCH 16/55] More readme fixups --- docs/DATA_FORMATS_INPUT.md | 1 - plugins/inputs/exec/README.md | 1 - 2 files changed, 2 deletions(-) diff --git a/docs/DATA_FORMATS_INPUT.md b/docs/DATA_FORMATS_INPUT.md index 3f970ec38..b282d1f8f 100644 --- a/docs/DATA_FORMATS_INPUT.md +++ b/docs/DATA_FORMATS_INPUT.md @@ -303,7 +303,6 @@ There are many more options available, name_suffix = "_mycollector" ## Data format to consume. - (line-protocol) ## Each data format has it's own unique set of configuration options, read ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md diff --git a/plugins/inputs/exec/README.md b/plugins/inputs/exec/README.md index 549d54d68..a75ae7856 100644 --- a/plugins/inputs/exec/README.md +++ b/plugins/inputs/exec/README.md @@ -16,7 +16,6 @@ are configured for ```[[inputs.exec]]``` in JSON format. commands = ["/tmp/test.sh", "/tmp/test2.sh"] # Data format to consume. - (line-protocol) # NOTE json only reads numerical measurements, strings and booleans are ignored. data_format = "json" From 64066c4ea87e52f6d15ff3e7a9783545834fcdaf Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Tue, 5 Apr 2016 16:25:50 -0600 Subject: [PATCH 17/55] Update input data format readme --- docs/DATA_FORMATS_INPUT.md | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/docs/DATA_FORMATS_INPUT.md b/docs/DATA_FORMATS_INPUT.md index b282d1f8f..6a916711b 100644 --- a/docs/DATA_FORMATS_INPUT.md +++ b/docs/DATA_FORMATS_INPUT.md @@ -2,11 +2,11 @@ Telegraf is able to parse the following input data formats into metrics: -1. InfluxDB Line Protocol -1. JSON -1. Graphite -1. Value, ie: 45 or "booyah" -1. Nagios (exec input only) +1. [InfluxDB Line Protocol](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#influx) +1. [JSON](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#json) +1. [Graphite](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#graphite) +1. [Value](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#value), ie: 45 or "booyah" +1. [Nagios](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#nagios) (exec input only) Telegraf metrics, like InfluxDB [points](https://docs.influxdata.com/influxdb/v0.10/write_protocols/line/), @@ -51,7 +51,7 @@ example, in the exec plugin: Each data_format has an additional set of configuration options available, which I'll go over below. -## Influx: +# Influx: There are no additional configuration options for InfluxDB line-protocol. The metrics are parsed directly into Telegraf metrics. @@ -73,7 +73,7 @@ metrics are parsed directly into Telegraf metrics. data_format = "influx" ``` -## JSON: +# JSON: The JSON data format flattens JSON into metric _fields_. For example, this JSON: @@ -142,21 +142,20 @@ Your Telegraf metrics would get tagged with "my_tag_1" exec_mycollector,my_tag_1=foo a=5,b_c=6 ``` -## Value: +# Value: The "value" data format translates single values into Telegraf metrics. This -is done by assigning a measurement name (which can be overridden using the -`name_override` config option), and setting a single field ("value") as the -parsed metric. +is done by assigning a measurement name and setting a single field ("value") +as the parsed metric. #### Value Configuration: -You can tell Telegraf what type of metric to collect by using the `data_type` -configuration option. +You **must** tell Telegraf what type of metric to collect by using the +`data_type` configuration option. -It is also recommended that you set `name_override` to a measurement name that -makes sense for your metric, otherwise it will just be set to the name of the -plugin. +**Note:** It is also recommended that you set `name_override` to a measurement +name that makes sense for your metric, otherwise it will just be set to the +name of the plugin. ```toml [[inputs.exec]] @@ -171,10 +170,10 @@ plugin. ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md data_format = "value" - data_type = "integer" + data_type = "integer" # required ``` -## Graphite: +# Graphite: The Graphite data format translates graphite _dot_ buckets directly into telegraf measurement names, with a single value field, and without any tags. For @@ -328,7 +327,7 @@ There are many more options available, ] ``` -## Nagios: +# Nagios: There are no additional configuration options for Nagios line-protocol. The metrics are parsed directly into Telegraf metrics. From 30464396d9750cadfdf301088d0d8c3905f6f576 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Tue, 5 Apr 2016 10:37:21 -0600 Subject: [PATCH 18/55] Make the UDP input buffer only once --- CHANGELOG.md | 1 + plugins/inputs/statsd/statsd.go | 17 ++++++++--------- plugins/inputs/udp_listener/udp_listener.go | 15 +++++++-------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a742541ab..9c7d0f507 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## v0.12.1 [unreleased] ### Features +- [#976](https://github.com/influxdata/telegraf/pull/976): Reduce allocations in the UDP and statsd inputs. ### Bugfixes - [#968](https://github.com/influxdata/telegraf/issues/968): Processes plugin gets unknown state when spaces are in (command name) diff --git a/plugins/inputs/statsd/statsd.go b/plugins/inputs/statsd/statsd.go index d31e6bfc9..84687511e 100644 --- a/plugins/inputs/statsd/statsd.go +++ b/plugins/inputs/statsd/statsd.go @@ -18,7 +18,9 @@ import ( ) const ( - UDP_PACKET_SIZE int = 1500 + // UDP packet limit, see + // https://en.wikipedia.org/wiki/User_Datagram_Protocol#Packet_structure + UDP_PACKET_SIZE int = 65507 defaultFieldName = "value" @@ -157,10 +159,6 @@ const sampleConfig = ` ## calculation of percentiles. Raising this limit increases the accuracy ## of percentiles but also increases the memory usage and cpu time. percentile_limit = 1000 - - ## UDP packet size for the server to listen for. This will depend on the size - ## of the packets that the client is sending, which is usually 1500 bytes. - udp_packet_size = 1500 ` func (_ *Statsd) SampleConfig() string { @@ -274,12 +272,12 @@ func (s *Statsd) udpListen() error { } log.Println("Statsd listener listening on: ", s.listener.LocalAddr().String()) + buf := make([]byte, s.UDPPacketSize) for { select { case <-s.done: return nil default: - buf := make([]byte, s.UDPPacketSize) n, _, err := s.listener.ReadFromUDP(buf) if err != nil && !strings.Contains(err.Error(), "closed network") { log.Printf("ERROR READ: %s\n", err.Error()) @@ -300,11 +298,12 @@ func (s *Statsd) udpListen() error { // single statsd metric into a struct. func (s *Statsd) parser() error { defer s.wg.Done() + var packet []byte for { select { case <-s.done: return nil - case packet := <-s.in: + case packet = <-s.in: lines := strings.Split(string(packet), "\n") for _, line := range lines { line = strings.TrimSpace(line) @@ -631,8 +630,8 @@ func (s *Statsd) Stop() { func init() { inputs.Add("statsd", func() telegraf.Input { return &Statsd{ - ConvertNames: true, - UDPPacketSize: UDP_PACKET_SIZE, + MetricSeparator: "_", + UDPPacketSize: UDP_PACKET_SIZE, } }) } diff --git a/plugins/inputs/udp_listener/udp_listener.go b/plugins/inputs/udp_listener/udp_listener.go index 794f1791d..442cf98b3 100644 --- a/plugins/inputs/udp_listener/udp_listener.go +++ b/plugins/inputs/udp_listener/udp_listener.go @@ -30,7 +30,9 @@ type UdpListener struct { listener *net.UDPConn } -const UDP_PACKET_SIZE int = 1500 +// UDP packet limit, see +// https://en.wikipedia.org/wiki/User_Datagram_Protocol#Packet_structure +const UDP_PACKET_SIZE int = 65507 var dropwarn = "ERROR: Message queue full. Discarding line [%s] " + "You may want to increase allowed_pending_messages in the config\n" @@ -43,11 +45,6 @@ const sampleConfig = ` ## UDP listener will start dropping packets. allowed_pending_messages = 10000 - ## UDP packet size for the server to listen for. This will depend - ## on the size of the packets that the client is sending, which is - ## usually 1500 bytes, but can be as large as 65,535 bytes. - udp_packet_size = 1500 - ## Data format to consume. ## Each data format has it's own unique set of configuration options, read ## more about them here: @@ -107,12 +104,12 @@ func (u *UdpListener) udpListen() error { } log.Println("UDP server listening on: ", u.listener.LocalAddr().String()) + buf := make([]byte, u.UDPPacketSize) for { select { case <-u.done: return nil default: - buf := make([]byte, u.UDPPacketSize) n, _, err := u.listener.ReadFromUDP(buf) if err != nil && !strings.Contains(err.Error(), "closed network") { log.Printf("ERROR: %s\n", err.Error()) @@ -130,11 +127,13 @@ func (u *UdpListener) udpListen() error { func (u *UdpListener) udpParser() error { defer u.wg.Done() + + var packet []byte for { select { case <-u.done: return nil - case packet := <-u.in: + case packet = <-u.in: metrics, err := u.parser.Parse(packet) if err == nil { u.storeMetrics(metrics) From 0f16c0f4cf58b8a7ebb096e26d541c5d896269e8 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Tue, 5 Apr 2016 14:35:14 -0600 Subject: [PATCH 19/55] Reduce TCP listener allocations --- CHANGELOG.md | 1 + plugins/inputs/tcp_listener/tcp_listener.go | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c7d0f507..5a4552698 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Features - [#976](https://github.com/influxdata/telegraf/pull/976): Reduce allocations in the UDP and statsd inputs. +- [#979](https://github.com/influxdata/telegraf/pull/979): Reduce allocations in the TCP listener. ### Bugfixes - [#968](https://github.com/influxdata/telegraf/issues/968): Processes plugin gets unknown state when spaces are in (command name) diff --git a/plugins/inputs/tcp_listener/tcp_listener.go b/plugins/inputs/tcp_listener/tcp_listener.go index a1b991058..4559a3bf5 100644 --- a/plugins/inputs/tcp_listener/tcp_listener.go +++ b/plugins/inputs/tcp_listener/tcp_listener.go @@ -39,7 +39,7 @@ type TcpListener struct { acc telegraf.Accumulator } -var dropwarn = "ERROR: Message queue full. Discarding line [%s] " + +var dropwarn = "ERROR: Message queue full. Discarding metric. " + "You may want to increase allowed_pending_messages in the config\n" const sampleConfig = ` @@ -202,11 +202,10 @@ func (t *TcpListener) handler(conn *net.TCPConn, id string) { if !scanner.Scan() { return } - buf := scanner.Bytes() select { - case t.in <- buf: + case t.in <- scanner.Bytes(): default: - log.Printf(dropwarn, string(buf)) + log.Printf(dropwarn) } } } @@ -215,11 +214,12 @@ func (t *TcpListener) handler(conn *net.TCPConn, id string) { // tcpParser parses the incoming tcp byte packets func (t *TcpListener) tcpParser() error { defer t.wg.Done() + var packet []byte for { select { case <-t.done: return nil - case packet := <-t.in: + case packet = <-t.in: if len(packet) == 0 { continue } From 9320a6e115b0bc2d7a832ae56ef0c8329df9db79 Mon Sep 17 00:00:00 2001 From: Ricard Clau Date: Sat, 2 Apr 2016 11:28:44 +0100 Subject: [PATCH 20/55] windows service docs closes #954 --- docs/WINDOWS_SERVICE.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 docs/WINDOWS_SERVICE.md diff --git a/docs/WINDOWS_SERVICE.md b/docs/WINDOWS_SERVICE.md new file mode 100644 index 000000000..679a41527 --- /dev/null +++ b/docs/WINDOWS_SERVICE.md @@ -0,0 +1,36 @@ +# Running Telegraf as a Windows Service + +If you have tried to install Go binaries as Windows Services with the **sc.exe** +tool you may have seen that the service errors and stops running after a while. + +**NSSM** (the Non-Sucking Service Manager) is a tool that helps you in a +[number of scenarios](http://nssm.cc/scenarios) including running Go binaries +that were not specifically designed to run only in Windows platforms. + +## NSSM Installation via Chocolatey + +You can install [Chocolatey](https://chocolatey.org/) and [NSSM](http://nssm.cc/) +with these commands + +```powershell +iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')) +choco install -y nssm +``` + +## Installing Telegraf as a Windows Service with NSSM + +You can download the latest Telegraf Windows binaries (still Experimental at +the moment) from [the Telegraf Github repo](https://github.com/influxdata/telegraf). + +Then you can create a C:\telegraf folder, unzip the binary there and modify the +**telegraf.conf** sample to allocate the metrics you want to send to **InfluxDB**. + +Once you have NSSM installed in your system, the process is quite straightforward. +You only need to type this command in your Windows shell + +```powershell +nssm install Telegraf c:\telegraf\telegraf.exe -config c:\telegraf\telegraf.config +``` + +And now your service will be installed in Windows and you will be able to start and +stop it gracefully \ No newline at end of file From 32213cad018efb46f8296d9927b15bfcd1195ea1 Mon Sep 17 00:00:00 2001 From: Sergio Jimenez Date: Sun, 3 Apr 2016 00:34:34 +0200 Subject: [PATCH 21/55] input(docker): docker/engine-api * Made required changes to get it to compile * First manual tests looking good, still unit tests need fixing * Made go linter happier --- plugins/inputs/docker/docker.go | 184 +++++++++++++++----------------- 1 file changed, 89 insertions(+), 95 deletions(-) diff --git a/plugins/inputs/docker/docker.go b/plugins/inputs/docker/docker.go index cdc8ec1e5..b1bbc63b3 100644 --- a/plugins/inputs/docker/docker.go +++ b/plugins/inputs/docker/docker.go @@ -3,6 +3,7 @@ package system import ( "encoding/json" "fmt" + "io" "log" "regexp" "strconv" @@ -10,12 +11,15 @@ import ( "sync" "time" + "golang.org/x/net/context" + + "github.com/docker/engine-api/client" + "github.com/docker/engine-api/types" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/inputs" - - "github.com/fsouza/go-dockerclient" ) +// Docker object type Docker struct { Endpoint string ContainerNames []string @@ -23,14 +27,14 @@ type Docker struct { client DockerClient } +// DockerClient interface, useful for testing type DockerClient interface { - // Docker Client wrapper - // Useful for test - Info() (*docker.Env, error) - ListContainers(opts docker.ListContainersOptions) ([]docker.APIContainers, error) - Stats(opts docker.StatsOptions) error + Info(ctx context.Context) (types.Info, error) + ContainerList(ctx context.Context, options types.ContainerListOptions) ([]types.Container, error) + ContainerStats(ctx context.Context, containerID string, stream bool) (io.ReadCloser, error) } +// KB, MB, GB, TB, PB...human friendly const ( KB = 1000 MB = 1000 * KB @@ -52,28 +56,32 @@ var sampleConfig = ` container_names = [] ` +// Description returns input description func (d *Docker) Description() string { return "Read metrics about docker containers" } +// SampleConfig prints sampleConfig func (d *Docker) SampleConfig() string { return sampleConfig } +// Gather starts stats collection func (d *Docker) Gather(acc telegraf.Accumulator) error { if d.client == nil { - var c *docker.Client + var c *client.Client var err error + defaultHeaders := map[string]string{"User-Agent": "engine-api-cli-1.0"} if d.Endpoint == "ENV" { - c, err = docker.NewClientFromEnv() + c, err = client.NewEnvClient() if err != nil { return err } } else if d.Endpoint == "" { - c, err = docker.NewClient("unix:///var/run/docker.sock") + c, err = client.NewClient("unix:///var/run/docker.sock", "", nil, defaultHeaders) if err != nil { return err } } else { - c, err = docker.NewClient(d.Endpoint) + c, err = client.NewClient(d.Endpoint, "", nil, defaultHeaders) if err != nil { return err } @@ -88,8 +96,8 @@ func (d *Docker) Gather(acc telegraf.Accumulator) error { } // List containers - opts := docker.ListContainersOptions{} - containers, err := d.client.ListContainers(opts) + opts := types.ContainerListOptions{} + containers, err := d.client.ContainerList(context.Background(), opts) if err != nil { return err } @@ -99,7 +107,7 @@ func (d *Docker) Gather(acc telegraf.Accumulator) error { wg.Add(len(containers)) for _, container := range containers { - go func(c docker.APIContainers) { + go func(c types.Container) { defer wg.Done() err := d.gatherContainer(c, acc) if err != nil { @@ -114,23 +122,22 @@ func (d *Docker) Gather(acc telegraf.Accumulator) error { func (d *Docker) gatherInfo(acc telegraf.Accumulator) error { // Init vars - var driverStatus [][]string dataFields := make(map[string]interface{}) metadataFields := make(map[string]interface{}) now := time.Now() // Get info from docker daemon - info, err := d.client.Info() + info, err := d.client.Info(context.Background()) if err != nil { return err } fields := map[string]interface{}{ - "n_cpus": info.GetInt64("NCPU"), - "n_used_file_descriptors": info.GetInt64("NFd"), - "n_containers": info.GetInt64("Containers"), - "n_images": info.GetInt64("Images"), - "n_goroutines": info.GetInt64("NGoroutines"), - "n_listener_events": info.GetInt64("NEventsListener"), + "n_cpus": info.NCPU, + "n_used_file_descriptors": info.NFd, + "n_containers": info.Containers, + "n_images": info.Images, + "n_goroutines": info.NGoroutines, + "n_listener_events": info.NEventsListener, } // Add metrics acc.AddFields("docker", @@ -138,13 +145,13 @@ func (d *Docker) gatherInfo(acc telegraf.Accumulator) error { nil, now) acc.AddFields("docker", - map[string]interface{}{"memory_total": info.GetInt64("MemTotal")}, + map[string]interface{}{"memory_total": info.MemTotal}, map[string]string{"unit": "bytes"}, now) // Get storage metrics - driverStatusRaw := []byte(info.Get("DriverStatus")) - json.Unmarshal(driverStatusRaw, &driverStatus) - for _, rawData := range driverStatus { + //driverStatusRaw := []byte(info.DriverStatus) + //json.Unmarshal(driverStatusRaw, &driverStatus) + for _, rawData := range info.DriverStatus { // Try to convert string to int (bytes) value, err := parseSize(rawData[1]) if err != nil { @@ -159,12 +166,12 @@ func (d *Docker) gatherInfo(acc telegraf.Accumulator) error { now) } else if strings.HasPrefix(name, "data_space_") { // data space - field_name := strings.TrimPrefix(name, "data_space_") - dataFields[field_name] = value + fieldName := strings.TrimPrefix(name, "data_space_") + dataFields[fieldName] = value } else if strings.HasPrefix(name, "metadata_space_") { // metadata space - field_name := strings.TrimPrefix(name, "metadata_space_") - metadataFields[field_name] = value + fieldName := strings.TrimPrefix(name, "metadata_space_") + metadataFields[fieldName] = value } } if len(dataFields) > 0 { @@ -183,9 +190,10 @@ func (d *Docker) gatherInfo(acc telegraf.Accumulator) error { } func (d *Docker) gatherContainer( - container docker.APIContainers, + container types.Container, acc telegraf.Accumulator, ) error { + var v *types.StatsJSON // Parse container name cname := "unknown" if len(container.Names) > 0 { @@ -204,28 +212,14 @@ func (d *Docker) gatherContainer( } } - statChan := make(chan *docker.Stats) - done := make(chan bool) - statOpts := docker.StatsOptions{ - Stream: false, - ID: container.ID, - Stats: statChan, - Done: done, - Timeout: time.Duration(time.Second * 5), + r, err := d.client.ContainerStats(context.Background(), container.ID, false) + if err != nil { + log.Printf("Error getting docker stats: %s\n", err.Error()) } - - go func() { - err := d.client.Stats(statOpts) - if err != nil { - log.Printf("Error getting docker stats: %s\n", err.Error()) - } - }() - - stat := <-statChan - close(done) - - if stat == nil { - return nil + defer r.Close() + dec := json.NewDecoder(r) + if err = dec.Decode(&v); err != nil { + log.Printf("Error decoding: %s\n", err.Error()) } // Add labels to tags @@ -233,13 +227,13 @@ func (d *Docker) gatherContainer( tags[k] = v } - gatherContainerStats(stat, acc, tags) + gatherContainerStats(v, acc, tags) return nil } func gatherContainerStats( - stat *docker.Stats, + stat *types.StatsJSON, acc telegraf.Accumulator, tags map[string]string, ) { @@ -250,35 +244,35 @@ func gatherContainerStats( "usage": stat.MemoryStats.Usage, "fail_count": stat.MemoryStats.Failcnt, "limit": stat.MemoryStats.Limit, - "total_pgmafault": stat.MemoryStats.Stats.TotalPgmafault, - "cache": stat.MemoryStats.Stats.Cache, - "mapped_file": stat.MemoryStats.Stats.MappedFile, - "total_inactive_file": stat.MemoryStats.Stats.TotalInactiveFile, - "pgpgout": stat.MemoryStats.Stats.Pgpgout, - "rss": stat.MemoryStats.Stats.Rss, - "total_mapped_file": stat.MemoryStats.Stats.TotalMappedFile, - "writeback": stat.MemoryStats.Stats.Writeback, - "unevictable": stat.MemoryStats.Stats.Unevictable, - "pgpgin": stat.MemoryStats.Stats.Pgpgin, - "total_unevictable": stat.MemoryStats.Stats.TotalUnevictable, - "pgmajfault": stat.MemoryStats.Stats.Pgmajfault, - "total_rss": stat.MemoryStats.Stats.TotalRss, - "total_rss_huge": stat.MemoryStats.Stats.TotalRssHuge, - "total_writeback": stat.MemoryStats.Stats.TotalWriteback, - "total_inactive_anon": stat.MemoryStats.Stats.TotalInactiveAnon, - "rss_huge": stat.MemoryStats.Stats.RssHuge, - "hierarchical_memory_limit": stat.MemoryStats.Stats.HierarchicalMemoryLimit, - "total_pgfault": stat.MemoryStats.Stats.TotalPgfault, - "total_active_file": stat.MemoryStats.Stats.TotalActiveFile, - "active_anon": stat.MemoryStats.Stats.ActiveAnon, - "total_active_anon": stat.MemoryStats.Stats.TotalActiveAnon, - "total_pgpgout": stat.MemoryStats.Stats.TotalPgpgout, - "total_cache": stat.MemoryStats.Stats.TotalCache, - "inactive_anon": stat.MemoryStats.Stats.InactiveAnon, - "active_file": stat.MemoryStats.Stats.ActiveFile, - "pgfault": stat.MemoryStats.Stats.Pgfault, - "inactive_file": stat.MemoryStats.Stats.InactiveFile, - "total_pgpgin": stat.MemoryStats.Stats.TotalPgpgin, + "total_pgmafault": stat.MemoryStats.Stats["total_pgmajfault"], + "cache": stat.MemoryStats.Stats["cache"], + "mapped_file": stat.MemoryStats.Stats["mapped_file"], + "total_inactive_file": stat.MemoryStats.Stats["total_inactive_file"], + "pgpgout": stat.MemoryStats.Stats["pagpgout"], + "rss": stat.MemoryStats.Stats["rss"], + "total_mapped_file": stat.MemoryStats.Stats["total_mapped_file"], + "writeback": stat.MemoryStats.Stats["writeback"], + "unevictable": stat.MemoryStats.Stats["unevictable"], + "pgpgin": stat.MemoryStats.Stats["pgpgin"], + "total_unevictable": stat.MemoryStats.Stats["total_unevictable"], + "pgmajfault": stat.MemoryStats.Stats["pgmajfault"], + "total_rss": stat.MemoryStats.Stats["total_rss"], + "total_rss_huge": stat.MemoryStats.Stats["total_rss_huge"], + "total_writeback": stat.MemoryStats.Stats["total_write_back"], + "total_inactive_anon": stat.MemoryStats.Stats["total_inactive_anon"], + "rss_huge": stat.MemoryStats.Stats["rss_huge"], + "hierarchical_memory_limit": stat.MemoryStats.Stats["hierarchical_memory_limit"], + "total_pgfault": stat.MemoryStats.Stats["total_pgfault"], + "total_active_file": stat.MemoryStats.Stats["total_active_file"], + "active_anon": stat.MemoryStats.Stats["active_anon"], + "total_active_anon": stat.MemoryStats.Stats["total_active_anon"], + "total_pgpgout": stat.MemoryStats.Stats["total_pgpgout"], + "total_cache": stat.MemoryStats.Stats["total_cache"], + "inactive_anon": stat.MemoryStats.Stats["inactive_anon"], + "active_file": stat.MemoryStats.Stats["active_file"], + "pgfault": stat.MemoryStats.Stats["pgfault"], + "inactive_file": stat.MemoryStats.Stats["inactive_file"], + "total_pgpgin": stat.MemoryStats.Stats["total_pgpgin"], "usage_percent": calculateMemPercent(stat), } acc.AddFields("docker_mem", memfields, tags, now) @@ -287,7 +281,7 @@ func gatherContainerStats( "usage_total": stat.CPUStats.CPUUsage.TotalUsage, "usage_in_usermode": stat.CPUStats.CPUUsage.UsageInUsermode, "usage_in_kernelmode": stat.CPUStats.CPUUsage.UsageInKernelmode, - "usage_system": stat.CPUStats.SystemCPUUsage, + "usage_system": stat.CPUStats.SystemUsage, "throttling_periods": stat.CPUStats.ThrottlingData.Periods, "throttling_throttled_periods": stat.CPUStats.ThrottlingData.ThrottledPeriods, "throttling_throttled_time": stat.CPUStats.ThrottlingData.ThrottledTime, @@ -323,7 +317,7 @@ func gatherContainerStats( gatherBlockIOMetrics(stat, acc, tags, now) } -func calculateMemPercent(stat *docker.Stats) float64 { +func calculateMemPercent(stat *types.StatsJSON) float64 { var memPercent = 0.0 if stat.MemoryStats.Limit > 0 { memPercent = float64(stat.MemoryStats.Usage) / float64(stat.MemoryStats.Limit) * 100.0 @@ -331,11 +325,11 @@ func calculateMemPercent(stat *docker.Stats) float64 { return memPercent } -func calculateCPUPercent(stat *docker.Stats) float64 { +func calculateCPUPercent(stat *types.StatsJSON) float64 { var cpuPercent = 0.0 // calculate the change for the cpu and system usage of the container in between readings cpuDelta := float64(stat.CPUStats.CPUUsage.TotalUsage) - float64(stat.PreCPUStats.CPUUsage.TotalUsage) - systemDelta := float64(stat.CPUStats.SystemCPUUsage) - float64(stat.PreCPUStats.SystemCPUUsage) + systemDelta := float64(stat.CPUStats.SystemUsage) - float64(stat.PreCPUStats.SystemUsage) if systemDelta > 0.0 && cpuDelta > 0.0 { cpuPercent = (cpuDelta / systemDelta) * float64(len(stat.CPUStats.CPUUsage.PercpuUsage)) * 100.0 @@ -344,7 +338,7 @@ func calculateCPUPercent(stat *docker.Stats) float64 { } func gatherBlockIOMetrics( - stat *docker.Stats, + stat *types.StatsJSON, acc telegraf.Accumulator, tags map[string]string, now time.Time, @@ -353,7 +347,7 @@ func gatherBlockIOMetrics( // Make a map of devices to their block io stats deviceStatMap := make(map[string]map[string]interface{}) - for _, metric := range blkioStats.IOServiceBytesRecursive { + for _, metric := range blkioStats.IoServiceBytesRecursive { device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor) _, ok := deviceStatMap[device] if !ok { @@ -364,7 +358,7 @@ func gatherBlockIOMetrics( deviceStatMap[device][field] = metric.Value } - for _, metric := range blkioStats.IOServicedRecursive { + for _, metric := range blkioStats.IoServicedRecursive { device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor) _, ok := deviceStatMap[device] if !ok { @@ -375,31 +369,31 @@ func gatherBlockIOMetrics( deviceStatMap[device][field] = metric.Value } - for _, metric := range blkioStats.IOQueueRecursive { + for _, metric := range blkioStats.IoQueuedRecursive { device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor) field := fmt.Sprintf("io_queue_recursive_%s", strings.ToLower(metric.Op)) deviceStatMap[device][field] = metric.Value } - for _, metric := range blkioStats.IOServiceTimeRecursive { + for _, metric := range blkioStats.IoServiceTimeRecursive { device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor) field := fmt.Sprintf("io_service_time_recursive_%s", strings.ToLower(metric.Op)) deviceStatMap[device][field] = metric.Value } - for _, metric := range blkioStats.IOWaitTimeRecursive { + for _, metric := range blkioStats.IoWaitTimeRecursive { device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor) field := fmt.Sprintf("io_wait_time_%s", strings.ToLower(metric.Op)) deviceStatMap[device][field] = metric.Value } - for _, metric := range blkioStats.IOMergedRecursive { + for _, metric := range blkioStats.IoMergedRecursive { device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor) field := fmt.Sprintf("io_merged_recursive_%s", strings.ToLower(metric.Op)) deviceStatMap[device][field] = metric.Value } - for _, metric := range blkioStats.IOTimeRecursive { + for _, metric := range blkioStats.IoTimeRecursive { device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor) field := fmt.Sprintf("io_time_recursive_%s", strings.ToLower(metric.Op)) deviceStatMap[device][field] = metric.Value From 708cbf937f737c14e2138b4712a94f26fa09f562 Mon Sep 17 00:00:00 2001 From: Sergio Jimenez Date: Sun, 3 Apr 2016 19:37:41 +0200 Subject: [PATCH 22/55] input(docker): Fixed tests to work with engine-api * Modified tests to work with engine-api --- plugins/inputs/docker/docker_test.go | 196 +++++++++++++++++---------- 1 file changed, 121 insertions(+), 75 deletions(-) diff --git a/plugins/inputs/docker/docker_test.go b/plugins/inputs/docker/docker_test.go index 23fd0bb34..c9fe6cea1 100644 --- a/plugins/inputs/docker/docker_test.go +++ b/plugins/inputs/docker/docker_test.go @@ -1,13 +1,18 @@ package system import ( - "encoding/json" + "io" + "io/ioutil" + "strings" "testing" "time" + "golang.org/x/net/context" + + "github.com/docker/engine-api/types" + "github.com/docker/engine-api/types/registry" "github.com/influxdata/telegraf/testutil" - "github.com/fsouza/go-dockerclient" "github.com/stretchr/testify/require" ) @@ -114,58 +119,58 @@ func TestDockerGatherContainerStats(t *testing.T) { acc.AssertContainsTaggedFields(t, "docker_cpu", cpu1fields, cputags) } -func testStats() *docker.Stats { - stats := &docker.Stats{ - Read: time.Now(), - Networks: make(map[string]docker.NetworkStats), - } +func testStats() *types.StatsJSON { + stats := &types.StatsJSON{} + stats.Read = time.Now() + stats.Networks = make(map[string]types.NetworkStats) stats.CPUStats.CPUUsage.PercpuUsage = []uint64{1, 1002} stats.CPUStats.CPUUsage.UsageInUsermode = 100 stats.CPUStats.CPUUsage.TotalUsage = 500 stats.CPUStats.CPUUsage.UsageInKernelmode = 200 - stats.CPUStats.SystemCPUUsage = 100 + stats.CPUStats.SystemUsage = 100 stats.CPUStats.ThrottlingData.Periods = 1 stats.PreCPUStats.CPUUsage.TotalUsage = 400 - stats.PreCPUStats.SystemCPUUsage = 50 + stats.PreCPUStats.SystemUsage = 50 - stats.MemoryStats.Stats.TotalPgmafault = 0 - stats.MemoryStats.Stats.Cache = 0 - stats.MemoryStats.Stats.MappedFile = 0 - stats.MemoryStats.Stats.TotalInactiveFile = 0 - stats.MemoryStats.Stats.Pgpgout = 0 - stats.MemoryStats.Stats.Rss = 0 - stats.MemoryStats.Stats.TotalMappedFile = 0 - stats.MemoryStats.Stats.Writeback = 0 - stats.MemoryStats.Stats.Unevictable = 0 - stats.MemoryStats.Stats.Pgpgin = 0 - stats.MemoryStats.Stats.TotalUnevictable = 0 - stats.MemoryStats.Stats.Pgmajfault = 0 - stats.MemoryStats.Stats.TotalRss = 44 - stats.MemoryStats.Stats.TotalRssHuge = 444 - stats.MemoryStats.Stats.TotalWriteback = 55 - stats.MemoryStats.Stats.TotalInactiveAnon = 0 - stats.MemoryStats.Stats.RssHuge = 0 - stats.MemoryStats.Stats.HierarchicalMemoryLimit = 0 - stats.MemoryStats.Stats.TotalPgfault = 0 - stats.MemoryStats.Stats.TotalActiveFile = 0 - stats.MemoryStats.Stats.ActiveAnon = 0 - stats.MemoryStats.Stats.TotalActiveAnon = 0 - stats.MemoryStats.Stats.TotalPgpgout = 0 - stats.MemoryStats.Stats.TotalCache = 0 - stats.MemoryStats.Stats.InactiveAnon = 0 - stats.MemoryStats.Stats.ActiveFile = 1 - stats.MemoryStats.Stats.Pgfault = 2 - stats.MemoryStats.Stats.InactiveFile = 3 - stats.MemoryStats.Stats.TotalPgpgin = 4 + stats.MemoryStats.Stats = make(map[string]uint64) + stats.MemoryStats.Stats["total_pgmajfault"] = 0 + stats.MemoryStats.Stats["cache"] = 0 + stats.MemoryStats.Stats["mapped_file"] = 0 + stats.MemoryStats.Stats["total_inactive_file"] = 0 + stats.MemoryStats.Stats["pagpgout"] = 0 + stats.MemoryStats.Stats["rss"] = 0 + stats.MemoryStats.Stats["total_mapped_file"] = 0 + stats.MemoryStats.Stats["writeback"] = 0 + stats.MemoryStats.Stats["unevictable"] = 0 + stats.MemoryStats.Stats["pgpgin"] = 0 + stats.MemoryStats.Stats["total_unevictable"] = 0 + stats.MemoryStats.Stats["pgmajfault"] = 0 + stats.MemoryStats.Stats["total_rss"] = 44 + stats.MemoryStats.Stats["total_rss_huge"] = 444 + stats.MemoryStats.Stats["total_write_back"] = 55 + stats.MemoryStats.Stats["total_inactive_anon"] = 0 + stats.MemoryStats.Stats["rss_huge"] = 0 + stats.MemoryStats.Stats["hierarchical_memory_limit"] = 0 + stats.MemoryStats.Stats["total_pgfault"] = 0 + stats.MemoryStats.Stats["total_active_file"] = 0 + stats.MemoryStats.Stats["active_anon"] = 0 + stats.MemoryStats.Stats["total_active_anon"] = 0 + stats.MemoryStats.Stats["total_pgpgout"] = 0 + stats.MemoryStats.Stats["total_cache"] = 0 + stats.MemoryStats.Stats["inactive_anon"] = 0 + stats.MemoryStats.Stats["active_file"] = 1 + stats.MemoryStats.Stats["pgfault"] = 2 + stats.MemoryStats.Stats["inactive_file"] = 3 + stats.MemoryStats.Stats["total_pgpgin"] = 4 stats.MemoryStats.MaxUsage = 1001 stats.MemoryStats.Usage = 1111 stats.MemoryStats.Failcnt = 1 stats.MemoryStats.Limit = 2000 - stats.Networks["eth0"] = docker.NetworkStats{ + stats.Networks["eth0"] = types.NetworkStats{ RxDropped: 1, RxBytes: 2, RxErrors: 3, @@ -176,23 +181,23 @@ func testStats() *docker.Stats { TxBytes: 4, } - sbr := docker.BlkioStatsEntry{ + sbr := types.BlkioStatEntry{ Major: 6, Minor: 0, Op: "read", Value: 100, } - sr := docker.BlkioStatsEntry{ + sr := types.BlkioStatEntry{ Major: 6, Minor: 0, Op: "write", Value: 101, } - stats.BlkioStats.IOServiceBytesRecursive = append( - stats.BlkioStats.IOServiceBytesRecursive, sbr) - stats.BlkioStats.IOServicedRecursive = append( - stats.BlkioStats.IOServicedRecursive, sr) + stats.BlkioStats.IoServiceBytesRecursive = append( + stats.BlkioStats.IoServiceBytesRecursive, sbr) + stats.BlkioStats.IoServicedRecursive = append( + stats.BlkioStats.IoServicedRecursive, sr) return stats } @@ -200,35 +205,78 @@ func testStats() *docker.Stats { type FakeDockerClient struct { } -func (d FakeDockerClient) Info() (*docker.Env, error) { - env := docker.Env{"Containers=108", "OomKillDisable=false", "SystemTime=2016-02-24T00:55:09.15073105-05:00", "NEventsListener=0", "ID=5WQQ:TFWR:FDNG:OKQ3:37Y4:FJWG:QIKK:623T:R3ME:QTKB:A7F7:OLHD", "Debug=false", "LoggingDriver=json-file", "KernelVersion=4.3.0-1-amd64", "IndexServerAddress=https://index.docker.io/v1/", "MemTotal=3840757760", "Images=199", "CpuCfsQuota=true", "Name=absol", "SwapLimit=false", "IPv4Forwarding=true", "ExecutionDriver=native-0.2", "InitSha1=23a51f3c916d2b5a3bbb31caf301fd2d14edd518", "ExperimentalBuild=false", "CpuCfsPeriod=true", "RegistryConfig={\"IndexConfigs\":{\"docker.io\":{\"Mirrors\":null,\"Name\":\"docker.io\",\"Official\":true,\"Secure\":true}},\"InsecureRegistryCIDRs\":[\"127.0.0.0/8\"],\"Mirrors\":null}", "OperatingSystem=Linux Mint LMDE (containerized)", "BridgeNfIptables=true", "HttpsProxy=", "Labels=null", "MemoryLimit=false", "DriverStatus=[[\"Pool Name\",\"docker-8:1-1182287-pool\"],[\"Pool Blocksize\",\"65.54 kB\"],[\"Backing Filesystem\",\"extfs\"],[\"Data file\",\"/dev/loop0\"],[\"Metadata file\",\"/dev/loop1\"],[\"Data Space Used\",\"17.3 GB\"],[\"Data Space Total\",\"107.4 GB\"],[\"Data Space Available\",\"36.53 GB\"],[\"Metadata Space Used\",\"20.97 MB\"],[\"Metadata Space Total\",\"2.147 GB\"],[\"Metadata Space Available\",\"2.127 GB\"],[\"Udev Sync Supported\",\"true\"],[\"Deferred Removal Enabled\",\"false\"],[\"Data loop file\",\"/var/lib/docker/devicemapper/devicemapper/data\"],[\"Metadata loop file\",\"/var/lib/docker/devicemapper/devicemapper/metadata\"],[\"Library Version\",\"1.02.115 (2016-01-25)\"]]", "NFd=19", "HttpProxy=", "Driver=devicemapper", "NGoroutines=39", "InitPath=/usr/lib/docker.io/dockerinit", "NCPU=4", "DockerRootDir=/var/lib/docker", "NoProxy=", "BridgeNfIp6tables=true"} - return &env, nil +func (d FakeDockerClient) Info(ctx context.Context) (types.Info, error) { + env := types.Info{ + Containers: 108, + OomKillDisable: false, + SystemTime: "2016-02-24T00:55:09.15073105-05:00", + NEventsListener: 0, + ID: "5WQQ:TFWR:FDNG:OKQ3:37Y4:FJWG:QIKK:623T:R3ME:QTKB:A7F7:OLHD", + Debug: false, + LoggingDriver: "json-file", + KernelVersion: "4.3.0-1-amd64", + IndexServerAddress: "https://index.docker.io/v1/", + MemTotal: 3840757760, + Images: 199, + CPUCfsQuota: true, + Name: "absol", + SwapLimit: false, + IPv4Forwarding: true, + ExecutionDriver: "native-0.2", + ExperimentalBuild: false, + CPUCfsPeriod: true, + RegistryConfig: ®istry.ServiceConfig{ + IndexConfigs: map[string]*registry.IndexInfo{ + "docker.io": { + Name: "docker.io", + Mirrors: []string{}, + Official: true, + Secure: true, + }, + }, InsecureRegistryCIDRs: []*registry.NetIPNet{{IP: []byte{127, 0, 0, 0}, Mask: []byte{255, 0, 0, 0}}}, Mirrors: []string{}}, + OperatingSystem: "Linux Mint LMDE (containerized)", + BridgeNfIptables: true, + HTTPSProxy: "", + Labels: []string{}, + MemoryLimit: false, + DriverStatus: [][2]string{{"Pool Name", "docker-8:1-1182287-pool"}, {"Pool Blocksize", "65.54 kB"}, {"Backing Filesystem", "extfs"}, {"Data file", "/dev/loop0"}, {"Metadata file", "/dev/loop1"}, {"Data Space Used", "17.3 GB"}, {"Data Space Total", "107.4 GB"}, {"Data Space Available", "36.53 GB"}, {"Metadata Space Used", "20.97 MB"}, {"Metadata Space Total", "2.147 GB"}, {"Metadata Space Available", "2.127 GB"}, {"Udev Sync Supported", "true"}, {"Deferred Removal Enabled", "false"}, {"Data loop file", "/var/lib/docker/devicemapper/devicemapper/data"}, {"Metadata loop file", "/var/lib/docker/devicemapper/devicemapper/metadata"}, {"Library Version", "1.02.115 (2016-01-25)"}}, + NFd: 19, + HTTPProxy: "", + Driver: "devicemapper", + NGoroutines: 39, + NCPU: 4, + DockerRootDir: "/var/lib/docker", + NoProxy: "", + BridgeNfIP6tables: true, + } + return env, nil } -func (d FakeDockerClient) ListContainers(opts docker.ListContainersOptions) ([]docker.APIContainers, error) { - container1 := docker.APIContainers{ +func (d FakeDockerClient) ContainerList(octx context.Context, options types.ContainerListOptions) ([]types.Container, error) { + container1 := types.Container{ ID: "e2173b9478a6ae55e237d4d74f8bbb753f0817192b5081334dc78476296b7dfb", + Names: []string{"/etcd"}, Image: "quay.io/coreos/etcd:v2.2.2", Command: "/etcd -name etcd0 -advertise-client-urls http://localhost:2379 -listen-client-urls http://0.0.0.0:2379", Created: 1455941930, Status: "Up 4 hours", - Ports: []docker.APIPort{ - docker.APIPort{ + Ports: []types.Port{ + types.Port{ PrivatePort: 7001, PublicPort: 0, Type: "tcp", }, - docker.APIPort{ + types.Port{ PrivatePort: 4001, PublicPort: 0, Type: "tcp", }, - docker.APIPort{ + types.Port{ PrivatePort: 2380, PublicPort: 0, Type: "tcp", }, - docker.APIPort{ + types.Port{ PrivatePort: 2379, PublicPort: 2379, Type: "tcp", @@ -237,31 +285,31 @@ func (d FakeDockerClient) ListContainers(opts docker.ListContainersOptions) ([]d }, SizeRw: 0, SizeRootFs: 0, - Names: []string{"/etcd"}, } - container2 := docker.APIContainers{ + container2 := types.Container{ ID: "b7dfbb9478a6ae55e237d4d74f8bbb753f0817192b5081334dc78476296e2173", + Names: []string{"/etcd2"}, Image: "quay.io/coreos/etcd:v2.2.2", Command: "/etcd -name etcd2 -advertise-client-urls http://localhost:2379 -listen-client-urls http://0.0.0.0:2379", Created: 1455941933, Status: "Up 4 hours", - Ports: []docker.APIPort{ - docker.APIPort{ + Ports: []types.Port{ + types.Port{ PrivatePort: 7002, PublicPort: 0, Type: "tcp", }, - docker.APIPort{ + types.Port{ PrivatePort: 4002, PublicPort: 0, Type: "tcp", }, - docker.APIPort{ + types.Port{ PrivatePort: 2381, PublicPort: 0, Type: "tcp", }, - docker.APIPort{ + types.Port{ PrivatePort: 2382, PublicPort: 2382, Type: "tcp", @@ -270,21 +318,19 @@ func (d FakeDockerClient) ListContainers(opts docker.ListContainersOptions) ([]d }, SizeRw: 0, SizeRootFs: 0, - Names: []string{"/etcd2"}, } - containers := []docker.APIContainers{container1, container2} + containers := []types.Container{container1, container2} return containers, nil //#{e6a96c84ca91a5258b7cb752579fb68826b68b49ff957487695cd4d13c343b44 titilambert/snmpsim /bin/sh -c 'snmpsimd --agent-udpv4-endpoint=0.0.0.0:31161 --process-user=root --process-group=user' 1455724831 Up 4 hours [{31161 31161 udp 0.0.0.0}] 0 0 [/snmp] map[]}]2016/02/24 01:05:01 Gathered metrics, (3s interval), from 1 inputs in 1.233836656s } -func (d FakeDockerClient) Stats(opts docker.StatsOptions) error { +func (d FakeDockerClient) ContainerStats(ctx context.Context, containerID string, stream bool) (io.ReadCloser, error) { + var stat io.ReadCloser jsonStat := `{"read":"2016-02-24T11:42:27.472459608-05:00","memory_stats":{"stats":{},"limit":18935443456},"blkio_stats":{"io_service_bytes_recursive":[{"major":252,"minor":1,"op":"Read","value":753664},{"major":252,"minor":1,"op":"Write"},{"major":252,"minor":1,"op":"Sync"},{"major":252,"minor":1,"op":"Async","value":753664},{"major":252,"minor":1,"op":"Total","value":753664}],"io_serviced_recursive":[{"major":252,"minor":1,"op":"Read","value":26},{"major":252,"minor":1,"op":"Write"},{"major":252,"minor":1,"op":"Sync"},{"major":252,"minor":1,"op":"Async","value":26},{"major":252,"minor":1,"op":"Total","value":26}]},"cpu_stats":{"cpu_usage":{"percpu_usage":[17871,4959158,1646137,1231652,11829401,244656,369972,0],"usage_in_usermode":10000000,"total_usage":20298847},"system_cpu_usage":24052607520000000,"throttling_data":{}},"precpu_stats":{"cpu_usage":{"percpu_usage":[17871,4959158,1646137,1231652,11829401,244656,369972,0],"usage_in_usermode":10000000,"total_usage":20298847},"system_cpu_usage":24052599550000000,"throttling_data":{}}}` - var stat docker.Stats - json.Unmarshal([]byte(jsonStat), &stat) - opts.Stats <- &stat - return nil + stat = ioutil.NopCloser(strings.NewReader(jsonStat)) + return stat, nil } func TestDockerGatherInfo(t *testing.T) { @@ -299,12 +345,12 @@ func TestDockerGatherInfo(t *testing.T) { acc.AssertContainsTaggedFields(t, "docker", map[string]interface{}{ - "n_listener_events": int64(0), - "n_cpus": int64(4), - "n_used_file_descriptors": int64(19), - "n_containers": int64(108), - "n_images": int64(199), - "n_goroutines": int64(39), + "n_listener_events": int(0), + "n_cpus": int(4), + "n_used_file_descriptors": int(19), + "n_containers": int(108), + "n_images": int(199), + "n_goroutines": int(39), }, map[string]string{}, ) From fd1f05c8e01706e388c5885f2c443ab927c2f817 Mon Sep 17 00:00:00 2001 From: Sergio Jimenez Date: Sun, 3 Apr 2016 19:40:12 +0200 Subject: [PATCH 23/55] input(docker): Fixed io sectors/io_time recursive * On engine-api sectors_recursive and io_time_recursive have no Op --- plugins/inputs/docker/docker.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/plugins/inputs/docker/docker.go b/plugins/inputs/docker/docker.go index b1bbc63b3..d753e8b63 100644 --- a/plugins/inputs/docker/docker.go +++ b/plugins/inputs/docker/docker.go @@ -395,14 +395,12 @@ func gatherBlockIOMetrics( for _, metric := range blkioStats.IoTimeRecursive { device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor) - field := fmt.Sprintf("io_time_recursive_%s", strings.ToLower(metric.Op)) - deviceStatMap[device][field] = metric.Value + deviceStatMap[device]["io_time_recursive"] = metric.Value } for _, metric := range blkioStats.SectorsRecursive { device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor) - field := fmt.Sprintf("sectors_recursive_%s", strings.ToLower(metric.Op)) - deviceStatMap[device][field] = metric.Value + deviceStatMap[device]["sectors_recursive"] = metric.Value } for device, fields := range deviceStatMap { From 741fb1181ff6f5453b85f7a0c8a3da5f1f53f4c1 Mon Sep 17 00:00:00 2001 From: Sergio Jimenez Date: Sun, 3 Apr 2016 20:06:01 +0200 Subject: [PATCH 24/55] godeps(): Updated Godeps file for engine-api * Added required deps for engine-api * Removed fsouza/go-dockerclient --- Godeps | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Godeps b/Godeps index 255b95ab5..b2b6b7e58 100644 --- a/Godeps +++ b/Godeps @@ -9,10 +9,12 @@ github.com/couchbase/gomemcached a5ea6356f648fec6ab89add00edd09151455b4b2 github.com/couchbase/goutils 5823a0cbaaa9008406021dc5daf80125ea30bba6 github.com/dancannon/gorethink e7cac92ea2bc52638791a021f212145acfedb1fc github.com/davecgh/go-spew 5215b55f46b2b919f50a1df0eaa5886afe4e3b3d +github.com/docker/engine-api 8924d6900370b4c7e7984be5adc61f50a80d7537 +github.com/docker/go-connections f549a9393d05688dff0992ef3efd8bbe6c628aeb +github.com/docker/go-units 5d2041e26a699eaca682e2ea41c8f891e1060444 github.com/eapache/go-resiliency b86b1ec0dd4209a588dc1285cdd471e73525c0b3 github.com/eapache/queue ded5959c0d4e360646dc9e9908cff48666781367 github.com/eclipse/paho.mqtt.golang 4ab3e867810d1ec5f35157c59e965054dbf43a0d -github.com/fsouza/go-dockerclient a49c8269a6899cae30da1f8a4b82e0ce945f9967 github.com/go-sql-driver/mysql 1fca743146605a172a266e1654e01e5cd5669bee github.com/golang/protobuf 552c7b9542c194800fd493123b3798ef0a832032 github.com/golang/snappy 427fb6fc07997f43afa32f35e850833760e489a7 From 5c688daff12d298fd902607d6f4018599620fb1d Mon Sep 17 00:00:00 2001 From: Sergio Jimenez Date: Sun, 3 Apr 2016 20:42:52 +0200 Subject: [PATCH 25/55] input(docker): Updated README * Replaced links to fsouza/go-dockerclient by docker/engine-api --- plugins/inputs/docker/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/docker/README.md b/plugins/inputs/docker/README.md index 97450e2aa..045e09a81 100644 --- a/plugins/inputs/docker/README.md +++ b/plugins/inputs/docker/README.md @@ -5,11 +5,11 @@ docker containers. You can read Docker's documentation for their remote API [here](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.20/#get-container-stats-based-on-resource-usage) The docker plugin uses the excellent -[fsouza go-dockerclient](https://github.com/fsouza/go-dockerclient) library to +[docker engine-api](https://github.com/docker/engine-api) library to gather stats. Documentation for the library can be found [here](https://godoc.org/github.com/fsouza/go-dockerclient) and documentation for the stat structure can be found -[here](https://godoc.org/github.com/fsouza/go-dockerclient#Stats) +[here](https://godoc.org/github.com/docker/engine-api/types#Stats) ### Configuration: From 9f68a329349f564809a610dd7ed0f391b75a04d8 Mon Sep 17 00:00:00 2001 From: Sergio Jimenez Date: Sun, 3 Apr 2016 20:57:53 +0200 Subject: [PATCH 26/55] fix(): Last link on README --- plugins/inputs/docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/docker/README.md b/plugins/inputs/docker/README.md index 045e09a81..c22e6af8e 100644 --- a/plugins/inputs/docker/README.md +++ b/plugins/inputs/docker/README.md @@ -7,7 +7,7 @@ docker containers. You can read Docker's documentation for their remote API The docker plugin uses the excellent [docker engine-api](https://github.com/docker/engine-api) library to gather stats. Documentation for the library can be found -[here](https://godoc.org/github.com/fsouza/go-dockerclient) and documentation +[here](https://godoc.org/github.com/docker/engine-api) and documentation for the stat structure can be found [here](https://godoc.org/github.com/docker/engine-api/types#Stats) From 8274798499b38145ba403484b4ddc49f639cb3a1 Mon Sep 17 00:00:00 2001 From: Sergio Jimenez Date: Mon, 4 Apr 2016 00:35:31 +0200 Subject: [PATCH 27/55] fix(Godeps): Added github.com/opencontainers/runc --- Godeps | 1 + 1 file changed, 1 insertion(+) diff --git a/Godeps b/Godeps index b2b6b7e58..aa5be999d 100644 --- a/Godeps +++ b/Godeps @@ -34,6 +34,7 @@ github.com/naoina/go-stringutil 6b638e95a32d0c1131db0e7fe83775cbea4a0d0b github.com/nats-io/nats b13fc9d12b0b123ebc374e6b808c6228ae4234a3 github.com/nats-io/nuid 4f84f5f3b2786224e336af2e13dba0a0a80b76fa github.com/nsqio/go-nsq 0b80d6f05e15ca1930e0c5e1d540ed627e299980 +github.com/opencontainers/runc 89ab7f2ccc1e45ddf6485eaa802c35dcf321dfc8 github.com/prometheus/client_golang 18acf9993a863f4c4b40612e19cdd243e7c86831 github.com/prometheus/client_model fa8ad6fec33561be4280a8f0514318c79d7f6cb6 github.com/prometheus/common e8eabff8812b05acf522b45fdcd725a785188e37 From e19c474a92999137b59e2410483540d6dca3d4bb Mon Sep 17 00:00:00 2001 From: Sergio Jimenez Date: Tue, 5 Apr 2016 01:03:28 +0200 Subject: [PATCH 28/55] input(docker): Cleanup * Removed leftovers, unused code closes #957 fixes #645 --- CHANGELOG.md | 1 + plugins/inputs/docker/docker.go | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a4552698..b5a2c9f78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - [#968](https://github.com/influxdata/telegraf/issues/968): Processes plugin gets unknown state when spaces are in (command name) - [#969](https://github.com/influxdata/telegraf/pull/969): ipmi_sensors: allow : in password. Thanks @awaw! - [#972](https://github.com/influxdata/telegraf/pull/972): dovecot: remove extra newline in dovecot command. Thanks @mrannanj! +- [#645](https://github.com/influxdata/telegraf/issues/645): docker plugin i/o error on closed pipe. Thanks @tripledes! ## v0.12.0 [2016-04-05] diff --git a/plugins/inputs/docker/docker.go b/plugins/inputs/docker/docker.go index d753e8b63..094bad8ca 100644 --- a/plugins/inputs/docker/docker.go +++ b/plugins/inputs/docker/docker.go @@ -149,8 +149,6 @@ func (d *Docker) gatherInfo(acc telegraf.Accumulator) error { map[string]string{"unit": "bytes"}, now) // Get storage metrics - //driverStatusRaw := []byte(info.DriverStatus) - //json.Unmarshal(driverStatusRaw, &driverStatus) for _, rawData := range info.DriverStatus { // Try to convert string to int (bytes) value, err := parseSize(rawData[1]) From d5b9e003fee44ff5276e7177e048cdc007ea95f8 Mon Sep 17 00:00:00 2001 From: Josh Hardy Date: Fri, 25 Mar 2016 15:16:23 -0700 Subject: [PATCH 29/55] Add CloudWatch input plugin Rebased commit of previously reviewed branch. Added cloudwatch client Mock and more rich unit tests. closes #935 closes #936 --- CHANGELOG.md | 1 + plugins/inputs/all/all.go | 1 + plugins/inputs/cloudwatch/README.md | 86 ++++++ plugins/inputs/cloudwatch/cloudwatch.go | 305 +++++++++++++++++++ plugins/inputs/cloudwatch/cloudwatch_test.go | 131 ++++++++ 5 files changed, 524 insertions(+) create mode 100644 plugins/inputs/cloudwatch/README.md create mode 100644 plugins/inputs/cloudwatch/cloudwatch.go create mode 100644 plugins/inputs/cloudwatch/cloudwatch_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index b5a2c9f78..09a00f069 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### Features - [#976](https://github.com/influxdata/telegraf/pull/976): Reduce allocations in the UDP and statsd inputs. - [#979](https://github.com/influxdata/telegraf/pull/979): Reduce allocations in the TCP listener. +- [#935](https://github.com/influxdata/telegraf/pull/935): AWS Cloudwatch input plugin. Thanks @joshhardy & @ljosa! ### Bugfixes - [#968](https://github.com/influxdata/telegraf/issues/968): Processes plugin gets unknown state when spaces are in (command name) diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index 4f7d45f60..52ee6c13d 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -4,6 +4,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/aerospike" _ "github.com/influxdata/telegraf/plugins/inputs/apache" _ "github.com/influxdata/telegraf/plugins/inputs/bcache" + _ "github.com/influxdata/telegraf/plugins/inputs/cloudwatch" _ "github.com/influxdata/telegraf/plugins/inputs/couchbase" _ "github.com/influxdata/telegraf/plugins/inputs/couchdb" _ "github.com/influxdata/telegraf/plugins/inputs/disque" diff --git a/plugins/inputs/cloudwatch/README.md b/plugins/inputs/cloudwatch/README.md new file mode 100644 index 000000000..04501161d --- /dev/null +++ b/plugins/inputs/cloudwatch/README.md @@ -0,0 +1,86 @@ +# Amazon CloudWatch Statistics Input + +This plugin will pull Metric Statistics from Amazon CloudWatch. + +### Amazon Authentication + +This plugin uses a credential chain for Authentication with the CloudWatch +API endpoint. In the following order the plugin will attempt to authenticate. +1. [IAMS Role](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) +2. [Environment Variables](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#environment-variables) +3. [Shared Credentials](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#shared-credentials-file) + +### Configuration: + +```toml +[[inputs.cloudwatch]] + ## Amazon Region (required) + region = 'us-east-1' + + ## Requested CloudWatch aggregation Period (required - must be a multiple of 60s) + period = '1m' + + ## Collection Delay (required - must account for metrics availability via CloudWatch API) + delay = '1m' + + ## Override global run interval (optional - defaults to global interval) + ## Recomended: use metric 'interval' that is a multiple of 'period' to avoid + ## gaps or overlap in pulled data + interval = '1m' + + ## Metric Statistic Namespace (required) + namespace = 'AWS/ELB' + + ## Metrics to Pull (optional) + ## Defaults to all Metrics in Namespace if nothing is provided + ## Refreshes Namespace available metrics every 1h + [[inputs.cloudwatch.metrics]] + names = ['Latency', 'RequestCount'] + + ## Dimension filters for Metric (optional) + [[inputs.cloudwatch.metrics.dimensions]] + name = 'LoadBalancerName' + value = 'p-example' +``` +#### Requirements and Terminology + +Plugin Configuration utilizes [CloudWatch concepts](http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/cloudwatch_concepts.html) and access pattern to allow monitoring of any CloudWatch Metric. + +- `region` must be a valid AWS [Region](http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/cloudwatch_concepts.html#CloudWatchRegions) value +- `period` must be a valid CloudWatch [Period](http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/cloudwatch_concepts.html#CloudWatchPeriods) value +- `namespace` must be a valid CloudWatch [Namespace](http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/cloudwatch_concepts.html#Namespace) value +- `names` must be valid CloudWatch [Metric](http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/cloudwatch_concepts.html#Metric) names +- `dimensions` must be valid CloudWatch [Dimension](http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/cloudwatch_concepts.html#Dimension) name/value pairs + +#### Restrictions and Limitations +- CloudWatch metrics are not available instantly via the CloudWatch API. You should adjust your collection `delay` to account for this lag in metrics availability based on your [monitoring subscription level](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html) +- CloudWatch API usage incurs cost - see [GetMetricStatistics Pricing](https://aws.amazon.com/cloudwatch/pricing/) + +### Measurements & Fields: + +Each CloudWatch Namespace monitored records a measurement with fields for each available Metric Statistic +Namespace and Metrics are represented in [snake case](https://en.wikipedia.org/wiki/Snake_case) + +- cloudwatch_{namespace} + - {metric}_sum (metric Sum value) + - {metric}_average (metric Average value) + - {metric}_minimum (metric Minimum value) + - {metric}_maximum (metric Maximum value) + - {metric}_sample_count (metric SampleCount value) + + +### Tags: +Each measurement is tagged with the following identifiers to uniquely identify the associated metric +Tag Dimension names are represented in [snake case](https://en.wikipedia.org/wiki/Snake_case) + +- All measurements have the following tags: + - region (CloudWatch Region) + - unit (CloudWatch Metric Unit) + - {dimension-name} (Cloudwatch Dimension value - one for each metric dimension) + +### Example Output: + +``` +$ ./telegraf -config telegraf.conf -input-filter cloudwatch -test +> cloudwatch_aws_elb,load_balancer_name=p-example,region=us-east-1,unit=seconds latency_average=0.004810798017284538,latency_maximum=0.1100282669067383,latency_minimum=0.0006084442138671875,latency_sample_count=4029,latency_sum=19.382705211639404 1459542420000000000 +``` diff --git a/plugins/inputs/cloudwatch/cloudwatch.go b/plugins/inputs/cloudwatch/cloudwatch.go new file mode 100644 index 000000000..e3fa74bad --- /dev/null +++ b/plugins/inputs/cloudwatch/cloudwatch.go @@ -0,0 +1,305 @@ +package cloudwatch + +import ( + "fmt" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds" + "github.com/aws/aws-sdk-go/aws/ec2metadata" + "github.com/aws/aws-sdk-go/aws/session" + + "github.com/aws/aws-sdk-go/service/cloudwatch" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/plugins/inputs" +) + +type ( + CloudWatch struct { + Region string `toml:"region"` + Period internal.Duration `toml:"period"` + Delay internal.Duration `toml:"delay"` + Namespace string `toml:"namespace"` + Metrics []*Metric `toml:"metrics"` + client cloudwatchClient + metricCache *MetricCache + } + + Metric struct { + MetricNames []string `toml:"names"` + Dimensions []*Dimension `toml:"dimensions"` + } + + Dimension struct { + Name string `toml:"name"` + Value string `toml:"value"` + } + + MetricCache struct { + TTL time.Duration + Fetched time.Time + Metrics []*cloudwatch.Metric + } + + cloudwatchClient interface { + ListMetrics(*cloudwatch.ListMetricsInput) (*cloudwatch.ListMetricsOutput, error) + GetMetricStatistics(*cloudwatch.GetMetricStatisticsInput) (*cloudwatch.GetMetricStatisticsOutput, error) + } +) + +func (c *CloudWatch) SampleConfig() string { + return ` + ## Amazon Region + region = 'us-east-1' + + ## Requested CloudWatch aggregation Period (required - must be a multiple of 60s) + period = '1m' + + ## Collection Delay (required - must account for metrics availability via CloudWatch API) + delay = '1m' + + ## Recomended: use metric 'interval' that is a multiple of 'period' to avoid + ## gaps or overlap in pulled data + interval = '1m' + + ## Metric Statistic Namespace (required) + namespace = 'AWS/ELB' + + ## Metrics to Pull (optional) + ## Defaults to all Metrics in Namespace if nothing is provided + ## Refreshes Namespace available metrics every 1h + #[[inputs.cloudwatch.metrics]] + # names = ['Latency', 'RequestCount'] + # + # ## Dimension filters for Metric (optional) + # [[inputs.cloudwatch.metrics.dimensions]] + # name = 'LoadBalancerName' + # value = 'p-example' +` +} + +func (c *CloudWatch) Description() string { + return "Pull Metric Statistics from Amazon CloudWatch" +} + +func (c *CloudWatch) Gather(acc telegraf.Accumulator) error { + if c.client == nil { + c.initializeCloudWatch() + } + + var metrics []*cloudwatch.Metric + + // check for provided metric filter + if c.Metrics != nil { + metrics = []*cloudwatch.Metric{} + for _, m := range c.Metrics { + dimensions := make([]*cloudwatch.Dimension, len(m.Dimensions)) + for k, d := range m.Dimensions { + dimensions[k] = &cloudwatch.Dimension{ + Name: aws.String(d.Name), + Value: aws.String(d.Value), + } + } + for _, name := range m.MetricNames { + metrics = append(metrics, &cloudwatch.Metric{ + Namespace: aws.String(c.Namespace), + MetricName: aws.String(name), + Dimensions: dimensions, + }) + } + } + } else { + var err error + metrics, err = c.fetchNamespaceMetrics() + if err != nil { + return err + } + } + + metricCount := len(metrics) + var errChan = make(chan error, metricCount) + + now := time.Now() + + // limit concurrency or we can easily exhaust user connection limit + semaphore := make(chan byte, 64) + + for _, m := range metrics { + semaphore <- 0x1 + go c.gatherMetric(acc, m, now, semaphore, errChan) + } + + for i := 1; i <= metricCount; i++ { + err := <-errChan + if err != nil { + return err + } + } + return nil +} + +func init() { + inputs.Add("cloudwatch", func() telegraf.Input { + return &CloudWatch{} + }) +} + +/* + * Initialize CloudWatch client + */ +func (c *CloudWatch) initializeCloudWatch() error { + config := &aws.Config{ + Region: aws.String(c.Region), + Credentials: credentials.NewChainCredentials( + []credentials.Provider{ + &ec2rolecreds.EC2RoleProvider{Client: ec2metadata.New(session.New())}, + &credentials.EnvProvider{}, + &credentials.SharedCredentialsProvider{}, + }), + } + + c.client = cloudwatch.New(session.New(config)) + return nil +} + +/* + * Fetch available metrics for given CloudWatch Namespace + */ +func (c *CloudWatch) fetchNamespaceMetrics() (metrics []*cloudwatch.Metric, err error) { + if c.metricCache != nil && c.metricCache.IsValid() { + metrics = c.metricCache.Metrics + return + } + + metrics = []*cloudwatch.Metric{} + + var token *string + for more := true; more; { + params := &cloudwatch.ListMetricsInput{ + Namespace: aws.String(c.Namespace), + Dimensions: []*cloudwatch.DimensionFilter{}, + NextToken: token, + MetricName: nil, + } + + resp, err := c.client.ListMetrics(params) + if err != nil { + return nil, err + } + + metrics = append(metrics, resp.Metrics...) + + token = resp.NextToken + more = token != nil + } + + cacheTTL, _ := time.ParseDuration("1hr") + c.metricCache = &MetricCache{ + Metrics: metrics, + Fetched: time.Now(), + TTL: cacheTTL, + } + + return +} + +/* + * Gather given Metric and emit any error + */ +func (c *CloudWatch) gatherMetric(acc telegraf.Accumulator, metric *cloudwatch.Metric, now time.Time, semaphore chan byte, errChan chan error) { + params := c.getStatisticsInput(metric, now) + resp, err := c.client.GetMetricStatistics(params) + if err != nil { + errChan <- err + <-semaphore + return + } + + for _, point := range resp.Datapoints { + tags := map[string]string{ + "region": c.Region, + "unit": snakeCase(*point.Unit), + } + + for _, d := range metric.Dimensions { + tags[snakeCase(*d.Name)] = *d.Value + } + + // record field for each statistic + fields := map[string]interface{}{} + + if point.Average != nil { + fields[formatField(*metric.MetricName, cloudwatch.StatisticAverage)] = *point.Average + } + if point.Maximum != nil { + fields[formatField(*metric.MetricName, cloudwatch.StatisticMaximum)] = *point.Maximum + } + if point.Minimum != nil { + fields[formatField(*metric.MetricName, cloudwatch.StatisticMinimum)] = *point.Minimum + } + if point.SampleCount != nil { + fields[formatField(*metric.MetricName, cloudwatch.StatisticSampleCount)] = *point.SampleCount + } + if point.Sum != nil { + fields[formatField(*metric.MetricName, cloudwatch.StatisticSum)] = *point.Sum + } + + acc.AddFields(formatMeasurement(c.Namespace), fields, tags, *point.Timestamp) + } + + errChan <- nil + <-semaphore +} + +/* + * Formatting helpers + */ +func formatField(metricName string, statistic string) string { + return fmt.Sprintf("%s_%s", snakeCase(metricName), snakeCase(statistic)) +} + +func formatMeasurement(namespace string) string { + namespace = strings.Replace(namespace, "/", "_", -1) + namespace = snakeCase(namespace) + return fmt.Sprintf("cloudwatch_%s", namespace) +} + +func snakeCase(s string) string { + s = internal.SnakeCase(s) + s = strings.Replace(s, "__", "_", -1) + return s +} + +/* + * Map Metric to *cloudwatch.GetMetricStatisticsInput for given timeframe + */ +func (c *CloudWatch) getStatisticsInput(metric *cloudwatch.Metric, now time.Time) *cloudwatch.GetMetricStatisticsInput { + end := now.Add(-c.Delay.Duration) + + input := &cloudwatch.GetMetricStatisticsInput{ + StartTime: aws.Time(end.Add(-c.Period.Duration)), + EndTime: aws.Time(end), + MetricName: metric.MetricName, + Namespace: metric.Namespace, + Period: aws.Int64(int64(c.Period.Duration.Seconds())), + Dimensions: metric.Dimensions, + Statistics: []*string{ + aws.String(cloudwatch.StatisticAverage), + aws.String(cloudwatch.StatisticMaximum), + aws.String(cloudwatch.StatisticMinimum), + aws.String(cloudwatch.StatisticSum), + aws.String(cloudwatch.StatisticSampleCount)}, + } + return input +} + +/* + * Check Metric Cache validity + */ +func (c *MetricCache) IsValid() bool { + return c.Metrics != nil && time.Since(c.Fetched) < c.TTL +} diff --git a/plugins/inputs/cloudwatch/cloudwatch_test.go b/plugins/inputs/cloudwatch/cloudwatch_test.go new file mode 100644 index 000000000..8f8a3ad0b --- /dev/null +++ b/plugins/inputs/cloudwatch/cloudwatch_test.go @@ -0,0 +1,131 @@ +package cloudwatch + +import ( + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/assert" +) + +type mockCloudWatchClient struct{} + +func (m *mockCloudWatchClient) ListMetrics(params *cloudwatch.ListMetricsInput) (*cloudwatch.ListMetricsOutput, error) { + metric := &cloudwatch.Metric{ + Namespace: params.Namespace, + MetricName: aws.String("Latency"), + Dimensions: []*cloudwatch.Dimension{ + &cloudwatch.Dimension{ + Name: aws.String("LoadBalancerName"), + Value: aws.String("p-example"), + }, + }, + } + + result := &cloudwatch.ListMetricsOutput{ + Metrics: []*cloudwatch.Metric{metric}, + } + return result, nil +} + +func (m *mockCloudWatchClient) GetMetricStatistics(params *cloudwatch.GetMetricStatisticsInput) (*cloudwatch.GetMetricStatisticsOutput, error) { + dataPoint := &cloudwatch.Datapoint{ + Timestamp: params.EndTime, + Minimum: aws.Float64(0.1), + Maximum: aws.Float64(0.3), + Average: aws.Float64(0.2), + Sum: aws.Float64(123), + SampleCount: aws.Float64(100), + Unit: aws.String("Seconds"), + } + result := &cloudwatch.GetMetricStatisticsOutput{ + Label: aws.String("Latency"), + Datapoints: []*cloudwatch.Datapoint{dataPoint}, + } + return result, nil +} + +func TestGather(t *testing.T) { + duration, _ := time.ParseDuration("1m") + internalDuration := internal.Duration{ + Duration: duration, + } + c := &CloudWatch{ + Region: "us-east-1", + Namespace: "AWS/ELB", + Delay: internalDuration, + Period: internalDuration, + } + + var acc testutil.Accumulator + c.client = &mockCloudWatchClient{} + + c.Gather(&acc) + + fields := map[string]interface{}{} + fields["latency_minimum"] = 0.1 + fields["latency_maximum"] = 0.3 + fields["latency_average"] = 0.2 + fields["latency_sum"] = 123.0 + fields["latency_sample_count"] = 100.0 + + tags := map[string]string{} + tags["unit"] = "seconds" + tags["region"] = "us-east-1" + tags["load_balancer_name"] = "p-example" + + assert.True(t, acc.HasMeasurement("cloudwatch_aws_elb")) + acc.AssertContainsTaggedFields(t, "cloudwatch_aws_elb", fields, tags) + +} + +func TestGenerateStatisticsInputParams(t *testing.T) { + d := &cloudwatch.Dimension{ + Name: aws.String("LoadBalancerName"), + Value: aws.String("p-example"), + } + + m := &cloudwatch.Metric{ + MetricName: aws.String("Latency"), + Dimensions: []*cloudwatch.Dimension{d}, + } + + duration, _ := time.ParseDuration("1m") + internalDuration := internal.Duration{ + Duration: duration, + } + + c := &CloudWatch{ + Namespace: "AWS/ELB", + Delay: internalDuration, + Period: internalDuration, + } + + c.initializeCloudWatch() + + now := time.Now() + + params := c.getStatisticsInput(m, now) + + assert.EqualValues(t, *params.EndTime, now.Add(-c.Delay.Duration)) + assert.EqualValues(t, *params.StartTime, now.Add(-c.Period.Duration).Add(-c.Delay.Duration)) + assert.Len(t, params.Dimensions, 1) + assert.Len(t, params.Statistics, 5) + assert.EqualValues(t, *params.Period, 60) +} + +func TestMetricsCacheTimeout(t *testing.T) { + ttl, _ := time.ParseDuration("5ms") + cache := &MetricCache{ + Metrics: []*cloudwatch.Metric{}, + Fetched: time.Now(), + TTL: ttl, + } + + assert.True(t, cache.IsValid()) + time.Sleep(ttl) + assert.False(t, cache.IsValid()) +} From 1a612bcae99e95045e70e9b00e9347c4d4e90b6b Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Wed, 6 Apr 2016 13:49:50 -0600 Subject: [PATCH 30/55] Update README and etc/telegraf.conf --- README.md | 1 + etc/telegraf.conf | 39 ++++++++++++++++++++++++++++++--------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 9813ca6d4..caa562a6d 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,7 @@ more information on each, please look at the directory of the same name in Currently implemented sources: +* [aws cloudwatch](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/cloudwatch) * [aerospike](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/aerospike) * [apache](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/apache) * [bcache](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/bcache) diff --git a/etc/telegraf.conf b/etc/telegraf.conf index 3d65aaf62..1b534d888 100644 --- a/etc/telegraf.conf +++ b/etc/telegraf.conf @@ -429,6 +429,36 @@ # bcacheDevs = ["bcache0"] +# # Pull Metric Statistics from Amazon CloudWatch +# [[inputs.cloudwatch]] +# ## Amazon Region +# region = 'us-east-1' +# +# ## Requested CloudWatch aggregation Period (required - must be a multiple of 60s) +# period = '1m' +# +# ## Collection Delay (required - must account for metrics availability via CloudWatch API) +# delay = '1m' +# +# ## Recomended: use metric 'interval' that is a multiple of 'period' to avoid +# ## gaps or overlap in pulled data +# interval = '1m' +# +# ## Metric Statistic Namespace (required) +# namespace = 'AWS/ELB' +# +# ## Metrics to Pull (optional) +# ## Defaults to all Metrics in Namespace if nothing is provided +# ## Refreshes Namespace available metrics every 1h +# #[[inputs.cloudwatch.metrics]] +# # names = ['Latency', 'RequestCount'] +# # +# # ## Dimension filters for Metric (optional) +# # [[inputs.cloudwatch.metrics.dimensions]] +# # name = 'LoadBalancerName' +# # value = 'p-example' + + # # Read metrics from one or many couchbase clusters # [[inputs.couchbase]] # ## specify servers via a url matching: @@ -1245,10 +1275,6 @@ # ## calculation of percentiles. Raising this limit increases the accuracy # ## of percentiles but also increases the memory usage and cpu time. # percentile_limit = 1000 -# -# ## UDP packet size for the server to listen for. This will depend on the size -# ## of the packets that the client is sending, which is usually 1500 bytes. -# udp_packet_size = 1500 # # Generic TCP listener @@ -1279,11 +1305,6 @@ # ## UDP listener will start dropping packets. # allowed_pending_messages = 10000 # -# ## UDP packet size for the server to listen for. This will depend -# ## on the size of the packets that the client is sending, which is -# ## usually 1500 bytes, but can be as large as 65,535 bytes. -# udp_packet_size = 1500 -# # ## Data format to consume. # ## Each data format has it's own unique set of configuration options, read # ## more about them here: From cce35da366ac95cb41568047b908d14fac5fe9ce Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Thu, 7 Apr 2016 09:26:35 -0600 Subject: [PATCH 31/55] Godeps_windows: update file --- Godeps_windows | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Godeps_windows b/Godeps_windows index f499fa915..ab3004bb8 100644 --- a/Godeps_windows +++ b/Godeps_windows @@ -1,3 +1,4 @@ +github.com/Microsoft/go-winio 9f57cbbcbcb41dea496528872a4f0e37a4f7ae98 github.com/Shopify/sarama 8aadb476e66ca998f2f6bb3c993e9a2daa3666b9 github.com/Sirupsen/logrus 219c8cb75c258c552e999735be6df753ffc7afdc github.com/StackExchange/wmi f3e2bae1e0cb5aef83e319133eabfee30013a4a5 @@ -9,24 +10,24 @@ github.com/couchbase/go-couchbase cb664315a324d87d19c879d9cc67fda6be8c2ac1 github.com/couchbase/gomemcached a5ea6356f648fec6ab89add00edd09151455b4b2 github.com/couchbase/goutils 5823a0cbaaa9008406021dc5daf80125ea30bba6 github.com/dancannon/gorethink e7cac92ea2bc52638791a021f212145acfedb1fc -github.com/davecgh/go-spew fc32781af5e85e548d3f1abaf0fa3dbe8a72495c +github.com/davecgh/go-spew 5215b55f46b2b919f50a1df0eaa5886afe4e3b3d +github.com/docker/engine-api 8924d6900370b4c7e7984be5adc61f50a80d7537 +github.com/docker/go-connections f549a9393d05688dff0992ef3efd8bbe6c628aeb +github.com/docker/go-units 5d2041e26a699eaca682e2ea41c8f891e1060444 github.com/eapache/go-resiliency b86b1ec0dd4209a588dc1285cdd471e73525c0b3 github.com/eapache/queue ded5959c0d4e360646dc9e9908cff48666781367 github.com/eclipse/paho.mqtt.golang 4ab3e867810d1ec5f35157c59e965054dbf43a0d -github.com/fsouza/go-dockerclient a49c8269a6899cae30da1f8a4b82e0ce945f9967 -github.com/go-ini/ini 776aa739ce9373377cd16f526cdf06cb4c89b40f github.com/go-ole/go-ole 50055884d646dd9434f16bbb5c9801749b9bafe4 github.com/go-sql-driver/mysql 1fca743146605a172a266e1654e01e5cd5669bee github.com/golang/protobuf 552c7b9542c194800fd493123b3798ef0a832032 -github.com/golang/snappy 5979233c5d6225d4a8e438cdd0b411888449ddab +github.com/golang/snappy 427fb6fc07997f43afa32f35e850833760e489a7 github.com/gonuts/go-shellquote e842a11b24c6abfb3dd27af69a17f482e4b483c2 github.com/gorilla/context 1ea25387ff6f684839d82767c1733ff4d4d15d0a github.com/gorilla/mux c9e326e2bdec29039a3761c07bece13133863e1e github.com/hailocab/go-hostpool e80d13ce29ede4452c43dea11e79b9bc8a15b478 github.com/influxdata/config b79f6829346b8d6e78ba73544b1e1038f1f1c9da -github.com/influxdata/influxdb c190778997f4154294e6160c41b90140641ac915 +github.com/influxdata/influxdb e3fef5593c21644f2b43af55d6e17e70910b0e48 github.com/influxdata/toml af4df43894b16e3fd2b788d01bd27ad0776ef2d0 -github.com/jmespath/go-jmespath 0b12d6b521d83fc7f755e7cfc1b1fbdd35a01a74 github.com/klauspost/crc32 19b0b332c9e4516a6370a0456e6182c3b5036720 github.com/lib/pq e182dc4027e2ded4b19396d638610f2653295f36 github.com/lxn/win 9a7734ea4db26bc593d52f6a8a957afdad39c5c1 @@ -37,7 +38,6 @@ github.com/naoina/go-stringutil 6b638e95a32d0c1131db0e7fe83775cbea4a0d0b github.com/nats-io/nats b13fc9d12b0b123ebc374e6b808c6228ae4234a3 github.com/nats-io/nuid 4f84f5f3b2786224e336af2e13dba0a0a80b76fa github.com/nsqio/go-nsq 0b80d6f05e15ca1930e0c5e1d540ed627e299980 -github.com/pmezard/go-difflib 792786c7400a136282c1664665ae0a8db921c6c2 github.com/prometheus/client_golang 18acf9993a863f4c4b40612e19cdd243e7c86831 github.com/prometheus/client_model fa8ad6fec33561be4280a8f0514318c79d7f6cb6 github.com/prometheus/common e8eabff8812b05acf522b45fdcd725a785188e37 @@ -47,9 +47,8 @@ github.com/shirou/gopsutil 1f32ce1bb380845be7f5d174ac641a2c592c0c42 github.com/shirou/w32 ada3ba68f000aa1b58580e45c9d308fe0b7fc5c5 github.com/soniah/gosnmp b1b4f885b12c5dcbd021c5cee1c904110de6db7d github.com/streadway/amqp b4f3ceab0337f013208d31348b578d83c0064744 -github.com/stretchr/objx 1a9d0bb9f541897e62256577b352fdbc1fb4fd94 github.com/stretchr/testify 1f4a1643a57e798696635ea4c126e9127adb7d3c -github.com/wvanbergen/kafka 1a8639a45164fcc245d5c7b4bd3ccfbd1a0ffbf3 +github.com/wvanbergen/kafka 46f9a1cf3f670edec492029fadded9c2d9e18866 github.com/wvanbergen/kazoo-go 0f768712ae6f76454f987c3356177e138df258f8 github.com/zensqlmonitor/go-mssqldb ffe5510c6fa5e15e6d983210ab501c815b56b363 golang.org/x/net 6acef71eb69611914f7a30939ea9f6e194c78172 From dfbe231a51bea200b45cd84d0c014223c6009fdd Mon Sep 17 00:00:00 2001 From: Luke Swithenbank Date: Thu, 31 Mar 2016 18:33:28 +1100 Subject: [PATCH 32/55] add http_response plugin --- plugins/inputs/http_response/README.md | 36 +++++++ plugins/inputs/http_response/http_response.go | 95 +++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 plugins/inputs/http_response/README.md create mode 100644 plugins/inputs/http_response/http_response.go diff --git a/plugins/inputs/http_response/README.md b/plugins/inputs/http_response/README.md new file mode 100644 index 000000000..13da76097 --- /dev/null +++ b/plugins/inputs/http_response/README.md @@ -0,0 +1,36 @@ +# Example Input Plugin + +This input plugin will test HTTP/HTTPS connections. + +### Configuration: + +``` +# List of UDP/TCP connections you want to check +[[inputs.http_response]] + # Server address (default http://localhost) + address = "http://github.com:80" + # Set http response timeout (default 1.0) + response_timeout = 1.0 + # HTTP Method (default "GET") + method = "GET" +``` + +### Measurements & Fields: + +- http_response + - response_time (float, seconds) + - http_response_code (int) #The code received + +### Tags: + +- All measurements have the following tags: + - server + - port + - protocol + +### Example Output: + +``` +$ ./telegraf -config telegraf.conf -input-filter http_response -test +http_response,server=http://192.168.2.2:2000,method=GET response_time=0.18070360500000002,http_response_code=200 1454785464182527094 +``` diff --git a/plugins/inputs/http_response/http_response.go b/plugins/inputs/http_response/http_response.go new file mode 100644 index 000000000..e19c698a8 --- /dev/null +++ b/plugins/inputs/http_response/http_response.go @@ -0,0 +1,95 @@ +package http_response + +import ( + "errors" + "net/http" + "net/url" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" +) + +// HttpResponses struct +type HttpResponse struct { + Address string + Method string + ResponseTimeout int +} + +func (_ *HttpResponse) Description() string { + return "HTTP/HTTPS request given an address a method and a timeout" +} + +var sampleConfig = ` + ## Server address (default http://localhost) + address = "http://github.com:80" + ## Set response_timeout (default 1 seconds) + response_timeout = 1 + ## HTTP Method + method = "GET" +` + +func (_ *HttpResponse) SampleConfig() string { + return sampleConfig +} + +func (h *HttpResponse) HttpGather() (map[string]interface{}, error) { + // Prepare fields + fields := make(map[string]interface{}) + + client := &http.Client{ + Timeout: time.Second * time.Duration(h.ResponseTimeout), + } + request, err := http.NewRequest(h.Method, h.Address, nil) + if err != nil { + return nil, err + } + // Start Timer + start := time.Now() + resp, err := client.Do(request) + if err != nil { + return nil, err + } + fields["response_time"] = time.Since(start).Seconds() + fields["http_response_code"] = resp.StatusCode + return fields, nil +} + +func (c *HttpResponse) Gather(acc telegraf.Accumulator) error { + // Set default values + if c.ResponseTimeout < 1 { + c.ResponseTimeout = 1 + } + // Check send and expected string + if c.Method == "" { + c.Method = "GET" + } + if c.Address == "" { + c.Address = "http://localhost" + } + addr, err := url.Parse(c.Address) + if err != nil { + return err + } + if addr.Scheme != "http" && addr.Scheme != "https" { + return errors.New("Only http and https are supported") + } + // Prepare data + tags := map[string]string{"server": c.Address, "method": c.Method} + var fields map[string]interface{} + // Gather data + fields, err = c.HttpGather() + if err != nil { + return err + } + // Add metrics + acc.AddFields("http_response", fields, tags) + return nil +} + +func init() { + inputs.Add("http_response", func() telegraf.Input { + return &HttpResponse{} + }) +} From 70aa0ef85da3bd67c0978c3cd23fca2ca9f06af4 Mon Sep 17 00:00:00 2001 From: Luke Swithenbank Date: Thu, 31 Mar 2016 19:47:10 +1100 Subject: [PATCH 33/55] add plugin to all --- plugins/inputs/all/all.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index 52ee6c13d..b8534fd6d 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -16,6 +16,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/github_webhooks" _ "github.com/influxdata/telegraf/plugins/inputs/haproxy" _ "github.com/influxdata/telegraf/plugins/inputs/httpjson" + _ "github.com/influxdata/telegraf/plugins/inputs/http_response" _ "github.com/influxdata/telegraf/plugins/inputs/influxdb" _ "github.com/influxdata/telegraf/plugins/inputs/ipmi_sensor" _ "github.com/influxdata/telegraf/plugins/inputs/jolokia" From 207ab5a0d1d9f1c33aa168c99b21e487a8d38462 Mon Sep 17 00:00:00 2001 From: Luke Swithenbank Date: Thu, 31 Mar 2016 19:54:08 +1100 Subject: [PATCH 34/55] update to make a working sample_config --- plugins/inputs/http_response/README.md | 6 +++--- plugins/inputs/http_response/http_response.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/inputs/http_response/README.md b/plugins/inputs/http_response/README.md index 13da76097..b70bbde72 100644 --- a/plugins/inputs/http_response/README.md +++ b/plugins/inputs/http_response/README.md @@ -8,9 +8,9 @@ This input plugin will test HTTP/HTTPS connections. # List of UDP/TCP connections you want to check [[inputs.http_response]] # Server address (default http://localhost) - address = "http://github.com:80" - # Set http response timeout (default 1.0) - response_timeout = 1.0 + address = "https://github.com" + # Set http response timeout (default 10) + response_timeout = 10 # HTTP Method (default "GET") method = "GET" ``` diff --git a/plugins/inputs/http_response/http_response.go b/plugins/inputs/http_response/http_response.go index e19c698a8..d2d35025a 100644 --- a/plugins/inputs/http_response/http_response.go +++ b/plugins/inputs/http_response/http_response.go @@ -23,9 +23,9 @@ func (_ *HttpResponse) Description() string { var sampleConfig = ` ## Server address (default http://localhost) - address = "http://github.com:80" + address = "https://github.com" ## Set response_timeout (default 1 seconds) - response_timeout = 1 + response_timeout = 10 ## HTTP Method method = "GET" ` From b7435b9cd18949b5291301b571fa32f473b434d2 Mon Sep 17 00:00:00 2001 From: Luke Swithenbank Date: Thu, 31 Mar 2016 19:55:17 +1100 Subject: [PATCH 35/55] fmt --- plugins/inputs/all/all.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index b8534fd6d..b28291c24 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -15,8 +15,8 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/exec" _ "github.com/influxdata/telegraf/plugins/inputs/github_webhooks" _ "github.com/influxdata/telegraf/plugins/inputs/haproxy" - _ "github.com/influxdata/telegraf/plugins/inputs/httpjson" _ "github.com/influxdata/telegraf/plugins/inputs/http_response" + _ "github.com/influxdata/telegraf/plugins/inputs/httpjson" _ "github.com/influxdata/telegraf/plugins/inputs/influxdb" _ "github.com/influxdata/telegraf/plugins/inputs/ipmi_sensor" _ "github.com/influxdata/telegraf/plugins/inputs/jolokia" From 7219efbdb76006fe52c475994b5608af19dcdaaf Mon Sep 17 00:00:00 2001 From: Luke Swithenbank Date: Thu, 31 Mar 2016 20:53:51 +1100 Subject: [PATCH 36/55] add the ability to parse http headers --- plugins/inputs/http_response/http_response.go | 52 +++++++++++++------ 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/plugins/inputs/http_response/http_response.go b/plugins/inputs/http_response/http_response.go index d2d35025a..df21311ae 100644 --- a/plugins/inputs/http_response/http_response.go +++ b/plugins/inputs/http_response/http_response.go @@ -1,23 +1,28 @@ package http_response import ( + "bufio" "errors" "net/http" + "net/textproto" "net/url" + "strings" "time" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/inputs" ) -// HttpResponses struct -type HttpResponse struct { +// HTTPResponse struct +type HTTPResponse struct { Address string Method string ResponseTimeout int + Headers string } -func (_ *HttpResponse) Description() string { +// Description returns the plugin Description +func (h *HTTPResponse) Description() string { return "HTTP/HTTPS request given an address a method and a timeout" } @@ -28,13 +33,19 @@ var sampleConfig = ` response_timeout = 10 ## HTTP Method method = "GET" + ## HTTP Request Headers + headers = ''' + Host: github.com + ''' ` -func (_ *HttpResponse) SampleConfig() string { +// SampleConfig returns the plugin SampleConfig +func (h *HTTPResponse) SampleConfig() string { return sampleConfig } -func (h *HttpResponse) HttpGather() (map[string]interface{}, error) { +// HTTPGather gathers all fields and returns any errors it encounters +func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { // Prepare fields fields := make(map[string]interface{}) @@ -45,6 +56,14 @@ func (h *HttpResponse) HttpGather() (map[string]interface{}, error) { if err != nil { return nil, err } + h.Headers = strings.TrimSpace(h.Headers) + "\n\n" + reader := bufio.NewReader(strings.NewReader(h.Headers)) + tp := textproto.NewReader(reader) + mimeHeader, err := tp.ReadMIMEHeader() + if err != nil { + return nil, err + } + request.Header = http.Header(mimeHeader) // Start Timer start := time.Now() resp, err := client.Do(request) @@ -56,19 +75,20 @@ func (h *HttpResponse) HttpGather() (map[string]interface{}, error) { return fields, nil } -func (c *HttpResponse) Gather(acc telegraf.Accumulator) error { +// Gather gets all metric fields and tags and returns any errors it encounters +func (h *HTTPResponse) Gather(acc telegraf.Accumulator) error { // Set default values - if c.ResponseTimeout < 1 { - c.ResponseTimeout = 1 + if h.ResponseTimeout < 1 { + h.ResponseTimeout = 1 } // Check send and expected string - if c.Method == "" { - c.Method = "GET" + if h.Method == "" { + h.Method = "GET" } - if c.Address == "" { - c.Address = "http://localhost" + if h.Address == "" { + h.Address = "http://localhost" } - addr, err := url.Parse(c.Address) + addr, err := url.Parse(h.Address) if err != nil { return err } @@ -76,10 +96,10 @@ func (c *HttpResponse) Gather(acc telegraf.Accumulator) error { return errors.New("Only http and https are supported") } // Prepare data - tags := map[string]string{"server": c.Address, "method": c.Method} + tags := map[string]string{"server": h.Address, "method": h.Method} var fields map[string]interface{} // Gather data - fields, err = c.HttpGather() + fields, err = h.HTTPGather() if err != nil { return err } @@ -90,6 +110,6 @@ func (c *HttpResponse) Gather(acc telegraf.Accumulator) error { func init() { inputs.Add("http_response", func() telegraf.Input { - return &HttpResponse{} + return &HTTPResponse{} }) } From f947fa86e37a1679eacb7ae98b06a8826d03a5d3 Mon Sep 17 00:00:00 2001 From: Luke Swithenbank Date: Thu, 31 Mar 2016 21:18:19 +1100 Subject: [PATCH 37/55] update to allow for following redirects --- plugins/inputs/http_response/README.md | 19 +++++++---- plugins/inputs/http_response/http_response.go | 33 ++++++++++++++++--- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/plugins/inputs/http_response/README.md b/plugins/inputs/http_response/README.md index b70bbde72..99770e526 100644 --- a/plugins/inputs/http_response/README.md +++ b/plugins/inputs/http_response/README.md @@ -7,12 +7,18 @@ This input plugin will test HTTP/HTTPS connections. ``` # List of UDP/TCP connections you want to check [[inputs.http_response]] - # Server address (default http://localhost) - address = "https://github.com" - # Set http response timeout (default 10) + ## Server address (default http://localhost) + address = "http://github.com" + ## Set response_timeout (default 10 seconds) response_timeout = 10 - # HTTP Method (default "GET") + ## HTTP Method method = "GET" + ## HTTP Request Headers + headers = ''' + Host: github.com + ''' + ## Whether to follow redirects from the server (defaults to false) + follow_redirects = true ``` ### Measurements & Fields: @@ -25,12 +31,11 @@ This input plugin will test HTTP/HTTPS connections. - All measurements have the following tags: - server - - port - - protocol + - method ### Example Output: ``` $ ./telegraf -config telegraf.conf -input-filter http_response -test -http_response,server=http://192.168.2.2:2000,method=GET response_time=0.18070360500000002,http_response_code=200 1454785464182527094 +http_response,method=GET,server=http://www.github.com http_response_code=200i,response_time=6.223266528 1459419354977857955 ``` diff --git a/plugins/inputs/http_response/http_response.go b/plugins/inputs/http_response/http_response.go index df21311ae..09569fe73 100644 --- a/plugins/inputs/http_response/http_response.go +++ b/plugins/inputs/http_response/http_response.go @@ -3,9 +3,11 @@ package http_response import ( "bufio" "errors" + "fmt" "net/http" "net/textproto" "net/url" + "os" "strings" "time" @@ -19,6 +21,7 @@ type HTTPResponse struct { Method string ResponseTimeout int Headers string + FollowRedirects bool } // Description returns the plugin Description @@ -28,8 +31,8 @@ func (h *HTTPResponse) Description() string { var sampleConfig = ` ## Server address (default http://localhost) - address = "https://github.com" - ## Set response_timeout (default 1 seconds) + address = "http://github.com" + ## Set response_timeout (default 10 seconds) response_timeout = 10 ## HTTP Method method = "GET" @@ -37,6 +40,8 @@ var sampleConfig = ` headers = ''' Host: github.com ''' + ## Whether to follow redirects from the server (defaults to false) + follow_redirects = true ` // SampleConfig returns the plugin SampleConfig @@ -44,6 +49,8 @@ func (h *HTTPResponse) SampleConfig() string { return sampleConfig } +var ErrRedirectAttempted = errors.New("redirect") + // HTTPGather gathers all fields and returns any errors it encounters func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { // Prepare fields @@ -52,6 +59,14 @@ func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { client := &http.Client{ Timeout: time.Second * time.Duration(h.ResponseTimeout), } + + if h.FollowRedirects == false { + fmt.Println(h.FollowRedirects) + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return ErrRedirectAttempted + } + } + request, err := http.NewRequest(h.Method, h.Address, nil) if err != nil { return nil, err @@ -66,9 +81,19 @@ func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { request.Header = http.Header(mimeHeader) // Start Timer start := time.Now() + request.Write(os.Stdout) resp, err := client.Do(request) if err != nil { - return nil, err + if h.FollowRedirects { + return nil, err + } + if urlError, ok := err.(*url.Error); ok && + urlError.Err == ErrRedirectAttempted { + fmt.Println(err) + err = nil + } else { + return nil, err + } } fields["response_time"] = time.Since(start).Seconds() fields["http_response_code"] = resp.StatusCode @@ -79,7 +104,7 @@ func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { func (h *HTTPResponse) Gather(acc telegraf.Accumulator) error { // Set default values if h.ResponseTimeout < 1 { - h.ResponseTimeout = 1 + h.ResponseTimeout = 10 } // Check send and expected string if h.Method == "" { From 73a7916ce3f7b43005097ba591157f43fda45d2e Mon Sep 17 00:00:00 2001 From: Luke Swithenbank Date: Thu, 31 Mar 2016 22:06:47 +1100 Subject: [PATCH 38/55] take a request body as a param --- plugins/inputs/http_response/README.md | 6 +++++- plugins/inputs/http_response/http_response.go | 20 ++++++++++++------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/plugins/inputs/http_response/README.md b/plugins/inputs/http_response/README.md index 99770e526..f2f45b2af 100644 --- a/plugins/inputs/http_response/README.md +++ b/plugins/inputs/http_response/README.md @@ -11,7 +11,7 @@ This input plugin will test HTTP/HTTPS connections. address = "http://github.com" ## Set response_timeout (default 10 seconds) response_timeout = 10 - ## HTTP Method + ## HTTP Request Method method = "GET" ## HTTP Request Headers headers = ''' @@ -19,6 +19,10 @@ This input plugin will test HTTP/HTTPS connections. ''' ## Whether to follow redirects from the server (defaults to false) follow_redirects = true + ## Optional HTTP Request Body + body = ''' + {'fake':'data'} + ''' ``` ### Measurements & Fields: diff --git a/plugins/inputs/http_response/http_response.go b/plugins/inputs/http_response/http_response.go index 09569fe73..dc4b2df60 100644 --- a/plugins/inputs/http_response/http_response.go +++ b/plugins/inputs/http_response/http_response.go @@ -3,11 +3,10 @@ package http_response import ( "bufio" "errors" - "fmt" + "io" "net/http" "net/textproto" "net/url" - "os" "strings" "time" @@ -18,6 +17,7 @@ import ( // HTTPResponse struct type HTTPResponse struct { Address string + Body string Method string ResponseTimeout int Headers string @@ -34,7 +34,7 @@ var sampleConfig = ` address = "http://github.com" ## Set response_timeout (default 10 seconds) response_timeout = 10 - ## HTTP Method + ## HTTP Request Method method = "GET" ## HTTP Request Headers headers = ''' @@ -42,6 +42,10 @@ var sampleConfig = ` ''' ## Whether to follow redirects from the server (defaults to false) follow_redirects = true + ## Optional HTTP Request Body + body = ''' + {'fake':'data'} + ''' ` // SampleConfig returns the plugin SampleConfig @@ -49,6 +53,7 @@ func (h *HTTPResponse) SampleConfig() string { return sampleConfig } +// ErrRedirectAttempted indicates that a redirect occurred var ErrRedirectAttempted = errors.New("redirect") // HTTPGather gathers all fields and returns any errors it encounters @@ -61,13 +66,16 @@ func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { } if h.FollowRedirects == false { - fmt.Println(h.FollowRedirects) client.CheckRedirect = func(req *http.Request, via []*http.Request) error { return ErrRedirectAttempted } } - request, err := http.NewRequest(h.Method, h.Address, nil) + var body io.Reader + if h.Body != "" { + body = strings.NewReader(h.Body) + } + request, err := http.NewRequest(h.Method, h.Address, body) if err != nil { return nil, err } @@ -81,7 +89,6 @@ func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { request.Header = http.Header(mimeHeader) // Start Timer start := time.Now() - request.Write(os.Stdout) resp, err := client.Do(request) if err != nil { if h.FollowRedirects { @@ -89,7 +96,6 @@ func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { } if urlError, ok := err.(*url.Error); ok && urlError.Err == ErrRedirectAttempted { - fmt.Println(err) err = nil } else { return nil, err From 437bd87d7c4994052def798b5e21e8d9ee1f7c28 Mon Sep 17 00:00:00 2001 From: Luke Swithenbank Date: Mon, 4 Apr 2016 12:20:07 +1000 Subject: [PATCH 39/55] added tests and did some refactoring --- plugins/inputs/http_response/http_response.go | 56 ++-- .../http_response/http_response_test.go | 245 ++++++++++++++++++ 2 files changed, 281 insertions(+), 20 deletions(-) create mode 100644 plugins/inputs/http_response/http_response_test.go diff --git a/plugins/inputs/http_response/http_response.go b/plugins/inputs/http_response/http_response.go index dc4b2df60..cee33795a 100644 --- a/plugins/inputs/http_response/http_response.go +++ b/plugins/inputs/http_response/http_response.go @@ -40,12 +40,12 @@ var sampleConfig = ` headers = ''' Host: github.com ''' - ## Whether to follow redirects from the server (defaults to false) - follow_redirects = true - ## Optional HTTP Request Body - body = ''' - {'fake':'data'} - ''' + ## Whether to follow redirects from the server (defaults to false) + follow_redirects = true + ## Optional HTTP Request Body + body = ''' + {'fake':'data'} + ''' ` // SampleConfig returns the plugin SampleConfig @@ -56,20 +56,40 @@ func (h *HTTPResponse) SampleConfig() string { // ErrRedirectAttempted indicates that a redirect occurred var ErrRedirectAttempted = errors.New("redirect") +// CreateHttpClient creates an http client which will timeout at the specified +// timeout period and can follow redirects if specified +func CreateHttpClient(followRedirects bool, ResponseTimeout time.Duration) *http.Client { + client := &http.Client{ + Timeout: time.Second * ResponseTimeout, + } + + if followRedirects == false { + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return ErrRedirectAttempted + } + } + return client +} + +// ParseHeaders takes a string of newline seperated http headers and returns a +// http.Header object. An error is returned if the headers cannot be parsed. +func ParseHeaders(headers string) (http.Header, error) { + headers = strings.TrimSpace(headers) + "\n\n" + reader := bufio.NewReader(strings.NewReader(headers)) + tp := textproto.NewReader(reader) + mimeHeader, err := tp.ReadMIMEHeader() + if err != nil { + return nil, err + } + return http.Header(mimeHeader), nil +} + // HTTPGather gathers all fields and returns any errors it encounters func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { // Prepare fields fields := make(map[string]interface{}) - client := &http.Client{ - Timeout: time.Second * time.Duration(h.ResponseTimeout), - } - - if h.FollowRedirects == false { - client.CheckRedirect = func(req *http.Request, via []*http.Request) error { - return ErrRedirectAttempted - } - } + client := CreateHttpClient(h.FollowRedirects, time.Duration(h.ResponseTimeout)) var body io.Reader if h.Body != "" { @@ -79,14 +99,10 @@ func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { if err != nil { return nil, err } - h.Headers = strings.TrimSpace(h.Headers) + "\n\n" - reader := bufio.NewReader(strings.NewReader(h.Headers)) - tp := textproto.NewReader(reader) - mimeHeader, err := tp.ReadMIMEHeader() + request.Header, err = ParseHeaders(h.Headers) if err != nil { return nil, err } - request.Header = http.Header(mimeHeader) // Start Timer start := time.Now() resp, err := client.Do(request) diff --git a/plugins/inputs/http_response/http_response_test.go b/plugins/inputs/http_response/http_response_test.go new file mode 100644 index 000000000..0f568e3b4 --- /dev/null +++ b/plugins/inputs/http_response/http_response_test.go @@ -0,0 +1,245 @@ +package http_response + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestParseHeaders(t *testing.T) { + fakeHeaders := ` +Accept: text/plain +Content-Type: application/json +Cache-Control: no-cache +` + headers, err := ParseHeaders(fakeHeaders) + require.NoError(t, err) + testHeaders := make(http.Header) + testHeaders.Add("Accept", "text/plain") + testHeaders.Add("Content-Type", "application/json") + testHeaders.Add("Cache-Control", "no-cache") + assert.Equal(t, testHeaders, headers) + + headers, err = ParseHeaders("Accept text/plain") + require.Error(t, err) +} + +func setUpTestMux() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/redirect", func(w http.ResponseWriter, req *http.Request) { + http.Redirect(w, req, "/good", http.StatusMovedPermanently) + }) + mux.HandleFunc("/good", func(w http.ResponseWriter, req *http.Request) { + fmt.Fprintf(w, "hit the good page!") + }) + mux.HandleFunc("/badredirect", func(w http.ResponseWriter, req *http.Request) { + http.Redirect(w, req, "/badredirect", http.StatusMovedPermanently) + }) + mux.HandleFunc("/mustbepostmethod", func(w http.ResponseWriter, req *http.Request) { + if req.Method != "POST" { + http.Error(w, "method wasn't post", http.StatusMethodNotAllowed) + return + } + fmt.Fprintf(w, "used post correctly!") + }) + mux.HandleFunc("/musthaveabody", func(w http.ResponseWriter, req *http.Request) { + body, err := ioutil.ReadAll(req.Body) + req.Body.Close() + if err != nil { + http.Error(w, "couldn't read request body", http.StatusBadRequest) + return + } + if string(body) == "" { + http.Error(w, "body was empty", http.StatusBadRequest) + return + } + fmt.Fprintf(w, "sent a body!") + }) + mux.HandleFunc("/twosecondnap", func(w http.ResponseWriter, req *http.Request) { + time.Sleep(time.Second * 2) + return + }) + return mux +} + +func TestFields(t *testing.T) { + mux := setUpTestMux() + ts := httptest.NewServer(mux) + defer ts.Close() + + h := &HTTPResponse{ + Address: ts.URL + "/good", + Body: "{ 'test': 'data'}", + Method: "GET", + ResponseTimeout: 20, + Headers: ` +Content-Type: application/json +`, + FollowRedirects: true, + } + fields, err := h.HTTPGather() + require.NoError(t, err) + assert.NotEmpty(t, fields) + if assert.NotNil(t, fields["http_response_code"]) { + assert.Equal(t, http.StatusOK, fields["http_response_code"]) + } + assert.NotNil(t, fields["response_time"]) + +} + +func TestRedirects(t *testing.T) { + mux := setUpTestMux() + ts := httptest.NewServer(mux) + defer ts.Close() + + h := &HTTPResponse{ + Address: ts.URL + "/redirect", + Body: "{ 'test': 'data'}", + Method: "GET", + ResponseTimeout: 20, + Headers: ` +Content-Type: application/json +`, + FollowRedirects: true, + } + fields, err := h.HTTPGather() + require.NoError(t, err) + assert.NotEmpty(t, fields) + if assert.NotNil(t, fields["http_response_code"]) { + assert.Equal(t, http.StatusOK, fields["http_response_code"]) + } + + h = &HTTPResponse{ + Address: ts.URL + "/badredirect", + Body: "{ 'test': 'data'}", + Method: "GET", + ResponseTimeout: 20, + Headers: ` +Content-Type: application/json +`, + FollowRedirects: true, + } + fields, err = h.HTTPGather() + require.Error(t, err) +} + +func TestMethod(t *testing.T) { + mux := setUpTestMux() + ts := httptest.NewServer(mux) + defer ts.Close() + + h := &HTTPResponse{ + Address: ts.URL + "/mustbepostmethod", + Body: "{ 'test': 'data'}", + Method: "POST", + ResponseTimeout: 20, + Headers: ` +Content-Type: application/json +`, + FollowRedirects: true, + } + fields, err := h.HTTPGather() + require.NoError(t, err) + assert.NotEmpty(t, fields) + if assert.NotNil(t, fields["http_response_code"]) { + assert.Equal(t, http.StatusOK, fields["http_response_code"]) + } + + h = &HTTPResponse{ + Address: ts.URL + "/mustbepostmethod", + Body: "{ 'test': 'data'}", + Method: "GET", + ResponseTimeout: 20, + Headers: ` +Content-Type: application/json +`, + FollowRedirects: true, + } + fields, err = h.HTTPGather() + require.NoError(t, err) + assert.NotEmpty(t, fields) + if assert.NotNil(t, fields["http_response_code"]) { + assert.Equal(t, http.StatusMethodNotAllowed, fields["http_response_code"]) + } + + //check that lowercase methods work correctly + h = &HTTPResponse{ + Address: ts.URL + "/mustbepostmethod", + Body: "{ 'test': 'data'}", + Method: "head", + ResponseTimeout: 20, + Headers: ` +Content-Type: application/json +`, + FollowRedirects: true, + } + fields, err = h.HTTPGather() + require.NoError(t, err) + assert.NotEmpty(t, fields) + if assert.NotNil(t, fields["http_response_code"]) { + assert.Equal(t, http.StatusMethodNotAllowed, fields["http_response_code"]) + } +} + +func TestBody(t *testing.T) { + mux := setUpTestMux() + ts := httptest.NewServer(mux) + defer ts.Close() + + h := &HTTPResponse{ + Address: ts.URL + "/musthaveabody", + Body: "{ 'test': 'data'}", + Method: "GET", + ResponseTimeout: 20, + Headers: ` +Content-Type: application/json +`, + FollowRedirects: true, + } + fields, err := h.HTTPGather() + require.NoError(t, err) + assert.NotEmpty(t, fields) + if assert.NotNil(t, fields["http_response_code"]) { + assert.Equal(t, http.StatusOK, fields["http_response_code"]) + } + + h = &HTTPResponse{ + Address: ts.URL + "/musthaveabody", + Method: "GET", + ResponseTimeout: 20, + Headers: ` +Content-Type: application/json +`, + FollowRedirects: true, + } + fields, err = h.HTTPGather() + require.NoError(t, err) + assert.NotEmpty(t, fields) + if assert.NotNil(t, fields["http_response_code"]) { + assert.Equal(t, http.StatusBadRequest, fields["http_response_code"]) + } +} + +func TestTimeout(t *testing.T) { + mux := setUpTestMux() + ts := httptest.NewServer(mux) + defer ts.Close() + + h := &HTTPResponse{ + Address: ts.URL + "/twosecondnap", + Body: "{ 'test': 'data'}", + Method: "GET", + ResponseTimeout: 1, + Headers: ` +Content-Type: application/json +`, + FollowRedirects: true, + } + _, err := h.HTTPGather() + require.Error(t, err) +} From 377b030d88d78a8046f7d1c155a240e7212d60a2 Mon Sep 17 00:00:00 2001 From: Luke Swithenbank Date: Thu, 7 Apr 2016 11:57:49 +1000 Subject: [PATCH 40/55] update to 5 second default and string map for headers --- plugins/inputs/http_response/README.md | 9 ++- plugins/inputs/http_response/http_response.go | 40 +++++------ .../http_response/http_response_test.go | 72 +++++++++---------- 3 files changed, 54 insertions(+), 67 deletions(-) diff --git a/plugins/inputs/http_response/README.md b/plugins/inputs/http_response/README.md index f2f45b2af..e2bf75b5f 100644 --- a/plugins/inputs/http_response/README.md +++ b/plugins/inputs/http_response/README.md @@ -9,14 +9,13 @@ This input plugin will test HTTP/HTTPS connections. [[inputs.http_response]] ## Server address (default http://localhost) address = "http://github.com" - ## Set response_timeout (default 10 seconds) - response_timeout = 10 + ## Set response_timeout (default 5 seconds) + response_timeout = 5 ## HTTP Request Method method = "GET" ## HTTP Request Headers - headers = ''' - Host: github.com - ''' + [inputs.http_response.headers] + Host = github.com ## Whether to follow redirects from the server (defaults to false) follow_redirects = true ## Optional HTTP Request Body diff --git a/plugins/inputs/http_response/http_response.go b/plugins/inputs/http_response/http_response.go index cee33795a..73533fed4 100644 --- a/plugins/inputs/http_response/http_response.go +++ b/plugins/inputs/http_response/http_response.go @@ -1,11 +1,9 @@ package http_response import ( - "bufio" "errors" "io" "net/http" - "net/textproto" "net/url" "strings" "time" @@ -20,7 +18,7 @@ type HTTPResponse struct { Body string Method string ResponseTimeout int - Headers string + Headers map[string]string FollowRedirects bool } @@ -32,14 +30,13 @@ func (h *HTTPResponse) Description() string { var sampleConfig = ` ## Server address (default http://localhost) address = "http://github.com" - ## Set response_timeout (default 10 seconds) - response_timeout = 10 + ## Set response_timeout (default 5 seconds) + response_timeout = 5 ## HTTP Request Method method = "GET" - ## HTTP Request Headers - headers = ''' - Host: github.com - ''' + ## HTTP Request Headers (all values must be strings) + [inputs.http_response.headers] + # Host = "github.com" ## Whether to follow redirects from the server (defaults to false) follow_redirects = true ## Optional HTTP Request Body @@ -71,17 +68,14 @@ func CreateHttpClient(followRedirects bool, ResponseTimeout time.Duration) *http return client } -// ParseHeaders takes a string of newline seperated http headers and returns a -// http.Header object. An error is returned if the headers cannot be parsed. -func ParseHeaders(headers string) (http.Header, error) { - headers = strings.TrimSpace(headers) + "\n\n" - reader := bufio.NewReader(strings.NewReader(headers)) - tp := textproto.NewReader(reader) - mimeHeader, err := tp.ReadMIMEHeader() - if err != nil { - return nil, err +// CreateHeaders takes a map of header strings and puts them +// into a http.Header Object +func CreateHeaders(headers map[string]string) http.Header { + httpHeaders := make(http.Header) + for key := range headers { + httpHeaders.Add(key, headers[key]) } - return http.Header(mimeHeader), nil + return httpHeaders } // HTTPGather gathers all fields and returns any errors it encounters @@ -99,10 +93,8 @@ func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { if err != nil { return nil, err } - request.Header, err = ParseHeaders(h.Headers) - if err != nil { - return nil, err - } + request.Header = CreateHeaders(h.Headers) + // Start Timer start := time.Now() resp, err := client.Do(request) @@ -126,7 +118,7 @@ func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { func (h *HTTPResponse) Gather(acc telegraf.Accumulator) error { // Set default values if h.ResponseTimeout < 1 { - h.ResponseTimeout = 10 + h.ResponseTimeout = 5 } // Check send and expected string if h.Method == "" { diff --git a/plugins/inputs/http_response/http_response_test.go b/plugins/inputs/http_response/http_response_test.go index 0f568e3b4..acdfeac75 100644 --- a/plugins/inputs/http_response/http_response_test.go +++ b/plugins/inputs/http_response/http_response_test.go @@ -11,22 +11,18 @@ import ( "time" ) -func TestParseHeaders(t *testing.T) { - fakeHeaders := ` -Accept: text/plain -Content-Type: application/json -Cache-Control: no-cache -` - headers, err := ParseHeaders(fakeHeaders) - require.NoError(t, err) +func TestCreateHeaders(t *testing.T) { + fakeHeaders := map[string]string{ + "Accept": "text/plain", + "Content-Type": "application/json", + "Cache-Control": "no-cache", + } + headers := CreateHeaders(fakeHeaders) testHeaders := make(http.Header) testHeaders.Add("Accept", "text/plain") testHeaders.Add("Content-Type", "application/json") testHeaders.Add("Cache-Control", "no-cache") assert.Equal(t, testHeaders, headers) - - headers, err = ParseHeaders("Accept text/plain") - require.Error(t, err) } func setUpTestMux() http.Handler { @@ -77,9 +73,9 @@ func TestFields(t *testing.T) { Body: "{ 'test': 'data'}", Method: "GET", ResponseTimeout: 20, - Headers: ` -Content-Type: application/json -`, + Headers: map[string]string{ + "Content-Type": "application/json", + }, FollowRedirects: true, } fields, err := h.HTTPGather() @@ -102,9 +98,9 @@ func TestRedirects(t *testing.T) { Body: "{ 'test': 'data'}", Method: "GET", ResponseTimeout: 20, - Headers: ` -Content-Type: application/json -`, + Headers: map[string]string{ + "Content-Type": "application/json", + }, FollowRedirects: true, } fields, err := h.HTTPGather() @@ -119,9 +115,9 @@ Content-Type: application/json Body: "{ 'test': 'data'}", Method: "GET", ResponseTimeout: 20, - Headers: ` -Content-Type: application/json -`, + Headers: map[string]string{ + "Content-Type": "application/json", + }, FollowRedirects: true, } fields, err = h.HTTPGather() @@ -138,9 +134,9 @@ func TestMethod(t *testing.T) { Body: "{ 'test': 'data'}", Method: "POST", ResponseTimeout: 20, - Headers: ` -Content-Type: application/json -`, + Headers: map[string]string{ + "Content-Type": "application/json", + }, FollowRedirects: true, } fields, err := h.HTTPGather() @@ -155,9 +151,9 @@ Content-Type: application/json Body: "{ 'test': 'data'}", Method: "GET", ResponseTimeout: 20, - Headers: ` -Content-Type: application/json -`, + Headers: map[string]string{ + "Content-Type": "application/json", + }, FollowRedirects: true, } fields, err = h.HTTPGather() @@ -173,9 +169,9 @@ Content-Type: application/json Body: "{ 'test': 'data'}", Method: "head", ResponseTimeout: 20, - Headers: ` -Content-Type: application/json -`, + Headers: map[string]string{ + "Content-Type": "application/json", + }, FollowRedirects: true, } fields, err = h.HTTPGather() @@ -196,9 +192,9 @@ func TestBody(t *testing.T) { Body: "{ 'test': 'data'}", Method: "GET", ResponseTimeout: 20, - Headers: ` -Content-Type: application/json -`, + Headers: map[string]string{ + "Content-Type": "application/json", + }, FollowRedirects: true, } fields, err := h.HTTPGather() @@ -212,9 +208,9 @@ Content-Type: application/json Address: ts.URL + "/musthaveabody", Method: "GET", ResponseTimeout: 20, - Headers: ` -Content-Type: application/json -`, + Headers: map[string]string{ + "Content-Type": "application/json", + }, FollowRedirects: true, } fields, err = h.HTTPGather() @@ -235,9 +231,9 @@ func TestTimeout(t *testing.T) { Body: "{ 'test': 'data'}", Method: "GET", ResponseTimeout: 1, - Headers: ` -Content-Type: application/json -`, + Headers: map[string]string{ + "Content-Type": "application/json", + }, FollowRedirects: true, } _, err := h.HTTPGather() From 90185dc6b369981e82ceedc844ce435a000fc094 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Thu, 7 Apr 2016 10:31:06 -0600 Subject: [PATCH 41/55] cleanup & comment http_response def config closes #332 --- CHANGELOG.md | 1 + README.md | 3 ++- etc/telegraf.conf | 19 +++++++++++++++++++ plugins/inputs/http_response/http_response.go | 12 ++++++------ 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09a00f069..699d0f602 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - [#976](https://github.com/influxdata/telegraf/pull/976): Reduce allocations in the UDP and statsd inputs. - [#979](https://github.com/influxdata/telegraf/pull/979): Reduce allocations in the TCP listener. - [#935](https://github.com/influxdata/telegraf/pull/935): AWS Cloudwatch input plugin. Thanks @joshhardy & @ljosa! +- [#943](https://github.com/influxdata/telegraf/pull/943): http_response input plugin. Thanks @Lswith! ### Bugfixes - [#968](https://github.com/influxdata/telegraf/issues/968): Processes plugin gets unknown state when spaces are in (command name) diff --git a/README.md b/README.md index caa562a6d..8621238dd 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,8 @@ Currently implemented sources: * [elasticsearch](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/elasticsearch) * [exec](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/exec ) (generic executable plugin, support JSON, influx, graphite and nagios) * [haproxy](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/haproxy) -* [httpjson ](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/httpjson ) (generic JSON-emitting http service plugin) +* [http_response](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/http_response) +* [httpjson](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/httpjson) (generic JSON-emitting http service plugin) * [influxdb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/influxdb) * [ipmi_sensor](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/ipmi_sensor) * [jolokia](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/jolokia) diff --git a/etc/telegraf.conf b/etc/telegraf.conf index 1b534d888..fa77a3a34 100644 --- a/etc/telegraf.conf +++ b/etc/telegraf.conf @@ -569,6 +569,25 @@ # ## servers = ["socket://run/haproxy/admin.sock"] +# # HTTP/HTTPS request given an address a method and a timeout +# [[inputs.http_response]] +# ## Server address (default http://localhost) +# address = "http://github.com" +# ## Set response_timeout (default 5 seconds) +# response_timeout = 5 +# ## HTTP Request Method +# method = "GET" +# ## Whether to follow redirects from the server (defaults to false) +# follow_redirects = true +# ## HTTP Request Headers (all values must be strings) +# # [inputs.http_response.headers] +# # Host = "github.com" +# ## Optional HTTP Request Body +# # body = ''' +# # {'fake':'data'} +# # ''' + + # # Read flattened metrics from one or more JSON HTTP endpoints # [[inputs.httpjson]] # ## NOTE This plugin only reads numerical measurements, strings and booleans diff --git a/plugins/inputs/http_response/http_response.go b/plugins/inputs/http_response/http_response.go index 73533fed4..69c8fcd06 100644 --- a/plugins/inputs/http_response/http_response.go +++ b/plugins/inputs/http_response/http_response.go @@ -34,15 +34,15 @@ var sampleConfig = ` response_timeout = 5 ## HTTP Request Method method = "GET" - ## HTTP Request Headers (all values must be strings) - [inputs.http_response.headers] - # Host = "github.com" ## Whether to follow redirects from the server (defaults to false) follow_redirects = true + ## HTTP Request Headers (all values must be strings) + # [inputs.http_response.headers] + # Host = "github.com" ## Optional HTTP Request Body - body = ''' - {'fake':'data'} - ''' + # body = ''' + # {'fake':'data'} + # ''' ` // SampleConfig returns the plugin SampleConfig From c4ea122d6679b72e7b0a4e66135438f2dffb4a0e Mon Sep 17 00:00:00 2001 From: Rene Zbinden Date: Tue, 29 Mar 2016 21:38:07 +0200 Subject: [PATCH 42/55] add sysstat plugin --- plugins/inputs/all/all.go | 1 + plugins/inputs/sysstat/README.md | 448 +++++++++++++++++++++++++ plugins/inputs/sysstat/sysstat.go | 315 +++++++++++++++++ plugins/inputs/sysstat/sysstat_test.go | 305 +++++++++++++++++ 4 files changed, 1069 insertions(+) create mode 100644 plugins/inputs/sysstat/README.md create mode 100644 plugins/inputs/sysstat/sysstat.go create mode 100644 plugins/inputs/sysstat/sysstat_test.go diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index b28291c24..2d784ca27 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -52,6 +52,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/snmp" _ "github.com/influxdata/telegraf/plugins/inputs/sqlserver" _ "github.com/influxdata/telegraf/plugins/inputs/statsd" + _ "github.com/influxdata/telegraf/plugins/inputs/sysstat" _ "github.com/influxdata/telegraf/plugins/inputs/system" _ "github.com/influxdata/telegraf/plugins/inputs/tcp_listener" _ "github.com/influxdata/telegraf/plugins/inputs/trig" diff --git a/plugins/inputs/sysstat/README.md b/plugins/inputs/sysstat/README.md new file mode 100644 index 000000000..7191a83d3 --- /dev/null +++ b/plugins/inputs/sysstat/README.md @@ -0,0 +1,448 @@ +# sysstat Input Plugin + +Collect [sysstat](https://github.com/sysstat/sysstat) metrics - requires the sysstat +package installed. + +This plugin collects system metrics with the sysstat collector utility `sadc` and parses +the created binary data file with the `sadf` utility. + +### Configuration: + +```toml +# Sysstat metrics collector +[[inputs.sysstat]] + ## Collect interval in seconds. This value has to be equal + ## to the telegraf collect interval. + collect_interval = 30 # required + # + # + ## Path to the sadc command. + sadc_path = "/usr/lib/sa/sadc" # required + # + # + ## Path to the sadf command, if it is not in PATH + # sadf_path = "/usr/bin/sadf" + # + # + ## Activities is a list of activities, that are passed as argument to the + ## sadc collector utility (e.g: DISK, SNMP etc...) + ## The more activities that are added, the more data is collected. + # activities = ["DISK"] + # + # + ## Group metrics to measurements. + ## + ## If group is false each metric will be prefixed with a description + ## and represents itself a measurement. + ## + ## If Group is true, corresponding metrics are grouped to a single measurement. + # group = false + # + # + ## Options for the sadf command. The values on the left represent the sadf options and + ## the values on the right their description (wich are used for grouping and prefixing metrics). + [inputs.sysstat.options] + -C = "cpu" + -B = "paging" + -b = "io" + -d = "disk" # requires DISK activity + -H = "hugepages" + "-I ALL" = "interrupts" # requires INT activity + "-n ALL" = "network" + "-P ALL" = "per_cpu" + -q = "queue" + -R = "mem" + "-r ALL" = "mem_util" + -S = "swap_util" + -u = "cpu_util" + -v = "inode" + -W = "swap" + -w = "task" + # + ## Device tags can be used to add additional tags for devices. For example the configuration below + ## adds a tag vg=rootvg for all metrics with sda devices. + # [[inputs.sysstat.device_tags.sda]] + # vg = "rootvg" +``` + +### Measurements & Fields: +#### group=true +- cpu + - pct_idle (float) + - pct_iowait (float) + - pct_nice (float) + - pct_steal (float) + - pct_system (float) + - pct_user (float) + +- disk + - avgqu-sz (float) + - avgrq-sz (float) + - await (float) + - pct_util (float) + - rd_sec_pers (float) + - svctm (float) + - tps (float) + +And much more, depending on the options you configure. + +#### group=false +- cpu_pct_idle + - value (float) +- cpu_pct_iowait + - value (float) +- cpu_pct_nice + - value (float) +- cpu_pct_steal + - value (float) +- cpu_pct_system + - value (float) +- cpu_pct_user + - value (float) +- disk_avgqu-sz + - value (float) +- disk_avgrq-sz + - value (float) +- disk_await + - value (float) +- disk_pct_util + - value (float) +- disk_rd_sec_per_s + - value (float) +- disk_svctm + - value (float) +- disk_tps + - value (float) + +And much more, depending on the options you configure. + +### Tags: + +- All measurements have the following tags: + - device + +And more if you define some `device_tags`. +### Example Output: + +With the configuration below: +```toml +[[inputs.sysstat]] + collect_interval = 30 + sadc_path = "/usr/lib/sa/sadc" # required + activities = ["DISK", "SNMP", "INT"] + group = true + [inputs.sysstat.options] + -C = "cpu" + -B = "paging" + -b = "io" + -d = "disk" # requires DISK activity + -H = "hugepages" + "-I ALL" = "interrupts" # requires INT activity + "-n ALL" = "network" + "-P ALL" = "per_cpu" + -q = "queue" + -R = "mem" + "-r ALL" = "mem_util" + -S = "swap_util" + -u = "cpu_util" + -v = "inode" + -W = "swap" + -w = "task" + [[inputs.sysstat.device_tags.sda]] + vg = "rootvg" +``` + +you get the following output: +``` +$ telegraf -config telegraf.conf -input-filter sysstat -test +* Plugin: sysstat, Collection 1 +> cpu_util,device=all pct_idle=98.85,pct_iowait=0,pct_nice=0.38,pct_steal=0,pct_system=0.64,pct_user=0.13 1459255626657883725 +> swap pswpin_per_s=0,pswpout_per_s=0 1459255626658387650 +> per_cpu,device=cpu1 pct_idle=98.98,pct_iowait=0,pct_nice=0.26,pct_steal=0,pct_system=0.51,pct_user=0.26 1459255626659630437 +> per_cpu,device=all pct_idle=98.85,pct_iowait=0,pct_nice=0.38,pct_steal=0,pct_system=0.64,pct_user=0.13 1459255626659670744 +> per_cpu,device=cpu0 pct_idle=98.73,pct_iowait=0,pct_nice=0.76,pct_steal=0,pct_system=0.51,pct_user=0 1459255626659697515 +> hugepages kbhugfree=0,kbhugused=0,pct_hugused=0 1459255626660057517 +> network,device=lo coll_per_s=0,pct_ifutil=0,rxcmp_per_s=0,rxdrop_per_s=0,rxerr_per_s=0,rxfifo_per_s=0,rxfram_per_s=0,rxkB_per_s=0.81,rxmcst_per_s=0,rxpck_per_s=16,txcarr_per_s=0,txcmp_per_s=0,txdrop_per_s=0,txerr_per_s=0,txfifo_per_s=0,txkB_per_s=0.81,txpck_per_s=16 1459255626661197666 +> network access_per_s=0,active_per_s=0,asmf_per_s=0,asmok_per_s=0,asmrq_per_s=0,atmptf_per_s=0,badcall_per_s=0,call_per_s=0,estres_per_s=0,fragcrt_per_s=0,fragf_per_s=0,fragok_per_s=0,fwddgm_per_s=0,getatt_per_s=0,hit_per_s=0,iadrerr_per_s=0,iadrmk_per_s=0,iadrmkr_per_s=0,idel_per_s=16,idgm_per_s=0,idgmerr_per_s=0,idisc_per_s=0,idstunr_per_s=0,iech_per_s=0,iechr_per_s=0,ierr_per_s=0,ihdrerr_per_s=0,imsg_per_s=0,ip-frag=0,iparmpb_per_s=0,irec_per_s=16,iredir_per_s=0,iseg_per_s=16,isegerr_per_s=0,isrcq_per_s=0,itm_per_s=0,itmex_per_s=0,itmr_per_s=0,iukwnpr_per_s=0,miss_per_s=0,noport_per_s=0,oadrmk_per_s=0,oadrmkr_per_s=0,odgm_per_s=0,odisc_per_s=0,odstunr_per_s=0,oech_per_s=0,oechr_per_s=0,oerr_per_s=0,omsg_per_s=0,onort_per_s=0,oparmpb_per_s=0,oredir_per_s=0,orq_per_s=16,orsts_per_s=0,oseg_per_s=16,osrcq_per_s=0,otm_per_s=0,otmex_per_s=0,otmr_per_s=0,packet_per_s=0,passive_per_s=0,rawsck=0,read_per_s=0,retrans_per_s=0,saccess_per_s=0,scall_per_s=0,sgetatt_per_s=0,sread_per_s=0,swrite_per_s=0,tcp-tw=7,tcp_per_s=0,tcpsck=1543,totsck=4052,udp_per_s=0,udpsck=2,write_per_s=0 1459255626661381788 +> network,device=ens33 coll_per_s=0,pct_ifutil=0,rxcmp_per_s=0,rxdrop_per_s=0,rxerr_per_s=0,rxfifo_per_s=0,rxfram_per_s=0,rxkB_per_s=0,rxmcst_per_s=0,rxpck_per_s=0,txcarr_per_s=0,txcmp_per_s=0,txdrop_per_s=0,txerr_per_s=0,txfifo_per_s=0,txkB_per_s=0,txpck_per_s=0 1459255626661533072 +> disk,device=sda,vg=rootvg avgqu-sz=0.01,avgrq-sz=8.5,await=3.31,pct_util=0.1,rd_sec_per_s=0,svctm=0.25,tps=4,wr_sec_per_s=34 1459255626663974389 +> queue blocked=0,ldavg-1=1.61,ldavg-15=1.34,ldavg-5=1.67,plist-sz=1415,runq-sz=0 1459255626664159054 +> paging fault_per_s=0.25,majflt_per_s=0,pct_vmeff=0,pgfree_per_s=19,pgpgin_per_s=0,pgpgout_per_s=17,pgscand_per_s=0,pgscank_per_s=0,pgsteal_per_s=0 1459255626664304249 +> mem_util kbactive=2206568,kbanonpg=1472208,kbbuffers=118020,kbcached=1035252,kbcommit=8717200,kbdirty=156,kbinact=418912,kbkstack=24672,kbmemfree=1744868,kbmemused=3610272,kbpgtbl=87116,kbslab=233804,kbvmused=0,pct_commit=136.13,pct_memused=67.42 1459255626664554981 +> io bread_per_s=0,bwrtn_per_s=34,rtps=0,tps=4,wtps=4 1459255626664596198 +> inode dentunusd=235039,file-nr=17120,inode-nr=94505,pty-nr=14 1459255626664663693 +> interrupts,device=i000 intr_per_s=0 1459255626664800109 +> interrupts,device=i003 intr_per_s=0 1459255626665255145 +> interrupts,device=i004 intr_per_s=0 1459255626665281776 +> interrupts,device=i006 intr_per_s=0 1459255626665297416 +> interrupts,device=i007 intr_per_s=0 1459255626665321008 +> interrupts,device=i010 intr_per_s=0 1459255626665339413 +> interrupts,device=i012 intr_per_s=0 1459255626665361510 +> interrupts,device=i013 intr_per_s=0 1459255626665381327 +> interrupts,device=i015 intr_per_s=1 1459255626665397313 +> interrupts,device=i001 intr_per_s=0.25 1459255626665412985 +> interrupts,device=i002 intr_per_s=0 1459255626665430475 +> interrupts,device=i005 intr_per_s=0 1459255626665453944 +> interrupts,device=i008 intr_per_s=0 1459255626665470650 +> interrupts,device=i011 intr_per_s=0 1459255626665486069 +> interrupts,device=i009 intr_per_s=0 1459255626665502913 +> interrupts,device=i014 intr_per_s=0 1459255626665518152 +> task cswch_per_s=722.25,proc_per_s=0 1459255626665849646 +> cpu,device=all pct_idle=98.85,pct_iowait=0,pct_nice=0.38,pct_steal=0,pct_system=0.64,pct_user=0.13 1459255626666639715 +> mem bufpg_per_s=0,campg_per_s=1.75,frmpg_per_s=-8.25 1459255626666770205 +> swap_util kbswpcad=0,kbswpfree=1048572,kbswpused=0,pct_swpcad=0,pct_swpused=0 1459255626667313276 +``` + +If you change the group value to false like below: +```toml +[[inputs.sysstat]] + collect_interval = 30 + sadc_path = "/usr/lib/sa/sadc" # required + activities = ["DISK", "SNMP", "INT"] + group = false + [inputs.sysstat.options] + -C = "cpu" + -B = "paging" + -b = "io" + -d = "disk" # requires DISK activity + -H = "hugepages" + "-I ALL" = "interrupts" # requires INT activity + "-n ALL" = "network" + "-P ALL" = "per_cpu" + -q = "queue" + -R = "mem" + "-r ALL" = "mem_util" + -S = "swap_util" + -u = "cpu_util" + -v = "inode" + -W = "swap" + -w = "task" + [[inputs.sysstat.device_tags.sda]] + vg = "rootvg" +``` + +you get the following output: +``` +$ telegraf -config telegraf.conf -input-filter sysstat -test +* Plugin: sysstat, Collection 1 +> io_tps value=0.5 1459255780126025822 +> io_rtps value=0 1459255780126025822 +> io_wtps value=0.5 1459255780126025822 +> io_bread_per_s value=0 1459255780126025822 +> io_bwrtn_per_s value=38 1459255780126025822 +> cpu_util_pct_user,device=all value=39.07 1459255780126025822 +> cpu_util_pct_nice,device=all value=0 1459255780126025822 +> cpu_util_pct_system,device=all value=47.94 1459255780126025822 +> cpu_util_pct_iowait,device=all value=0 1459255780126025822 +> cpu_util_pct_steal,device=all value=0 1459255780126025822 +> cpu_util_pct_idle,device=all value=12.98 1459255780126025822 +> swap_pswpin_per_s value=0 1459255780126025822 +> cpu_pct_user,device=all value=39.07 1459255780126025822 +> cpu_pct_nice,device=all value=0 1459255780126025822 +> cpu_pct_system,device=all value=47.94 1459255780126025822 +> cpu_pct_iowait,device=all value=0 1459255780126025822 +> cpu_pct_steal,device=all value=0 1459255780126025822 +> cpu_pct_idle,device=all value=12.98 1459255780126025822 +> per_cpu_pct_user,device=all value=39.07 1459255780126025822 +> per_cpu_pct_nice,device=all value=0 1459255780126025822 +> per_cpu_pct_system,device=all value=47.94 1459255780126025822 +> per_cpu_pct_iowait,device=all value=0 1459255780126025822 +> per_cpu_pct_steal,device=all value=0 1459255780126025822 +> per_cpu_pct_idle,device=all value=12.98 1459255780126025822 +> per_cpu_pct_user,device=cpu0 value=33.5 1459255780126025822 +> per_cpu_pct_nice,device=cpu0 value=0 1459255780126025822 +> per_cpu_pct_system,device=cpu0 value=65.25 1459255780126025822 +> per_cpu_pct_iowait,device=cpu0 value=0 1459255780126025822 +> per_cpu_pct_steal,device=cpu0 value=0 1459255780126025822 +> per_cpu_pct_idle,device=cpu0 value=1.25 1459255780126025822 +> per_cpu_pct_user,device=cpu1 value=44.85 1459255780126025822 +> per_cpu_pct_nice,device=cpu1 value=0 1459255780126025822 +> per_cpu_pct_system,device=cpu1 value=29.55 1459255780126025822 +> per_cpu_pct_iowait,device=cpu1 value=0 1459255780126025822 +> per_cpu_pct_steal,device=cpu1 value=0 1459255780126025822 +> per_cpu_pct_idle,device=cpu1 value=25.59 1459255780126025822 +> hugepages_kbhugfree value=0 1459255780126025822 +> hugepages_kbhugused value=0 1459255780126025822 +> hugepages_pct_hugused value=0 1459255780126025822 +> interrupts_intr_per_s,device=i000 value=0 1459255780126025822 +> inode_dentunusd value=252876 1459255780126025822 +> mem_util_kbmemfree value=1613612 1459255780126025822 +> disk_tps,device=sda,vg=rootvg value=0.5 1459255780126025822 +> swap_pswpout_per_s value=0 1459255780126025822 +> network_rxpck_per_s,device=ens33 value=0 1459255780126025822 +> queue_runq-sz value=4 1459255780126025822 +> task_proc_per_s value=0 1459255780126025822 +> task_cswch_per_s value=2019 1459255780126025822 +> mem_frmpg_per_s value=0 1459255780126025822 +> mem_bufpg_per_s value=0.5 1459255780126025822 +> mem_campg_per_s value=1.25 1459255780126025822 +> interrupts_intr_per_s,device=i001 value=0 1459255780126025822 +> inode_file-nr value=19104 1459255780126025822 +> mem_util_kbmemused value=3741528 1459255780126025822 +> disk_rd_sec_per_s,device=sda,vg=rootvg value=0 1459255780126025822 +> network_txpck_per_s,device=ens33 value=0 1459255780126025822 +> queue_plist-sz value=1512 1459255780126025822 +> paging_pgpgin_per_s value=0 1459255780126025822 +> paging_pgpgout_per_s value=19 1459255780126025822 +> paging_fault_per_s value=0.25 1459255780126025822 +> paging_majflt_per_s value=0 1459255780126025822 +> paging_pgfree_per_s value=34.25 1459255780126025822 +> paging_pgscank_per_s value=0 1459255780126025822 +> paging_pgscand_per_s value=0 1459255780126025822 +> paging_pgsteal_per_s value=0 1459255780126025822 +> paging_pct_vmeff value=0 1459255780126025822 +> interrupts_intr_per_s,device=i002 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i003 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i004 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i005 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i006 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i007 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i008 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i009 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i010 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i011 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i012 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i013 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i014 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i015 value=1 1459255780126025822 +> inode_inode-nr value=94709 1459255780126025822 +> inode_pty-nr value=14 1459255780126025822 +> mem_util_pct_memused value=69.87 1459255780126025822 +> mem_util_kbbuffers value=118252 1459255780126025822 +> mem_util_kbcached value=1045240 1459255780126025822 +> mem_util_kbcommit value=9628152 1459255780126025822 +> mem_util_pct_commit value=150.35 1459255780126025822 +> mem_util_kbactive value=2303752 1459255780126025822 +> mem_util_kbinact value=428340 1459255780126025822 +> mem_util_kbdirty value=104 1459255780126025822 +> mem_util_kbanonpg value=1568676 1459255780126025822 +> mem_util_kbslab value=240032 1459255780126025822 +> mem_util_kbkstack value=26224 1459255780126025822 +> mem_util_kbpgtbl value=98056 1459255780126025822 +> mem_util_kbvmused value=0 1459255780126025822 +> disk_wr_sec_per_s,device=sda,vg=rootvg value=38 1459255780126025822 +> disk_avgrq-sz,device=sda,vg=rootvg value=76 1459255780126025822 +> disk_avgqu-sz,device=sda,vg=rootvg value=0 1459255780126025822 +> disk_await,device=sda,vg=rootvg value=2 1459255780126025822 +> disk_svctm,device=sda,vg=rootvg value=2 1459255780126025822 +> disk_pct_util,device=sda,vg=rootvg value=0.1 1459255780126025822 +> network_rxkB_per_s,device=ens33 value=0 1459255780126025822 +> network_txkB_per_s,device=ens33 value=0 1459255780126025822 +> network_rxcmp_per_s,device=ens33 value=0 1459255780126025822 +> network_txcmp_per_s,device=ens33 value=0 1459255780126025822 +> network_rxmcst_per_s,device=ens33 value=0 1459255780126025822 +> network_pct_ifutil,device=ens33 value=0 1459255780126025822 +> network_rxpck_per_s,device=lo value=10.75 1459255780126025822 +> network_txpck_per_s,device=lo value=10.75 1459255780126025822 +> network_rxkB_per_s,device=lo value=0.77 1459255780126025822 +> network_txkB_per_s,device=lo value=0.77 1459255780126025822 +> network_rxcmp_per_s,device=lo value=0 1459255780126025822 +> network_txcmp_per_s,device=lo value=0 1459255780126025822 +> network_rxmcst_per_s,device=lo value=0 1459255780126025822 +> network_pct_ifutil,device=lo value=0 1459255780126025822 +> network_rxerr_per_s,device=ens33 value=0 1459255780126025822 +> network_txerr_per_s,device=ens33 value=0 1459255780126025822 +> network_coll_per_s,device=ens33 value=0 1459255780126025822 +> network_rxdrop_per_s,device=ens33 value=0 1459255780126025822 +> network_txdrop_per_s,device=ens33 value=0 1459255780126025822 +> network_txcarr_per_s,device=ens33 value=0 1459255780126025822 +> network_rxfram_per_s,device=ens33 value=0 1459255780126025822 +> network_rxfifo_per_s,device=ens33 value=0 1459255780126025822 +> network_txfifo_per_s,device=ens33 value=0 1459255780126025822 +> network_rxerr_per_s,device=lo value=0 1459255780126025822 +> network_txerr_per_s,device=lo value=0 1459255780126025822 +> network_coll_per_s,device=lo value=0 1459255780126025822 +> network_rxdrop_per_s,device=lo value=0 1459255780126025822 +> network_txdrop_per_s,device=lo value=0 1459255780126025822 +> network_txcarr_per_s,device=lo value=0 1459255780126025822 +> network_rxfram_per_s,device=lo value=0 1459255780126025822 +> network_rxfifo_per_s,device=lo value=0 1459255780126025822 +> network_txfifo_per_s,device=lo value=0 1459255780126025822 +> network_call_per_s value=0 1459255780126025822 +> network_retrans_per_s value=0 1459255780126025822 +> network_read_per_s value=0 1459255780126025822 +> network_write_per_s value=0 1459255780126025822 +> network_access_per_s value=0 1459255780126025822 +> network_getatt_per_s value=0 1459255780126025822 +> network_scall_per_s value=0 1459255780126025822 +> network_badcall_per_s value=0 1459255780126025822 +> network_packet_per_s value=0 1459255780126025822 +> network_udp_per_s value=0 1459255780126025822 +> network_tcp_per_s value=0 1459255780126025822 +> network_hit_per_s value=0 1459255780126025822 +> network_miss_per_s value=0 1459255780126025822 +> network_sread_per_s value=0 1459255780126025822 +> network_swrite_per_s value=0 1459255780126025822 +> network_saccess_per_s value=0 1459255780126025822 +> network_sgetatt_per_s value=0 1459255780126025822 +> network_totsck value=4234 1459255780126025822 +> network_tcpsck value=1637 1459255780126025822 +> network_udpsck value=2 1459255780126025822 +> network_rawsck value=0 1459255780126025822 +> network_ip-frag value=0 1459255780126025822 +> network_tcp-tw value=4 1459255780126025822 +> network_irec_per_s value=10.75 1459255780126025822 +> network_fwddgm_per_s value=0 1459255780126025822 +> network_idel_per_s value=10.75 1459255780126025822 +> network_orq_per_s value=10.75 1459255780126025822 +> network_asmrq_per_s value=0 1459255780126025822 +> network_asmok_per_s value=0 1459255780126025822 +> network_fragok_per_s value=0 1459255780126025822 +> network_fragcrt_per_s value=0 1459255780126025822 +> network_ihdrerr_per_s value=0 1459255780126025822 +> network_iadrerr_per_s value=0 1459255780126025822 +> network_iukwnpr_per_s value=0 1459255780126025822 +> network_idisc_per_s value=0 1459255780126025822 +> network_odisc_per_s value=0 1459255780126025822 +> network_onort_per_s value=0 1459255780126025822 +> network_asmf_per_s value=0 1459255780126025822 +> network_fragf_per_s value=0 1459255780126025822 +> network_imsg_per_s value=0 1459255780126025822 +> network_omsg_per_s value=0 1459255780126025822 +> network_iech_per_s value=0 1459255780126025822 +> network_iechr_per_s value=0 1459255780126025822 +> network_oech_per_s value=0 1459255780126025822 +> network_oechr_per_s value=0 1459255780126025822 +> network_itm_per_s value=0 1459255780126025822 +> network_itmr_per_s value=0 1459255780126025822 +> network_otm_per_s value=0 1459255780126025822 +> network_otmr_per_s value=0 1459255780126025822 +> network_iadrmk_per_s value=0 1459255780126025822 +> network_iadrmkr_per_s value=0 1459255780126025822 +> network_oadrmk_per_s value=0 1459255780126025822 +> network_oadrmkr_per_s value=0 1459255780126025822 +> network_ierr_per_s value=0 1459255780126025822 +> network_oerr_per_s value=0 1459255780126025822 +> network_idstunr_per_s value=0 1459255780126025822 +> network_odstunr_per_s value=0 1459255780126025822 +> network_itmex_per_s value=0 1459255780126025822 +> network_otmex_per_s value=0 1459255780126025822 +> network_iparmpb_per_s value=0 1459255780126025822 +> network_oparmpb_per_s value=0 1459255780126025822 +> network_isrcq_per_s value=0 1459255780126025822 +> network_osrcq_per_s value=0 1459255780126025822 +> network_iredir_per_s value=0 1459255780126025822 +> network_oredir_per_s value=0 1459255780126025822 +> network_active_per_s value=0 1459255780126025822 +> network_passive_per_s value=0 1459255780126025822 +> network_iseg_per_s value=10.75 1459255780126025822 +> network_oseg_per_s value=9.5 1459255780126025822 +> network_atmptf_per_s value=0 1459255780126025822 +> network_estres_per_s value=0 1459255780126025822 +> network_retrans_per_s value=1.5 1459255780126025822 +> network_isegerr_per_s value=0.25 1459255780126025822 +> network_orsts_per_s value=0 1459255780126025822 +> network_idgm_per_s value=0 1459255780126025822 +> network_odgm_per_s value=0 1459255780126025822 +> network_noport_per_s value=0 1459255780126025822 +> network_idgmerr_per_s value=0 1459255780126025822 +> queue_ldavg-1 value=2.1 1459255780126025822 +> queue_ldavg-5 value=1.82 1459255780126025822 +> queue_ldavg-15 value=1.44 1459255780126025822 +> queue_blocked value=0 1459255780126025822 +> swap_util_kbswpfree value=1048572 1459255780126025822 +> swap_util_kbswpused value=0 1459255780126025822 +> swap_util_pct_swpused value=0 1459255780126025822 +> swap_util_kbswpcad value=0 1459255780126025822 +> swap_util_pct_swpcad value=0 1459255780126025822 +``` diff --git a/plugins/inputs/sysstat/sysstat.go b/plugins/inputs/sysstat/sysstat.go new file mode 100644 index 000000000..2b605b3ad --- /dev/null +++ b/plugins/inputs/sysstat/sysstat.go @@ -0,0 +1,315 @@ +// build +linux + +package sysstat + +import ( + "bufio" + "encoding/csv" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path" + "strconv" + "strings" + "sync" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" +) + +var ( + execCommand = exec.Command // execCommand is used to mock commands in tests. + dfltActivities = []string{"DISK"} +) + +const parseInterval = 1 // parseInterval is the interval (in seconds) where the parsing takes place. + +type Sysstat struct { + // Interval that defines how long data is collected by Sadc cmd. + // + // This value has to be the same as the thelegraf collection interval. + Interval int `toml:"collect_interval"` + + // Sadc represents the path to the sadc collector utility. + Sadc string `toml:"sadc_path"` + + // Sadf represents the path to the sadf cmd. + Sadf string `toml:"sadf_path"` + + // Activities is a list of activities that are passed as argument to the + // collector utility (e.g: DISK, SNMP etc...) + // The more activities that are added, the more data is collected. + Activities []string + + // Options is a map of options. + // + // The key represents the actual option that the Sadf command is called with and + // the value represents the description for that option. + // + // For example, if you have the following options map: + // map[string]string{"-C": "cpu", "-d": "disk"} + // The Sadf command is run with the options -C and -d to extract cpu and + // disk metrics from the collected binary file. + // + // If Group is false (see below), each metric will be prefixed with the corresponding description + // and represents itself a measurement. + // + // If Group is true, metrics are grouped to a single measurement with the corresponding description as name. + Options map[string]string + + // Group determines if metrics are grouped or not. + Group bool + + // DeviceTags adds the possibility to add additional tags for devices. + DeviceTags map[string][]map[string]string `toml:"device_tags"` + tmpFile string +} + +func (*Sysstat) Description() string { + return "Sysstat metrics collector" +} + +var sampleConfig = ` + ## Collect interval in seconds. This value has to be equal + ## to the telegraf collect interval. + collect_interval = 5 # required + # + # + ## Path to the sadc command. + sadc_path = "/usr/lib/sa/sadc" # required + # + # + ## Path to the sadf command, if it is not in PATH + # sadf_path = "/usr/bin/sadf" + # + # + ## Activities is a list of activities, that are passed as argument to the + ## sadc collector utility (e.g: DISK, SNMP etc...) + ## The more activities that are added, the more data is collected. + # activities = ["DISK"] + # + # + ## Group metrics to measurements. + ## + ## If group is false each metric will be prefixed with a description + ## and represents itself a measurement. + ## + ## If Group is true, corresponding metrics are grouped to a single measurement. + # group = false + # + # + ## Options for the sasf command. The values on the left represent the sadf options and + ## the values on the right their description (wich are used for grouping and prefixing metrics). + [inputs.sysstat.options] + -C = "cpu" + -B = "paging" + -b = "io" + -d = "disk" # requires DISK activity + -H = "hugepages" + "-I ALL" = "interrupts" # requires INT activity + "-n ALL" = "network" + "-P ALL" = "per_cpu" + -q = "queue" + -R = "mem" + "-r ALL" = "mem_util" + -S = "swap_util" + -u = "cpu_util" + -v = "inode" + -W = "swap" + -w = "task" + # + # + ## Device tags can be used to add additional tags for devices. For example the configuration below + ## adds a tag vg with value rootvg for all metrics with sda devices. + # [[inputs.sysstat.device_tags.sda]] + # vg = "rootvg" +` + +func (*Sysstat) SampleConfig() string { + return sampleConfig +} + +func (s *Sysstat) Gather(acc telegraf.Accumulator) error { + ts := time.Now().Add(time.Duration(s.Interval) * time.Second) + if err := s.collect(); err != nil { + return err + } + var wg sync.WaitGroup + errorChannel := make(chan error, len(s.Options)*2) + for option := range s.Options { + wg.Add(1) + go func(acc telegraf.Accumulator, option string) { + defer wg.Done() + if err := s.parse(acc, option, ts); err != nil { + errorChannel <- err + } + }(acc, option) + } + wg.Wait() + close(errorChannel) + + errorStrings := []string{} + for err := range errorChannel { + errorStrings = append(errorStrings, err.Error()) + } + + if _, err := os.Stat(s.tmpFile); err == nil { + if err := os.Remove(s.tmpFile); err != nil { + errorStrings = append(errorStrings, err.Error()) + } + } + + if len(errorStrings) == 0 { + return nil + } + return errors.New(strings.Join(errorStrings, "\n")) +} + +// collect collects sysstat data with the collector utility sadc. It runs the following command: +// Sadc -S -S ... 2 tmpFile +// The above command collects system metrics during and saves it in binary form to tmpFile. +func (s *Sysstat) collect() error { + if len(s.Activities) == 0 { + s.Activities = dfltActivities + } + options := []string{} + for _, act := range s.Activities { + options = append(options, "-S", act) + } + s.tmpFile = path.Join("/tmp", fmt.Sprintf("sysstat-%d", time.Now().Unix())) + collectInterval := s.Interval - parseInterval // collectInterval has to be smaller than the telegraf data collection interval + options = append(options, strconv.Itoa(collectInterval), "2", s.tmpFile) + cmd := execCommand(s.Sadc, options...) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to run command %s: %s", strings.Join(cmd.Args, " "), string(out)) + } + return nil +} + +// parse runs Sadf on the previously saved tmpFile: +// Sadf -p -- -p